From 4b82c3cab84bab656a802b2d81865d6abacafa42 Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Sun, 22 Dec 2024 18:11:04 +0100 Subject: [PATCH 01/10] initial --- optimum/exporters/onnx/base.py | 1 + optimum/exporters/onnx/model_configs.py | 44 +++++++++++++++++++++++++ optimum/exporters/tasks.py | 5 +++ optimum/utils/__init__.py | 1 + optimum/utils/normalized_config.py | 4 +++ 5 files changed, 55 insertions(+) diff --git a/optimum/exporters/onnx/base.py b/optimum/exporters/onnx/base.py index 7e35691d54..0bd99a585e 100644 --- a/optimum/exporters/onnx/base.py +++ b/optimum/exporters/onnx/base.py @@ -177,6 +177,7 @@ class OnnxConfig(ExportConfig, ABC): "text2text-generation": OrderedDict({"logits": {0: "batch_size", 1: "decoder_sequence_length"}}), "text-classification": OrderedDict({"logits": {0: "batch_size"}}), "text-generation": OrderedDict({"logits": {0: "batch_size", 1: "sequence_length"}}), + "time-series-forecasting": OrderedDict({"prediction_outputs": {0: "batch_size"}}), "token-classification": OrderedDict({"logits": {0: "batch_size", 1: "sequence_length"}}), "visual-question-answering": OrderedDict({"logits": {0: "batch_size", 1: "sequence_length"}}), "zero-shot-image-classification": OrderedDict( diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index 1c83840880..a1d11728a4 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -59,6 +59,7 @@ NormalizedTextAndVisionConfig, NormalizedTextConfig, NormalizedTextConfigWithGQA, + NormalizedTimeSeriesForecastingConfig, NormalizedVisionConfig, check_if_diffusers_greater, check_if_transformers_greater, @@ -2499,3 +2500,46 @@ class EncoderDecoderOnnxConfig(EncoderDecoderBaseOnnxConfig): NORMALIZED_CONFIG_CLASS = NormalizedEncoderDecoderConfig DEFAULT_ONNX_OPSET = 14 # uses SDPA in Transformers, hence opset>=14. + + + + +class TimesFMDummyInputGenerator(DummyInputGenerator): + SUPPORTED_INPUT_NAMES = ("inputs",) + + def __init__( + self, + task: str, + normalized_config: NormalizedConfig, + batch_size: int = DEFAULT_DUMMY_SHAPES["batch_size"], + **kwargs, + ): + self.task = task + self.normalized_config = normalized_config + + self.batch_size = batch_size + self.context_len = normalized_config.context_len + + def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): + return self.random_float_tensor( + shape=[self.batch_size, self.context_len], + min_value=-1, + max_value=1, + framework=framework, + dtype=float_dtype, + ) + + +class TimesFMOnnxConfig(OnnxConfig): + NORMALIZED_CONFIG_CLASS = NormalizedTimeSeriesForecastingConfig + MIN_TRANSFORMERS_VERSION = version.parse("4.47.0") + DUMMY_INPUT_GENERATOR_CLASSES = (TimesFMDummyInputGenerator,) + + + @property + def inputs(self) -> Dict[str, Dict[int, str]]: + return {"inputs": {0: "batch_size", 1: "sequence_length"}} + + @property + def outputs(self) -> Dict[str, Dict[int, str]]: + return super().outputs diff --git a/optimum/exporters/tasks.py b/optimum/exporters/tasks.py index 4db4130302..a9abf6556f 100644 --- a/optimum/exporters/tasks.py +++ b/optimum/exporters/tasks.py @@ -320,6 +320,7 @@ class TasksManager: ("pt", "visual-bert", "question-answering"): ("transformers", "VisualBertForQuestionAnswering"), # VisionEncoderDecoderModel is not registered in AutoModelForDocumentQuestionAnswering ("pt", "vision-encoder-decoder", "document-question-answering"): ("transformers", "VisionEncoderDecoderModel"), + ("pt", "timesfm", "time-series-forecasting"): ("transformers", "TimesFMModelForPrediction"), } _ENCODER_DECODER_TASKS = ( @@ -939,6 +940,10 @@ class TasksManager: "text-classification", onnx="Qwen2OnnxConfig", ), + "timesfm": supported_tasks_mapping( + "time-series-forecasting", + onnx="TimesFMOnnxConfig", + ), "llama": supported_tasks_mapping( "feature-extraction", "feature-extraction-with-past", diff --git a/optimum/utils/__init__.py b/optimum/utils/__init__.py index 2aa90253d0..23ac9545ad 100644 --- a/optimum/utils/__init__.py +++ b/optimum/utils/__init__.py @@ -91,5 +91,6 @@ NormalizedTextAndVisionConfig, NormalizedTextConfig, NormalizedTextConfigWithGQA, + NormalizedTimeSeriesForecastingConfig, NormalizedVisionConfig, ) diff --git a/optimum/utils/normalized_config.py b/optimum/utils/normalized_config.py index 9ceed24c2d..3dc63e5471 100644 --- a/optimum/utils/normalized_config.py +++ b/optimum/utils/normalized_config.py @@ -96,6 +96,10 @@ class NormalizedSeq2SeqConfig(NormalizedTextConfig): DECODER_NUM_ATTENTION_HEADS = NormalizedTextConfig.NUM_ATTENTION_HEADS +class NormalizedTimeSeriesForecastingConfig(NormalizedConfig): + CONTEXT_LEN = "context_len" + + class NormalizedVisionConfig(NormalizedConfig): IMAGE_SIZE = "image_size" NUM_CHANNELS = "num_channels" From f5509ff898fd00ba1fea9912dfb514a1b989c39c Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Fri, 28 Feb 2025 17:18:24 +0100 Subject: [PATCH 02/10] fix style --- optimum/exporters/onnx/model_configs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index 3f63010329..1454c5f830 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -2631,7 +2631,7 @@ class EncoderDecoderOnnxConfig(EncoderDecoderBaseOnnxConfig): class TimesFMDummyInputGenerator(DummyInputGenerator): SUPPORTED_INPUT_NAMES = ("inputs",) - + def __init__( self, task: str, @@ -2660,7 +2660,6 @@ class TimesFMOnnxConfig(OnnxConfig): MIN_TRANSFORMERS_VERSION = version.parse("4.47.0") DUMMY_INPUT_GENERATOR_CLASSES = (TimesFMDummyInputGenerator,) - @property def inputs(self) -> Dict[str, Dict[int, str]]: return {"inputs": {0: "batch_size", 1: "sequence_length"}} From 5c2fee550b4c3d7826fc25b9c53173342c80f66f Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Fri, 28 Feb 2025 17:52:40 +0100 Subject: [PATCH 03/10] add fixes --- optimum/exporters/onnx/model_configs.py | 7 ++++++- optimum/exporters/tasks.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index 1454c5f830..4dedde0664 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -2659,6 +2659,7 @@ class TimesFMOnnxConfig(OnnxConfig): NORMALIZED_CONFIG_CLASS = NormalizedTimeSeriesForecastingConfig MIN_TRANSFORMERS_VERSION = version.parse("4.47.0") DUMMY_INPUT_GENERATOR_CLASSES = (TimesFMDummyInputGenerator,) + DEFAULT_ONNX_OPSET = 14 # uses SDPA in Transformers, needs opset>=14 @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -2666,7 +2667,11 @@ def inputs(self) -> Dict[str, Dict[int, str]]: @property def outputs(self) -> Dict[str, Dict[int, str]]: - return super().outputs + return { + "last_hidden_state": {0: "batch_size"}, + "mean_predictions": {0: "batch_size"}, + "full_predictions": {0: "batch_size"}, + } class PatchTSTOnnxConfig(OnnxConfig): diff --git a/optimum/exporters/tasks.py b/optimum/exporters/tasks.py index 7ddadcea00..4494fc2fc5 100644 --- a/optimum/exporters/tasks.py +++ b/optimum/exporters/tasks.py @@ -329,7 +329,7 @@ class TasksManager: ("pt", "visual-bert", "question-answering"): ("transformers", "VisualBertForQuestionAnswering"), # VisionEncoderDecoderModel is not registered in AutoModelForDocumentQuestionAnswering ("pt", "vision-encoder-decoder", "document-question-answering"): ("transformers", "VisionEncoderDecoderModel"), - ("pt", "timesfm", "time-series-forecasting"): ("transformers", "TimesFMModelForPrediction"), + ("pt", "timesfm", "time-series-forecasting"): ("transformers", "TimesFmModelForPrediction"), ("pt", "vitpose", "keypoint-detection"): ("transformers", "VitPoseForPoseEstimation"), } From c373ae053c1980d13fda9c00c5ab4e957a8a7258 Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Fri, 28 Feb 2025 18:01:12 +0100 Subject: [PATCH 04/10] remove time series normalized config --- optimum/utils/normalized_config.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/optimum/utils/normalized_config.py b/optimum/utils/normalized_config.py index 35b21d74f3..3f497b5920 100644 --- a/optimum/utils/normalized_config.py +++ b/optimum/utils/normalized_config.py @@ -101,10 +101,6 @@ class NormalizedSeq2SeqConfig(NormalizedTextConfig): DECODER_NUM_ATTENTION_HEADS = NormalizedTextConfig.NUM_ATTENTION_HEADS -class NormalizedTimeSeriesForecastingConfig(NormalizedConfig): - CONTEXT_LEN = "context_len" - - class NormalizedVisionConfig(NormalizedConfig): IMAGE_SIZE = "image_size" NUM_CHANNELS = "num_channels" From 9329cb03d535f474af6b5aba62b0cbf90e823e9c Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Fri, 28 Feb 2025 18:09:08 +0100 Subject: [PATCH 05/10] update normalized config --- optimum/exporters/onnx/model_configs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index 4dedde0664..8986d4de11 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -2641,9 +2641,8 @@ def __init__( ): self.task = task self.normalized_config = normalized_config - self.batch_size = batch_size - self.context_len = normalized_config.context_len + self.context_len = normalized_config.context_length def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): return self.random_float_tensor( @@ -2656,7 +2655,7 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int class TimesFMOnnxConfig(OnnxConfig): - NORMALIZED_CONFIG_CLASS = NormalizedTimeSeriesForecastingConfig + NORMALIZED_CONFIG_CLASS = NormalizedTimeSeriesForecastingConfig.with_args(context_length="context_len") MIN_TRANSFORMERS_VERSION = version.parse("4.47.0") DUMMY_INPUT_GENERATOR_CLASSES = (TimesFMDummyInputGenerator,) DEFAULT_ONNX_OPSET = 14 # uses SDPA in Transformers, needs opset>=14 From 5a50917f4a4b9667cf06b41bad8995a8effe0baa Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Thu, 24 Apr 2025 15:42:05 +0200 Subject: [PATCH 06/10] remove mapping context len --- optimum/exporters/onnx/model_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index 45359d68b2..2e8bad3f59 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -2656,7 +2656,7 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int class TimesFMOnnxConfig(OnnxConfig): - NORMALIZED_CONFIG_CLASS = NormalizedTimeSeriesForecastingConfig.with_args(context_length="context_len") + NORMALIZED_CONFIG_CLASS = NormalizedTimeSeriesForecastingConfig MIN_TRANSFORMERS_VERSION = version.parse("4.47.0") DUMMY_INPUT_GENERATOR_CLASSES = (TimesFMDummyInputGenerator,) DEFAULT_ONNX_OPSET = 14 # uses SDPA in Transformers, needs opset>=14 From af6a2d313c73604b23b4053f68860364f41bc87f Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Thu, 24 Apr 2025 16:21:08 +0200 Subject: [PATCH 07/10] fix export --- optimum/exporters/onnx/model_configs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index 2e8bad3f59..81a8ca299b 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -2631,7 +2631,7 @@ class EncoderDecoderOnnxConfig(EncoderDecoderBaseOnnxConfig): class TimesFMDummyInputGenerator(DummyInputGenerator): - SUPPORTED_INPUT_NAMES = ("inputs",) + SUPPORTED_INPUT_NAMES = ("past_values",) def __init__( self, @@ -2643,11 +2643,11 @@ def __init__( self.task = task self.normalized_config = normalized_config self.batch_size = batch_size - self.context_len = normalized_config.context_length + self.context_length = normalized_config.context_length def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): return self.random_float_tensor( - shape=[self.batch_size, self.context_len], + shape=[self.batch_size, self.context_length], min_value=-1, max_value=1, framework=framework, @@ -2663,7 +2663,7 @@ class TimesFMOnnxConfig(OnnxConfig): @property def inputs(self) -> Dict[str, Dict[int, str]]: - return {"inputs": {0: "batch_size", 1: "sequence_length"}} + return {"past_values": {0: "batch_size", 1: "sequence_length"}} @property def outputs(self) -> Dict[str, Dict[int, str]]: From 0ece0c5f9fb11c74f5b997171a85120550399f16 Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Thu, 24 Apr 2025 16:21:15 +0200 Subject: [PATCH 08/10] add test --- tests/exporters/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/exporters/utils.py b/tests/exporters/utils.py index 1efa9b8a51..90acd44ff0 100644 --- a/tests/exporters/utils.py +++ b/tests/exporters/utils.py @@ -170,6 +170,7 @@ "swin2sr": "hf-internal-testing/tiny-random-Swin2SRModel", "t5": "hf-internal-testing/tiny-random-t5", "table-transformer": "hf-internal-testing/tiny-random-TableTransformerModel", + "timesfm": "huggingface/timesfm-tourism-monthly", "vit": "hf-internal-testing/tiny-random-vit", "vit-mae": "hf-internal-testing/tiny-random-ViTMAEModel", "vit-msn": "hf-internal-testing/tiny-random-ViTMSNForImageClassification", From 891512ad9f57cdc24f54f0264086f9ea2b98070a Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Thu, 24 Apr 2025 16:25:54 +0200 Subject: [PATCH 09/10] move dummy input generator to utils --- optimum/exporters/onnx/model_configs.py | 28 ++----------------------- optimum/utils/__init__.py | 1 + optimum/utils/input_generators.py | 25 ++++++++++++++++++++++ 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index 81a8ca299b..8b2935c211 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -66,6 +66,7 @@ NormalizedTimeSeriesForecastingConfig, NormalizedVisionConfig, PerceiverDummyInputGenerator, + TimesFMDummyInputGenerator, VitPoseDummyInputGenerator, is_diffusers_available, is_diffusers_version, @@ -2630,34 +2631,9 @@ class EncoderDecoderOnnxConfig(EncoderDecoderBaseOnnxConfig): DEFAULT_ONNX_OPSET = 14 # uses SDPA in Transformers, hence opset>=14. -class TimesFMDummyInputGenerator(DummyInputGenerator): - SUPPORTED_INPUT_NAMES = ("past_values",) - - def __init__( - self, - task: str, - normalized_config: NormalizedConfig, - batch_size: int = DEFAULT_DUMMY_SHAPES["batch_size"], - **kwargs, - ): - self.task = task - self.normalized_config = normalized_config - self.batch_size = batch_size - self.context_length = normalized_config.context_length - - def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): - return self.random_float_tensor( - shape=[self.batch_size, self.context_length], - min_value=-1, - max_value=1, - framework=framework, - dtype=float_dtype, - ) - - class TimesFMOnnxConfig(OnnxConfig): NORMALIZED_CONFIG_CLASS = NormalizedTimeSeriesForecastingConfig - MIN_TRANSFORMERS_VERSION = version.parse("4.47.0") + MIN_TRANSFORMERS_VERSION = version.parse("4.51.0") # TODO: update to 4.52 DUMMY_INPUT_GENERATOR_CLASSES = (TimesFMDummyInputGenerator,) DEFAULT_ONNX_OPSET = 14 # uses SDPA in Transformers, needs opset>=14 diff --git a/optimum/utils/__init__.py b/optimum/utils/__init__.py index f945ac14f3..0dcf4c93bc 100644 --- a/optimum/utils/__init__.py +++ b/optimum/utils/__init__.py @@ -92,6 +92,7 @@ MistralDummyPastKeyValuesGenerator, MultiQueryPastKeyValuesGenerator, PerceiverDummyInputGenerator, + TimesFMDummyInputGenerator, VitPoseDummyInputGenerator, ) from .modeling_utils import recurse_getattr, recurse_setattr diff --git a/optimum/utils/input_generators.py b/optimum/utils/input_generators.py index 40cf2cde49..2fae73288e 100644 --- a/optimum/utils/input_generators.py +++ b/optimum/utils/input_generators.py @@ -1658,3 +1658,28 @@ class PerceiverDummyInputGenerator(DummyVisionStaticInputGenerator): class VitPoseDummyInputGenerator(DummyVisionStaticInputGenerator): pass + + +class TimesFMDummyInputGenerator(DummyInputGenerator): + SUPPORTED_INPUT_NAMES = ("past_values",) + + def __init__( + self, + task: str, + normalized_config: NormalizedConfig, + batch_size: int = DEFAULT_DUMMY_SHAPES["batch_size"], + **kwargs, + ): + self.task = task + self.normalized_config = normalized_config + self.batch_size = batch_size + self.context_length = normalized_config.context_length + + def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): + return self.random_float_tensor( + shape=[self.batch_size, self.context_length], + min_value=-1, + max_value=1, + framework=framework, + dtype=float_dtype, + ) From 6d216001d3009344df33298775ccc22a9af2777e Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Thu, 24 Apr 2025 16:48:45 +0200 Subject: [PATCH 10/10] upgrade min transformers version --- optimum/exporters/onnx/model_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index 8b2935c211..9b115d6d1c 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -2633,7 +2633,7 @@ class EncoderDecoderOnnxConfig(EncoderDecoderBaseOnnxConfig): class TimesFMOnnxConfig(OnnxConfig): NORMALIZED_CONFIG_CLASS = NormalizedTimeSeriesForecastingConfig - MIN_TRANSFORMERS_VERSION = version.parse("4.51.0") # TODO: update to 4.52 + MIN_TRANSFORMERS_VERSION = version.parse("4.52.0") DUMMY_INPUT_GENERATOR_CLASSES = (TimesFMDummyInputGenerator,) DEFAULT_ONNX_OPSET = 14 # uses SDPA in Transformers, needs opset>=14