Draft: [TRTC-1934][feat] Initial SA recipe database.

tensorrt_llm/configure/database.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional
+
+import pandas as pd
+import yaml
+from pydantic import BaseModel, Field, RootModel
+
+DATABASE_LIST_PATH = Path(__file__).parent / "database" / "scenario_list.yaml"
+
+
+class RecipeConstraints(BaseModel):
+    """Recipe record for scenario list."""
+
+    model: str = Field(description="Model name")
+    gpu: str = Field(description="GPU name")
+    isl: int = Field(description="Input sequence length")
+    osl: int = Field(description="Output sequence length")
+    concurrency: int = Field(description="Concurrency")
+    config_path: str = Field(description="Configuration path")
+    tps_per_user: Optional[int] = Field(description="TPS per user", default=None)
+    tps_per_gpu: Optional[int] = Field(description="TPS per GPU", default=None)
+    num_gpus: int = Field(description="Number of GPUs")
+
+    def to_pandas_row(self) -> pd.Series:
+        """Convert recipe record to pandas Series.
+
+        Returns:
+            pd.Series: Pandas Series containing the recipe record.
+        """
+        return pd.Series(self.model_dump())
+
+    def load_config(self) -> Dict[str, Any]:
+        """Load configuration from the configuration path."""
+        with open(self.config_path, "r") as f:
+            return Recipe(**yaml.load(f, Loader=yaml.FullLoader))
+
+
+class Recipe(BaseModel):
+    """Recipe that describes a single scenario."""
+
+    constraints: RecipeConstraints = Field(description="Recipe constraints")
+    env_overrides: Dict[str, Any] = Field(description="Environment overrides", default_factory=dict)
+    config: Dict[str, Any] = Field(description="Configuration overrides", default_factory=dict)
+
+
+class RecipeList(RootModel[List[RecipeConstraints]]):
+    @classmethod
+    def from_yaml(cls, yaml_path: Path) -> "RecipeList":
+        """Load recipe list from YAML file and validate the data.
+
+        Args:
+            yaml_path (Path): Path to the YAML file containing the recipe list.
+
+        Returns:
+            RecipeList: Recipe list object.
+        """
+        with open(yaml_path, "r") as f:
+            data = yaml.load(f, Loader=yaml.FullLoader)
+        return cls(data)
+
+    def to_pandas_dataframe(self) -> pd.DataFrame:
+        """Convert recipe list to pandas DataFrame.
+
+        Returns:
+            pd.DataFrame: Pandas DataFrame where each row contains a recipe record.
+        """
+        return pd.DataFrame([record.to_pandas_row() for record in self.root])
+
+    def __iter__(self) -> Iterator[RecipeConstraints]:
+        """Iterate over the recipe list.
+
+        Returns:
+            Iterator[RecipeRecord]: Iterator over the recipe list.
+        """
+        for record in self.root:
+            yield record
+
+    def __len__(self) -> int:
+        """Get the number of recipes in the list.
+
+        Returns:
+            int: Number of recipes in the list.
+        """
+        return len(self.root)

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml

@@ -0,0 +1,22 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: B200_NVL
+  num_gpus: 8
+  target_isl: 1024
+  target_osl: 1024
+  target_concurrency: 16
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 256
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.8
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: DEEPGEMM

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml

@@ -0,0 +1,22 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: B200_NVL
+  num_gpus: 8
+  target_isl: 1024
+  target_osl: 1024
+  target_concurrency: 32
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 256
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.8
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: DEEPGEMM

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml

@@ -0,0 +1,23 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: B200_NVL
+  num_gpus: 8
+  target_isl: 1024
+  target_osl: 1024
+  target_concurrency: 4
+
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 256
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.8
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: DEEPGEMM

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml

@@ -0,0 +1,22 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: B200_NVL
+  num_gpus: 8
+  target_isl: 1024
+  target_osl: 1024
+  target_concurrency: 64
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 256
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.8
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: DEEPGEMM

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml

@@ -0,0 +1,22 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: B200_NVL
+  num_gpus: 8
+  target_isl: 1024
+  target_osl: 1024
+  target_concurrency: 8
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 256
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.8
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: DEEPGEMM

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml

@@ -0,0 +1,22 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: B200_NVL
+  num_gpus: 8
+  target_isl: 8192
+  target_osl: 1024
+  target_concurrency: 16
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 256
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.8
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: DEEPGEMM

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml

@@ -0,0 +1,22 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: B200_NVL
+  num_gpus: 8
+  target_isl: 8192
+  target_osl: 1024
+  target_concurrency: 32
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 256
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.8
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: DEEPGEMM

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml

@@ -0,0 +1,22 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: B200_NVL
+  num_gpus: 8
+  target_isl: 8192
+  target_osl: 1024
+  target_concurrency: 4
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 256
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.8
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: DEEPGEMM

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml

@@ -0,0 +1,26 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: B200_NVL
+  num_gpus: 8
+  target_isl: 8192
+  target_osl: 1024
+  target_concurrency: 64
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 256
+  enable_attention_dp: true
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.8
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: DEEPGEMM
+  attention_dp_config:
+    batching_wait_iters: 0
+    enable_balance: true
+    timeout_iters: 60

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml

@@ -0,0 +1,22 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: B200_NVL
+  num_gpus: 8
+  target_isl: 8192
+  target_osl: 1024
+  target_concurrency: 8
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 256
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.8
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: DEEPGEMM

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml

@@ -0,0 +1,22 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: H200_SXM
+  num_gpus: 8
+  target_isl: 1024
+  target_osl: 1024
+  target_concurrency: 16
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 128
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.75
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: CUTLASS

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml

@@ -0,0 +1,22 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: H200_SXM
+  num_gpus: 8
+  target_isl: 1024
+  target_osl: 1024
+  target_concurrency: 32
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 128
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.75
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: CUTLASS

tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml

@@ -0,0 +1,22 @@
+constraints:
+  model: deepseek-ai/DeepSeek-R1-0528
+  gpu: H200_SXM
+  num_gpus: 8
+  target_isl: 1024
+  target_osl: 1024
+  target_concurrency: 4
+env_overrides:
+
+config:
+  cuda_graph_config:
+    enable_padding: true
+    max_batch_size: 128
+  enable_attention_dp: false
+  print_iter_log: true
+  kv_cache_config:
+    dtype: fp8
+    free_gpu_memory_fraction: 0.75
+    enable_block_reuse: false
+  stream_interval: 10
+  moe_config:
+    backend: CUTLASS