From b7665d369fbac47473e37badca1c325467e8aca2 Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 09:15:05 -0800 Subject: [PATCH 1/6] Initial SA recipe database. Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- tensorrt_llm/configure/database.py | 38 + ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml | 22 + ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml | 22 + ...k_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml | 22 + ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml | 22 + ...k_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml | 22 + ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml | 22 + ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml | 22 + ...k_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml | 22 + ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml | 26 + ...k_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml | 22 + ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml | 22 + ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml | 22 + ...k_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml | 22 + ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml | 22 + ...k_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml | 22 + ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml | 22 + ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml | 22 + ...k_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml | 22 + ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml | 26 + ...k_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml | 22 + ..._fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml | 22 + ..._fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml | 22 + ..._fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml | 22 + ..._fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml | 22 + ..._fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml | 22 + ..._fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml | 26 + ...28_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml | 22 + ..._fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml | 22 + ..._fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml | 26 + ...28_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml | 22 + ...oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml | 24 + ...oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml | 23 + .../configure/database/scenario_list.yaml | 1512 +++++++++++++++++ 170 files changed, 5466 insertions(+) create mode 100644 tensorrt_llm/configure/database.py create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/scenario_list.yaml diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py new file mode 100644 index 000000000000..e093d17fde90 --- /dev/null +++ b/tensorrt_llm/configure/database.py @@ -0,0 +1,38 @@ +from pathlib import Path +from typing import List, Optional + +import pandas as pd +import yaml +from pydantic import BaseModel + +DATABASE_LIST_PATH = Path(__file__).parent / "database" / "scenario_list.yaml" + + +class RecipeRecord(BaseModel): + model: str + gpu: str + isl: int + osl: int + concurrency: int + config_path: str + tps_per_user: Optional[int] = None + tps_per_gpu: Optional[int] = None + num_gpus: int + + @classmethod + def from_yaml(cls, yaml_path: Path) -> List["RecipeRecord"]: + with open(yaml_path, "r") as f: + data = yaml.load(f, Loader=yaml.FullLoader) + return [cls(**item) for item in data] + + def to_pandas_row(self) -> pd.Series: + return pd.Series(self.model_dump()) + + +def get_recipe_dataframe(yaml_path: Path) -> pd.DataFrame: + records = RecipeRecord.from_yaml(yaml_path) + return pd.DataFrame([record.to_pandas_row() for record in records]) + + +def load_recipe_database() -> pd.DataFrame: + return get_recipe_dataframe(DATABASE_LIST_PATH) diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..4725c335a2fc --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..778dba5027c8 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..1724b0b8af4b --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..cc943a4e5ad9 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..f06e5f4b1d74 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..e31b303f7337 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..dbb06f58e467 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..f7eccec94125 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..d813f6fbf0d0 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml @@ -0,0 +1,26 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..c70a7dd6c6ee --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..e6581080c029 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..0826c51771ef --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..39ecdca068ff --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..3b346249fc45 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..786ec7fe456c --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..a4e5eabe851a --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..932933361a74 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..ff7bffb1da18 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..dcc1392d2fd7 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml @@ -0,0 +1,26 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..2f17c33de980 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml new file mode 100644 index 000000000000..a422e84c1955 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 128 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..18b25097ea28 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml new file mode 100644 index 000000000000..1b8401a825f9 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 256 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..1b530b5f5f6e --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..551e8b25cb37 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..38b625426ba8 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..ec59a1bd5052 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml new file mode 100644 index 000000000000..deb3f13ec516 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 128 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..69c72d4dafb0 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml new file mode 100644 index 000000000000..d57c515f196b --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 256 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..320198425c91 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..e001ce83a9a7 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..c5c393bf1051 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..2242bc618914 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml new file mode 100644 index 000000000000..bf1a026cd2db --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 128 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..56fc955f2b9a --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml new file mode 100644 index 000000000000..46ef51eb8e29 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 256 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..f29b6e2bd350 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..efdf16ff5e09 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..0041a26c747e --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..7151a5a2f33c --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml new file mode 100644 index 000000000000..705aefdc6602 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 128 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..adfdfb5cefb6 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml new file mode 100644 index 000000000000..fa01711d9df7 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 256 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..d650e3feb927 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..23e7cd45ac7f --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..99620affd547 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..dec1388712f7 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..f6ff78bcddae --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..13952e7762a5 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..67c8cf3ee558 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..2eabed9bcf94 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..246decd0fd96 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..cfbb10e76771 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..fd9218e8ae32 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..b4991a90e933 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..9527b6f0bc30 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..b57df2745953 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..820fc7ae0db1 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..ea0821a0f86f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..70e6eb7210ff --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..c39784c47845 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..c1ef5051906c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..a521f7ada743 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..b9f33be7e853 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..d9f21c5bd1cd --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..1399bcd8d79f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..d26d7d09586c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml new file mode 100644 index 000000000000..24fe38a91547 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml new file mode 100644 index 000000000000..4aec4e67aa2c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml new file mode 100644 index 000000000000..828907414fb9 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml new file mode 100644 index 000000000000..c937c1790f4e --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml new file mode 100644 index 000000000000..e0705dd791e6 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml new file mode 100644 index 000000000000..189f9ffa2772 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml new file mode 100644 index 000000000000..86d778e36820 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml new file mode 100644 index 000000000000..d45e090b19e8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml new file mode 100644 index 000000000000..c7c6b64ae0ca --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml new file mode 100644 index 000000000000..d6db00bad77d --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml new file mode 100644 index 000000000000..0c6894a4aa49 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml new file mode 100644 index 000000000000..cf83f8b20273 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml new file mode 100644 index 000000000000..c151d91c550c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml new file mode 100644 index 000000000000..b77e9a76702f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml new file mode 100644 index 000000000000..b225cbec3ef3 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml new file mode 100644 index 000000000000..12671456b7c8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml new file mode 100644 index 000000000000..1ac9a817c3d6 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml new file mode 100644 index 000000000000..ad803d1b735b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml new file mode 100644 index 000000000000..0828c8c7b7a3 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml new file mode 100644 index 000000000000..14e8ce0e7487 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..1a641445db0f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..8235cb5cfcef --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..6eb6bc9c941a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..15642e38b5b0 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..6cfbf7fd3814 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..47c8a10c41af --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..b1e4c1e7a248 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..d800736568ec --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..231f46619ae8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..a2785728d660 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..da1eb7ba274f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..a1afef408818 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..fd7f6010bc42 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..817b4edd1f4a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..180b978f373c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..c325516359d0 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..a71283a4141f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..bb6bb88818b8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..03b5d975965b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..ea085d3b566a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..a7f028528add --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..50ec01cf3d9e --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..5095f52556a2 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..9ab79f6d4c8d --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..ee96a94e02ae --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..386944f8df14 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..6fbcab4deb34 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..b5dc97dc1e87 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..296bc44a45c5 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..04974c0ae5e4 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..208e378dfd26 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..1f4539c5f5f8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..0c7412cdcb61 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..83ca2b959214 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..917a5302ae5f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..fe202ea3a74a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..35ce1c525e21 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..090b845f106e --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..45e3e3578e4c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..0a2c9f03e7e9 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml new file mode 100644 index 000000000000..c8dd741573f4 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml new file mode 100644 index 000000000000..ba5550457957 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml new file mode 100644 index 000000000000..2e65b4c58aed --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml new file mode 100644 index 000000000000..237fbcbca6de --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml new file mode 100644 index 000000000000..74f1eb81e185 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml new file mode 100644 index 000000000000..49806dc006cd --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml new file mode 100644 index 000000000000..8a20bacb3746 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml new file mode 100644 index 000000000000..781cf8c54629 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml new file mode 100644 index 000000000000..8c48d05d31f7 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml new file mode 100644 index 000000000000..e384f100e5a2 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml new file mode 100644 index 000000000000..e7bfd019857c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml new file mode 100644 index 000000000000..f84382235406 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml new file mode 100644 index 000000000000..23adf309293a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml new file mode 100644 index 000000000000..2ccfba511de7 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml new file mode 100644 index 000000000000..8150379dd83b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml new file mode 100644 index 000000000000..cab722643fc7 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml new file mode 100644 index 000000000000..bdad6740434f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml new file mode 100644 index 000000000000..750c34944182 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml new file mode 100644 index 000000000000..bac10aff8956 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml new file mode 100644 index 000000000000..7bd32c84151a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..d71b30b0a630 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..20fce080a34c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..487f189b4d1a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..9561df3add9a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..1e2ddb8bea41 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..378972ee0fb8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..a1dd4d0b8d83 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..76b5ac58593b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..8aafc181e1f1 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..c86014915bde --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..777747a68c37 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..14a0b5a8437c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..274c4a5657df --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..93ea732ea408 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..d47c769b604a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..0f48bb8e1439 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..98e6f59141ed --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..27238bb87641 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..663fbc3e3793 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..6e3f80882b7e --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/scenario_list.yaml b/tensorrt_llm/configure/database/scenario_list.yaml new file mode 100644 index 000000000000..dacb09069ccb --- /dev/null +++ b/tensorrt_llm/configure/database/scenario_list.yaml @@ -0,0 +1,1512 @@ +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 128 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 256 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 128 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 256 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 128 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 256 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 128 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 256 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 From d427dda11ec766282f1a2e25e4fa7d60629bf2df Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 18:43:31 -0800 Subject: [PATCH 2/6] Rename scenaro to constraints Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml | 2 +- 168 files changed, 168 insertions(+), 168 deletions(-) diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml index 4725c335a2fc..c32260a8b365 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml index 778dba5027c8..3b0b031c53e4 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml index 1724b0b8af4b..e48e9516ab1a 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml index cc943a4e5ad9..d46158b4ef2e 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml index f06e5f4b1d74..dafe1f8d5985 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml index e31b303f7337..22c518dd0dc7 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml index dbb06f58e467..04875f5f354f 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml index f7eccec94125..9e8c7bf54bf5 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml index d813f6fbf0d0..717b211778ee 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml index c70a7dd6c6ee..ddeb7068476d 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml index e6581080c029..bb392c21ae2f 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml index 0826c51771ef..13c556051e85 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml index 39ecdca068ff..0379939797bd 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml index 3b346249fc45..897af3f38c18 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml index 786ec7fe456c..a06a7927aebc 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml index a4e5eabe851a..1de38df7ce56 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml index 932933361a74..6e805c2c9ae9 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml index ff7bffb1da18..586e24ad4a43 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml index dcc1392d2fd7..8023959ef8fc 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml index 2f17c33de980..d4c9c6a6f476 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml index a422e84c1955..eb803a65ce7a 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml index 18b25097ea28..ca329b80af8e 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml index 1b8401a825f9..45efb8cf53a3 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml index 1b530b5f5f6e..24c49d146bd2 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml index 551e8b25cb37..3262f411d32d 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml index 38b625426ba8..bed3c77816df 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml index ec59a1bd5052..0a98ec0becc0 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml index deb3f13ec516..f69f3f777301 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml index 69c72d4dafb0..d9d64e3a389e 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml index d57c515f196b..9e4eccd4971f 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml index 320198425c91..cc77ecb9ae06 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml index e001ce83a9a7..9ccb55f1f0a4 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml index c5c393bf1051..d99ed3d65a9d 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml index 2242bc618914..aab1a81302a9 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml index bf1a026cd2db..0430562fa590 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml index 56fc955f2b9a..f0ce6a6a3531 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml index 46ef51eb8e29..8a82d06ab8ed 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml index f29b6e2bd350..d168277f2930 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml index efdf16ff5e09..189c6add635f 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml index 0041a26c747e..989dd9ff357d 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml index 7151a5a2f33c..4dbd12b3c63c 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml index 705aefdc6602..ce5975fbed4e 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml index adfdfb5cefb6..7440d24f2603 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml index fa01711d9df7..63cb11dc39a0 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml index d650e3feb927..5162ce288808 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml index 23e7cd45ac7f..a54b9f135e5f 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml index 99620affd547..9089abde72b4 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml index dec1388712f7..a469fb03f243 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml index f6ff78bcddae..d5f0550ea1b0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml index 13952e7762a5..89b55714267c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml index 67c8cf3ee558..1613da6b48f8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml index 2eabed9bcf94..f6f9e8c3f649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml index 246decd0fd96..c82b6d942499 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml index cfbb10e76771..1ce17af3fff8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml index fd9218e8ae32..835418d65518 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml index b4991a90e933..f41a5509f55e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml index 9527b6f0bc30..5c46254934cd 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml index b57df2745953..bf21a4008393 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml index 820fc7ae0db1..5f4e4f3d0917 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml index ea0821a0f86f..b308fae2b635 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml index 70e6eb7210ff..57415b1ac30e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml index c39784c47845..99a2dec59c77 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml index c1ef5051906c..929c29419173 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml index a521f7ada743..51b4c0ef4260 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml index b9f33be7e853..f21524912fc5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml index d9f21c5bd1cd..14a8731ccd71 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml index 1399bcd8d79f..0a95a868a2c7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml index d26d7d09586c..277c40c69638 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml index 24fe38a91547..ac4d6e378192 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml index 4aec4e67aa2c..82930e426934 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml index 828907414fb9..1c4f51de5e27 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml index c937c1790f4e..5fd46eedbec9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml index e0705dd791e6..efb4aa2b72a2 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml index 189f9ffa2772..bb19699c4e0b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml index 86d778e36820..97abd8b3b32e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml index d45e090b19e8..815c6f330dbc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml index c7c6b64ae0ca..0c66a691f4d2 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml index d6db00bad77d..530e113b7642 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml index 0c6894a4aa49..e81dc73007a9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml index cf83f8b20273..6039f66643c5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml index c151d91c550c..87cf5aee57e5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml index b77e9a76702f..ab86c90b0f94 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml index b225cbec3ef3..0d36f3d5b947 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml index 12671456b7c8..9c0b40f3f622 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml index 1ac9a817c3d6..41831f3b101e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml index ad803d1b735b..4d7c08e07ed0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml index 0828c8c7b7a3..454fb7f7d092 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml index 14e8ce0e7487..c4d8d751f9ef 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml index 1a641445db0f..9e4c63ad5244 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml index 8235cb5cfcef..c0f07ae30c2e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml index 6eb6bc9c941a..c63ce43af3ea 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml index 15642e38b5b0..2870aef58ce4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml index 6cfbf7fd3814..eeaed03658c1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml index 47c8a10c41af..4825fd4f104b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml index b1e4c1e7a248..6962e2cd5bb3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml index d800736568ec..f6fab914bf4f 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml index 231f46619ae8..9728183b72ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml index a2785728d660..f72199de217c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml index da1eb7ba274f..f45b3ba06497 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml index a1afef408818..25152fe7193e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml index fd7f6010bc42..7e8b593ce809 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml index 817b4edd1f4a..1b6e1a065cc1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml index 180b978f373c..c8af20ef5ac5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml index c325516359d0..7988b5983161 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml index a71283a4141f..75c4072286cf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml index bb6bb88818b8..9bcef306ea24 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml index 03b5d975965b..1f651261c4bf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml index ea085d3b566a..d1a8583a35fd 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml index a7f028528add..88b0d04ccbc7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml index 50ec01cf3d9e..49edff89a581 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml index 5095f52556a2..efb12b2990cd 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml index 9ab79f6d4c8d..97e5c74b6d96 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml index ee96a94e02ae..6ff750c395c1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml index 386944f8df14..e8297ab21eb4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml index 6fbcab4deb34..28f755db1b25 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml index b5dc97dc1e87..88713c60c6b9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml index 296bc44a45c5..5d037f31c48c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml index 04974c0ae5e4..412dc2caf305 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml index 208e378dfd26..1ae5fb2bd5a7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml index 1f4539c5f5f8..0be3059dc658 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml index 0c7412cdcb61..734dddd62f34 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml index 83ca2b959214..66993b96fa3f 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml index 917a5302ae5f..e5a3d155cb97 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml index fe202ea3a74a..00025fe7e36b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml index 35ce1c525e21..0399f460e166 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml index 090b845f106e..6ba67335e3de 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml index 45e3e3578e4c..67829867b918 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml index 0a2c9f03e7e9..3f2c9b7ae225 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml index c8dd741573f4..f03a6a17d31f 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml index ba5550457957..74cedf07a862 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml index 2e65b4c58aed..320cd4b678b5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml index 237fbcbca6de..d6f4b02c7743 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml index 74f1eb81e185..f23993712fec 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml index 49806dc006cd..f9f20e1bb8fe 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml index 8a20bacb3746..8894a2b85476 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml index 781cf8c54629..045cf268f942 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml index 8c48d05d31f7..1c386c7186f0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml index e384f100e5a2..da88c0116597 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml index e7bfd019857c..5bc5999fac4f 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml index f84382235406..2d156af01d7e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml index 23adf309293a..452105fa700c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml index 2ccfba511de7..fc6d4b9357c9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml index 8150379dd83b..bebdda511b6a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml index cab722643fc7..97062cac5759 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml index bdad6740434f..b262cee2fc7c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml index 750c34944182..2799aaa47328 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml index bac10aff8956..b1f3f9a3e7d6 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml index 7bd32c84151a..774005a2b9d4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml index d71b30b0a630..04bc59300e64 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml index 20fce080a34c..3ef38bbf1071 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml index 487f189b4d1a..1f7d8f55eeb0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml index 9561df3add9a..6eebb3597dfb 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml index 1e2ddb8bea41..83eed2725443 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml index 378972ee0fb8..42fce0a487c8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml index a1dd4d0b8d83..2da5debb90cc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml index 76b5ac58593b..4ba1c56d1411 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml index 8aafc181e1f1..56fe2397d584 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml index c86014915bde..d7f5958b5cbf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml index 777747a68c37..af86bbff53e0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml index 14a0b5a8437c..ef0df13921fd 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml index 274c4a5657df..e8d20a5911ec 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml index 93ea732ea408..1e5a5b8abe51 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml index d47c769b604a..941ccfaaea97 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml index 0f48bb8e1439..b265a2be47f0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml index 98e6f59141ed..84f5d2664499 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml index 27238bb87641..1f2eeb238d55 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml index 663fbc3e3793..3467cc360761 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml index 6e3f80882b7e..de54bc31d23c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 From adc08b20d4a1d66a2b7464d67c6d6077976530d2 Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 19:59:41 -0800 Subject: [PATCH 3/6] Port over to RootModel Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- tensorrt_llm/configure/database.py | 71 +++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py index e093d17fde90..74e1fff7e2ba 100644 --- a/tensorrt_llm/configure/database.py +++ b/tensorrt_llm/configure/database.py @@ -1,38 +1,69 @@ from pathlib import Path -from typing import List, Optional +from typing import Iterator, List, Optional import pandas as pd import yaml -from pydantic import BaseModel +from pydantic import BaseModel, Field, RootModel DATABASE_LIST_PATH = Path(__file__).parent / "database" / "scenario_list.yaml" class RecipeRecord(BaseModel): - model: str - gpu: str - isl: int - osl: int - concurrency: int - config_path: str - tps_per_user: Optional[int] = None - tps_per_gpu: Optional[int] = None - num_gpus: int + model: str = Field(description="Model name") + gpu: str = Field(description="GPU name") + isl: int = Field(description="Input sequence length") + osl: int = Field(description="Output sequence length") + concurrency: int = Field(description="Concurrency") + config_path: str = Field(description="Configuration path") + tps_per_user: Optional[int] = Field(description="TPS per user", default=None) + tps_per_gpu: Optional[int] = Field(description="TPS per GPU", default=None) + num_gpus: int = Field(description="Number of GPUs") + def to_pandas_row(self) -> pd.Series: + """Convert recipe record to pandas Series. + + Returns: + pd.Series: Pandas Series containing the recipe record. + """ + return pd.Series(self.model_dump()) + + +class RecipeList(RootModel[List[RecipeRecord]]): @classmethod - def from_yaml(cls, yaml_path: Path) -> List["RecipeRecord"]: + def from_yaml(cls, yaml_path: Path) -> "RecipeList": + """Load recipe list from YAML file and validate the data. + + Args: + yaml_path (Path): Path to the YAML file containing the recipe list. + + Returns: + RecipeList: Recipe list object. + """ with open(yaml_path, "r") as f: data = yaml.load(f, Loader=yaml.FullLoader) - return [cls(**item) for item in data] + return cls(data) - def to_pandas_row(self) -> pd.Series: - return pd.Series(self.model_dump()) + def to_pandas_dataframe(self) -> pd.DataFrame: + """Convert recipe list to pandas DataFrame. + + Returns: + pd.DataFrame: Pandas DataFrame where each row contains a recipe record. + """ + return pd.DataFrame([record.to_pandas_row() for record in self.root]) + def __iter__(self) -> Iterator[RecipeRecord]: + """Iterate over the recipe list. -def get_recipe_dataframe(yaml_path: Path) -> pd.DataFrame: - records = RecipeRecord.from_yaml(yaml_path) - return pd.DataFrame([record.to_pandas_row() for record in records]) + Returns: + Iterator[RecipeRecord]: Iterator over the recipe list. + """ + for record in self.root: + yield record + def __len__(self) -> int: + """Get the number of recipes in the list. -def load_recipe_database() -> pd.DataFrame: - return get_recipe_dataframe(DATABASE_LIST_PATH) + Returns: + int: Number of recipes in the list. + """ + return len(self.root) From b79365b21997f885b1a54eab650753f48ae40687 Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 22:16:39 -0800 Subject: [PATCH 4/6] Update recipe classes. Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- tensorrt_llm/configure/database.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py index 74e1fff7e2ba..9582e801ea54 100644 --- a/tensorrt_llm/configure/database.py +++ b/tensorrt_llm/configure/database.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Iterator, List, Optional +from typing import Any, Dict, Iterator, List, Optional import pandas as pd import yaml @@ -8,7 +8,9 @@ DATABASE_LIST_PATH = Path(__file__).parent / "database" / "scenario_list.yaml" -class RecipeRecord(BaseModel): +class RecipeConstraints(BaseModel): + """Recipe record for scenario list.""" + model: str = Field(description="Model name") gpu: str = Field(description="GPU name") isl: int = Field(description="Input sequence length") @@ -28,7 +30,15 @@ def to_pandas_row(self) -> pd.Series: return pd.Series(self.model_dump()) -class RecipeList(RootModel[List[RecipeRecord]]): +class Recipe(BaseModel): + """Recipe that describes a single scenario.""" + + constraints: RecipeConstraints = Field(description="Recipe constraints") + env_overrides: Dict[str, Any] = Field(description="Environment overrides", default_factory=dict) + config: Dict[str, Any] = Field(description="Configuration overrides", default_factory=dict) + + +class RecipeList(RootModel[List[RecipeConstraints]]): @classmethod def from_yaml(cls, yaml_path: Path) -> "RecipeList": """Load recipe list from YAML file and validate the data. @@ -51,7 +61,7 @@ def to_pandas_dataframe(self) -> pd.DataFrame: """ return pd.DataFrame([record.to_pandas_row() for record in self.root]) - def __iter__(self) -> Iterator[RecipeRecord]: + def __iter__(self) -> Iterator[RecipeConstraints]: """Iterate over the recipe list. Returns: From 414a1c0b0d674dc19414e9310503324784244789 Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 22:27:20 -0800 Subject: [PATCH 5/6] Update env to env_overrides. Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml | 3 ++- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml | 2 +- 168 files changed, 169 insertions(+), 168 deletions(-) diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml index c32260a8b365..bc11985520fa 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml index 3b0b031c53e4..007485c83102 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml index e48e9516ab1a..cfad1d587d33 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml @@ -5,7 +5,8 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: + +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml index d46158b4ef2e..77977855bdd7 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml index dafe1f8d5985..dcd3dc25ca71 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml index 22c518dd0dc7..f01f36528e0b 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml index 04875f5f354f..c1ffca06dfd1 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml index 9e8c7bf54bf5..4f3f547628d3 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml index 717b211778ee..e0308d9c4b6d 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml index ddeb7068476d..fee6bb4c0bc8 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml index bb392c21ae2f..e9b11f0b9865 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml index 13c556051e85..ece060c8d50e 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml index 0379939797bd..f87be9b97437 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml index 897af3f38c18..42c88d66c5e3 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml index a06a7927aebc..9abe8950140b 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml index 1de38df7ce56..8045307da664 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml index 6e805c2c9ae9..d90b6207bf05 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml index 586e24ad4a43..ef58a2020a02 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml index 8023959ef8fc..891784943e1f 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml index d4c9c6a6f476..8fd9790fca3e 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml index eb803a65ce7a..1ce823ebce33 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 128 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml index ca329b80af8e..eb9f9e67fc40 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml index 45efb8cf53a3..2f0bd54767f1 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 256 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml index 24c49d146bd2..a6bca1b4f8ec 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml index 3262f411d32d..6daee6414106 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml index bed3c77816df..618fe65a1edd 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml index 0a98ec0becc0..454ac155fd0e 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml index f69f3f777301..e4fd062546eb 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 128 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml index d9d64e3a389e..1ca81b3f047a 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml index 9e4eccd4971f..fa9adde1fc73 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 256 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml index cc77ecb9ae06..622934a54543 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml index 9ccb55f1f0a4..962fab4f8ff3 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml index d99ed3d65a9d..2fbff7bd5932 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml index aab1a81302a9..754ecd630e34 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml index 0430562fa590..696a10ede199 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 128 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml index f0ce6a6a3531..336ecb1375de 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml index 8a82d06ab8ed..9eeb9fe78fd2 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 256 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml index d168277f2930..38518b790dc9 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml index 189c6add635f..5d3a01c77a10 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml index 989dd9ff357d..1c6ca3c7d03c 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml index 4dbd12b3c63c..e880f71dacb9 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml index ce5975fbed4e..08ae474cb1c9 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 128 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml index 7440d24f2603..ed8dc45b999b 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml index 63cb11dc39a0..db542084c7ac 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 256 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml index 5162ce288808..2398a785a9c6 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml index a54b9f135e5f..39797e4fd262 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml index 9089abde72b4..51e4cf3be355 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml index a469fb03f243..6f6ac3c4efbc 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml index d5f0550ea1b0..07ab42c83dc7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml index 89b55714267c..0bc68ec73e0b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml index 1613da6b48f8..0e7cdcb0fed0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml index f6f9e8c3f649..5b33b0061109 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml index c82b6d942499..ddde04632fef 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml index 1ce17af3fff8..e2423f02b31b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml index 835418d65518..5eb831f7b771 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml index f41a5509f55e..25af6251f57f 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml index 5c46254934cd..27ea9bdc2c0c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml index bf21a4008393..151c830b7114 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml index 5f4e4f3d0917..34c577437f6d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml index b308fae2b635..cf36077532f4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml index 57415b1ac30e..a67b9048aacc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml index 99a2dec59c77..5c03bc1288b5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml index 929c29419173..08d62899be07 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml index 51b4c0ef4260..139b661d9523 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml index f21524912fc5..c528c248e40e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml index 14a8731ccd71..8e64d3238545 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml index 0a95a868a2c7..2102a649c6d8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml index 277c40c69638..638e74ce66f9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml index ac4d6e378192..a1f54c1bed4c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml index 82930e426934..c2941b4776c4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml index 1c4f51de5e27..5d22acc2b3ef 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml index 5fd46eedbec9..2a23217ff87c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml index efb4aa2b72a2..b9b567e3c6cf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml index bb19699c4e0b..c103c1c8d2bc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml index 97abd8b3b32e..544adfd99e38 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml index 815c6f330dbc..188bf67cbb2c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml index 0c66a691f4d2..8cb27e0a60c3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml index 530e113b7642..7682c3aae59b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml index e81dc73007a9..6b8a5c99a24d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml index 6039f66643c5..29e7dd016023 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml index 87cf5aee57e5..81c963bc3d66 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml index ab86c90b0f94..d43af26ceb14 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml index 0d36f3d5b947..70a2a883cbd1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml index 9c0b40f3f622..087811f62ea7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml index 41831f3b101e..9dd73aa1bc65 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml index 4d7c08e07ed0..c5d51031dc7a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml index 454fb7f7d092..9f67d733f83d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml index c4d8d751f9ef..b9fa200f76fa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml index 9e4c63ad5244..cc70292eba00 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml index c0f07ae30c2e..c0c2048c2e26 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml index c63ce43af3ea..734f90ab3239 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml index 2870aef58ce4..b1bf229643e8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml index eeaed03658c1..1c0a5181dcac 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml index 4825fd4f104b..0305e02939e7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml index 6962e2cd5bb3..0753dfb129a4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml index f6fab914bf4f..5b4e0deaedd7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml index 9728183b72ca..d40f00d6f983 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml index f72199de217c..6f8e8db1458c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml index f45b3ba06497..cb2e735be596 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml index 25152fe7193e..bb6a98714c40 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml index 7e8b593ce809..45f749c8fcf9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml index 1b6e1a065cc1..16915e230b1b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml index c8af20ef5ac5..a0043a10f702 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml index 7988b5983161..4b5eac7996ff 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml index 75c4072286cf..fa2cac24e6fe 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml index 9bcef306ea24..910d9b621f85 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml index 1f651261c4bf..74802561ad1d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml index d1a8583a35fd..c3ac98310da5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml index 88b0d04ccbc7..19ed66323e75 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml index 49edff89a581..cf38aa9547d6 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml index efb12b2990cd..1584f4231915 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml index 97e5c74b6d96..59f7e59a0c66 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml index 6ff750c395c1..bebec6d64c29 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml index e8297ab21eb4..89b43993793c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml index 28f755db1b25..ff90474ca631 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml index 88713c60c6b9..6b0f346b957e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml index 5d037f31c48c..7639a842953a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml index 412dc2caf305..055050456b38 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml index 1ae5fb2bd5a7..da7decb5c24c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml index 0be3059dc658..74bdd4ec6941 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml index 734dddd62f34..ba10fa27f660 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml index 66993b96fa3f..75560253a0ed 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml index e5a3d155cb97..26706dc2a9a3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml index 00025fe7e36b..710c4e7f3d08 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml index 0399f460e166..be72abced5e5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml index 6ba67335e3de..bd53e3a2251a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml index 67829867b918..4b8e06cd73d0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml index 3f2c9b7ae225..8b277eb06773 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml index f03a6a17d31f..d8b2d4cb7111 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml index 74cedf07a862..a909b624ecad 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml index 320cd4b678b5..acfc7da97178 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml index d6f4b02c7743..c1f8c66f9ca0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml index f23993712fec..574634491b44 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml index f9f20e1bb8fe..5954a22ea253 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml index 8894a2b85476..b5e20ed3f0a1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml index 045cf268f942..c23e61e524c1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml index 1c386c7186f0..b3d6f3522cda 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml index da88c0116597..7689682459e8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml index 5bc5999fac4f..09799ac87bd3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml index 2d156af01d7e..ee43cfa08998 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml index 452105fa700c..5d2f9278deae 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml index fc6d4b9357c9..cb6b902eae03 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml index bebdda511b6a..cda2dbd4be30 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml index 97062cac5759..a56724079357 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml index b262cee2fc7c..3abcf5b7f657 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml index 2799aaa47328..cfe49a3f53fc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml index b1f3f9a3e7d6..1dbd831b09b3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml index 774005a2b9d4..fa3ac5f75756 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml index 04bc59300e64..51e7db06098c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml index 3ef38bbf1071..20c70f3ad997 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml index 1f7d8f55eeb0..3b31277ea799 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml index 6eebb3597dfb..7b48e384c981 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml index 83eed2725443..4ba80525301e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml index 42fce0a487c8..951ca48e5217 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml index 2da5debb90cc..16e05ba27766 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml index 4ba1c56d1411..e759749e4eb8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml index 56fe2397d584..a3ef256793ce 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml index d7f5958b5cbf..45db5ad5dc9b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml index af86bbff53e0..3d751ee390da 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml index ef0df13921fd..165df6c51b99 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml index e8d20a5911ec..ed9ac653da27 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml index 1e5a5b8abe51..e8ea41b65258 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml index 941ccfaaea97..6c58809f77d7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml index b265a2be47f0..32052150a463 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml index 84f5d2664499..9c9115251f1d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml index 1f2eeb238d55..e175fadc5ca9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml index 3467cc360761..f1bacc77c3fa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml index de54bc31d23c..fa76ed2f6111 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: From f5203b55831e602220683cbdf41fe44f9d358f06 Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 22:27:48 -0800 Subject: [PATCH 6/6] Update to add config recipe + load. Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- tensorrt_llm/configure/database.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py index 9582e801ea54..47655e2a5f87 100644 --- a/tensorrt_llm/configure/database.py +++ b/tensorrt_llm/configure/database.py @@ -29,6 +29,11 @@ def to_pandas_row(self) -> pd.Series: """ return pd.Series(self.model_dump()) + def load_config(self) -> Dict[str, Any]: + """Load configuration from the configuration path.""" + with open(self.config_path, "r") as f: + return Recipe(**yaml.load(f, Loader=yaml.FullLoader)) + class Recipe(BaseModel): """Recipe that describes a single scenario."""