[None][ci] Test waives for the main branch 09/14

jenkins/scripts/slurm_run.sh

@@ -105,5 +105,5 @@ trap - ERR
 
 eval $fullCmd
 exitCode=$?
-echo "Pytest exit code: $exitCode"
+echo "Rank${SLURM_LOCALID} Pytest exit code: $exitCode"
 exit $exitCode

jenkins/scripts/trtllm-llmapi-launch

@@ -47,7 +47,7 @@ log_stderr "tllm_mpi_size: $tllm_mpi_size"
 export_free_tcp_addr_for_spawn_proxy_process
 
 if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]; then
-    log_stderr "rank${mpi_rank} run ${task_with_command[@]} in background"
+    log_stderr "Rank${mpi_rank} run ${task_with_command[@]} in background"
 
     # MPI doesn't allow spawn a process sharing the MPI environment in a MPI
     # process, or duplicate MPI_Init in the child process will cause undefined
@@ -76,39 +76,39 @@ if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]; then
         # Execute the task with cleaned environment
         "${task_with_command[@]}"
         task_exit_code=$?
-        echo "Task exit code: $task_exit_code"
+        log_stderr "Rank${mpi_rank} Task exit code: $task_exit_code"
 
         # Stop the MPI Comm server
         python3 -m tensorrt_llm.llmapi.mgmn_leader_node --action stop
         mpi_exit_code=$?
-        echo "MPI Comm server exit code: $mpi_exit_code"
+        log_stderr "Rank${mpi_rank} MPI Comm server exit code: $mpi_exit_code"
 
         # Propagate task exit status
         if [ $task_exit_code -ne 0 ]; then
             exit $task_exit_code
         else
             exit $mpi_exit_code
         fi
-    ) 1>&2 &
+    ) &
 
     # Turn off "exit on error" so the following lines always run
     set +e
 
     # Capture subshell PID
     subshell_pid=$!
-    echo "Subshell PID: $subshell_pid"
+    log_stderr "Rank${mpi_rank} Subshell PID: $subshell_pid"
 
-    log_stderr "rank${mpi_rank} run mgmn leader node with mpi_world_size: $(mpi_world_size) ..."
-    log_stderr "rank0 host: $HOSTNAME"
+    log_stderr "Rank${mpi_rank} run mgmn leader node with mpi_world_size: $(mpi_world_size) ..."
+    log_stderr "Rank0 host: $HOSTNAME"
     python3 -m tensorrt_llm.llmapi.mgmn_leader_node
     mgmn_leader_node_exit_code=$?
-    echo "MGMN leader node exit code: $mgmn_leader_node_exit_code"
+    log_stderr "Rank${mpi_rank} MGMN leader node exit code: $mgmn_leader_node_exit_code"
 
     # Wait for subshell
     wait $subshell_pid
     # This is subshell's exit code
     subshell_exit_code=$?
-    echo "Subshell exit code: $subshell_exit_code"
+    log_stderr "Rank${mpi_rank} Subshell exit code: $subshell_exit_code"
 
     # Propagate subshell exit status
     if [ $subshell_exit_code -ne 0 ]; then
@@ -120,10 +120,10 @@ else
     # Turn off "exit on error" so the following lines always run
     set +e
 
-    log_stderr "rank${mpi_rank} run mgmn worker node with mpi_world_size: $(mpi_world_size) ..."
+    log_stderr "Rank${mpi_rank} run mgmn worker node with mpi_world_size: $(mpi_world_size) ..."
     python3 -m tensorrt_llm.llmapi.mgmn_worker_node
     mgmn_worker_node_exit_code=$?
-    echo "MGMN worker node exit code: $mgmn_worker_node_exit_code"
+    log_stderr "Rank${mpi_rank} MGMN worker node exit code: $mgmn_worker_node_exit_code"
 
     exit $mgmn_worker_node_exit_code
 fi

tests/integration/test_lists/waives.txt

@@ -341,3 +341,7 @@ full:L20/perf/test_perf.py::test_perf[phi_4_mini_instruct-bench-bfloat16-maxbs:3
 full:RTXPro6000/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/5512556)
 full:RTXPro6000/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/5512556)
 accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=False] SKIP (https://nvbugs/5513237)
+accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[mtp=disable-fp8kv=True-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/5516661)
+accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/5516665)
+accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/5516665)
+accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm] SKIP (https://nvbugs/5516666)

tests/unittest/llmapi/test_llm.py

@@ -1877,6 +1877,7 @@ async def main():
         asyncio.run(main())
 
 
+@pytest.mark.skip(reason="https://nvbugs/5516660")
 @force_ampere
 @pytest.mark.parametrize(
     "prompt_logprobs, logprobs, return_context_logits, return_generation_logits",
@@ -1890,6 +1891,7 @@ def test_llm_return_logprobs(prompt_logprobs: Optional[int],
                                      return_generation_logits)
 
 
+@pytest.mark.skip(reason="https://nvbugs/5516660")
 @force_ampere
 def test_llm_return_logprobs_streaming():
     llm_return_logprobs_test_harness(2, 2, False, True, streaming=True)