diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index f09760fbf286..49340dc4edcc 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -84,6 +84,21 @@ def get_ucx_tls(): return "^ib,gdr_copy" +def apply_pre_hopper_nccl_workaround(env: dict) -> None: + """Keep NCCL off PCIe peer-to-peer on pre-Hopper CI nodes. + + The 8xL40S CI nodes run with the AMD IOMMU in DMA translation mode, so + PCIe P2P writes across the host bridge fault (``AMD-Vi IO_PAGE_FAULT``) + and never land. NCCL still picks its P2P/CUMEM transport there, and the + first PP send/recv or TP collective in the warmup forward hangs the + worker before it can register with the disagg server + (https://nvbugs/6728119). Restricting P2P to PCIe-switch peers makes NCCL + fall back to SHM on those nodes; NVLink platforms are unaffected. + """ + if get_sm_version() < 90: + env.setdefault("NCCL_P2P_LEVEL", "PXB") + + def cleanup_output_files(): """Clean up output files from previous runs.""" for file in ['output.json', 'output_streaming.json']: @@ -1059,6 +1074,7 @@ def run_disaggregated_test(example_dir, run_env = env.copy() if env else os.environ.copy() run_env["UCX_TLS"] = get_ucx_tls() + apply_pre_hopper_nccl_workaround(run_env) ctx_run_env = run_env.copy() if ctx_env: ctx_run_env.update(ctx_env) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index e315f8698421..0e7e23dd451b 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -156,11 +156,7 @@ full:H20/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_au full:H20/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) full:H20/accuracy/test_llm_api_pytorch_multimodal.py::TestNanoV3Omni::test_auto_dtype[bf16] SKIP (https://nvbugs/6618649) full:L40S/accuracy/test_llm_api_pytorch_encode.py::TestEncoderEncode::test_encoder_encode_cuda_graph_matches_eager_logits[bert-yelp] SKIP (https://nvbugs/6802114) -full:L40S/disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2 SKIP (https://nvbugs/6728119) -full:L40S/disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_gentp2 SKIP (https://nvbugs/6728119) -full:L40S/disaggregated/test_disaggregated.py::test_disaggregated_ctxtp2_genpp2 SKIP (https://nvbugs/6728119) full:L40S/disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap[DeepSeek-V3-Lite-fp8] SKIP (https://nvbugs/6581064) -full:L40S/disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu SKIP (https://nvbugs/6728119) full:RTXPro6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True] SKIP (https://nvbugs/5929339) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False] SKIP (https://nvbugs/6616033) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True] SKIP (https://nvbugs/6616033)