Prepare Blazor workloads before dispatching to Helix

docs/sdk-scenarios.md

@@ -47,6 +47,28 @@ does not download arbitrary Helix uploads into pipeline artifacts. Asynchronous
 skip the submitter's `artifacts/helix-results` downloads, which were not published by
 this pipeline. Local `--send-to-helix` runs still wait and download performance reports.
 
+## Blazor workload preparation
+
+SDK Blazor jobs install `wasm-tools` once on the build agent through
+`PreparePayloadWorkItems`, using the SDK in the correlation payload, its bundled
+manifests (`--skip-manifest-update`), and the payload's `NuGet.config`. An installation
+failure stops payload preparation before the job is sent to Helix.
+
+Helix work items use `pre.py publish --has-workload --readonly-dotnet` and
+`post.py --readonly-dotnet`. These flags prevent the scenario helpers from changing
+the shared workload installation; they do not set filesystem permissions. Application
+restore, publish, size measurements, and cleanup of work-item-local outputs still run
+on Helix. Standalone scenario scripts retain their existing workload-management
+defaults.
+
+Workload availability does not select the benchmark's compilation mode. The Minimum
+Template explicitly enables native relinking, while the non-AOT New Template, Pizza,
+and Localized scenarios disable it with `WasmNativeWorkload=false`. AOT scenarios
+enable `RunAOTCompilation=true`. The Localized AOT command passes this property directly
+instead of referencing item metadata as an MSBuild property, which previously dropped
+the AOT argument. Its reported sizes may therefore change when it first runs with AOT
+actually enabled.
+
 ## SDK Build Throughput Scenario
 
 **SDK Build Throughput** is a scenario test that measures the throughput of SDK build process. To be more specific, our test *implicitly calls*

eng/performance/blazor_scenarios.proj

@@ -7,9 +7,13 @@
     <PerfLabTargetFrameworksEnvVar Condition="'$(TargetsWindows)' == 'true'">%PERFLAB_TARGET_FRAMEWORKS%</PerfLabTargetFrameworksEnvVar>
     <PerfLabTargetFrameworksEnvVar Condition="'$(TargetsWindows)' != 'true'">%24{PERFLAB_TARGET_FRAMEWORKS}</PerfLabTargetFrameworksEnvVar>
     <_MSBuildArgs>/p:_TrimmerDumpDependencies=true;/warnaserror:NU1602,NU1604</_MSBuildArgs>
-    <PublishArgs>--msbuild &quot;$(_MSBuildArgs)&quot; --msbuild-static AdditionalMonoLinkerOptions=%27&quot;%24(AdditionalMonoLinkerOptions) --dump-dependencies&quot;%27 --binlog $(LogDirectory)blazor_publish.binlog</PublishArgs>
+    <PublishArgs>--has-workload --readonly-dotnet --msbuild &quot;$(_MSBuildArgs)&quot; --msbuild-static AdditionalMonoLinkerOptions=%27&quot;%24(AdditionalMonoLinkerOptions) --dump-dependencies&quot;%27 --binlog $(LogDirectory)blazor_publish.binlog</PublishArgs>
     <PublishCommand>$(Python) pre.py publish $(PublishArgs)</PublishCommand>
 
+    <_BlazorDotNet>$([MSBuild]::NormalizePath('$(CorrelationPayloadDirectory)', 'dotnet', 'dotnet'))</_BlazorDotNet>
+    <_BlazorDotNet Condition="'$(TargetsWindows)' == 'true'">$(_BlazorDotNet).exe</_BlazorDotNet>
+    <_BlazorNuGetConfig>$([MSBuild]::NormalizePath('$(CorrelationPayloadDirectory)', 'performance', 'NuGet.config'))</_BlazorNuGetConfig>
+
     <PizzaAppPubLocation Condition="'$(TargetsWindows)' == 'true'">pub\wwwroot</PizzaAppPubLocation>
     <PizzaAppPubLocation Condition="'$(TargetsWindows)' != 'true'">pub/wwwroot</PizzaAppPubLocation>
   </PropertyGroup>
@@ -27,6 +31,11 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PreparePayloadWorkItem Include="Blazor wasm-tools workload">
+      <Command>&quot;$(_BlazorDotNet)&quot; workload install wasm-tools --skip-manifest-update --configfile &quot;$(_BlazorNuGetConfig)&quot;</Command>
+      <WorkingDirectory>$(CorrelationPayloadDirectory)performance</WorkingDirectory>
+    </PreparePayloadWorkItem>
+
     <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
       <PayloadDirectory>%(Identity)</PayloadDirectory>
     </HelixCorrelationPayload>
@@ -35,7 +44,7 @@
 
   <ItemDefinitionGroup>
       <HelixWorkItem>
-      <PostCommands>$(Python) post.py</PostCommands>
+      <PostCommands>$(Python) post.py --readonly-dotnet</PostCommands>
       <Timeout>4:00</Timeout>
     </HelixWorkItem>
   </ItemDefinitionGroup>
@@ -78,17 +87,19 @@
       <Command>$(PublishCommand) -f $(PerfLabTargetFrameworksEnvVar) &amp;&amp; $(Python) test.py sod --scenario-name &quot;%(Identity)&quot; --dirs $(PizzaAppPubLocation) $(ScenarioArgs)</Command>
     </HelixWorkItem>
 
+    <!-- Unlike the other scenarios, both Localized variants use the same pre.py.
+         Select the mode here: disable native relinking for non-AOT even with wasm-tools
+         installed, and explicitly enable AOT for the AOT work item. -->
     <HelixWorkItem Include="SOD - Localized App - Publish">
       <PayloadDirectory>$(ScenariosDir)blazorlocalized</PayloadDirectory>
       <!-- Specifying both linker dump msbuild properties in case linker version is not updated -->
-      <Command>$(PublishCommand) -f $(PerfLabTargetFrameworksEnvVar) &amp;&amp; $(Python) test.py sod --scenario-name &quot;%(Identity)&quot; --dirs $(PizzaAppPubLocation) $(ScenarioArgs)</Command>
+      <Command>$(PublishCommand) --msbuild &quot;$(_MSBuildArgs);/p:WasmNativeWorkload=false&quot; -f $(PerfLabTargetFrameworksEnvVar) &amp;&amp; $(Python) test.py sod --scenario-name &quot;%(Identity)&quot; --dirs $(PizzaAppPubLocation) $(ScenarioArgs)</Command>
     </HelixWorkItem>
 
     <HelixWorkItem Include="SOD - Localized App - Publish - AOT"> 
-      <_PublishArgsWithAOT>--msbuild &quot;$(_MSBuildArgs);/p:RunAOTCompilation=true&quot;</_PublishArgsWithAOT>
       <PayloadDirectory>$(ScenariosDir)blazorlocalized</PayloadDirectory>
       <!-- Specifying both linker dump msbuild properties in case linker version is not updated -->
-      <Command>$(PublishCommand) $(_PublishArgsWithAOT) -f $(PerfLabTargetFrameworksEnvVar) &amp;&amp; $(Python) test.py sod --scenario-name &quot;%(Identity)&quot; --dirs $(PizzaAppPubLocation) $(ScenarioArgs)</Command>
+      <Command>$(PublishCommand) --msbuild &quot;$(_MSBuildArgs);/p:RunAOTCompilation=true&quot; -f $(PerfLabTargetFrameworksEnvVar) &amp;&amp; $(Python) test.py sod --scenario-name &quot;%(Identity)&quot; --dirs $(PizzaAppPubLocation) $(ScenarioArgs)</Command>
     </HelixWorkItem>
   </ItemGroup>
   <Import Project="PreparePayloadWorkItems.targets" />

scripts/tests/test_blazor_workload.py

@@ -0,0 +1,181 @@
+import importlib
+import json
+import os
+from pathlib import Path
+import shlex
+import shutil
+import subprocess
+import sys
+from unittest.mock import Mock
+import xml.etree.ElementTree as ET
+
+import pytest
+
+
+repo_root = Path(__file__).resolve().parents[2]
+
+
+@pytest.fixture
+def scenario_commands(monkeypatch):
+    monkeypatch.syspath_prepend(str(repo_root / "scripts"))
+    monkeypatch.syspath_prepend(str(repo_root / "src" / "scenarios"))
+    return (
+        importlib.import_module("shared.precommands").PreCommands,
+        importlib.import_module("shared.postcommands").PostCommands,
+    )
+
+
+@pytest.fixture(params=["true", "false"], ids=["windows", "unix"])
+def blazor_items(request, tmp_path):
+    dotnet = shutil.which("dotnet")
+    if dotnet is None:
+        pytest.skip("MSBuild evaluation requires a .NET SDK")
+
+    project = ET.parse(repo_root / "eng" / "performance" / "blazor_scenarios.proj")
+    # Evaluate the real work items without downloading the Helix SDK.
+    project.getroot().attrib.pop("Sdk")
+    project.find("Import").set(
+        "Project", str(repo_root / "eng" / "performance" / "PreparePayloadWorkItems.targets")
+    )
+    project_path = tmp_path / "blazor.proj"
+    project.write(project_path, encoding="utf-8")
+    payload = tmp_path / "correlation payload"
+    result = subprocess.run(
+        [
+            dotnet, "msbuild", str(project_path), "-nologo",
+            "-getItem:PreparePayloadWorkItem,HelixWorkItem",
+            f"-p:CorrelationPayloadDirectory={payload}{os.sep}",
+            f"-p:TargetsWindows={request.param}",
+            "-p:Python=python",
+        ],
+        cwd=tmp_path,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return json.loads(result.stdout)["Items"], payload, request.param == "true"
+
+
+def test_workload_is_prepared_once_in_payload(blazor_items):
+    items, payload, windows = blazor_items
+    preparation, = items["PreparePayloadWorkItem"]
+    executable = payload / "dotnet" / ("dotnet.exe" if windows else "dotnet")
+    config = payload / "performance" / "NuGet.config"
+
+    assert preparation["Command"] == (
+        f'"{executable}" workload install wasm-tools --skip-manifest-update '
+        f'--configfile "{config}"'
+    )
+    assert Path(preparation["WorkingDirectory"]) == payload / "performance"
+
+
+@pytest.mark.parametrize("exit_code", [0, 23])
+def test_preparation_propagates_workload_exit_code(blazor_items, tmp_path, exit_code):
+    _, payload, windows = blazor_items
+    working_directory = payload / "performance"
+    working_directory.mkdir(parents=True)
+    # Substitute only the executable; run the real preparation target and arguments.
+    (working_directory / "workload").write_text(
+        "import sys\n"
+        "assert sys.argv[1:4] == ['install', 'wasm-tools', '--skip-manifest-update']\n"
+        "print('Workload preparation executed')\n"
+        f"sys.exit({exit_code})\n",
+        encoding="utf-8",
+    )
+    result = subprocess.run(
+        [
+            shutil.which("dotnet"), "msbuild", str(tmp_path / "blazor.proj"), "-nologo",
+            "-t:PreparePayloadWorkItems",
+            f"-p:CorrelationPayloadDirectory={payload}{os.sep}",
+            f"-p:TargetsWindows={str(windows).lower()}",
+            f"-p:_BlazorDotNet={sys.executable}",
+        ],
+        cwd=tmp_path,
+        capture_output=True,
+        text=True,
+    )
+
+    assert "Workload preparation executed" in result.stdout
+    if exit_code:
+        assert result.returncode != 0
+        assert "MSB3073" in result.stdout
+    else:
+        assert result.returncode == 0, result.stdout + result.stderr
+
+
+def test_all_work_items_leave_shared_workload_untouched(
+    blazor_items, scenario_commands, monkeypatch
+):
+    items, _, _ = blazor_items
+    PreCommands, PostCommands = scenario_commands
+    run = Mock(side_effect=AssertionError("A work item must not modify the workload"))
+    monkeypatch.setattr(subprocess, "run", run)
+
+    assert len(items["HelixWorkItem"]) == 8
+    for item in items["HelixWorkItem"]:
+        command = item["Command"].split(" && ", 1)[0]
+        monkeypatch.setattr(sys, "argv", shlex.split(command)[1:])
+        pre = PreCommands()
+        assert pre.has_workload
+        assert pre.readonly_dotnet
+        pre.install_workload("wasm-tools")
+        pre.uninstall_workload("wasm-tools")
+
+        monkeypatch.setattr(sys, "argv", shlex.split(item["PostCommands"])[1:])
+        post = PostCommands()
+        assert post.readonly_dotnet
+        post.uninstall_workload("wasm-tools")
+
+    run.assert_not_called()
+
+
+@pytest.mark.parametrize(
+    ("name", "property_argument"),
+    [
+        ("SOD - Localized App - Publish", "/p:WasmNativeWorkload=false"),
+        ("SOD - Localized App - Publish - AOT", "/p:RunAOTCompilation=true"),
+    ],
+)
+def test_localized_publish_mode_is_forwarded(
+    blazor_items, scenario_commands, monkeypatch, name, property_argument
+):
+    items, _, _ = blazor_items
+    PreCommands, _ = scenario_commands
+    item = next(item for item in items["HelixWorkItem"] if item["Identity"] == name)
+    command = item["Command"].split(" && ", 1)[0]
+    monkeypatch.setattr(sys, "argv", shlex.split(command)[1:])
+    pre = PreCommands()
+    pre.project = Mock()
+    pre.execute()
+
+    assert pre.project.publish.call_args.args[6] == [
+        "/p:_TrimmerDumpDependencies=true",
+        "/warnaserror:NU1602,NU1604",
+        property_argument,
+    ]
+
+
+def test_standalone_workload_install_is_unchanged(scenario_commands, monkeypatch):
+    PreCommands, _ = scenario_commands
+    monkeypatch.setattr(sys, "argv", ["pre.py", "publish"])
+    run = Mock()
+    monkeypatch.setattr(subprocess, "run", run)
+
+    PreCommands().install_workload("wasm-tools")
+
+    run.assert_called_once_with(
+        ["dotnet", "workload", "install", "wasm-tools", "--skip-manifest-update"],
+        check=True,
+    )
+
+
+def test_readonly_sdk_requires_preinstalled_workload(scenario_commands, monkeypatch):
+    PreCommands, _ = scenario_commands
+    monkeypatch.setattr(sys, "argv", ["pre.py", "publish", "--readonly-dotnet"])
+    run = Mock()
+    monkeypatch.setattr(subprocess, "run", run)
+
+    with pytest.raises(Exception, match="has_workload=false"):
+        PreCommands().install_workload("wasm-tools")
+
+    run.assert_not_called()