diff --git a/.github/scripts/shard-assign.sh b/.github/scripts/shard-assign.sh new file mode 100755 index 0000000000..631945b860 --- /dev/null +++ b/.github/scripts/shard-assign.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# +# Prints, one per line, the test .csproj paths assigned to shard $SHARD_INDEX of +# $SHARD_TOTAL. +# +# 🚨 Used by BOTH jobs in dotnet-test.yml and they MUST agree: +# * `build` calls it once per shard to pack that shard's test bins into its OWN +# artifact (so a shard downloads ~1/6 of the output instead of all 3.3 GB); +# * `test` calls it to decide which projects to run. +# If the two ever disagreed, a shard would run a project whose bin it never +# downloaded. That is why the assignment lives in one file instead of being +# duplicated in two `run:` blocks. +# +# Portable on purpose: no `declare -A`, so it runs on the runner's bash 5 AND on +# a stock macOS bash 3.2 — you can verify a change locally before pushing. +# SHARD_INDEX=0 SHARD_TOTAL=6 .github/scripts/shard-assign.sh +# +# ── Balancing ──────────────────────────────────────────────────────────────── +# Greedy LPT: heaviest first onto the currently least-loaded shard, ties to the +# lowest shard id. Explicit and ORDER-INDEPENDENT — the original +# `counter % SHARD_TOTAL` over a sorted csproj list silently depended on locale +# byte order (C-locale sorts MeshWeaver.AI.Test BEFORE MeshWeaver.Acme.Test, +# 'I' 0x49 < 'c' 0x63), which stacked AI + Orleans — two of the three +# ALC-heaviest projects — onto shard 0 and made it the ~19-minute long pole of +# every run. +# +# Weights are wall-clock SECONDS, the MAX over two green runs (30903428924 and +# 30903158378, both 2026-08-04) so a shard is sized for the bad case, not the +# average. The per-project `[CI] exit=` markers in the test job's log ARE +# the measurement: take the delta between each `::group::` and its marker. +# Re-measure when a project's runtime changes materially — the previous table +# (2026-07-13) had drifted far enough to cost ~100 s of long pole (Monolith +# 200→345, Data 25→73, PluginTester 60→8, Persistence 165→97). +# +# The floor for ANY sharding is the heaviest single project: today +# Hosting.Monolith.Test at 345 s. Adding shards below that buys nothing. +# +# Unlisted projects get DEFAULT_WEIGHT — a deliberate over-estimate, since an +# unlisted project is a NEW one whose cost nobody has measured yet. +# +# ── Splitting a heavyweight across shards ──────────────────────────────────── +# An optional THIRD column splits one project into N parts, each weighted 1/N and +# scheduled independently — so the halves land on DIFFERENT shards, i.e. different +# runners. That matters twice over: it lowers the floor (no amount of sharding can +# beat the heaviest single project), and it introduces no shared-resource contention, +# because two runners share nothing. `maxParallelThreads: 1` is per-process and stays +# intact. +# +# A split entry prints as `#/`; the test job turns that into an +# `-class` filter list by enumerating the assembly's classes and taking every Nth. +# The BUILD job strips the suffix, so a split project's bin is packed into every shard +# that runs one of its parts. +set -euo pipefail + +: "${SHARD_INDEX:?SHARD_INDEX must be set}" +: "${SHARD_TOTAL:?SHARD_TOTAL must be set}" + +DEFAULT_WEIGHT=10 + +# " ", heaviest first. +WEIGHTS=$(cat <<'EOF' +345 MeshWeaver.Hosting.Monolith.Test 2 +263 MeshWeaver.AI.Test 2 +239 MeshWeaver.Hosting.Orleans.Test +130 MeshWeaver.Hosting.PostgreSql.Test +104 MeshWeaver.Threading.Test +97 MeshWeaver.Persistence.Test +91 MeshWeaver.Content.Test +73 MeshWeaver.FutuRe.Test +73 MeshWeaver.Data.Test +62 MeshWeaver.Acme.Test +53 MeshWeaver.Security.Test +51 MeshWeaver.GitSync.Test +49 MeshWeaver.Autocomplete.Test +49 MeshWeaver.Query.Test +33 MeshWeaver.Auth.Test +31 MeshWeaver.Graph.Test +30 MeshWeaver.PluginCatalog.Test +27 MeshWeaver.NodeOperations.Test +26 MeshWeaver.InstanceSync.Test +22 MeshWeaver.Messaging.Hub.Test +18 MeshWeaver.Layout.Test +15 MeshWeaver.Markdown.Test +15 MeshWeaver.Import.Test +14 MeshWeaver.Courses.Test +12 MeshWeaver.PathResolution.Test +12 MeshWeaver.ContentCollections.Indexing.Graph.Test +12 MeshWeaver.MemexTemplate.Test +9 MeshWeaver.Northwind.Test +8 MeshWeaver.PluginTester.Test +8 MeshWeaver.PythonDemo.Test +7 MeshWeaver.ContentCollections.Indexing.PostgreSql.Test +6 MeshWeaver.AccessControl.Test +5 MeshWeaver.Hosting.Grpc.Test +5 MeshWeaver.Todo.Test +4 MeshWeaver.MathDemo.Test +4 MeshWeaver.Hosting.Test +3 Memex.Portal.Shared.Test +3 MeshWeaver.Social.Test +3 MeshWeaver.Hosting.Sqlite.Test +3 MeshWeaver.Hosting.Blazor.Test +3 MeshWeaver.ContentCollections.Test +3 MeshWeaver.Documentation.Test +2 MeshWeaver.Serialization.Test +2 MeshWeaver.Markdown.Export.Test +1 MeshWeaver.TestDomain +1 MeshWeaver.Data.TestDomain +1 MeshWeaver.Hub.Fixture +1 MeshWeaver.AI.Test.FakeCli +1 MeshWeaver.Portal.E2E.Test +1 MeshWeaver.Search.Test +1 MeshWeaver.Reactive.Assertions.Test +1 MeshWeaver.DataSetReader.Test +1 MeshWeaver.Markdown.Collaboration.Test +1 MeshWeaver.Maui.E2E.Test +1 MeshWeaver.Maui.Abstractions.Test +1 MeshWeaver.Maui.Integration.Test +1 MeshWeaver.PluginImage.Test +1 MeshWeaver.Hosting.Snowflake.Test +1 MeshWeaver.Connection.SignalR.Test +1 MeshWeaver.Speech.Test +1 MeshWeaver.ContentCollections.Indexing.Test +1 MeshWeaver.Kernel.Test +EOF +) + +weights_file=$(mktemp) +trap 'rm -f "$weights_file"' EXIT +printf '%s\n' "$WEIGHTS" > "$weights_file" + +# Only Cosmos is excluded — it needs the Cosmos emulator, which is heavy to set +# up on a runner. Everything else runs (PostgreSql via Testcontainers on the +# pre-installed Docker, Orleans via in-proc TestCluster, Acme/FutuRe via dynamic +# Code-piece compilation). +# +# The weights arrive as a FILE, not `awk -v` — macOS's awk rejects a -v value +# containing newlines ("newline in string"), so the -v form silently worked on +# the runner while failing for anyone verifying a change locally. +find test -name '*.csproj' ! -path '*Cosmos*' ! -path '*/bin/*' \ + | awk -v dflt="$DEFAULT_WEIGHT" ' + FNR == NR { if (NF >= 2) { weight[$2] = $1; if (NF >= 3) parts[$2] = $3 } next } + { + name = $0; sub(/.*\//, "", name); sub(/\.csproj$/, "", name) + w = (name in weight ? weight[name] : dflt) + n = (name in parts ? parts[name] : 1) + if (n <= 1) { printf "%06d %s %s\n", w, name, $0; next } + # Each part is its own schedulable unit at 1/N the weight. + for (i = 1; i <= n; i++) + printf "%06d %s#%d/%d %s#%d/%d\n", int(w / n), name, i, n, $0, i, n + }' "$weights_file" - \ + | sort -k1,1nr -k2,2 \ + | awk -v idx="$SHARD_INDEX" -v total="$SHARD_TOTAL" ' + BEGIN { for (i = 0; i < total; i++) loads[i] = 0 } + { + best = 0 + for (i = 1; i < total; i++) if (loads[i] < loads[best]) best = i + loads[best] += $1 + 0 + if (best == idx) print $3 + } + END { + # Diagnostics on stderr — stdout is the machine-readable csproj list. + s = "" + for (i = 0; i < total; i++) s = s " " loads[i] + printf "shard-assign: shard %s/%s, weighted loads:%s\n", idx, total, s > "/dev/stderr" + }' diff --git a/.github/workflows/dotnet-test.yml b/.github/workflows/dotnet-test.yml index b07997ff99..3670ee88cb 100644 --- a/.github/workflows/dotnet-test.yml +++ b/.github/workflows/dotnet-test.yml @@ -208,11 +208,18 @@ jobs: with: tool-cache: false dotnet: false - android: true - haskell: true - large-packages: false - docker-images: true - swap-storage: true + android: true # 11 GiB, the bulk of the reclaim + haskell: true # 3.6 GiB + large-packages: false # slowest removal (~40-60 s of apt), not needed + # docker-images OFF: it prunes the runner's PRE-BAKED images (node, + # buildpack-deps — 2 GiB), none of which is the pgvector image the + # Testcontainers tests then pull anyway. Pure cost. + docker-images: false + # swap-storage OFF: deleting the swapfile on a job that runs + # memory-heavy hosts is how you turn a slow test into an OOM kill + # (Persistence.Test already carries a memory watchdog). It reclaimed + # 0 B on the measured run. + swap-storage: false - name: Setup .NET uses: actions/setup-dotnet@v5 with: @@ -226,33 +233,81 @@ jobs: # the gate explicit at the CI build step. NuGet restore advisories (NU19xx vulnerabilities) stay # warnings (WarningsNotAsErrors in Directory.Build.props) — handled via dependency overrides. run: dotnet build --no-restore -c Release -p:CIRun=true -warnaserror - - name: Package test outputs + - name: Package test outputs (one tarball per shard) + env: + SHARD_TOTAL: 6 run: | set -euo pipefail - # Only the TEST projects' bin trees travel to the shards. Each test bin - # is self-contained (dependency closure + TestData fixtures copied in), - # and the native xUnit v3 hosts need no obj/, no src/samples bins, and - # no NuGet global-packages folder (runtime NuGet-resolver tests declare - # themselves network-based and restore into the default cache). - find test -mindepth 2 -maxdepth 2 -type d -name bin > .build-paths.txt - echo "Packaging $(wc -l < .build-paths.txt) test bin trees" - # One zstd tarball (-T0 = all cores) moves through upload/download-artifact - # FAR faster than millions of loose files would. --long=27 (128 MiB match - # window) dedupes across files: the ~60 self-contained test bins each carry - # near-identical copies of the same framework + MeshWeaver DLLs, which a - # default window (8 MiB) can't see. The EXTRACT side must pass --long=27 - # too, or zstd refuses with a window-size error. - tar -I 'zstd -T0 --long=27' -cf build-output.tar.zst -T .build-paths.txt - ls -lh build-output.tar.zst - - name: "Upload artifact: build output" + # ── Why PER SHARD and not one tarball for everyone ────────────────── + # The single 3.3 GB artifact was downloaded by all 6 shards — 20 GB of + # transfer per run, and the artifact service's throughput varies ~10x + # run to run (measured 35 s … 318 s for the same bytes). That download + # WAS the shard imbalance: on run 30903428924 the long-pole shard spent + # 318 s downloading and a perfectly ordinary 370 s testing, while the + # fastest shard downloaded the same bytes in 35 s. + # + # Packing per shard costs the same compression work (same bytes, split + # six ways) plus the cross-shard dedupe we give up, and it turns a 3.3 GB + # download into ~1/6 of one. + # + # Only the TEST projects' bin trees travel. Each test bin is + # self-contained (dependency closure + TestData fixtures copied in — e.g. + # AI.Test's CopyFakeCli target puts FakeCli under its OWN bin), and the + # native xUnit v3 hosts need no obj/, no src/samples bins, and no NuGet + # global-packages folder (runtime NuGet-resolver tests declare themselves + # network-based and restore into the default cache). + # + # 🚨 The SAME script decides the assignment here and in the test job. If + # they ever diverged, a shard would run a project whose bin it never + # downloaded. + for ((i=0; i#/`. Strip the suffix: the + # SAME bin has to travel to every shard running one of its parts. + csproj="${entry%%#*}" + d="$(dirname "$csproj")/bin" + # A project that produced no output (helper with nothing to + # build) simply contributes nothing to its shard's tarball. + [ -d "$d" ] && echo "$d" + done | sort -u > ".build-paths-$i.txt" + # zstd -T0 = all cores. --long=27 (128 MiB match window) dedupes ACROSS + # files: the self-contained test bins each carry near-identical copies + # of the same framework + MeshWeaver DLLs, which the default 8 MiB + # window can't see. The EXTRACT side must pass --long=27 too, or zstd + # refuses with a window-size error. + # 🚨 An empty path list would produce an empty tarball, and a shard that + # extracts nothing finds no .dll — which the run loop would classify + # as "helper project, nothing to run", exit=0, GREEN. A whole suite could + # pass having executed zero tests. Fail here instead. + if [ ! -s ".build-paths-$i.txt" ]; then + echo "::error::shard $i got no bin trees to pack — shard-assign.sh failed or the build produced no test output." + exit 1 + fi + tar -I 'zstd -T0 --long=27' -cf "build-output-$i.tar.zst" -T ".build-paths-$i.txt" + echo "shard $i: $(wc -l < ".build-paths-$i.txt") bin trees, $(du -h "build-output-$i.tar.zst" | cut -f1)" + done + # One artifact per shard. Six near-identical steps because a step cannot be + # generated in a loop — keep them in sync with SHARD_TOTAL above and with the + # test job's matrix. + - name: "Upload artifact: shard 0" uses: actions/upload-artifact@v6 - with: - name: build-output - path: build-output.tar.zst - # Intermediate, consumed within this run only. - retention-days: 1 - # Already zstd-compressed — skip the redundant zip pass. - compression-level: 0 + with: { name: build-output-0, path: build-output-0.tar.zst, retention-days: 1, compression-level: 0 } + - name: "Upload artifact: shard 1" + uses: actions/upload-artifact@v6 + with: { name: build-output-1, path: build-output-1.tar.zst, retention-days: 1, compression-level: 0 } + - name: "Upload artifact: shard 2" + uses: actions/upload-artifact@v6 + with: { name: build-output-2, path: build-output-2.tar.zst, retention-days: 1, compression-level: 0 } + - name: "Upload artifact: shard 3" + uses: actions/upload-artifact@v6 + with: { name: build-output-3, path: build-output-3.tar.zst, retention-days: 1, compression-level: 0 } + - name: "Upload artifact: shard 4" + uses: actions/upload-artifact@v6 + with: { name: build-output-4, path: build-output-4.tar.zst, retention-days: 1, compression-level: 0 } + - name: "Upload artifact: shard 5" + uses: actions/upload-artifact@v6 + with: { name: build-output-5, path: build-output-5.tar.zst, retention-days: 1, compression-level: 0 } # ───────────────────────────── TEST ────────────────────────────── # Matrix-sharded: the ~45 test projects are split across SHARD_TOTAL parallel @@ -262,7 +317,11 @@ jobs: test: name: "Run tests (shard ${{ matrix.shard }})" needs: build - timeout-minutes: 30 + # HARD CAP on a shard. Each project already has its own `timeout 8m` below, but that + # only bounds a project that is RUNNING — it cannot bound a shard that wedges between + # projects. 20 min is ~1.5x the slowest observed shard (14m36s) with the rebalanced + # weights, so a shard that hits this is stuck, not slow. + timeout-minutes: 20 runs-on: ubuntu-latest strategy: # Don't cancel sibling shards when one fails — we want every shard's @@ -280,25 +339,36 @@ jobs: with: tool-cache: false dotnet: false - android: true - haskell: true - large-packages: false - docker-images: true - swap-storage: true + android: true # 11 GiB, the bulk of the reclaim + haskell: true # 3.6 GiB + large-packages: false # slowest removal (~40-60 s of apt), not needed + # docker-images OFF: it prunes the runner's PRE-BAKED images (node, + # buildpack-deps — 2 GiB), none of which is the pgvector image the + # Testcontainers tests then pull anyway. Pure cost. + docker-images: false + # swap-storage OFF: deleting the swapfile on a job that runs + # memory-heavy hosts is how you turn a slow test into an OOM kill + # (Persistence.Test already carries a memory watchdog). It reclaimed + # 0 B on the measured run. + swap-storage: false - name: Setup .NET uses: actions/setup-dotnet@v5 with: dotnet-version: 10.0.x - - name: "Download artifact: build output" + - name: "Download artifact: this shard's build output" uses: actions/download-artifact@v6 with: - name: build-output + # Only THIS shard's test bins (~1/6 of the output). The old single + # 3.3 GB artifact was pulled by every shard and its 35–318 s transfer + # variance, not the tests, decided which shard was the long pole. + name: build-output-${{ matrix.shard }} - name: Extract build output run: | set -euo pipefail # --long=27 matches the pack side (window > 8 MiB default; required to decode). - tar -I 'zstd --long=27' -xf build-output.tar.zst - rm -f build-output.tar.zst + tar -I 'zstd --long=27' -xf "build-output-${{ matrix.shard }}.tar.zst" + rm -f "build-output-${{ matrix.shard }}.tar.zst" + df -h / | tail -1 | awk '{print "disk after extract: " $4 " free of " $2}' - name: Run Tests env: DOTNET_ENVIRONMENT: Development @@ -326,83 +396,47 @@ jobs: : > test/test-results.log mkdir -p /tmp/coredumps - # ── Shard assignment: measured weights + greedy LPT ────────────────── - # Weights are wall-clock seconds measured on ubuntu-latest (run - # 29213734083, 2026-07-13). Explicit and ORDER-INDEPENDENT: the previous - # `counter % SHARD_TOTAL` over a sorted csproj list silently depended on - # locale byte order — C-locale sorts MeshWeaver.AI.Test BEFORE - # MeshWeaver.Acme.Test ('I' 0x49 < 'c' 0x63), which stacked AI + Orleans - # (two of the three ALC-heaviest projects) onto shard 0, the exact - # pairing the scheme existed to prevent, making shard 0 the ~19-minute - # long pole of every run. Greedy LPT (heaviest first onto the currently - # least-loaded shard, ties to the lowest shard id) balances by measured - # cost regardless of enumeration order. - # Unlisted projects default to 10 s. When a project's runtime changes - # materially, re-measure (the per-project markers in this step's log ARE - # the measurement) and update the number here. - declare -A WEIGHT=( - [MeshWeaver.AI.Test]=340 - [MeshWeaver.Hosting.Orleans.Test]=200 - [MeshWeaver.Hosting.Monolith.Test]=200 - [MeshWeaver.Persistence.Test]=165 - [MeshWeaver.Threading.Test]=105 - [MeshWeaver.Hosting.PostgreSql.Test]=105 - [MeshWeaver.Security.Test]=90 - [MeshWeaver.FutuRe.Test]=60 - [MeshWeaver.Content.Test]=60 - [MeshWeaver.Acme.Test]=55 - [MeshWeaver.Query.Test]=55 - [MeshWeaver.Autocomplete.Test]=50 - [MeshWeaver.Auth.Test]=40 - [MeshWeaver.GitSync.Test]=35 - [MeshWeaver.NodeOperations.Test]=35 - [MeshWeaver.InstanceSync.Test]=30 - [MeshWeaver.Graph.Test]=30 - [MeshWeaver.Data.Test]=25 - [MeshWeaver.Messaging.Hub.Test]=25 - [MeshWeaver.Courses.Test]=20 - [MeshWeaver.Layout.Test]=20 - [MeshWeaver.PluginCatalog.Test]=20 - [MeshWeaver.PluginTester.Test]=60 - [MeshWeaver.Markdown.Test]=20 - [MeshWeaver.PathResolution.Test]=15 - [MeshWeaver.PensionFund.Test]=15 - [MeshWeaver.ContentCollections.Indexing.PostgreSql.Test]=15 - [MeshWeaver.ContentCollections.Indexing.Graph.Test]=15 - [MeshWeaver.MemexTemplate.Test]=15 - ) - DEFAULT_WEIGHT=10 - - assignment=$(mktemp) - for csproj in $(find test -name '*.csproj' \ - ! -path '*Cosmos*' \ - ! -path '*/bin/*' \ - | sort); do - name=$(basename "$csproj" .csproj) - w=${WEIGHT[$name]:-$DEFAULT_WEIGHT} - printf '%05d %s %s\n' "$w" "$name" "$csproj" - done | sort -k1,1nr -k2,2 > "$assignment" - - declare -a loads - for ((i=0; i#/`. + # Each part runs in its own process on its own runner, so the halves share + # nothing — no ports, no containers, no state. maxParallelThreads:1 is + # per-process and is unaffected. + csproj="${entry%%#*}" + partspec="${entry#*#}" + if [ "$partspec" = "$entry" ]; then part=1; parts=1; else + part="${partspec%%/*}"; parts="${partspec##*/}" + fi name=$(basename "$csproj" .csproj) + label="$name" + [ "$parts" -gt 1 ] && label="$name (part $part/$parts)" dir=$(dirname "$csproj")/bin/Release/net10.0 # Helper projects (TestDomain, Hub.Fixture, AI.Test.FakeCli) ship no # xunit.v3 host — nothing to run. Recorded so the log shows they were # consciously skipped, not lost. + # 🚨 Distinguish the two "nothing to run" cases. A missing bin TREE means this + # shard's artifact never carried the project (packaging or assignment + # mismatch) — reporting exit=0 there would let a shard read GREEN having run + # nothing. Only a present bin with no xunit host is a genuine helper project. + if [ ! -d "$(dirname "$csproj")/bin" ]; then + echo "::error::$name: bin tree absent — this shard's artifact did not contain it (build/test shard-assign mismatch)." + echo "[CI] $name exit=97 MISSING_BIN (artifact did not contain this project)" >> test/test-results.log + continue + fi if [ ! -f "$dir/$name.dll" ] || [ ! -f "$dir/xunit.v3.core.dll" ]; then echo "[CI] $name exit=0 (helper project — no xUnit v3 test host)" >> test/test-results.log continue @@ -420,11 +454,30 @@ jobs: # CWD = the project's bin dir, exactly like `dotnet test` ran the # host: appsettings.json + xunit.runner.json + TestData resolve there. mkdir -p "$dir/TestResults" + # For a split project, enumerate the assembly's classes and take every Nth from + # the SORTED list. Deterministic, and self-maintaining: a newly added test class + # is distributed automatically without anyone editing the weight table. + classargs=() + trxname="$name" + if [ "$parts" -gt 1 ]; then + mapfile -t allclasses < <( cd "$dir" && dotnet "$name.dll" -list classes 2>/dev/null \ + | grep -E "^[A-Za-z_][A-Za-z0-9_.]*$" | sort ) + if [ "${#allclasses[@]}" -eq 0 ]; then + echo "::error::$label: could not enumerate test classes — refusing to run a partial shard silently." + echo "[CI] $name exit=96 NO_CLASSES (class enumeration failed for a split project)" >> test/test-results.log + continue + fi + for ((ci=part-1; ci<${#allclasses[@]}; ci+=parts)); do + classargs+=(-class "${allclasses[$ci]}") + done + trxname="$name.part$part" + echo "$label: ${#classargs[@]} of ${#allclasses[@]} classes" + fi ( cd "$dir" && timeout --signal=TERM --kill-after=30s 8m \ - dotnet "$name.dll" -trx "TestResults/$name.trx" ) + dotnet "$name.dll" -trx "TestResults/$trxname.trx" "${classargs[@]}" ) rc=$? if [ "$rc" = "124" ] || [ "$rc" = "137" ]; then - marker="[CI] $name exit=$rc TIMEOUT (6m wall-clock cap hit — likely fixture/init hang)" + marker="[CI] $label exit=$rc TIMEOUT (6m wall-clock cap hit — likely fixture/init hang)" else marker="[CI] $name exit=$rc" fi @@ -437,6 +490,29 @@ jobs: if: always() run: | mkdir -p collected-logs + # ── Symbols for any host that died on a SIGNAL ────────────────────────── + # A core dump without symbols answers "it segfaulted" and almost nothing + # else: gdb/lldb cannot name a single frame, so the fault site stays + # unknown and every hypothesis stays unfalsifiable. That is exactly where + # the 2026-08-04 FutuRe SIGSEGV investigation ran out of road — the dump + # was collected, but nothing to symbolise it against. + # + # Only for projects whose marker shows a signal death (exit=13x), because + # a test bin is ~100-800 MB and we must not pay that on every green run. + # dll + pdb only (no runtimeconfig/deps/content) keeps it to the managed + # symbols a stack walk actually needs. Also record the runtime patch the + # dump was produced by — the matching DAC is what makes SOS work at all. + while read -r sigproj; do + [ -z "$sigproj" ] && continue + bindir=$(find test -type d -path "*/$sigproj/bin/Release/net10.0" | head -1) + [ -z "$bindir" ] && continue + echo "Collecting symbols for signal-killed $sigproj from $bindir" + mkdir -p "collected-logs/symbols-$sigproj" + find "$bindir" -maxdepth 1 \( -name '*.dll' -o -name '*.pdb' \) \ + -exec cp {} "collected-logs/symbols-$sigproj/" \; 2>/dev/null || true + dotnet --list-runtimes > "collected-logs/symbols-$sigproj/_runtimes.txt" 2>/dev/null || true + done < <(grep -oE '\[CI\] [A-Za-z0-9._]+ exit=13[0-9]' test/test-results.log 2>/dev/null \ + | awk '{print $2}' | sort -u) # Find all test-logs directories and collect their contents without renaming find . -path "*/bin/*/test-logs/*.log" -type f -exec cp {} collected-logs/ \; 2>/dev/null || true # The MonolithMeshTestBase phase trace + dispose trace live in the diff --git a/AGENTS.md b/AGENTS.md index 1896701203..38d5768aed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,6 +107,41 @@ gh pr merge PR_NUMBER --merge **If `FORBIDDEN`**: re-authenticate with `! gh auth login`. +### 🚨 "Is the build finished?" — filter by WORKFLOW, never wait for all check suites + +The merge gate is "CI green", so you must be able to tell when CI is actually *done*. **Never wait +for every check suite on the commit to reach `COMPLETED` — that never happens.** GitHub creates a +check suite for *every* installed App holding the Checks permission, and an App that posts no check +runs leaves its suite at `queued` forever. This repo has exactly that: the **Azure Boards** App +(installed 2026-08-03, `latest_check_runs_count: 0`, zero `AB#` references in the history) puts a +permanently-`queued` suite on every commit. It never blocks a merge — `mergeStateStatus` stays +`CLEAN` — but a naive "all suites complete" poll hangs until its timeout and then looks like CI +never finished. + +**Poll the `MeshWeaver Build and Test` suite specifically:** + +```bash +gh api graphql -f query='query($o:String!,$r:String!,$p:Int!){repository(owner:$o,name:$r){pullRequest(number:$p){ + mergeStateStatus commits(last:1){nodes{commit{checkSuites(first:20){nodes{ + status conclusion workflowRun{workflow{name}}}}}}}}}}' \ + -f o=Systemorph -f r=MeshWeaver -F p=PR_NUMBER \ + --jq '[.data.repository.pullRequest.commits.nodes[0].commit.checkSuites.nodes[] + | select(.workflowRun.workflow.name=="MeshWeaver Build and Test") + | "\(.status)/\(.conclusion)"]' +``` + +Merge only on `COMPLETED/SUCCESS` for that suite. Two further gotchas: + +- **`Consolidate test results` is the required check** — and the ONLY one to require. `Build solution + (once)` and the shards are legitimately *skipped* when a run reuses an already-green tree, and a + skipped required check blocks the merge forever. +- **Don't poll with `gh run watch` / `gh pr checks --watch`.** They hammer REST and burn the shared + token budget into 403s that masquerade as CI-red. Use the GraphQL query above, or a `Monitor` that + filters on the workflow name. + +**Also check the clock before declaring a job stuck.** GitHub timestamps are UTC; a local-time +comparison makes a healthy 7-minute build look like a 33-minute hang. `date -u` first. + ## 🚨 Postgres: One Schema Per Partition **`public.mesh_nodes` is empty by design.** Data lives in per-partition schemas (`acme.mesh_nodes`, `rbuergi.mesh_nodes`, etc.). @@ -543,7 +578,23 @@ Before building NodeTypes, data models, layout areas, or CSV loaders — read th **Never assert "exactly N change events"** on a stream backed by pg_notify or any change feed that can race the initial-snapshot path. Filter on the emission shape (e.g. `.Where(c => c.ChangeType == QueryChangeType.Initial)`), not the count. -xUnit v3 config (`xunit.runner.json`): `parallelizeAssembly: false`, `maxParallelThreads: 1`, `methodTimeout: 60000ms`. +xUnit v3 config (`test/xunit.runner.json`): `parallelizeAssembly: false`, +`parallelizeTestCollections: false`, `maxParallelThreads: 1`, `methodTimeout: 30000` (30 s). + +**🚨 `methodTimeout` only bounds time spent INSIDE a test method.** A wedge in fixture +construction, class init, or teardown is outside it and runs unbounded — on 2026-08-04 an +orphaned local run sat at a pegged core for 25+ minutes, and together with a leaking e2e +container drove the colima VM to 128 MB free / load average 195, OOM-killing unrelated +containers. So **always give a local test run its own wall-clock cap**: + +```bash +timeout 20m dotnet test test/MeshWeaver.Data.Test --no-restore # hard cap, always +``` + +CI has the equivalent at two levels: `timeout 8m` per project inside the shard loop, and +`timeout-minutes: 20` on the shard job itself for a wedge BETWEEN projects (which the +per-project cap cannot see). If a run hits either, it is stuck — do not raise the bound, +find what is not completing (AGENTS.md → "No band-aids"). Full guidance: [WritingTests.md](src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md) diff --git a/src/MeshWeaver.Hosting.Monolith.TestBase/MonolithMeshTestBase.cs b/src/MeshWeaver.Hosting.Monolith.TestBase/MonolithMeshTestBase.cs index 85957d244b..df1ba305f5 100644 --- a/src/MeshWeaver.Hosting.Monolith.TestBase/MonolithMeshTestBase.cs +++ b/src/MeshWeaver.Hosting.Monolith.TestBase/MonolithMeshTestBase.cs @@ -290,7 +290,13 @@ private static void TestPhaseTrace(string testClass, string phase, long? elapsed { try { - var line = $"{DateTime.UtcNow:HH:mm:ss.fff} [{testClass}] {phase}" + // pid on every line. The file is shared by EVERY test host in a CI shard (one + // process per project, all appending to the same fixed path), so without it the + // tail of the log belongs to whichever project finished last — not necessarily + // the one that crashed. A core dump is named `dotnet-.dmp`, so `grep pid=` + // is what ties the dump to the trace. Diagnosing the 2026-08-04 FutuRe SIGSEGV + // started by reading another project's lines for exactly this reason. + var line = $"{DateTime.UtcNow:HH:mm:ss.fff} pid={Environment.ProcessId} [{testClass}] {phase}" + (elapsedMs.HasValue ? $" elapsed={elapsedMs}ms" : "") + (extra is null ? "" : $" {extra}"); lock (TestTraceLogLock) diff --git a/src/MeshWeaver.Messaging.Hub/MessageHub.cs b/src/MeshWeaver.Messaging.Hub/MessageHub.cs index 218b3c227c..fdbd73e505 100644 --- a/src/MeshWeaver.Messaging.Hub/MessageHub.cs +++ b/src/MeshWeaver.Messaging.Hub/MessageHub.cs @@ -119,9 +119,11 @@ static MessageHub() private static void DisposeTrace(Address address, string phase, long elapsedMs, string? extra = null) { if (DisposeTraceChannel is null) return; + // pid: this file is shared by every test host in a CI shard, and a core dump is named + // `dotnet-.dmp` — without it you cannot tell which process's lines you are reading. var line = extra is null - ? $"{DateTime.UtcNow:HH:mm:ss.fff} {address} {phase} elapsed={elapsedMs}ms" - : $"{DateTime.UtcNow:HH:mm:ss.fff} {address} {phase} elapsed={elapsedMs}ms {extra}"; + ? $"{DateTime.UtcNow:HH:mm:ss.fff} pid={Environment.ProcessId} {address} {phase} elapsed={elapsedMs}ms" + : $"{DateTime.UtcNow:HH:mm:ss.fff} pid={Environment.ProcessId} {address} {phase} elapsed={elapsedMs}ms {extra}"; // Non-blocking: drops on full so a stuck writer never delays dispose. DisposeTraceChannel.Writer.TryWrite(line); }