From 7c87709a1a4c63fca31aa44f64baf8f966d43660 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:07:11 -0400 Subject: [PATCH 1/3] The nightly release publishes only after the Darling PostgreSQL tests pass Moves every release-publishing step (the GitHub release, the linux tarball upload, the ghcr container push, cosign signing, the build provenance attestation) out of the build and linux jobs and into one new publish job that needs build, linux and the Darling PostgreSQL tests job (darling-pg) and runs only when all three succeed. build and linux keep building and now upload their outputs as workflow artifacts; publish downloads them and publishes exactly what those jobs built. Adds a workflow_dispatch publish boolean input (default true) so a diagnostic re-run can skip publishing. darling-pg's failure artifact now also collects the gated upgrade tests' own temp-cluster server logs, not just the fixed cluster's log. Refs #4336 --- .github/workflows/nightly.yml | 224 +++++++++++++++++++++++++++------- 1 file changed, 179 insertions(+), 45 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3e86913a02..d06b6e8380 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,16 +11,17 @@ on: type: boolean required: false default: false + publish: + description: 'Publish the release artifacts (gh release, the ghcr image push) once build, linux and the Darling PostgreSQL tests all succeed. Set false for a diagnostic re-run that should build and test without publishing.' + type: boolean + required: false + default: true +# Read-only by default; the release-publishing steps (gh release, the ghcr push, cosign, the +# provenance attestation) run only in the publish job below, which declares its own write +# permissions scoped to itself. No other job needs to write anything. permissions: - contents: write - # #1804: the linux job pushes the nightly container image to ghcr. - packages: write - # Sigstore keyless signing + GitHub provenance for the linux artifacts: id-token lets the job - # obtain its OIDC identity (Fulcio issues the short-lived signing cert against it), attestations - # lets attest-build-provenance store the tarball's provenance. Neither grants anything else. - id-token: write - attestations: write + contents: read jobs: # Scheduled workflows always execute the DEFAULT branch's copy of this file, while nightly @@ -304,36 +305,32 @@ jobs: Write-Host "Checksums:" $checksums | ForEach-Object { Write-Host $_ } - - name: Delete previous nightly release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh release delete nightly --yes --cleanup-tag 2>$null; exit 0 - shell: pwsh - - - name: Create nightly release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: pwsh - run: | - $version = "${{ steps.version.outputs.VERSION }}" - $sha = git rev-parse --short HEAD - $body = @" - Automated nightly build from ``dev`` branch. - - **Version:** ``$version`` - **Commit:** ``$sha`` - **Built:** $(Get-Date -Format "yyyy-MM-dd HH:mm UTC") + # The release-publishing steps (gh release create, the linux upload, the ghcr push) moved to + # the publish job below, which waits on build, linux AND darling-pg. This job hands off its + # windows release assets (the zips, the Setup.exe, SHA256SUMS.txt) and the version string as a + # workflow artifact instead of publishing them itself, so a nightly can no longer publish + # while the Darling PostgreSQL tests are red: publish downloads exactly what this job built + # and cannot run until darling-pg reports success. + - name: Upload windows release assets + uses: actions/upload-artifact@v6 + with: + name: windows-release-assets + path: | + releases/*.zip + releases/*.exe + releases/SHA256SUMS.txt + if-no-files-found: error - > These builds include the latest changes and may be unstable. - > For production use, download the [latest stable release](https://github.com/erikdarlingdata/PerformanceMonitor/releases/latest). - "@ + - name: Save the nightly version string + shell: bash + run: echo "${{ steps.version.outputs.VERSION }}" > windows-version.txt - gh release create nightly ` - --target dev ` - --title "Nightly Build ($version)" ` - --notes $body ` - --prerelease ` - releases/*.zip releases/*.exe releases/SHA256SUMS.txt + - name: Upload the nightly version string + uses: actions/upload-artifact@v6 + with: + name: windows-version + path: windows-version.txt + if-no-files-found: error # Darling.Tests has live-PostgreSQL tests gated on env vars that the per-PR/push build never # sets, so regular CI runs only the ungated subset. This job stands up a throwaway PostgreSQL @@ -497,6 +494,12 @@ jobs: if (Test-Path "$bin\pg_ctl.exe") { & "$bin\pg_ctl.exe" -D $dataDir -m fast -w stop } exit 0 + # The gated upgrade-path tests stand up their OWN throwaway clusters under the runner's temp + # directory (Directory.CreateTempSubdirectory, prefixes like "darling-pg17boot-" and + # "darling-managedconf-"), separate from the fixed darling-pgdata cluster this job starts + # above. When one of those tests fails, its cluster's own server log (pg_ctl's -l file, + # named "logfile" inside each temp root, plus any "log\\" ring directory a managed-conf boot + # writes) is the only record of what postgres itself said, and it wasn't captured before. - name: Upload PG log and test results on failure if: failure() uses: actions/upload-artifact@v6 @@ -504,6 +507,9 @@ jobs: name: darling-pg-failure path: | ${{ runner.temp }}/darling-pg.log + ${{ runner.temp }}/darling-*/**/logfile + ${{ runner.temp }}/darling-*/**/log/** + ${{ runner.temp }}/darling-*/**/pg_log/** TestResults/ if-no-files-found: ignore @@ -560,12 +566,146 @@ jobs: tar -C publish/DarlingService-linux -czf "releases/PerformanceMonitorDarling-linux-x64-${version}.tar.gz" . (cd releases && sha256sum "PerformanceMonitorDarling-linux-x64-${version}.tar.gz" > SHA256SUMS-linux.txt && cat SHA256SUMS-linux.txt) + # The container image build needs THIS checkout (docker build's context is the repo root, + # #1804), so it stays here rather than moving to publish, which has no source tree. Only the + # PUSH, the signing and the release upload are release-publishing steps, and those move to + # publish below: this job saves the built image to a tar (docker save) and hands it, the + # linux tarball and its checksum file to publish as workflow artifacts. Nothing here talks to + # ghcr or the nightly release. + - name: Build container image (ghcr tags, not pushed) + shell: bash + run: | + set -euo pipefail + version="${{ steps.version.outputs.VERSION }}" + image="ghcr.io/${{ github.repository_owner }}/performancemonitor-darling" + # VERSION reaches the publish that runs INSIDE the image build (#3455): an image tagged + # with the nightly string whose service answers the clean release string is the zip + # defect again, one artifact over. The Dockerfile defaults the arg to empty, so + # build.yml's release image and any hand-run docker build keep publishing against the + # clean declared version. + docker build -f Darling/Dockerfile --build-arg VERSION="${version}" -t "${image}:nightly" -t "${image}:${version}" . + mkdir -p releases + docker save "${image}:nightly" "${image}:${version}" -o releases/darling-container-image.tar + + - name: Upload linux release assets + uses: actions/upload-artifact@v6 + with: + name: linux-release-assets + path: | + releases/PerformanceMonitorDarling-linux-x64-*.tar.gz + releases/SHA256SUMS-linux.txt + if-no-files-found: error + + - name: Upload container image + uses: actions/upload-artifact@v6 + with: + name: darling-container-image + path: releases/darling-container-image.tar + if-no-files-found: error + + # ── Publish (#4336 follow-up) ───────────────────────────────────────────────────────────────── + # The one job that talks to the outside world: the GitHub release, the ghcr push, the signature, + # the provenance attestation. It downloads exactly what build and linux built above — nothing + # here rebuilds or republishes — and it needs all three of build, linux and darling-pg to have + # SUCCEEDED, so a red Darling PostgreSQL run can no longer ship a release built on top of it. The + # workflow_dispatch `publish` input is a second, independent gate for a diagnostic re-run that + # wants the build and test coverage without touching the release. + publish: + needs: [build, linux, darling-pg] + if: | + needs.build.result == 'success' && + needs.linux.result == 'success' && + needs.darling-pg.result == 'success' && + (github.event_name != 'workflow_dispatch' || inputs.publish != false) + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: write + # #1804: pushes the nightly container image to ghcr. + packages: write + # Sigstore keyless signing + GitHub provenance for the linux artifact: id-token lets this job + # obtain its OIDC identity (Fulcio issues the short-lived signing cert against it), + # attestations lets attest-build-provenance store the tarball's provenance. + id-token: write + attestations: write + + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.ref_name || 'dev' }} + + - name: Download windows release assets + uses: actions/download-artifact@v6 + with: + name: windows-release-assets + path: releases + + - name: Download the nightly version string + uses: actions/download-artifact@v6 + with: + name: windows-version + path: . + + - name: Download linux release assets + uses: actions/download-artifact@v6 + with: + name: linux-release-assets + path: releases + + - name: Download container image + uses: actions/download-artifact@v6 + with: + name: darling-container-image + path: releases + + - name: Read the nightly version string + id: version + shell: bash + run: | + set -euo pipefail + version="$(cat windows-version.txt)" + [ -n "${version}" ] || { echo "::error::windows-version.txt was empty."; exit 1; } + echo "VERSION=${version}" >> "$GITHUB_OUTPUT" + echo "Nightly version: ${version}" + + - name: Delete previous nightly release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release delete nightly --yes --cleanup-tag || true + + - name: Create nightly release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + version="${{ steps.version.outputs.VERSION }}" + sha="$(git rev-parse --short HEAD)" + body=$(cat < These builds include the latest changes and may be unstable. + > For production use, download the [latest stable release](https://github.com/erikdarlingdata/PerformanceMonitor/releases/latest). + EOF + ) + + gh release create nightly \ + --target dev \ + --title "Nightly Build (${version})" \ + --notes "${body}" \ + --prerelease \ + releases/*.zip releases/*.exe releases/SHA256SUMS.txt + - name: Upload linux artifact to the nightly release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh release upload nightly releases/PerformanceMonitorDarling-linux-x64-*.tar.gz releases/SHA256SUMS-linux.txt --clobber - - name: Build and push container image (ghcr, :nightly) + - name: Push container image (ghcr, :nightly) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash @@ -574,20 +714,14 @@ jobs: version="${{ steps.version.outputs.VERSION }}" image="ghcr.io/${{ github.repository_owner }}/performancemonitor-darling" echo "$GH_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - # VERSION reaches the publish that runs INSIDE the image build (#3455): an image tagged - # with the nightly string whose service answers the clean release string is the zip - # defect again, one artifact over. The Dockerfile defaults the arg to empty, so - # build.yml's release image and any hand-run docker build keep publishing against the - # clean declared version. - docker build -f Darling/Dockerfile --build-arg VERSION="${version}" -t "${image}:nightly" -t "${image}:${version}" . + docker load -i releases/darling-container-image.tar docker push "${image}:nightly" docker push "${image}:${version}" # Keyless Sigstore signing: Fulcio issues a short-lived certificate against this job's OIDC # identity and the signature lands in ghcr next to the image, logged in Rekor. No keys exist # anywhere to manage or leak — the signature attests "built by this repository's workflow", - # which is the claim a container consumer actually wants verified. (SignPath's cosign support - # is edition-gated; this path has no subscription dependency.) Verify: + # which is the claim a container consumer actually wants verified. Verify: # cosign verify ghcr.io/erikdarlingdata/performancemonitor-darling:nightly \ # --certificate-identity-regexp 'github.com/erikdarlingdata/PerformanceMonitor' \ # --certificate-oidc-issuer https://token.actions.githubusercontent.com From 666cc501dc7d86066b6c022711625f8d638acabe Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:10:00 -0400 Subject: [PATCH 2/3] Collect the gated tests' cluster logs from the runner's user temp directory --- .github/workflows/nightly.yml | 36 ++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d06b6e8380..e8f6fc54e6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -500,6 +500,38 @@ jobs: # above. When one of those tests fails, its cluster's own server log (pg_ctl's -l file, # named "logfile" inside each temp root, plus any "log\\" ring directory a managed-conf boot # writes) is the only record of what postgres itself said, and it wasn't captured before. + # Those clusters land under %TEMP% (the Windows per-user temp directory), NOT + # ${{ runner.temp }} (RUNNER_TEMP) - the two are different directories on this runner - so + # this step copies them into RUNNER_TEMP first, preserving relative paths, before the + # upload step below (which can only reach RUNNER_TEMP). A test's own "finally" block may + # already have deleted its temp root by the time this runs, so this only catches clusters + # that survive; the test's own failure message (the in-place upgrade test appends the + # upgraded server's own log to its assertion) remains the primary record. + - name: Collect the gated tests' own cluster logs + if: failure() + shell: pwsh + run: | + $dest = "$env:RUNNER_TEMP\darling-cluster-logs" + New-Item -ItemType Directory -Force -Path $dest | Out-Null + $maxBytes = 50MB + $copied = 0 + $roots = Get-ChildItem -Path $env:TEMP -Directory -Filter "darling-*" -ErrorAction SilentlyContinue + foreach ($root in $roots) { + $files = Get-ChildItem -Path $root.FullName -Recurse -File -ErrorAction SilentlyContinue | Where-Object { + $_.Name -eq "logfile" -or $_.Name -like "*.log" -or + $_.FullName -match "\\log\\" -or $_.FullName -match "\\pg_log\\" + } + foreach ($file in $files) { + if ($file.Length -gt $maxBytes) { continue } + $relative = $file.FullName.Substring($env:TEMP.Length).TrimStart("\") + $target = Join-Path $dest $relative + New-Item -ItemType Directory -Force -Path (Split-Path $target -Parent) | Out-Null + Copy-Item -Path $file.FullName -Destination $target -ErrorAction SilentlyContinue + $copied++ + } + } + Write-Host "Collected $copied gated-test cluster log file(s) into $dest" + - name: Upload PG log and test results on failure if: failure() uses: actions/upload-artifact@v6 @@ -507,9 +539,7 @@ jobs: name: darling-pg-failure path: | ${{ runner.temp }}/darling-pg.log - ${{ runner.temp }}/darling-*/**/logfile - ${{ runner.temp }}/darling-*/**/log/** - ${{ runner.temp }}/darling-*/**/pg_log/** + ${{ runner.temp }}/darling-cluster-logs/** TestResults/ if-no-files-found: ignore From 7908818ea1a500839466dbc2ed3307666563b1b3 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:35:14 -0400 Subject: [PATCH 3/3] Name the built commit in the nightly release, and harden the publish job --- .github/workflows/nightly.yml | 88 +++++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 15 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e8f6fc54e6..56dd7688bd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -320,17 +320,31 @@ jobs: releases/*.exe releases/SHA256SUMS.txt if-no-files-found: error + retention-days: 3 - name: Save the nightly version string shell: bash run: echo "${{ steps.version.outputs.VERSION }}" > windows-version.txt + # The commit build actually checked out, so publish can name the release after the tree it + # built rather than dev's tip at publish time (20-60 minutes later, once darling-pg is done). + - name: Save the built commit SHA + shell: bash + run: | + set -euo pipefail + sha="$(git rev-parse HEAD)" + echo "${sha}" > windows-commit-sha.txt + echo "Built commit: ${sha}" + - name: Upload the nightly version string uses: actions/upload-artifact@v6 with: name: windows-version - path: windows-version.txt + path: | + windows-version.txt + windows-commit-sha.txt if-no-files-found: error + retention-days: 3 # Darling.Tests has live-PostgreSQL tests gated on env vars that the per-PR/push build never # sets, so regular CI runs only the ungated subset. This job stands up a throwaway PostgreSQL @@ -352,8 +366,8 @@ jobs: steps: # Scheduled runs always test dev (schedules execute from the default branch, so ref_name # would be main). A manual dispatch tests the DISPATCHED ref — the only way to validate a - # branch's gated-pg test changes before merge; the artifact-publishing build job stays - # pinned to dev either way, so a branch dispatch can never ship branch binaries. + # branch's gated-pg test changes before merge; the build job stays pinned to dev either + # way, so a branch dispatch can never ship branch binaries. - uses: actions/checkout@v7 with: ref: ${{ github.event_name == 'workflow_dispatch' && github.ref_name || 'dev' }} @@ -542,17 +556,21 @@ jobs: ${{ runner.temp }}/darling-cluster-logs/** TestResults/ if-no-files-found: ignore + overwrite: true # ── Linux artifact + container image (#1804) ───────────────────────────────────────────────────── - # Runs AFTER the windows build job so the nightly release exists to upload into. Publishes the - # linux-x64 service tar.gz with its own checksum file (the windows job owns SHA256SUMS.txt; a - # cross-job rewrite of one file is a race), and pushes the service image to ghcr tagged :nightly. - # The bundled pg-runtime is deliberately absent from the linux artifact — the compose distribution - # pairs the service with the official timescale/timescaledb image, and managed mode stays Windows. + # Runs in parallel with the windows build job — it shares only the check job's nightly stamp, + # not any windows artifact. Publishes the linux-x64 service tar.gz with its own checksum file + # (the windows job owns SHA256SUMS.txt; a cross-job rewrite of one file is a race), and builds + # the service container image, saving it to a tar for the publish job below to push — this job + # talks to neither the release nor ghcr. The bundled pg-runtime is deliberately absent from the + # linux artifact — the compose distribution pairs the service with the official + # timescale/timescaledb image, and managed mode stays Windows. linux: - # check as well as build: build already depends on check, so this adds no ordering, only - # access to the one stamp both jobs name their artifacts with (#3390). - needs: [check, build] + # Only check: this job reads no output of build's beyond the shared stamp (#3390), and + # nothing here consumes a windows artifact, so it runs in parallel with build rather than + # waiting on it. + needs: check runs-on: ubuntu-latest timeout-minutes: 30 @@ -595,6 +613,7 @@ jobs: mkdir -p releases tar -C publish/DarlingService-linux -czf "releases/PerformanceMonitorDarling-linux-x64-${version}.tar.gz" . (cd releases && sha256sum "PerformanceMonitorDarling-linux-x64-${version}.tar.gz" > SHA256SUMS-linux.txt && cat SHA256SUMS-linux.txt) + echo "${version}" > releases/linux-version.txt # The container image build needs THIS checkout (docker build's context is the repo root, # #1804), so it stays here rather than moving to publish, which has no source tree. Only the @@ -624,7 +643,9 @@ jobs: path: | releases/PerformanceMonitorDarling-linux-x64-*.tar.gz releases/SHA256SUMS-linux.txt + releases/linux-version.txt if-no-files-found: error + retention-days: 3 - name: Upload container image uses: actions/upload-artifact@v6 @@ -632,6 +653,7 @@ jobs: name: darling-container-image path: releases/darling-container-image.tar if-no-files-found: error + retention-days: 3 # ── Publish (#4336 follow-up) ───────────────────────────────────────────────────────────────── # The one job that talks to the outside world: the GitHub release, the ghcr push, the signature, @@ -646,7 +668,13 @@ jobs: needs.build.result == 'success' && needs.linux.result == 'success' && needs.darling-pg.result == 'success' && - (github.event_name != 'workflow_dispatch' || inputs.publish != false) + (github.event_name != 'workflow_dispatch' || format('{0}', inputs.publish) != 'false') + # A scheduled re-dispatch and a manual dispatch both land the same tag (`nightly`); two + # close dispatches otherwise race on this job's delete-then-create of the release, so only + # one publish runs at a time and a queued second one waits rather than colliding. + concurrency: + group: nightly-publish + cancel-in-progress: false runs-on: ubuntu-latest timeout-minutes: 20 permissions: @@ -698,6 +726,34 @@ jobs: echo "VERSION=${version}" >> "$GITHUB_OUTPUT" echo "Nightly version: ${version}" + # The full SHA build actually checked out (written by the build job's own `git rev-parse + # HEAD`), so the release below names the tree that was built rather than dev's tip at + # publish time — 20-60 minutes later, once darling-pg finishes. + - name: Read the built commit SHA + id: commit + shell: bash + run: | + set -euo pipefail + sha="$(cat windows-commit-sha.txt)" + [ -n "${sha}" ] || { echo "::error::windows-commit-sha.txt was empty."; exit 1; } + echo "SHA=${sha}" >> "$GITHUB_OUTPUT" + echo "Built commit: ${sha}" + + # publish takes its release version from the windows handoff but ships linux's tag and + # container image too; a mismatch would mean the two build jobs disagreed about which + # nightly stamp they were building, and shipping under one label anyway is the wrong call. + - name: Verify windows and linux versions match + shell: bash + run: | + set -euo pipefail + windows_version="${{ steps.version.outputs.VERSION }}" + linux_version="$(cat releases/linux-version.txt)" + if [ "${windows_version}" != "${linux_version}" ]; then + echo "::error::Windows and Linux nightly versions do not match. windows=${windows_version} linux=${linux_version}" + exit 1 + fi + echo "Windows and Linux versions match: ${windows_version}" + - name: Delete previous nightly release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -710,12 +766,14 @@ jobs: run: | set -euo pipefail version="${{ steps.version.outputs.VERSION }}" - sha="$(git rev-parse --short HEAD)" + sha="${{ steps.commit.outputs.SHA }}" + short_sha="${sha:0:7}" + echo "Naming the release after the built commit: ${sha}" body=$(cat < These builds include the latest changes and may be unstable. @@ -724,7 +782,7 @@ jobs: ) gh release create nightly \ - --target dev \ + --target "${sha}" \ --title "Nightly Build (${version})" \ --notes "${body}" \ --prerelease \