From a43043d2d4b2d1a760da3125d96a2ce1b45e2f36 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 14:13:04 -0600 Subject: [PATCH 1/3] fix(release): bundle the managed runtime dependency closure --- .github/workflows/ci.yml | 1 + .github/workflows/pylon-preview-release.yml | 9 +- .github/workflows/pylon-stable-release.yml | 14 +- .pylon/features.yaml | 2 +- .pylon/upstream-review.md | 7 + docs/pylon-publication.md | 8 + package.json | 1 + .../self-contained-managed-artifact.md | 1 + scripts/build-pylon-prime-agent-release.mjs | 2 +- scripts/hydrate-pylon-runtime-inputs.mjs | 11 ++ scripts/lib/pylon-release.mjs | 1 + scripts/lib/pylon-runtime-dependencies.mjs | 176 ++++++++++++++++++ scripts/pack-pylon-prime-agent-release.mjs | 10 +- scripts/pylon-prime-agent-release.test.mjs | 20 +- scripts/pylon-prime-release-v1.json | 2 +- ...on-prime-supported-release-recipes-v1.json | 14 ++ scripts/pylon-publication.test.mjs | 37 ++-- scripts/pylon-runtime-dependencies.test.mjs | 125 +++++++++++++ scripts/smoke-pylon-prime-agent-release.mjs | 45 ++--- scripts/verify-pylon-prime-agent-release.mjs | 2 + 20 files changed, 434 insertions(+), 54 deletions(-) create mode 100644 packages/coding-agent/.changes/self-contained-managed-artifact.md create mode 100644 scripts/hydrate-pylon-runtime-inputs.mjs create mode 100644 scripts/lib/pylon-runtime-dependencies.mjs create mode 100644 scripts/pylon-runtime-dependencies.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb94b45dad2..5a102071d83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -254,6 +254,7 @@ jobs: test "$(node --version)" = "v${PYLON_RELEASE_NODE}" test "$(npm --version)" = "${PYLON_RELEASE_NPM}" npm ci + npm run release:pylon:hydrate-runtime - name: Test release contract run: npm run test:pylon-release diff --git a/.github/workflows/pylon-preview-release.yml b/.github/workflows/pylon-preview-release.yml index 02f8ca24dc4..e519cd4b658 100644 --- a/.github/workflows/pylon-preview-release.yml +++ b/.github/workflows/pylon-preview-release.yml @@ -77,6 +77,7 @@ jobs: test "$(node --version)" = "v${PYLON_RELEASE_NODE}" test "$(npm --version)" = "${PYLON_RELEASE_NPM}" npm ci + npm run release:pylon:hydrate-runtime - name: Test publication contract run: | @@ -96,7 +97,7 @@ jobs: - name: Verify and prepare six exact subjects run: | npm run release:pylon:verify - npm run release:pylon:preview -- --publication-policy-revision 3 + npm run release:pylon:preview -- --publication-policy-revision 4 npm run release:pylon:verify-preview - name: Upload isolated preview subjects @@ -281,7 +282,7 @@ jobs: release.source?.commit !== context.sha || release.source?.tree !== preview.build?.source?.tree || release.build?.id !== tag || preview.build?.tag !== tag || preview.build?.releaseManifest?.sha256 !== crypto.createHash("sha256").update(releaseBytes).digest("hex") || - preview.publicationPolicyRevision !== 3 || preview.sequenceEpoch !== 1 || + preview.publicationPolicyRevision !== 4 || preview.sequenceEpoch !== 1 || preview.sequence !== Number(process.env.GITHUB_RUN_NUMBER) || preview.workflowRunId !== process.env.GITHUB_RUN_ID ) throw new Error("Preview tag plan is not bound to the exact source and workflow sequence."); core.setOutput("tag", tag); @@ -469,7 +470,7 @@ jobs: if ( release.source?.commit !== context.sha || release.source?.tree !== preview.build?.source?.tree || release.build?.id !== tag || preview.build?.tag !== tag || preview.build?.releaseManifest?.sha256 !== sha256(releaseBytes) || - preview.publicationPolicyRevision !== 3 || preview.sequenceEpoch !== 1 || + preview.publicationPolicyRevision !== 4 || preview.sequenceEpoch !== 1 || preview.sequence !== Number(process.env.GITHUB_RUN_NUMBER) || preview.workflowRunId !== process.env.GITHUB_RUN_ID ) throw new Error("Preview draft manifests do not bind the exact source and workflow sequence."); const expectedNames = [...release.assets.map((asset) => asset.file), "pylon-prime-agent-release-v1.json", "pylon-preview-channel-v1.json"].sort(); @@ -795,7 +796,7 @@ jobs: previewManifest.build.source.commit !== sourceSha || previewManifest.build.source.tree !== releaseManifest.source.tree || previewManifest.build.releaseManifest.sha256 !== sha256(releaseBytes) || - previewManifest.publicationPolicyRevision !== 3 || previewManifest.sequenceEpoch !== 1 || + previewManifest.publicationPolicyRevision !== 4 || previewManifest.sequenceEpoch !== 1 || previewManifest.sequence !== Number(process.env.GITHUB_RUN_NUMBER) || previewManifest.workflowRunId !== process.env.GITHUB_RUN_ID ) { throw new Error("Downloaded preview metadata is not bound to this exact push and workflow sequence."); diff --git a/.github/workflows/pylon-stable-release.yml b/.github/workflows/pylon-stable-release.yml index bcad28067e5..ba7ad8771c9 100644 --- a/.github/workflows/pylon-stable-release.yml +++ b/.github/workflows/pylon-stable-release.yml @@ -473,7 +473,7 @@ jobs: --operation "$OPERATION" --policy-sha "${{ github.sha }}" --policy-tree "$policy_tree" - --publication-policy-revision 3 + --publication-policy-revision 4 ) if [ "$OPERATION" = withdraw ]; then args+=(--revoke-tag "$REVOKE_STABLE_TAG" --reason "$REASON") @@ -557,7 +557,7 @@ jobs: if ( manifest.schemaVersion !== 1 || manifest.channel !== "stable" || manifest.repository !== "https://github.com/pylon-code/prime-agent" || - manifest.promotion?.publicationPolicyRevision !== 3 || + manifest.promotion?.publicationPolicyRevision !== 4 || !/^pylon-stable-[0-9]{6}-g[0-9a-f]{12}-r[1-9][0-9]*$/.test(manifest.tag) ) throw new Error("Stable manifest identity is malformed."); NODE @@ -663,8 +663,8 @@ jobs: if ( !parsed || !preview || Number(parsed[1]) !== manifest.sequence || parsed[2] !== manifest.build.source?.commit?.slice(0, 12) || Number(parsed[3]) !== manifest.build.recipeRevision || preview[1] !== parsed[2] || Number(preview[2]) !== manifest.build.recipeRevision || - manifest.build.previewTag !== manifest.build.id || ![1, 2, 3].includes(manifest.build.publicationPolicyRevision) || - manifest.promotion?.policyCommit !== context.sha || manifest.promotion?.publicationPolicyRevision !== 3 + manifest.build.previewTag !== manifest.build.id || ![1, 2, 3, 4].includes(manifest.build.publicationPolicyRevision) || + manifest.promotion?.policyCommit !== context.sha || manifest.promotion?.publicationPolicyRevision !== 4 ) throw new Error("Stable draft identity is malformed or not signed by this policy commit."); const name = `Pylon Prime stable ${tag}`; const encoded = bytes.toString("base64"); @@ -840,10 +840,10 @@ jobs: Number(previewMatch[2]) !== manifest.build?.recipeRevision || manifest.build.previewTag !== manifest.build.id || manifest.build.previewSequence?.sequenceEpoch !== 1 || !Number.isSafeInteger(manifest.build.previewSequence?.sequence) || manifest.build.previewSequence.sequence < 1 || !/^[1-9][0-9]*$/.test(manifest.build.previewSequence?.workflowRunId ?? "") || - manifest.build.previewTag !== process.env.PREVIEW_TAG || ![1, 2, 3].includes(manifest.build.publicationPolicyRevision) || + manifest.build.previewTag !== process.env.PREVIEW_TAG || ![1, 2, 3, 4].includes(manifest.build.publicationPolicyRevision) || manifest.promotion?.kind !== operation || - (mode === "normal" ? manifest.promotion?.publicationPolicyRevision !== 3 : - ![1, 2, 3].includes(manifest.promotion?.publicationPolicyRevision)) || + (mode === "normal" ? manifest.promotion?.publicationPolicyRevision !== 4 : + ![1, 2, 3, 4].includes(manifest.promotion?.publicationPolicyRevision)) || manifest.promotion?.policyCommit !== process.env.POLICY_SHA || manifest.promotion?.policyTree !== process.env.POLICY_TREE ) throw new Error("Stable manifest, preview recipe, operator request, or policy identity differs."); if (operation === "withdraw") { diff --git a/.pylon/features.yaml b/.pylon/features.yaml index 9c6994327e9..520c84f370f 100644 --- a/.pylon/features.yaml +++ b/.pylon/features.yaml @@ -394,7 +394,7 @@ decisions: - https://github.com/pylon-code/pylon/issues/114 - https://github.com/pylon-code/pylon/issues/193 - https://github.com/pylon-code/pylon/issues/194 - fork_change: deterministic-pylon-release-artifacts-v1 + fork_change: deterministic-self-contained-pylon-release-artifacts-v2 upstream_support: Prime's R2 packer accepts mutable channel and archive inputs, refreshes live model catalogs during normal builds, retains upstream package provenance, and does not produce a Pylon-owned deterministic manifest or integrity-complete internal artifact graph. revisit_when: - Pylon no longer maintains a Prime distribution or the upstream artifact recipe can emit the exact fork-owned deterministic contract without publication side effects. diff --git a/.pylon/upstream-review.md b/.pylon/upstream-review.md index fdf54fb9f07..7c4ad2203fa 100644 --- a/.pylon/upstream-review.md +++ b/.pylon/upstream-review.md @@ -310,3 +310,10 @@ Follow-up: Task10 builds/packs the exact merged tree into a private prefix and r - Publication policy 3 appends exact preview/stable workflow digests. Historical policies 1/2 remain immutable, artifact recipe 1 and runtime 0.9.4 are unchanged, and stable recovery still accepts only explicitly registered historical policies. The matching Pylon consumer entry is required before managed installation. - Validation: all three inline upload regressions fail on the old generic request and pass with the specialized method. Preview covers six binary subjects, complete/partial draft joins, and altered receipt refusal. Stable normal staging retains crash-after-create recovery; zero-asset recovery verifies body bytes and rejects altered metadata/downloads before policy proof or reservation. An independent offline probe against the exact pinned action bundle proves the upload host, filename encoding and binary body preservation. The full focused publication suite and required repository checks accompany the PR. - A live corrected protected workflow remains the publication acceptance gate. Preserve the prior source's valid tag and empty draft; do not manually replace assets, expand credentials, or relax approval/CAS controls. Revisit when upstream provides an equivalent immutable, byte-preserving publication primitive. + +## 2026-09-11 — self-contained managed runtime archive + +- Reviewed compatibility remains Prime v0.9.4 at `1eee2938b4eeb7a4d72e17035adda669a89b63de`. Current upstream release/issue/PR searches found no standalone managed package replacing this contract. Upstream's CLI bundler deliberately externalizes native and interop-sensitive packages, and its SDK remains an unbundled module graph. +- `deterministic-pylon-release-artifacts`: **redesign** as recipe 2. The root archive contains the committed production dependency closure, including nested resolution, installed peers and every locked optional platform package. A separate build-input step acquires exact SHA-512-locked tarballs; offline packaging rechecks them, extracts regular files without lifecycle scripts, and copies the separately packed internal workspace artifacts. No semver resolution, host node_modules copying or installation-time dependency download is permitted. +- The installed smoke consumes the root archive directly and loads the public SDK, native/WASM dependencies and owned daemon contract. Historical recipe 1 retains its explicitly historical npm smoke. Final bundle verification requires every locked runtime package, and Pylon's actual managed parser remains a separate acceptance gate. The frozen upstream range, daemon protocol and four SDK feature tokens are unchanged. +- Publication policy 4 binds the added input hydration and exact current-policy guards; policies 1–3 and recipe 1 remain immutable. Recipe 2 requires its matching Pylon consumer. Revisit only when upstream supplies equivalent self-contained, deterministic installation without weakening source, integrity, platform or ownership guarantees. diff --git a/docs/pylon-publication.md b/docs/pylon-publication.md index 8cda986eeca..4d7f2498388 100644 --- a/docs/pylon-publication.md +++ b/docs/pylon-publication.md @@ -243,3 +243,11 @@ Use `--initialize` once, then omit it. The CLI requires the complete contiguous - **Invalid immutable release:** preserve evidence first. GitHub may require an administrator to temporarily disable immutable releases before exact-id deletion. Delete only the proven invalid release, restore/read back immutable releases immediately, and link every API response in the incident. Never alter a valid published sequence. Run offline policy tests with `npm run test:pylon-publication` and App-acceptance unit tests with `npm run test:pylon-ruleset-auditor-app`. They cover exact current/historical workflow digests and registry closure, immutable signed attempt evidence, zero-asset crash recovery, deterministic stale recovery, active heartbeats, transaction crash convergence, path-boundary checks, exact required-check paths/apps, preview/stable tag squats and CAS order, withdrawal tuples, rollback state, approval DAGs, every contents writer, pinned actions, no source/download execution in publication writers, mocked App scopes and endpoints, REST and GraphQL redaction, the nonzero canary, and token revocation. + +## Self-contained managed artifacts (recipe 2) + +Recipe 2 bundles the full production runtime into the root tarball. Before entering the offline build namespace, run `npm run release:pylon:hydrate-runtime` with the pinned toolchain. It acquires only URLs and SHA-512 identities already committed in `package-lock.json`, including every optional platform package. The offline pack rechecks these inputs, retains nested and peer resolution, includes the exact built internal workspaces, and never runs dependency lifecycle scripts. Ambient `node_modules` is not a runtime packaging input. + +The final root tarball is checked for every expected runtime package. Both installed-artifact jobs extract that same root directly, with no npm installation, and prove CLI, SDK, native/WASM loading and caller-owned daemon behavior. Historical recipe 1 smoke remains available only for historical publication verification. Pylon also verifies the final archive through its bounded managed installer; signed provenance alone does not prove a runnable package. + +The root retains SDK declarations and source maps. npm emits per-file PAX headers for a few dependency basenames longer than 100 bytes; the matching Pylon consumer accepts only bounded path/size/mtime metadata immediately followed by a regular file, preserving path, collision, link and size restrictions. Stable approval still requires successful protected artifact graduation for the exact preview. diff --git a/package.json b/package.json index 287b40727a2..27a52406854 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "release:pack": "node scripts/pack-prime-agent-release.mjs", "release:pylon:build": "node scripts/build-pylon-prime-agent-release.mjs", "release:pylon:hydrate-lock": "node scripts/hydrate-pylon-release-lock.mjs", + "release:pylon:hydrate-runtime": "node scripts/hydrate-pylon-runtime-inputs.mjs", "release:pylon:pack": "node scripts/build-pylon-prime-agent-release.mjs --pack", "release:pylon:verify": "node scripts/verify-pylon-prime-agent-release.mjs", "release:pylon:preview": "node scripts/prepare-pylon-preview-manifest.mjs", diff --git a/packages/coding-agent/.changes/self-contained-managed-artifact.md b/packages/coding-agent/.changes/self-contained-managed-artifact.md new file mode 100644 index 00000000000..909d7cb06a6 --- /dev/null +++ b/packages/coding-agent/.changes/self-contained-managed-artifact.md @@ -0,0 +1 @@ +- Fixed Pylon managed builds to include their exact runtime dependencies for offline installation on Linux and macOS. diff --git a/scripts/build-pylon-prime-agent-release.mjs b/scripts/build-pylon-prime-agent-release.mjs index e447b5e14be..5f5f9b5b01a 100644 --- a/scripts/build-pylon-prime-agent-release.mjs +++ b/scripts/build-pylon-prime-agent-release.mjs @@ -78,7 +78,7 @@ try { assertCleanSource(root, { rejectIgnoredInputs: true }); writeReleaseBuildReceipt(root, source, toolchain, lockfileSha256); if (args.pack) { - packPylonPrimeAgentRelease(args.outDir ? ["--out-dir", args.outDir] : [], environment); + await packPylonPrimeAgentRelease(args.outDir ? ["--out-dir", args.outDir] : [], environment); } } catch (error) { console.error(error instanceof Error ? error.message : String(error)); diff --git a/scripts/hydrate-pylon-runtime-inputs.mjs b/scripts/hydrate-pylon-runtime-inputs.mjs new file mode 100644 index 00000000000..b1fc8038485 --- /dev/null +++ b/scripts/hydrate-pylon-runtime-inputs.mjs @@ -0,0 +1,11 @@ +#!/usr/bin/env node +import { resolve } from "node:path"; +import { hydrateRuntimeDependencies } from "./lib/pylon-runtime-dependencies.mjs"; + +try { + const count = await hydrateRuntimeDependencies(resolve(import.meta.dirname, "..")); + console.log(`Verified inputs for ${count} locked runtime packages, including all platforms.`); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} diff --git a/scripts/lib/pylon-release.mjs b/scripts/lib/pylon-release.mjs index 0e1fcf1ec2f..59da3ac91de 100644 --- a/scripts/lib/pylon-release.mjs +++ b/scripts/lib/pylon-release.mjs @@ -310,6 +310,7 @@ export function createReleasePackageJson({ delete packageJson.private; if (releasePackage.publicPackage) { + packageJson.bundleDependencies = Object.keys({ ...packageJson.dependencies, ...packageJson.optionalDependencies }).sort(); packageJson.bin = { "prime-agent": "dist/bundle/cli.js" }; packageJson.piConfig = { ...(packageJson.piConfig || {}), name: "prime-agent", configDir: ".prime/agent" }; } diff --git a/scripts/lib/pylon-runtime-dependencies.mjs b/scripts/lib/pylon-runtime-dependencies.mjs new file mode 100644 index 00000000000..7055ceaccb7 --- /dev/null +++ b/scripts/lib/pylon-runtime-dependencies.mjs @@ -0,0 +1,176 @@ +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { join, posix, resolve } from "node:path"; +import { PYLON_RELEASE_PACKAGES, assertPinnedToolchain, isCanonicalSha512Integrity, readCompletePackageLock } from "./pylon-release.mjs"; + +const MAX_TARBALL_BYTES = 128 * 1024 * 1024; +const internalPaths = new Map(PYLON_RELEASE_PACKAGES.map((entry) => [`packages/${entry.packageDir}`, entry])); + +/** Follow the committed installation layout, including installed peers and every platform optional. */ +export function runtimeDependencyClosure(lock) { + const selected = new Map(); + function dependencyPath(from, name, optional) { + if (!/^(?:@[a-z0-9._-]+\/)?[a-z0-9._-]+$/i.test(name)) throw new Error(`Invalid dependency name: ${name}`); + let directory = from; + for (;;) { + const candidate = posix.join(directory, "node_modules", name); + if (lock.packages[candidate]) return candidate; + if (!directory) break; + directory = posix.dirname(directory); + if (directory === ".") directory = ""; + } + if (!optional) throw new Error(`Missing locked runtime dependency ${name} from ${from}`); + } + function visit(location) { + let entry = lock.packages[location]; + if (entry?.link) { + location = entry.resolved; + entry = lock.packages[location]; + } + if (!entry || (!location.startsWith("node_modules/") && !internalPaths.has(location))) { + throw new Error(`Unsupported runtime lock location: ${location}`); + } + if (selected.has(location)) return; + if (!internalPaths.has(location) && (!isCanonicalSha512Integrity(entry.integrity) || + !entry.resolved?.startsWith("https://registry.npmjs.org/"))) { + throw new Error(`Runtime dependency lacks locked registry integrity: ${location}`); + } + selected.set(location, entry); + for (const name of Object.keys({ ...entry.dependencies, ...entry.optionalDependencies, ...entry.peerDependencies }).sort()) { + const optional = name in (entry.optionalDependencies ?? {}) || entry.peerDependenciesMeta?.[name]?.optional === true; + const target = dependencyPath(location, name, optional); + if (target) visit(target); + } + } + visit("packages/coding-agent"); + return new Map([...selected].sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0)); +} + +function archivePath(root, entry) { + const digest = Buffer.from(entry.integrity.slice("sha512-".length), "base64").toString("hex"); + return join(root, ".npm", "pylon-runtime-inputs", `${digest}.tgz`); +} + +export function verifyRuntimeArchive(bytes, entry) { + if (bytes.byteLength < 1 || bytes.byteLength > MAX_TARBALL_BYTES || + `sha512-${createHash("sha512").update(bytes).digest("base64")}` !== entry.integrity) { + throw new Error(`Runtime tarball failed its locked size/integrity check: ${entry.resolved}`); + } +} + +/** Network is permitted only while acquiring build inputs; packaging rechecks every byte offline. */ +export async function hydrateRuntimeDependencies(root) { + assertPinnedToolchain(root); + const closure = runtimeDependencyClosure(readCompletePackageLock(root)); + mkdirSync(join(root, ".npm", "pylon-runtime-inputs"), { recursive: true }); + for (const [location, entry] of closure) { + if (internalPaths.has(location)) continue; + const destination = archivePath(root, entry); + if (existsSync(destination)) { + verifyRuntimeArchive(readFileSync(destination), entry); + continue; + } + const response = await fetch(entry.resolved, { redirect: "error", signal: AbortSignal.timeout(60_000) }); + if (!response.ok || !response.body) throw new Error(`Cannot acquire locked runtime input: ${entry.resolved}`); + const chunks = []; + let size = 0; + for await (const chunk of response.body) { + size += chunk.byteLength; + if (size > MAX_TARBALL_BYTES) throw new Error(`Runtime input exceeds bounded size: ${entry.resolved}`); + chunks.push(chunk); + } + const bytes = Buffer.concat(chunks); + verifyRuntimeArchive(bytes, entry); + writeFileSync(destination, bytes, { flag: "wx", mode: 0o600 }); + } + return closure.size; +} + +function pinnedTar(environment) { + const cli = environment.PYLON_RELEASE_NPM_CLI || environment.npm_execpath; + if (!cli?.endsWith("npm-cli.js")) throw new Error("Runtime packaging must run through the pinned npm CLI."); + return createRequire(resolve(cli))("tar"); +} + +/** Materialize only regular package files; never invoke a package manager or lifecycle script. */ +export async function extractRuntimeArchive(archive, destination, environment = process.env) { + const tar = pinnedTar(environment); + const seen = new Map(); + let archiveRoot; + let expandedBytes = 0; + tar.t({ file: archive, strict: true, sync: true, onReadEntry(entry) { + const name = entry.path.replace(/\/$/, ""); + const normalized = posix.normalize(name); + const first = name.split("/")[0]; + archiveRoot ??= first; + expandedBytes += entry.size; + if (name.includes("\\") || name.split("/").some((part) => part === ".." || part === "") || + first !== archiveRoot || /^[A-Za-z]:/.test(name) || + !["File", "Directory"].includes(entry.type) || (entry.mode & ~0o777) !== 0 || + expandedBytes > 768 * 1024 * 1024 || entry.size > MAX_TARBALL_BYTES || seen.size >= 30_000) { + throw new Error(`Unsafe runtime archive entry: ${name} in ${archive}`); + } + const digest = createHash("sha256"); + entry.on("data", (bytes) => digest.update(bytes)); + entry.on("end", () => { + const identity = `${entry.type}:${entry.mode}:${entry.size}:${digest.digest("hex")}`; + if (seen.has(normalized) && seen.get(normalized) !== identity) throw new Error(`Conflicting duplicate runtime entry: ${name}`); + seen.set(normalized, identity); + }); + } }); + mkdirSync(destination, { recursive: true }); + const extracted = new Set(); + await tar.x({ file: archive, cwd: destination, strip: 1, strict: true, noMtime: true, preserveOwner: false, filter(name) { + const normalized = posix.normalize(name.replace(/\/$/, "")); + if (extracted.has(normalized)) return false; + extracted.add(normalized); + return true; + } }); +} + +export async function materializeRuntimeDependencies({ root, stagingDir, internalStaging, environment }) { + assertPinnedToolchain(root, environment); + const closure = runtimeDependencyClosure(readCompletePackageLock(root)); + for (const [location, entry] of closure) { + if (internalPaths.has(location)) continue; + const archive = archivePath(root, entry); + verifyRuntimeArchive(readFileSync(archive), entry); + await extractRuntimeArchive(archive, join(stagingDir, location), environment); + const installed = JSON.parse(readFileSync(join(stagingDir, location, "package.json"), "utf8")); + if (installed.version !== entry.version) throw new Error(`Runtime package version differs from lock: ${location}`); + } + for (const [location, packageInfo] of internalPaths) { + if (!closure.has(location) || packageInfo.publicPackage) continue; + const source = internalStaging.get(packageInfo.packageName); + if (!source) throw new Error(`Missing built runtime workspace: ${location}`); + // Copy through the same package archive used by external consumers, retaining its exact provenance. + await extractRuntimeArchive(source, join(stagingDir, "node_modules", packageInfo.packageName), environment); + } + return [...closure.keys()]; +} + +/** Check npm's final bundle, including optional packages that the pack host cannot load. */ +export function verifyBundledRuntimeDependencies(root, archive, environment = process.env) { + const expected = new Map(); + for (const [location, entry] of runtimeDependencyClosure(readCompletePackageLock(root))) { + const internal = internalPaths.get(location); + if (internal?.publicPackage) continue; + const installed = internal ? `node_modules/${internal.packageName}` : location; + expected.set(`package/${installed}/package.json`, entry.version); + } + const tar = pinnedTar(environment); + tar.t({ file: archive, sync: true, strict: true, onReadEntry(entry) { + const version = expected.get(entry.path); + if (version === undefined) return; + if (entry.type !== "File" || entry.size > 256 * 1024) throw new Error(`Invalid bundled manifest: ${entry.path}`); + const chunks = []; + entry.on("data", (bytes) => chunks.push(bytes)); + entry.on("end", () => { + const manifest = JSON.parse(Buffer.concat(chunks).toString("utf8")); + if (manifest.version !== version) throw new Error(`Bundled runtime version differs from lock: ${entry.path}`); + expected.delete(entry.path); + }); + } }); + if (expected.size) throw new Error(`Root archive omitted locked runtime packages: ${[...expected.keys()].join(", ")}`); +} diff --git a/scripts/pack-pylon-prime-agent-release.mjs b/scripts/pack-pylon-prime-agent-release.mjs index 9ff92a4ca7d..bb97a1d729e 100644 --- a/scripts/pack-pylon-prime-agent-release.mjs +++ b/scripts/pack-pylon-prime-agent-release.mjs @@ -3,6 +3,7 @@ import { existsSync, readFileSync, rmSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { materializeRuntimeDependencies } from "./lib/pylon-runtime-dependencies.mjs"; import { assertCleanSource, assertPinnedToolchain, @@ -58,7 +59,7 @@ function requireBuiltPackages() { } } -export function packPylonPrimeAgentRelease(args = process.argv.slice(2), environment = process.env) { +export async function packPylonPrimeAgentRelease(args = process.argv.slice(2), environment = process.env) { const { outDir } = parseArgs(args); assertCleanSource(root, { rejectIgnoredInputs: true }); assertPylonRepository(root); @@ -78,6 +79,7 @@ export function packPylonPrimeAgentRelease(args = process.argv.slice(2), environ const stagingRoot = join(output, "staging"); const artifactsDir = join(output, "artifacts"); const internalArtifacts = new Map(); + const internalStaging = new Map(); const artifacts = []; for (const releasePackage of PYLON_RELEASE_PACKAGES) { @@ -96,10 +98,14 @@ export function packPylonPrimeAgentRelease(args = process.argv.slice(2), environ validateLockedRegistryGraph(shrinkwrap, packageLock.packages, internalArtifacts); const stagingDir = join(stagingRoot, releasePackage.packageDir); copyReleasePackage(packageDirectory(root, releasePackage.packageDir), stagingDir, packageJson, shrinkwrap); + if (releasePackage.publicPackage) { + await materializeRuntimeDependencies({ root, stagingDir, internalStaging, environment }); + } const file = releaseAssetFile(releasePackage.assetStem, version); const packed = packStagingPackage({ root, stagingDir, artifactsDir, assetFile: file, environment }); const artifact = { package: packageJson.name, file, version, ...packed }; artifacts.push(artifact); + internalStaging.set(sourcePackage.name, packed.path); internalArtifacts.set(sourcePackage.name, { packageName: packageJson.name, file, @@ -121,7 +127,7 @@ export function packPylonPrimeAgentRelease(args = process.argv.slice(2), environ if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { try { - packPylonPrimeAgentRelease(); + await packPylonPrimeAgentRelease(); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); diff --git a/scripts/pylon-prime-agent-release.test.mjs b/scripts/pylon-prime-agent-release.test.mjs index c505b3cfbf6..d2484eb1825 100644 --- a/scripts/pylon-prime-agent-release.test.mjs +++ b/scripts/pylon-prime-agent-release.test.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import "./pylon-runtime-dependencies.test.mjs"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -113,10 +114,10 @@ function fakeArtifacts() { } test("uses channel-neutral immutable Pylon asset identity", () => { - assert.equal(releaseBuildId(source.commit), "pylon-build-g0123456789ab-r1"); + assert.equal(releaseBuildId(source.commit), "pylon-build-g0123456789ab-r2"); assert.equal( releaseAssetUrl(source.commit, "pylon-prime-agent-0.8.1.tgz"), - "https://github.com/pylon-code/prime-agent/releases/download/pylon-build-g0123456789ab-r1/pylon-prime-agent-0.8.1.tgz", + "https://github.com/pylon-code/prime-agent/releases/download/pylon-build-g0123456789ab-r2/pylon-prime-agent-0.8.1.tgz", ); assert.throws( () => @@ -354,3 +355,18 @@ test("offline release scripts cannot invoke live model generation", () => { /PYLON_RELEASE_OFFLINE/, ); }); + +test("both fresh pack workflows acquire locked runtime inputs before the offline namespace", () => { + for (const [file, start, end] of [ + ["ci.yml", " pylon-artifact-pack:", " pylon-artifact-reproducibility:"], + ["pylon-preview-release.yml", " pack:", " reproducibility:"], + ]) { + const source = readFileSync(join(root, ".github", "workflows", file), "utf8"); + const job = source.slice(source.indexOf(start), source.indexOf(end)); + const install = job.indexOf("npm ci"); + const hydrate = job.indexOf("npm run release:pylon:hydrate-runtime"); + const offline = job.indexOf("unshare --net -- npm run release:pylon:pack"); + assert.ok(install > 0 && hydrate > install && offline > hydrate, file); + assert.equal(job.split("npm run release:pylon:hydrate-runtime").length, 2); + } +}); diff --git a/scripts/pylon-prime-release-v1.json b/scripts/pylon-prime-release-v1.json index f9a3ea32c7d..7a44fd3104c 100644 --- a/scripts/pylon-prime-release-v1.json +++ b/scripts/pylon-prime-release-v1.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "recipeRevision": 1, + "recipeRevision": 2, "sourceRepository": "https://github.com/pylon-code/prime-agent", "nodeVersion": "22.23.2", "npmVersion": "11.10.1", diff --git a/scripts/pylon-prime-supported-release-recipes-v1.json b/scripts/pylon-prime-supported-release-recipes-v1.json index e5559513b4e..3893b46e54e 100644 --- a/scripts/pylon-prime-supported-release-recipes-v1.json +++ b/scripts/pylon-prime-supported-release-recipes-v1.json @@ -7,6 +7,13 @@ "nodeVersion": "22.23.2", "npmVersion": "11.10.1", "minimumNodeVersion": "22.8.0" + }, + { + "recipeRevision": 2, + "manifestSchemaVersion": 1, + "nodeVersion": "22.23.2", + "npmVersion": "11.10.1", + "minimumNodeVersion": "22.8.0" } ], "publicationPolicies": [ @@ -30,6 +37,13 @@ "previewWorkflowSha256": "16f68e46801eccca5e7e99736f346b5ffd96ce7188792d4ac8fbc4580408a736", "stableWorkflowPath": ".github/workflows/pylon-stable-release.yml", "stableWorkflowSha256": "96e4f3ccd889a17a391f00b4398d132a4234b5bc337045fedd3605f4eb74c331" + }, + { + "publicationPolicyRevision": 4, + "previewWorkflowPath": ".github/workflows/pylon-preview-release.yml", + "previewWorkflowSha256": "8ad08abd8004b7205b04dc3cd9f7cc0e720649c9c6814431622f5b4dfb6e4c32", + "stableWorkflowPath": ".github/workflows/pylon-stable-release.yml", + "stableWorkflowSha256": "52c72e078b08c0a2774ab3dc512283c46ab31c4a87738d87007f55f955662703" } ] } diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index c8871c8972f..98ec1031d41 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -389,7 +389,7 @@ test("preview manifest binds the full source tree, build, recipe, and build-mani for (const mutate of [ (value) => (value.build.source.commit = "f".repeat(40)), (value) => (value.build.source.tree = "f".repeat(40)), - (value) => (value.build.recipeRevision = 2), + (value) => (value.build.recipeRevision = 3), (value) => (value.build.releaseManifest.sha256 = "f".repeat(64)), ]) { const changed = structuredClone(preview); @@ -3133,7 +3133,7 @@ test("stable manifest nested schema rejects extras, malformed identities, unsafe (value) => (value.build.source.repository = "https://github.com/fork/prime-agent"), (value) => (value.build.source.commit = "abc"), (value) => (value.build.source.tree = "abc"), - (value) => (value.build.recipeRevision = 2), + (value) => (value.build.recipeRevision = 3), (value) => delete value.build.publicationPolicyRevision, (value) => (value.build.publicationPolicyRevision = 999), (value) => (value.build.releaseManifest.file = "other.json"), @@ -3595,7 +3595,7 @@ test("recipe and publication policy registries close independent immutable ident const policies = registry.publicationPolicies; const historicalPolicy = policies[0]; const policy = policies.at(-1); - assert.deepEqual(policies.map((candidate) => candidate.publicationPolicyRevision), [1, 2, 3]); + assert.deepEqual(policies.map((candidate) => candidate.publicationPolicyRevision), [1, 2, 3, 4]); assert.equal(historicalPolicy.previewWorkflowSha256, "e790a5da7063bd40fbd886e84945c3200291194fdbd5b002079349e45356a41d"); assert.equal(historicalPolicy.stableWorkflowSha256, "dfcecdf6b58f143f9b7a543eadd124c190350ae29ac9eadccb907f1398b0958a"); const preview = readFileSync(join(root, policy.previewWorkflowPath), "utf8"); @@ -3676,11 +3676,11 @@ test("recipe and publication policy registries close independent immutable ident /Administration-gated branch-protection read/, ); - assert.match(preview, /--publication-policy-revision 3/); - assert.match(preview, /preview(?:Manifest)?\.publicationPolicyRevision !== 3/); - assert.match(stable, /--publication-policy-revision 3/); - assert.match(stable, /promotion\?\.publicationPolicyRevision !== 3/); - assert.match(stable, /!\[1, 2, 3\]\.includes\(manifest\.build\.publicationPolicyRevision\)/); + assert.match(preview, /--publication-policy-revision 4/); + assert.match(preview, /preview(?:Manifest)?\.publicationPolicyRevision !== 4/); + assert.match(stable, /--publication-policy-revision 4/); + assert.match(stable, /promotion\?\.publicationPolicyRevision !== 4/); + assert.match(stable, /!\[1, 2, 3, 4\]\.includes\(manifest\.build\.publicationPolicyRevision\)/); assert.throws( () => validateApprovedWorkflowBytes(historicalPolicy.previewWorkflowPath, preview, "preview", 1, policies), /bytes differ/, @@ -3692,7 +3692,7 @@ test("recipe and publication policy registries close independent immutable ident const promotedByR2 = firstStable(); promotedByR2.promotion.publicationPolicyRevision = 2; assert.equal(validateStableManifest(promotedByR2, registry.recipes, policies), promotedByR2); - assert.equal(promotedByR2.build.recipeRevision, 1); + assert.equal(promotedByR2.build.recipeRevision, 2); assert.equal(promotedByR2.build.publicationPolicyRevision, 1); assert.equal(promotedByR2.promotion.publicationPolicyRevision, 2); assert.equal(policies[1].previewWorkflowSha256, "9f4e3f38fb0bdb9c11662310c5369fb792765a3090e0f74b0ec0b34127b43ed8"); @@ -3702,8 +3702,23 @@ test("recipe and publication policy registries close independent immutable ident promotedByR3.build.publicationPolicyRevision = buildPolicy; promotedByR3.promotion.publicationPolicyRevision = 3; assert.equal(validateStableManifest(promotedByR3, registry.recipes, policies), promotedByR3); - assert.equal(promotedByR3.build.recipeRevision, 1); + assert.equal(promotedByR3.build.recipeRevision, 2); } + const promotedByR4 = firstStable(); + promotedByR4.promotion.publicationPolicyRevision = 4; + for (const recipeRevision of [1, 2]) { + const historical = structuredClone(promotedByR4); + historical.build.recipeRevision = recipeRevision; + historical.build.id = historical.build.id.replace(/-r[0-9]+$/, `-r${recipeRevision}`); + historical.build.previewTag = historical.build.id; + historical.tag = historical.tag.replace(/-r[0-9]+$/, `-r${recipeRevision}`); + for (const buildPolicy of [1, 2, 3, 4]) { + historical.build.publicationPolicyRevision = buildPolicy; + assert.equal(validateStableManifest(historical, registry.recipes, policies), historical); + } + } + assert.equal((stable.match(/!\[1, 2, 3, 4\]\.includes\(manifest\.build\.publicationPolicyRevision\)/g) ?? []).length, 2); + assert.match(stable, /!\[1, 2, 3, 4\]\.includes\(manifest\.promotion\?\.publicationPolicyRevision\)/); const stableVerifier = readFileSync(join(root, "scripts/verify-pylon-stable-attestation.mjs"), "utf8"); assert.match(stableVerifier, /manifest\.promotion\.publicationPolicyRevision/); assert.doesNotMatch(stableVerifier, /"stable",\s*manifest\.build\.recipeRevision/); @@ -6820,7 +6835,7 @@ test("stable zero-asset recovery uploads and verifies body bytes before policy c const draft = { id: 51, draft: true, immutable: false, assets: [], body: publicationReleaseBody({ channel: "stable", tag: manifest.tag, source: source.commit, tree: source.tree, - recipeRevision: 1, policyCommit: source.commit, policyTree: source.tree, stableManifestBytes: bytes }), + recipeRevision: manifest.build.recipeRevision, policyCommit: source.commit, policyTree: source.tree, stableManifestBytes: bytes }), }; const order = []; const github = { diff --git a/scripts/pylon-runtime-dependencies.test.mjs b/scripts/pylon-runtime-dependencies.test.mjs new file mode 100644 index 00000000000..89108dd37e5 --- /dev/null +++ b/scripts/pylon-runtime-dependencies.test.mjs @@ -0,0 +1,125 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { gzipSync } from "node:zlib"; +import { test } from "node:test"; +import { extractRuntimeArchive, runtimeDependencyClosure, verifyRuntimeArchive } from "./lib/pylon-runtime-dependencies.mjs"; + +function registry(name, fields = {}) { + return { version: "1.0.0", resolved: `https://registry.npmjs.org/${name}/-/package.tgz`, integrity: `sha512-${Buffer.alloc(64, 1).toString("base64")}`, ...fields }; +} + +function fixture() { + return { packages: { + "packages/coding-agent": { dependencies: { client: "1", "@earendil-works/pi-ai": "1" }, devDependencies: { testOnly: "1" } }, + "node_modules/@earendil-works/pi-ai": { link: true, resolved: "packages/ai" }, + "packages/ai": { dependencies: { shared: "1" } }, + "node_modules/client": registry("client", { dependencies: { shared: "2" }, peerDependencies: { peer: "1", absent: "1" }, peerDependenciesMeta: { absent: { optional: true } }, optionalDependencies: { linux: "1", macos: "1" } }), + "node_modules/shared": registry("shared"), + "node_modules/client/node_modules/shared": registry("shared", { version: "2.0.0" }), + "node_modules/peer": registry("peer"), + "node_modules/linux": registry("linux", { os: ["linux"] }), + "node_modules/macos": registry("macos", { os: ["darwin"] }), + "node_modules/testOnly": registry("testOnly"), + } }; +} + +test("runtime closure retains exact nested, workspace, peer and cross-platform optional dependencies", () => { + const closure = runtimeDependencyClosure(fixture()); + assert.deepEqual([...closure.keys()], [ + "node_modules/client", "node_modules/client/node_modules/shared", "node_modules/linux", "node_modules/macos", + "node_modules/peer", "node_modules/shared", "packages/ai", "packages/coding-agent", + ]); + assert.equal(closure.get("node_modules/client/node_modules/shared").version, "2.0.0"); +}); + +test("runtime closure rejects missing required and peer dependencies or incomplete integrity", () => { + for (const missing of ["node_modules/client", "node_modules/peer"]) { + const lock = fixture(); + delete lock.packages[missing]; + assert.throws(() => runtimeDependencyClosure(lock), /Missing locked runtime dependency/); + } + const lock = fixture(); + delete lock.packages["node_modules/peer"].integrity; + assert.throws(() => runtimeDependencyClosure(lock), /lacks locked registry integrity/); + lock.packages["node_modules/client"] = { link: true, resolved: "../../foreign" }; + assert.throws(() => runtimeDependencyClosure(lock), /Unsupported runtime lock location/); +}); + +test("runtime inputs accept only bytes authenticated by the frozen lock", () => { + const bytes = Buffer.from("locked archive"); + const entry = registry("fixture", { integrity: `sha512-${createHash("sha512").update(bytes).digest("base64")}` }); + assert.doesNotThrow(() => verifyRuntimeArchive(bytes, entry)); + assert.throws(() => verifyRuntimeArchive(Buffer.from("changed archive"), entry), /size\/integrity check/); + assert.throws(() => verifyRuntimeArchive(Buffer.alloc(0), entry), /size\/integrity check/); +}); + +test("current runtime closure includes every locked clipboard platform and required schema peer", () => { + const lock = JSON.parse(readFileSync(resolve(import.meta.dirname, "../package-lock.json"), "utf8")); + const closure = runtimeDependencyClosure(lock); + const clipboard = lock.packages["node_modules/@mariozechner/clipboard"]; + for (const name of Object.keys(clipboard.optionalDependencies)) assert.ok(closure.has(`node_modules/${name}`), name); + assert.ok(closure.has("node_modules/zod")); + assert.ok(closure.has("node_modules/undici")); + assert.ok(closure.has("node_modules/koffi")); + assert.ok(closure.has("node_modules/cli-highlight/node_modules/chalk")); + assert.ok(!closure.has("node_modules/@biomejs/biome")); +}); + +function tarball(entries) { + const blocks = []; + for (const { name, value = "x", type = "0", mode = 0o644 } of entries) { + const data = Buffer.from(type === "1" || type === "2" ? "" : value); + const header = Buffer.alloc(512); + header.write(name); + for (const [offset, size, number] of [[100, 8, mode], [108, 8, 0], [116, 8, 0], [124, 12, data.length], [136, 12, 0]]) { + header.write(`${number.toString(8).padStart(size - 1, "0")}\0`, offset); + } + header.fill(0x20, 148, 156); + header.write(type, 156); + if (type === "1" || type === "2") header.write("outside", 157); + header.write("ustar\0", 257); + header.write("00", 263); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + header.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148); + blocks.push(header, data, Buffer.alloc((512 - data.length % 512) % 512)); + } + return gzipSync(Buffer.concat([...blocks, Buffer.alloc(1024)])); +} + +test("runtime extraction folds byte-identical normalized entries without running package scripts", async () => { + const root = mkdtempSync(join(tmpdir(), "pylon-runtime-extract-")); + try { + const archive = join(root, "input.tgz"); + writeFileSync(archive, tarball([ + { name: "legacy-root/package.json", value: JSON.stringify({ scripts: { postinstall: "exit 91" } }) }, + { name: "legacy-root/./file.js", value: "export {};" }, + { name: "legacy-root/file.js", value: "export {};" }, + ])); + await extractRuntimeArchive(archive, join(root, "output")); + assert.equal(readFileSync(join(root, "output/file.js"), "utf8"), "export {};"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("runtime extraction rejects traversal, links, privileges and conflicting normalized duplicates", async () => { + const root = mkdtempSync(join(tmpdir(), "pylon-runtime-reject-")); + try { + const archive = join(root, "input.tgz"); + for (const entries of [ + [{ name: "package/../escape" }], [{ name: "/package/escape" }], + [{ name: "package/link", type: "2" }], [{ name: "package/link", type: "1" }], + [{ name: "package/file", mode: 0o4755 }], + [{ name: "package/file", value: "first" }, { name: "package/./file", value: "second" }], + [{ name: "package/file" }, { name: "foreign/file" }], + ]) { + writeFileSync(archive, tarball(entries)); + await assert.rejects(extractRuntimeArchive(archive, join(root, "output")), /Unsafe runtime|Conflicting duplicate/); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/smoke-pylon-prime-agent-release.mjs b/scripts/smoke-pylon-prime-agent-release.mjs index 22e2ee1edd1..6ae3fdf5f1c 100644 --- a/scripts/smoke-pylon-prime-agent-release.mjs +++ b/scripts/smoke-pylon-prime-agent-release.mjs @@ -13,6 +13,7 @@ import { } from "./lib/pylon-release.mjs"; import { verifyPylonPrimeAgentRelease } from "./verify-pylon-prime-agent-release.mjs"; import { verifyPreviewPublication } from "./verify-pylon-preview-publication.mjs"; +import { extractRuntimeArchive } from "./lib/pylon-runtime-dependencies.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const defaultArtifacts = join(root, ".npm", "pylon-release", "artifacts"); @@ -227,6 +228,7 @@ function cleanRuntimeEnv({ home, agentDir, sessionDir, packageDir }) { name.startsWith("PRIME_AGENT_INTERNAL_DAEMON_") || name.startsWith("RLM_") || name === "FORCE_COLOR" || + name === "NODE_PATH" || name === "NO_COLOR" ) { delete env[name]; @@ -315,11 +317,16 @@ function staticSdkImport() { return 'import * as sdk from "prime-agent";\n'; } -async function smokeSdkIdentity({ probesDir, cwd, env }) { +async function smokeSdkIdentity({ probesDir, cwd, env, standalone }) { const probePath = join(probesDir, "sdk-identity.mjs"); writeFileSync( probePath, `${staticSdkImport()} +import { createRequire } from "node:module"; +const require = createRequire(new URL("../node_modules/prime-agent/package.json", import.meta.url)); +if (${JSON.stringify(standalone)}) { + for (const name of ["koffi", "@silvia-odwyer/photon-node", "@mariozechner/clipboard"]) require(name); +} const expected = ${JSON.stringify(PYLON_RELEASE_EXPECTED_SDK_FEATURES)}; if (!Object.isFrozen(sdk.PRIME_AGENT_SDK_FEATURES)) throw new Error("SDK feature tokens are not frozen."); if (JSON.stringify([...sdk.PRIME_AGENT_SDK_FEATURES]) !== JSON.stringify(expected)) throw new Error("Unexpected SDK feature tokens."); @@ -646,38 +653,26 @@ export async function smokePylonPrimeAgentRelease(artifactsDir, { historical = f try { const prefix = join(tempRoot, "install"); mkdirSync(prefix, { recursive: true }); - createLocalAssetConsumer(prefix, artifactsDir, manifest); - const npm = npmInvocation(); - run( - npm.command, - [ - ...npm.prefixArgs, - "install", - "--ignore-scripts", - "--no-audit", - "--no-fund", - "--package-lock=false", - "--loglevel=verbose", - ], - { - cwd: prefix, - env: process.env, - stdio: "inherit", - timeoutMs: releaseInstallTimeoutMs(), - }, - ); const packageDir = join(prefix, "node_modules", "prime-agent"); + if (manifest.build.recipeRevision === 1 && historical) { + createLocalAssetConsumer(prefix, artifactsDir, manifest); + const npm = npmInvocation(); + run(npm.command, [...npm.prefixArgs, "install", "--ignore-scripts", "--no-audit", "--no-fund", "--package-lock=false", "--loglevel=verbose"], { + cwd: prefix, env: process.env, stdio: "inherit", timeoutMs: releaseInstallTimeoutMs(), + }); + } else { + await extractRuntimeArchive(join(artifactsDir, releaseAssetFile("pylon-prime-agent", manifest.package.version)), packageDir); + } const installedPackage = JSON.parse(readFileSync(join(packageDir, "package.json"), "utf8")); const cliEntry = join(packageDir, installedPackage.bin["prime-agent"]); const probesDir = join(prefix, "runtime-probes"); mkdirSync(probesDir, { recursive: true }); const fixture = writeRuntimeFixture(tempRoot, packageDir); - await smokeSdkIdentity({ probesDir, cwd: fixture.projectDir, env: fixture.env }); + await smokeSdkIdentity({ probesDir, cwd: fixture.projectDir, env: fixture.env, standalone: manifest.build.recipeRevision >= 2 }); - const installedNpm = npmInvocation(fixture.env); const version = runCli( - installedNpm.command, - [...installedNpm.prefixArgs, "exec", "--offline", "--yes=false", "--", "prime-agent", "--version"], + process.execPath, + [cliEntry, "--version"], { cwd: prefix, env: fixture.env }, ); const versionOutput = `${version.stdout}${version.stderr}`.trim(); diff --git a/scripts/verify-pylon-prime-agent-release.mjs b/scripts/verify-pylon-prime-agent-release.mjs index 05773adcfb6..900efb93176 100644 --- a/scripts/verify-pylon-prime-agent-release.mjs +++ b/scripts/verify-pylon-prime-agent-release.mjs @@ -3,6 +3,7 @@ import { readdirSync, readFileSync, statSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { verifyBundledRuntimeDependencies } from "./lib/pylon-runtime-dependencies.mjs"; import { assertCleanSource, assertPylonRepository, @@ -102,6 +103,7 @@ export function verifyPylonPrimeAgentRelease(artifactsDir) { ) { throw new Error("Public package or command identity changed."); } + verifyBundledRuntimeDependencies(root, join(artifactsDir, packageArtifacts.get("prime-agent").file)); const internalPackageNames = new Set([...packageArtifacts.keys()].filter((name) => name !== "prime-agent")); for (const artifact of packageArtifacts.values()) { From 69c6b7465de2cd2595617e2634c69fc731833a9e Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 14:17:16 -0600 Subject: [PATCH 2/3] fix(release): preserve bundled dependencies through symlinked staging paths --- scripts/lib/pylon-release.mjs | 4 +++- scripts/pylon-prime-agent-release.test.mjs | 28 +++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/scripts/lib/pylon-release.mjs b/scripts/lib/pylon-release.mjs index 59da3ac91de..7956ebfd78c 100644 --- a/scripts/lib/pylon-release.mjs +++ b/scripts/lib/pylon-release.mjs @@ -7,6 +7,7 @@ import { mkdirSync, readFileSync, readdirSync, + realpathSync, renameSync, rmSync, statSync, @@ -481,7 +482,8 @@ export function prepareOutputDirectory(outDir) { } export function packStagingPackage({ root, stagingDir, artifactsDir, assetFile, environment }) { - const output = runNpm(["pack", stagingDir, "--pack-destination", artifactsDir, "--silent"], { + // npm omits bundled dependencies when an ancestor of the package path is a symlink. + const output = runNpm(["pack", realpathSync(stagingDir), "--pack-destination", artifactsDir, "--silent"], { cwd: root, env: environment, forwardStderr: true, diff --git a/scripts/pylon-prime-agent-release.test.mjs b/scripts/pylon-prime-agent-release.test.mjs index d2484eb1825..c14f62ca7c8 100644 --- a/scripts/pylon-prime-agent-release.test.mjs +++ b/scripts/pylon-prime-agent-release.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import "./pylon-runtime-dependencies.test.mjs"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { test } from "node:test"; @@ -12,6 +12,8 @@ import { createReleasePackageJson, isCanonicalSha512Integrity, npmInvocation, + packStagingPackage, + readTarJson, pylonDistribution, PYLON_RELEASE_MANIFEST, PYLON_RELEASE_NPM_VERSION, @@ -370,3 +372,27 @@ test("both fresh pack workflows acquire locked runtime inputs before the offline assert.equal(job.split("npm run release:pylon:hydrate-runtime").length, 2); } }); + +test("packing through a symlinked staging parent retains bundled runtime files", () => { + const fixture = mkdtempSync(join(tmpdir(), "pylon-release-pack-link-")); + try { + const stage = join(fixture, "actual", "package"); + const artifactsDir = join(fixture, "artifacts"); + mkdirSync(join(stage, "node_modules", "runtime-fixture"), { recursive: true }); + mkdirSync(artifactsDir); + writeFileSync(join(stage, "package.json"), JSON.stringify({ + name: "pylon-pack-fixture", version: "1.0.0", + dependencies: { "runtime-fixture": "1.0.0" }, bundleDependencies: ["runtime-fixture"], + })); + const dependency = { name: "runtime-fixture", version: "1.0.0" }; + writeFileSync(join(stage, "node_modules", "runtime-fixture", "package.json"), JSON.stringify(dependency)); + symlinkSync(join(fixture, "actual"), join(fixture, "linked"), "dir"); + const packed = packStagingPackage({ + root: fixture, stagingDir: join(fixture, "linked", "package"), artifactsDir, assetFile: "fixture.tgz", + environment: { ...process.env, npm_config_offline: "true" }, + }); + assert.deepEqual(readTarJson(packed.path, "package/node_modules/runtime-fixture/package.json"), dependency); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); From 92344360b068dcd7eaf0b60cbb4423694d5f5c64 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 11 Sep 2026 14:32:44 -0600 Subject: [PATCH 3/3] test(publication): prove losing-builder cleanup refusal under contention --- scripts/fixtures/publication-crash/worker.mjs | 8 ++++ scripts/pylon-publication-stress.test.mjs | 41 ++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/scripts/fixtures/publication-crash/worker.mjs b/scripts/fixtures/publication-crash/worker.mjs index 1ece03bfb57..36f3150ef67 100644 --- a/scripts/fixtures/publication-crash/worker.mjs +++ b/scripts/fixtures/publication-crash/worker.mjs @@ -45,6 +45,14 @@ const options = { stateMaxBytes: config.stateMaxBytes ?? 1024, now: () => config beforeCommitDecision: () => barrier("semantic", { phase: "before", operation: "commit-decision", path: state }), afterCommitDecision: () => barrier("semantic", { phase: "after", operation: "commit-decision", path: state }), } }; +if (config.pauseLosingBuilderRead) options.afterInitialPathStat = async ({ path, stat }) => { + if (readPaused || !path.includes("/.building-") || basename(path) !== "checkpoint.json") return; + readPaused = true; + await send({ type: "cut", pid: process.pid, phase: "losing-builder-before-open", path, identity: { dev: stat.dev, ino: stat.ino } }); + await new Promise((resolve, reject) => process.once("message", (message) => { + if (message?.type === "release") resolve(); else reject(new Error("Unexpected losing-builder barrier release.")); + })); +}; if (config.pauseRead === "receipt-") options.lstatEntry = async (path) => { const stat = await lstat(path); if (basename(path).startsWith("receipt-")) await pauseRead(path, "receipt-"); diff --git a/scripts/pylon-publication-stress.test.mjs b/scripts/pylon-publication-stress.test.mjs index db5497024ca..df3b5bb432e 100644 --- a/scripts/pylon-publication-stress.test.mjs +++ b/scripts/pylon-publication-stress.test.mjs @@ -9,8 +9,9 @@ const rounds = 10; const refusals = /actively locked|changed|disappeared|ENOENT|EEXIST|inode|receipt|publication|authority|conflicting|writer|ownership|claim|checkpoint|namespace|unfinished|incomplete|unsafe type, owner or exact permissions/; const migrationScenarios = new Set(families.map((family) => `migrate-${family}`)); const staleMigrationTarget = "Migration temporary has an unknown immutable target."; +const lostBuilderCheckpoint = "Generation builder conflicts with the independently installed winner."; function assertRefusal(scenario, message) { - if (migrationScenarios.has(scenario) && message === staleMigrationTarget) return; + if (migrationScenarios.has(scenario) && [staleMigrationTarget, lostBuilderCheckpoint].includes(message)) return; assert.match(message, refusals); } async function competition(t, f, scenarios, round, historyProof) { @@ -160,6 +161,44 @@ for (const family of families) test(`publication stale pre-intent migration refu } finally { await Promise.all(workers.map((worker) => worker.stop())); await cleanup(f); } }); +test("publication losing-builder reader refuses native open loss after peer cleanup", { timeout: 60000 }, async (t) => { + const f = await fixture("loser-cleanup"); const workers = []; + try { + const names = await readdir(f.root); + const winner = names.find((name) => name.startsWith("generation-")); + const loser = names.find((name) => name.startsWith(".building-")); + assert.ok(winner && loser); + const winnerPath = join(f.root, winner, "checkpoint.json"); + const winnerBytes = await readFile(winnerPath); const winnerStat = await lstat(winnerPath); + assert.deepEqual(await readFile(join(f.root, loser, "checkpoint.json")), winnerBytes); + const reader = child(f, "loser-cleanup", { pauseLosingBuilderRead: true, marker: "stale-builder" }); workers.push(reader); + const cut = await reader.wait("cut"); + assert.equal(cut.pid, reader.process.pid); assert.equal(cut.phase, "losing-builder-before-open"); + assert.equal(cut.path, join(f.root, loser, "checkpoint.json")); + const pinned = await lstat(cut.path); + assert.deepEqual(cut.identity, { dev: pinned.dev, ino: pinned.ino }); + const peer = child(f, "loser-cleanup", { marker: "cleanup-peer" }); workers.push(peer); + const cleaned = await peer.finish(); assert.deepEqual(cleaned.entries, [winner]); + await assert.rejects(lstat(cut.path), { code: "ENOENT" }); + reader.process.send({ type: "release" }); + assert.deepEqual(await reader.exit, { code: 1, signal: null }); + const refusal = reader.messages.find((message) => message.type === "error"); + assert.equal(refusal.pid, reader.process.pid); + assert.equal(refusal.message, lostBuilderCheckpoint); + const recovery = child(f, "loser-cleanup", { marker: "fresh-builder" }); workers.push(recovery); + const recovered = await recovery.finish(); + assert.deepEqual(recovered.root, cleaned.root); assert.deepEqual(recovered.entries, [winner]); + assert.equal(await projection(f), null); + assert.deepEqual(await readFile(winnerPath), winnerBytes); + const finalStat = await lstat(winnerPath); + assert.deepEqual([finalStat.dev, finalStat.ino, finalStat.nlink], [winnerStat.dev, winnerStat.ino, winnerStat.nlink]); + for (const marker of ["stale-builder", "cleanup-peer", "fresh-builder"]) await assert.rejects(lstat(join(f.directory, `${marker}.callbacks`)), { code: "ENOENT" }); + t.diagnostic(JSON.stringify({ pids: workers.map((worker) => worker.process.pid), refusal: refusal.message, winnerSha256: digest(winnerBytes), root: recovered.root, callbacks: 0 })); + assertRefusal("migrate-v1-v2", refusal.message); + for (const denied of ["commit", "rotate", "migrate-unknown"]) assert.throws(() => assertRefusal(denied, refusal.message), assert.AssertionError); + } finally { await Promise.all(workers.map((worker) => worker.stop())); await cleanup(f); } +}); + for (const kind of ["checkpoint.json", "claim-index-", "heartbeat-", "terminal-", "transition-", "applied-", "receipt-"]) test(`publication pinned ${kind} reader versus process rotation`, { timeout: 60000 }, async () => { const f = await fixture("commit"); const reader = child(f, "commit", { recover: true, pauseRead: kind, marker: "reader", stateMaxBytes: 8192 }); let rotator; let recovery; try {