diff --git a/containers/enclave/Dockerfile b/containers/enclave/Dockerfile index 1f24fdb00..4e5c9bbaf 100644 --- a/containers/enclave/Dockerfile +++ b/containers/enclave/Dockerfile @@ -17,7 +17,7 @@ ARG NPM_CONFIG_REGISTRY=https://registry.npmjs.org/ RUN npm_config_ignore_scripts=false npm install -g "@github/copilot@${COPILOT_CLI_VERSION}" \ && npm_config_ignore_scripts=true npm install \ --prefix /usr/local/lib/node_modules/@github/copilot \ - --omit=dev adm-zip@0.6.0 \ + --omit=dev adm-zip@0.6.1 \ && find /usr/local/lib/node_modules/@github/copilot \ -path '*/foundry-local-sdk/node_modules/adm-zip' -prune -type d \ -exec sh -c 'rm -rf "$1" && cp -a /usr/local/lib/node_modules/@github/copilot/node_modules/adm-zip "$1"' sh {} \; \ diff --git a/docs/nvx-security-design.md b/docs/nvx-security-design.md index ba8f12f86..b1795f3b4 100644 --- a/docs/nvx-security-design.md +++ b/docs/nvx-security-design.md @@ -312,3 +312,33 @@ Phase 1 is complete when maintainers have reviewed and accepted: Phase 2 may then implement the deterministic filesystem builder and one-shot execution adapter. Managed execution is not an exit criterion. + +## Phase 2 implementation boundary + +Phase 2 adds reusable foundations without making `nvx` a selectable AWF +runtime: + +- `src/nvx/filesystem-builder.ts` creates ordered deterministic EROFS layers + from explicitly selected source roots, excludes known credential paths at + any depth, rejects escaping links and special files, records source and image + identities, and creates one fresh bounded ext4 scratch image per invocation. + Staging requires Linux `/proc/self/fd` and no-follow directory descriptors to + prevent source-tree path races; +- `src/nvx/one-shot-adapter.ts` constructs only `nvx.py sandbox run`, launches + without a shell or inherited credential environment, applies a host + wall-clock timeout or cancellation to the full process group, filters + workflow-command syntax as a byte stream, and retains bounded raw output + tails; and +- `src/nvx/outcome.ts` requires the exact one-shot outcome and teardown schema, + preserves guest status codes including `125`, and verifies that NVX reports + the requested fail-closed network policy. + +The adapter intentionally requires a prebuilt absolute entrypoint and +whitespace-free NVX arguments. Complex agent commands must be placed in an +immutable layer as an entrypoint script because the pinned NVX kernel-command +line ABI rejects whitespace-bearing `--arg` values. + +Phase 2 does not add `nvx` to the runtime registry or CLI. Selection remains +blocked until Phase 3 supplies the dedicated VMM identity, ACL-only device +access, filesystem jail, cgroup, host-enforced network namespace, live +confinement verification, artifact attestation, and durable stale cleanup. diff --git a/scripts/ci/self-hosted-runner-doctor-workflow.test.ts b/scripts/ci/self-hosted-runner-doctor-workflow.test.ts index b2807a831..23bffba52 100644 --- a/scripts/ci/self-hosted-runner-doctor-workflow.test.ts +++ b/scripts/ci/self-hosted-runner-doctor-workflow.test.ts @@ -289,7 +289,7 @@ describe('self-hosted runner doctor workflow config', () => { expect(portableAgent).toMatch(/^\| B33 \|.*github\/gh-aw-firewall#8624 \|$/m); expect(portableAgent).toMatch(/^\| D14 \|.*github\/gh-aw-firewall#8622 \|$/m); expect(portableAgent).toContain('| `Direct sbx egress reached 1.1.1.1 without proxy environment variables` that clears after a brief retry while Squid starts | D13 update'); - expect(portableAgent).toContain('| `docker compose up -d` fails after AWF topology recreates Squid mid-run (new `extra_hosts`), with stale/unwritable Squid log files | B33 update'); + expect(portableAgent).toContain('- `docker compose up -d` fails after AWF topology recreates Squid mid-run (new `extra_hosts`), with stale/unwritable Squid log files → B33 update'); expect(portableAgent).toContain('| `"cloud-hypervisor --version" exited with code undefined` under `--container-runtime cloud-hypervisor` | D14'); expect(portableAgent).toContain('D11 / github/gh-aw-firewall#6558 — gVisor + Node.js v22 V8 ESM startup crash root cause remains unresolved (`SIGABRT` `StringBytes::Encode` assertion and occasional exit 139).'); expect(portableAgent).toContain('D12 / github/gh-aw-firewall#6810, github/gh-aw-firewall#6811 — Copilot runs using `model: auto` under isolated runtimes (`--container-runtime gvisor` or `sbx`) could fail before agent start with `awf-reflect: request failed: fetch failed` plus `Model "auto" has no AI credits pricing and no default pricing is configured` when `apiProxy.maxAiCredits` was enabled.'); diff --git a/src/nvx/filesystem-builder.test.ts b/src/nvx/filesystem-builder.test.ts new file mode 100644 index 000000000..d1c6ca3e0 --- /dev/null +++ b/src/nvx/filesystem-builder.test.ts @@ -0,0 +1,234 @@ +import { promises as fs } from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + deriveNvxLayerUuid, + normalizeScratchBytes, + NvxFilesystemBuilder, + NVX_MIN_SCRATCH_BYTES, + type NvxFilesystemBuilderDependencies, +} from './filesystem-builder'; + +describe('NVX deterministic filesystem builder', () => { + it('builds ordered deterministic EROFS layers and a private ext4 scratch image', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'awf-nvx-images-')); + const distro = path.join(root, 'distro-source'); + const custom = path.join(root, 'custom-source'); + await fs.mkdir(path.join(distro, 'bin'), { recursive: true }); + await fs.writeFile(path.join(distro, 'bin', 'tool'), 'binary'); + await fs.chmod(path.join(distro, 'bin', 'tool'), 0o4755); + await fs.symlink('bin/tool', path.join(distro, 'tool')); + await fs.mkdir(path.join(custom, '.config', 'gh'), { recursive: true }); + await fs.writeFile(path.join(custom, '.config', 'gh', 'hosts.yml'), 'secret'); + await fs.mkdir(path.join(custom, '.ssh'), { recursive: true }); + await fs.writeFile(path.join(custom, '.ssh', 'deploy_key'), 'secret'); + await fs.writeFile(path.join(custom, 'README'), 'safe'); + + const commands: Array<{ command: string; args: readonly string[] }> = []; + const dependencies: NvxFilesystemBuilderDependencies = { + runTool: jest.fn(async (command, args) => { + commands.push({ command, args: [...args] }); + if (command === 'mkfs.erofs') { + await fs.writeFile(args[args.length - 2], `${command}\n${args.join('\n')}`); + } + }), + randomUuid: () => '11111111-2222-4333-8444-555555555555', + sha256: jest.fn(async (filePath) => `digest:${path.basename(filePath)}`), + }; + const builder = new NvxFilesystemBuilder({ + runId: 'phase-2', + workDir: root, + layers: [ + { role: 'custom', sourcePath: custom }, + { role: 'distro', sourcePath: distro }, + ], + sourceDateEpoch: 1_700_000_000, + }, dependencies); + + try { + const bundle = await builder.prepare(); + + expect(bundle.layers.map(({ role }) => role)).toEqual(['distro', 'custom']); + expect(bundle.scratch).toMatchObject({ + uuid: '11111111-2222-4333-8444-555555555555', + sizeBytes: NVX_MIN_SCRATCH_BYTES, + }); + expect(commands[0]).toEqual({ + command: 'mkfs.erofs', + args: expect.arrayContaining([ + '--workers=1', + '--sort=path', + '--all-root', + '-T', + '1700000000', + '-U', + bundle.layers[0].uuid, + ]), + }); + expect(commands.map(({ command }) => command)).toEqual([ + 'mkfs.erofs', + 'mkfs.erofs', + 'mke2fs', + ]); + expect(commands[2].args).toEqual(expect.arrayContaining([ + '-O', + '^has_journal', + '-E', + 'lazy_itable_init=0,lazy_journal_init=0,root_owner=65534:65534', + ])); + expect((await fs.stat(bundle.layers[0].path)).mode & 0o777).toBe(0o400); + expect((await fs.stat(bundle.scratch.path)).mode & 0o777).toBe(0o600); + expect(await fs.readFile( + path.join(builder.stagingDirectory, 'distro', 'tool'), + 'utf8', + )).toBe('binary'); + expect((await fs.stat( + path.join(builder.stagingDirectory, 'distro', 'bin', 'tool'), + )).mode & 0o777).toBe(0o755); + await expect(fs.access( + path.join(builder.stagingDirectory, 'custom', '.config', 'gh', 'hosts.yml'), + )).rejects.toThrow(); + await expect(fs.access( + path.join(builder.stagingDirectory, 'custom', '.ssh', 'deploy_key'), + )).rejects.toThrow(); + expect(await fs.readFile( + path.join(builder.stagingDirectory, 'custom', 'README'), + 'utf8', + )).toBe('safe'); + + const manifest = JSON.parse(await fs.readFile(bundle.manifestPath, 'utf8')); + expect(manifest).toMatchObject({ + schemaVersion: 1, + sourceDateEpoch: 1_700_000_000, + layers: [ + { role: 'distro', file: 'distro.erofs' }, + { role: 'custom', file: 'custom.erofs' }, + ], + scratch: { + file: 'scratch.ext4', + uuid: '11111111-2222-4333-8444-555555555555', + uid: 65534, + gid: 65534, + }, + }); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('derives stable role-bound UUIDs and enforces scratch limits', () => { + const digest = 'a'.repeat(64); + expect(deriveNvxLayerUuid('distro', digest)) + .toBe(deriveNvxLayerUuid('distro', digest)); + expect(deriveNvxLayerUuid('distro', digest)) + .not.toBe(deriveNvxLayerUuid('runtime', digest)); + expect(normalizeScratchBytes(1)).toBe(NVX_MIN_SCRATCH_BYTES); + expect(normalizeScratchBytes(NVX_MIN_SCRATCH_BYTES + 1) % 4096).toBe(0); + expect(() => normalizeScratchBytes( + NVX_MIN_SCRATCH_BYTES * 2, + NVX_MIN_SCRATCH_BYTES, + )).toThrow(/exceeding cap/); + }); + + it('rejects escaping symlinks, duplicate roles, and unsafe run identifiers', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'awf-nvx-images-')); + const source = path.join(root, 'source'); + await fs.mkdir(source); + await fs.symlink('../outside', path.join(source, 'escape')); + const dependencies: NvxFilesystemBuilderDependencies = { + runTool: jest.fn(), + randomUuid: () => '11111111-2222-4333-8444-555555555555', + sha256: jest.fn(), + }; + try { + await expect(new NvxFilesystemBuilder({ + runId: 'safe', + workDir: root, + layers: [{ role: 'distro', sourcePath: source }], + }, dependencies).prepare()).rejects.toThrow(/symlink target.*escapes/); + expect(() => new NvxFilesystemBuilder({ + runId: '../unsafe', + workDir: root, + layers: [{ role: 'distro', sourcePath: source }], + }, dependencies)).toThrow(/Unsafe NVX run id/); + expect(() => new NvxFilesystemBuilder({ + runId: 'duplicate', + workDir: root, + layers: [ + { role: 'distro', sourcePath: source }, + { role: 'distro', sourcePath: source }, + ], + }, dependencies)).toThrow(/Duplicate NVX layer role/); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('excludes credential paths nested below a rootfs home directory', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'awf-nvx-images-')); + const source = path.join(root, 'source'); + const nestedHome = path.join(source, 'home', 'runner'); + await fs.mkdir(path.join(nestedHome, '.config', 'gh'), { recursive: true }); + await fs.writeFile(path.join(nestedHome, '.config', 'gh', 'hosts.yml'), 'secret'); + await fs.writeFile(path.join(nestedHome, 'safe'), 'visible'); + const dependencies: NvxFilesystemBuilderDependencies = { + runTool: jest.fn(async (command, args) => { + if (command === 'mkfs.erofs') { + await fs.writeFile(args[args.length - 2], 'layer'); + } + }), + randomUuid: () => '11111111-2222-4333-8444-555555555555', + sha256: jest.fn(async () => 'a'.repeat(64)), + }; + const builder = new NvxFilesystemBuilder({ + runId: 'nested-credentials', + workDir: root, + layers: [{ role: 'distro', sourcePath: source }], + }, dependencies); + try { + await builder.prepare(); + await expect(fs.access(path.join( + builder.stagingDirectory, + 'distro', + 'home', + 'runner', + '.config', + 'gh', + ))).rejects.toThrow(); + await expect(fs.readFile(path.join( + builder.stagingDirectory, + 'distro', + 'home', + 'runner', + 'safe', + ), 'utf8')).resolves.toBe('visible'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('never removes a pre-existing run directory it does not own', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'awf-nvx-images-')); + const source = path.join(root, 'source'); + const existingRun = path.join(root, 'nvx-images', 'collision'); + await fs.mkdir(source); + await fs.mkdir(existingRun, { recursive: true }); + await fs.writeFile(path.join(existingRun, 'owned-by-other-run'), 'keep'); + const dependencies: NvxFilesystemBuilderDependencies = { + runTool: jest.fn(), + randomUuid: () => '11111111-2222-4333-8444-555555555555', + sha256: jest.fn(), + }; + try { + await expect(new NvxFilesystemBuilder({ + runId: 'collision', + workDir: root, + layers: [{ role: 'distro', sourcePath: source }], + }, dependencies).prepare()).rejects.toMatchObject({ code: 'EEXIST' }); + await expect(fs.readFile(path.join(existingRun, 'owned-by-other-run'), 'utf8')) + .resolves.toBe('keep'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/nvx/filesystem-builder.ts b/src/nvx/filesystem-builder.ts new file mode 100644 index 000000000..82b5c59a2 --- /dev/null +++ b/src/nvx/filesystem-builder.ts @@ -0,0 +1,641 @@ +import { createHash, randomUUID } from 'crypto'; +import { + constants, + createReadStream, + createWriteStream, + promises as fs, + type Stats, +} from 'fs'; +import type { FileHandle } from 'fs/promises'; +import * as path from 'path'; +import execa from 'execa'; +import { pipeline } from 'stream/promises'; +import { CREDENTIAL_ENTRIES, HOME_FORBIDDEN_SUBDIRS } from '../config/mount-policy'; + +const MIB = 1024 * 1024; +const NVX_BLOCK_BYTES = 4096; +export const NVX_MIN_SCRATCH_BYTES = 128 * MIB; +export const NVX_DEFAULT_MAX_SCRATCH_BYTES = 8 * 1024 * MIB; +export const NVX_LAYER_ROLES = ['distro', 'runtime', 'custom'] as const; +export type NvxLayerRole = typeof NVX_LAYER_ROLES[number]; + +export interface NvxLayerSource { + readonly role: NvxLayerRole; + readonly sourcePath: string; + readonly exclude?: readonly string[]; +} + +export interface NvxLayerSourceManifestEntry { + readonly path: string; + readonly type: 'file' | 'directory' | 'symlink'; + readonly mode: number; + readonly size: number; + readonly sha256?: string; + readonly target?: string; +} + +export interface NvxLayerArtifact { + readonly role: NvxLayerRole; + readonly path: string; + readonly uuid: string; + readonly sha256: string; + readonly sourceManifestSha256: string; + readonly sourceEntries: number; + readonly excludedPaths: readonly string[]; +} + +export interface NvxScratchArtifact { + readonly path: string; + readonly uuid: string; + readonly sizeBytes: number; + readonly uid: number; + readonly gid: number; +} + +export interface NvxFilesystemBundle { + readonly runDirectory: string; + readonly manifestPath: string; + readonly sourceDateEpoch: number; + readonly layers: readonly NvxLayerArtifact[]; + readonly scratch: NvxScratchArtifact; +} + +export interface NvxFilesystemBuilderConfig { + readonly runId: string; + readonly workDir: string; + readonly layers: readonly NvxLayerSource[]; + readonly scratchBytes?: number; + readonly maxScratchBytes?: number; + readonly scratchUid?: number; + readonly scratchGid?: number; + readonly sourceDateEpoch?: number; +} + +export interface NvxFilesystemBuilderDependencies { + runTool(command: string, args: readonly string[]): Promise; + randomUuid(): string; + sha256(filePath: string): Promise; +} + +const defaultDependencies: NvxFilesystemBuilderDependencies = { + runTool: async (command, args) => { + const result = await execa(command, [...args], { + reject: false, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + }); + if (result.exitCode === 0) return; + throw new Error( + `${command} exited with code ${result.exitCode}: ` + + `${result.stderr.trim() || result.stdout.trim()}`, + ); + }, + randomUuid: randomUUID, + sha256: sha256File, +}; + +interface StagedLayer { + readonly role: NvxLayerRole; + readonly stagingPath: string; + readonly sourceManifestSha256: string; + readonly sourceEntries: number; + readonly excludedPaths: readonly string[]; +} + +const DEFAULT_EXCLUDED_PATHS = [ + ...CREDENTIAL_ENTRIES.map(({ path: entryPath }) => normalizeRelative(entryPath)), + ...HOME_FORBIDDEN_SUBDIRS.map(normalizeRelative), +]; + +/** + * Builds immutable deterministic EROFS layers and a fresh private ext4 scratch + * image for one NVX invocation. + * + * This module does not launch NVX. Runtime selection remains disabled until + * the Phase 3 host-confinement and network-policy gates are implemented. + */ +export class NvxFilesystemBuilder { + readonly runDirectory: string; + readonly stagingDirectory: string; + readonly manifestPath: string; + + private prepared = false; + private ownsRunDirectory = false; + + constructor( + private readonly config: NvxFilesystemBuilderConfig, + private readonly dependencies: NvxFilesystemBuilderDependencies = defaultDependencies, + ) { + assertSafeRunId(config.runId); + assertLayerSources(config.layers); + this.runDirectory = path.join(config.workDir, 'nvx-images', config.runId); + this.stagingDirectory = path.join(this.runDirectory, 'staging'); + this.manifestPath = path.join(this.runDirectory, 'manifest.json'); + } + + async prepare(): Promise { + if (this.prepared) throw new Error('NVX filesystem bundle is already prepared'); + const sourceDateEpoch = this.config.sourceDateEpoch ?? 0; + assertSourceDateEpoch(sourceDateEpoch); + const scratchUid = this.config.scratchUid ?? 65534; + const scratchGid = this.config.scratchGid ?? 65534; + assertNonRootIdentity(scratchUid, scratchGid); + const scratchBytes = normalizeScratchBytes( + this.config.scratchBytes ?? NVX_MIN_SCRATCH_BYTES, + this.config.maxScratchBytes ?? NVX_DEFAULT_MAX_SCRATCH_BYTES, + ); + + const imageRoot = path.dirname(this.runDirectory); + await fs.mkdir(imageRoot, { recursive: true, mode: 0o700 }); + const imageRootStat = await fs.lstat(imageRoot); + if (!imageRootStat.isDirectory() || imageRootStat.isSymbolicLink()) { + throw new Error(`NVX image root must be a real directory: ${imageRoot}`); + } + await fs.chmod(imageRoot, 0o700); + try { + await fs.mkdir(this.runDirectory, { mode: 0o700 }); + this.ownsRunDirectory = true; + await fs.mkdir(this.stagingDirectory, { mode: 0o700 }); + const layers: NvxLayerArtifact[] = []; + for (const source of orderedLayerSources(this.config.layers)) { + const staged = await this.stageLayer(source); + layers.push(await this.buildLayer(staged, sourceDateEpoch)); + } + const scratch = await this.buildScratch(scratchBytes, scratchUid, scratchGid); + const manifest = { + schemaVersion: 1, + sourceDateEpoch, + layers: layers.map((layer) => ({ + role: layer.role, + file: path.basename(layer.path), + uuid: layer.uuid, + sha256: layer.sha256, + sourceManifestSha256: layer.sourceManifestSha256, + sourceEntries: layer.sourceEntries, + excludedPaths: layer.excludedPaths, + })), + scratch: { + file: path.basename(scratch.path), + uuid: scratch.uuid, + sizeBytes: scratch.sizeBytes, + uid: scratch.uid, + gid: scratch.gid, + }, + }; + await writeExclusiveJson(this.manifestPath, manifest, 0o400); + this.prepared = true; + return { + runDirectory: this.runDirectory, + manifestPath: this.manifestPath, + sourceDateEpoch, + layers, + scratch, + }; + } catch (error) { + if (this.ownsRunDirectory) { + await fs.rm(this.runDirectory, { recursive: true, force: true }); + this.ownsRunDirectory = false; + } + throw error; + } + } + + async cleanup(): Promise { + if (!this.ownsRunDirectory) return; + await fs.rm(this.runDirectory, { recursive: true, force: true }); + this.ownsRunDirectory = false; + } + + private async stageLayer(source: NvxLayerSource): Promise { + const sourcePath = path.resolve(source.sourcePath); + const sourceStat = await fs.lstat(sourcePath); + if (!sourceStat.isDirectory() || sourceStat.isSymbolicLink()) { + throw new Error(`NVX ${source.role} layer source must be a real directory: ${sourcePath}`); + } + const stagingPath = path.join(this.stagingDirectory, source.role); + await fs.mkdir(stagingPath, { mode: 0o700 }); + const excludedPaths = normalizeExcludedPaths([ + ...DEFAULT_EXCLUDED_PATHS, + ...(source.exclude ?? []), + ]); + const entries: NvxLayerSourceManifestEntry[] = []; + await copyDeterministicTree( + sourcePath, + stagingPath, + excludedPaths, + entries, + this.config.sourceDateEpoch ?? 0, + ); + const sourceManifestSha256 = sha256Text(JSON.stringify({ + role: source.role, + excludedPaths, + entries, + })); + return { + role: source.role, + stagingPath, + sourceManifestSha256, + sourceEntries: entries.length, + excludedPaths, + }; + } + + private async buildLayer( + staged: StagedLayer, + sourceDateEpoch: number, + ): Promise { + const uuid = deriveNvxLayerUuid(staged.role, staged.sourceManifestSha256); + const imagePath = path.join(this.runDirectory, `${staged.role}.erofs`); + await this.dependencies.runTool('mkfs.erofs', [ + '--quiet', + '--workers=1', + '--sort=path', + '--all-root', + '-E', 'force-inode-compact', + '-T', String(sourceDateEpoch), + '-U', uuid, + imagePath, + staged.stagingPath, + ]); + await assertRegularFile(imagePath, `NVX ${staged.role} layer image`); + await fs.chmod(imagePath, 0o400); + return { + role: staged.role, + path: imagePath, + uuid, + sha256: await this.dependencies.sha256(imagePath), + sourceManifestSha256: staged.sourceManifestSha256, + sourceEntries: staged.sourceEntries, + excludedPaths: staged.excludedPaths, + }; + } + + private async buildScratch( + sizeBytes: number, + uid: number, + gid: number, + ): Promise { + const scratchPath = path.join(this.runDirectory, 'scratch.ext4'); + const handle = await fs.open(scratchPath, 'wx', 0o600); + try { + await handle.truncate(sizeBytes); + } finally { + await handle.close(); + } + const uuid = this.dependencies.randomUuid(); + assertUuid(uuid, 'NVX scratch UUID'); + await this.dependencies.runTool('mke2fs', [ + '-t', 'ext4', + '-F', + '-q', + '-b', String(NVX_BLOCK_BYTES), + '-m', '0', + '-L', 'awf-nvx-scratch', + '-U', uuid, + '-O', '^has_journal', + '-E', `lazy_itable_init=0,lazy_journal_init=0,root_owner=${uid}:${gid}`, + scratchPath, + String(sizeBytes / NVX_BLOCK_BYTES), + ]); + await fs.chmod(scratchPath, 0o600); + return { + path: scratchPath, + uuid, + sizeBytes, + uid, + gid, + }; + } +} + +export function deriveNvxLayerUuid( + role: NvxLayerRole, + sourceManifestSha256: string, +): string { + if (!/^[a-f0-9]{64}$/.test(sourceManifestSha256)) { + throw new Error('NVX layer source manifest digest must be a lowercase SHA-256 value'); + } + const bytes = createHash('sha256') + .update(`awf-nvx-layer-v1\0${role}\0${sourceManifestSha256}`) + .digest() + .subarray(0, 16); + bytes[6] = (bytes[6] & 0x0f) | 0x50; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + const hex = bytes.toString('hex'); + return [ + hex.slice(0, 8), + hex.slice(8, 12), + hex.slice(12, 16), + hex.slice(16, 20), + hex.slice(20), + ].join('-'); +} + +export function normalizeScratchBytes( + requestedBytes: number, + maximumBytes = NVX_DEFAULT_MAX_SCRATCH_BYTES, +): number { + if ( + !Number.isSafeInteger(maximumBytes) || + maximumBytes < NVX_MIN_SCRATCH_BYTES + ) { + throw new Error( + `NVX scratch cap must be at least ${NVX_MIN_SCRATCH_BYTES} bytes`, + ); + } + if (!Number.isSafeInteger(requestedBytes) || requestedBytes <= 0) { + throw new Error(`Invalid NVX scratch size: ${requestedBytes}`); + } + const normalized = Math.max(NVX_MIN_SCRATCH_BYTES, requestedBytes); + const aligned = Math.ceil(normalized / NVX_BLOCK_BYTES) * NVX_BLOCK_BYTES; + if (aligned > maximumBytes) { + throw new Error(`NVX scratch requires ${aligned} bytes, exceeding cap ${maximumBytes}`); + } + return aligned; +} + +async function copyDeterministicTree( + sourceRoot: string, + destinationRoot: string, + excludedPaths: readonly string[], + manifest: NvxLayerSourceManifestEntry[], + sourceDateEpoch: number, +): Promise { + await assertDescriptorTraversalSupport(); + const root = await openDirectoryNoFollow(sourceRoot); + try { + await copyDirectory(root, ''); + } finally { + await root.close(); + } + + async function copyDirectory(directory: FileHandle, relativePath: string): Promise { + const current = descriptorPath(directory); + if (relativePath) { + const destination = stagingPath(destinationRoot, relativePath); + const stat = await directory.stat(); + const mode = normalizedMode(stat); + await fs.mkdir(destination, { recursive: true, mode }); + await fs.chmod(destination, mode); + manifest.push({ path: relativePath, type: 'directory', mode, size: 0 }); + } + const children = await fs.readdir(current); + children.sort(); + for (const child of children) { + const childPath = path.join(current, child); + const childRelativePath = relativePath ? `${relativePath}/${child}` : child; + if (isExcluded(childRelativePath, excludedPaths)) continue; + const stat = await fs.lstat(childPath); + const destination = stagingPath(destinationRoot, childRelativePath); + if (stat.isSymbolicLink()) { + const target = await fs.readlink(childPath); + if (path.isAbsolute(target)) { + throw new Error(`Absolute symlink is not safe for NVX layer: ${childRelativePath}`); + } + assertContained( + sourceRoot, + path.resolve(sourceRoot, path.dirname(childRelativePath), target), + `NVX layer symlink target for ${childRelativePath}`, + ); + await fs.mkdir(path.dirname(destination), { recursive: true, mode: 0o700 }); + await fs.symlink(target, destination); + await fs.lutimes(destination, sourceDateEpoch, sourceDateEpoch); + manifest.push({ + path: childRelativePath, + type: 'symlink', + mode: stat.mode & 0o777, + size: stat.size, + target, + }); + } else if (stat.isDirectory()) { + const childDirectory = await openDirectoryNoFollow(childPath); + try { + await copyDirectory(childDirectory, childRelativePath); + } finally { + await childDirectory.close(); + } + } else if (stat.isFile()) { + await copyRegularFile(childPath, destination, stat, childRelativePath); + } else { + throw new Error(`Special filesystem entry is not safe for NVX layer: ${childRelativePath}`); + } + } + if (relativePath) { + await fs.utimes(stagingPath(destinationRoot, relativePath), sourceDateEpoch, sourceDateEpoch); + } + } + + async function copyRegularFile( + source: string, + destination: string, + lstat: Stats, + relativePath: string, + ): Promise { + const handle = await fs.open(source, constants.O_RDONLY | noFollowFlag()); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.dev !== lstat.dev || stat.ino !== lstat.ino) { + throw new Error(`NVX layer source changed while opening file: ${relativePath}`); + } + const mode = normalizedMode(stat); + await fs.mkdir(path.dirname(destination), { recursive: true, mode: 0o700 }); + await pipeline( + createReadStream(descriptorPath(handle)), + createWriteStream(destination, { flags: 'wx', mode }), + ); + await fs.chmod(destination, mode); + await fs.utimes(destination, sourceDateEpoch, sourceDateEpoch); + const copied = await fs.stat(destination); + manifest.push({ + path: relativePath, + type: 'file', + mode, + size: copied.size, + sha256: await sha256File(destination), + }); + } finally { + await handle.close(); + } + } +} + +async function openDirectoryNoFollow(directoryPath: string): Promise { + const handle = await fs.open( + directoryPath, + constants.O_RDONLY | directoryFlag() | noFollowFlag(), + ); + const stat = await handle.stat(); + if (!stat.isDirectory()) { + await handle.close(); + throw new Error(`NVX layer source must be a real directory: ${directoryPath}`); + } + return handle; +} + +function descriptorPath(handle: FileHandle): string { + return `/proc/self/fd/${handle.fd}`; +} + +function noFollowFlag(): number { + return constants.O_NOFOLLOW as number; +} + +function directoryFlag(): number { + return constants.O_DIRECTORY as number; +} + +async function assertDescriptorTraversalSupport(): Promise { + if ( + process.platform !== 'linux' || + constants.O_NOFOLLOW === undefined || + constants.O_DIRECTORY === undefined + ) { + throw new Error('NVX filesystem staging requires Linux descriptor traversal support'); + } + try { + await fs.access('/proc/self/fd'); + } catch { + throw new Error('NVX filesystem staging requires mounted /proc/self/fd'); + } +} + +function stagingPath(root: string, relativePath: string): string { + const destination = path.join(root, relativePath); + assertContained(root, destination, 'NVX layer staging destination'); + return destination; +} + +function normalizedMode(stat: Stats): number { + return stat.mode & 0o777; +} + +function orderedLayerSources( + sources: readonly NvxLayerSource[], +): readonly NvxLayerSource[] { + const byRole = new Map(sources.map((source) => [source.role, source])); + return NVX_LAYER_ROLES.flatMap((role) => { + const source = byRole.get(role); + return source ? [source] : []; + }); +} + +function assertLayerSources(sources: readonly NvxLayerSource[]): void { + if (sources.length < 1 || sources.length > NVX_LAYER_ROLES.length) { + throw new Error('NVX requires one to three deterministic EROFS layer sources'); + } + const roles = sources.map(({ role }) => role); + for (const role of roles) { + if (!NVX_LAYER_ROLES.includes(role)) { + throw new Error(`Unsupported NVX layer role: ${String(role)}`); + } + } + const duplicates = roles.filter((role, index) => roles.indexOf(role) !== index); + if (duplicates.length > 0) { + throw new Error(`Duplicate NVX layer role: ${duplicates[0]}`); + } +} + +function normalizeExcludedPaths(values: readonly string[]): readonly string[] { + const normalized = new Set(); + for (const value of values) { + if ( + !value || + path.isAbsolute(value) || + value.includes('\0') + ) { + throw new Error(`NVX excluded path must be a non-empty relative path: ${value}`); + } + const relative = normalizeRelative(path.normalize(value)); + if (relative === '..' || relative.startsWith('../')) { + throw new Error(`NVX excluded path escapes its layer source: ${value}`); + } + normalized.add(relative.replace(/^\.\//, '').replace(/\/$/, '')); + } + return [...normalized].sort(); +} + +function isExcluded(relativePath: string, excludedPaths: readonly string[]): boolean { + return excludedPaths.some((excluded) => + relativePath === excluded || + relativePath.startsWith(`${excluded}/`) || + relativePath.endsWith(`/${excluded}`) || + relativePath.includes(`/${excluded}/`) + ); +} + +function assertContained(root: string, candidate: string, label: string): void { + const relative = path.relative(path.resolve(root), path.resolve(candidate)); + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new Error(`${label} escapes ${root}: ${candidate}`); + } +} + +function normalizeRelative(value: string): string { + return value.split(path.sep).join('/'); +} + +function assertSafeRunId(runId: string): void { + if (!/^[A-Za-z0-9-]{1,64}$/.test(runId)) { + throw new Error(`Unsafe NVX run id: ${runId}`); + } +} + +function assertSourceDateEpoch(value: number): void { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`Invalid NVX SOURCE_DATE_EPOCH value: ${value}`); + } +} + +function assertNonRootIdentity(uid: number, gid: number): void { + if ( + !Number.isInteger(uid) || + !Number.isInteger(gid) || + uid < 1 || + gid < 1 || + uid > 0xffff_ffff || + gid > 0xffff_ffff + ) { + throw new Error(`NVX scratch identity must be a non-root UID:GID: ${uid}:${gid}`); + } +} + +function assertUuid(value: string, label: string): void { + if ( + !/^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test( + value, + ) + ) { + throw new Error(`${label} must be a canonical lowercase UUID: ${value}`); + } +} + +async function assertRegularFile(filePath: string, label: string): Promise { + const stat = await fs.lstat(filePath); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error(`${label} must be a regular file: ${filePath}`); + } +} + +async function writeExclusiveJson( + filePath: string, + value: unknown, + mode: number, +): Promise { + const handle = await fs.open(filePath, 'wx', mode); + try { + await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`, 'utf8'); + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function sha256File(filePath: string): Promise { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(filePath)) hash.update(chunk as Buffer); + return hash.digest('hex'); +} + +function sha256Text(value: string): string { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} diff --git a/src/nvx/index.ts b/src/nvx/index.ts new file mode 100644 index 000000000..054e1f7f7 --- /dev/null +++ b/src/nvx/index.ts @@ -0,0 +1,38 @@ +export { + NvxFilesystemBuilder, + deriveNvxLayerUuid, + normalizeScratchBytes, + NVX_DEFAULT_MAX_SCRATCH_BYTES, + NVX_LAYER_ROLES, + NVX_MIN_SCRATCH_BYTES, +} from './filesystem-builder'; +export type { + NvxFilesystemBuilderConfig, + NvxFilesystemBuilderDependencies, + NvxFilesystemBundle, + NvxLayerArtifact, + NvxLayerRole, + NvxLayerSource, + NvxLayerSourceManifestEntry, + NvxScratchArtifact, +} from './filesystem-builder'; +export { + NvxOneShotAdapter, + NvxOneShotExecutionError, + buildNvxOneShotArguments, +} from './one-shot-adapter'; +export type { + NvxOneShotAdapterDependencies, + NvxOneShotExecutionRequest, + NvxOneShotExecutionResult, + NvxOneShotNetworkPlan, +} from './one-shot-adapter'; +export { + NVX_OUTCOME_SCHEMA_VERSION, + NVX_TEARDOWN_STAGES, + parseNvxOneShotOutcome, +} from './outcome'; +export type { + NvxOneShotOutcome, + NvxOutcomeCategory, +} from './outcome'; diff --git a/src/nvx/one-shot-adapter.test.ts b/src/nvx/one-shot-adapter.test.ts new file mode 100644 index 000000000..08ec943e9 --- /dev/null +++ b/src/nvx/one-shot-adapter.test.ts @@ -0,0 +1,478 @@ +import { createHash } from 'crypto'; +import { promises as fs } from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { PassThrough } from 'stream'; +import type { NvxFilesystemBundle } from './filesystem-builder'; +import { + buildNvxOneShotArguments, + NvxOneShotAdapter, + testHelpers, + type NvxOneShotAdapterDependencies, + type NvxOneShotExecutionRequest, +} from './one-shot-adapter'; +import { NVX_TEARDOWN_STAGES } from './outcome'; + +async function sha256(filePath: string): Promise { + return createHash('sha256').update(await fs.readFile(filePath)).digest('hex'); +} + +async function fixture(): Promise<{ + root: string; + bundle: NvxFilesystemBundle; + request: NvxOneShotExecutionRequest; +}> { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'awf-nvx-run-')); + const nvxRoot = path.join(root, 'nvx'); + const runDirectory = path.join(root, 'run'); + await fs.mkdir(path.join(nvxRoot, 'scripts'), { recursive: true }); + await fs.mkdir(runDirectory); + await fs.chmod(runDirectory, 0o700); + await fs.writeFile(path.join(nvxRoot, 'scripts', 'nvx.py'), '#!/usr/bin/env python3\n'); + const layerPath = path.join(runDirectory, 'distro.erofs'); + const scratchPath = path.join(runDirectory, 'scratch.ext4'); + await fs.writeFile(layerPath, 'layer'); + await fs.writeFile(scratchPath, 'scratch', { mode: 0o600 }); + await fs.chmod(layerPath, 0o400); + await fs.chmod(scratchPath, 0o600); + const bundle: NvxFilesystemBundle = { + runDirectory, + manifestPath: path.join(runDirectory, 'manifest.json'), + sourceDateEpoch: 0, + layers: [{ + role: 'distro', + path: layerPath, + uuid: '11111111-2222-5333-8444-555555555555', + sha256: await sha256(layerPath), + sourceManifestSha256: 'a'.repeat(64), + sourceEntries: 1, + excludedPaths: [], + }], + scratch: { + path: scratchPath, + uuid: 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee', + sizeBytes: (await fs.stat(scratchPath)).size, + uid: 65534, + gid: 65534, + }, + }; + await fs.writeFile(bundle.manifestPath, `${JSON.stringify({ + schemaVersion: 1, + sourceDateEpoch: bundle.sourceDateEpoch, + layers: bundle.layers.map((layer) => ({ + role: layer.role, + file: path.basename(layer.path), + uuid: layer.uuid, + sha256: layer.sha256, + sourceManifestSha256: layer.sourceManifestSha256, + sourceEntries: layer.sourceEntries, + excludedPaths: layer.excludedPaths, + })), + scratch: { + file: path.basename(bundle.scratch.path), + uuid: bundle.scratch.uuid, + sizeBytes: bundle.scratch.sizeBytes, + uid: bundle.scratch.uid, + gid: bundle.scratch.gid, + }, + })}\n`, { mode: 0o400 }); + await fs.chmod(bundle.manifestPath, 0o400); + return { + root, + bundle, + request: { + nvxRoot, + filesystem: bundle, + entrypoint: '/usr/local/bin/awf-agent', + args: ['--run'], + memoryMaxBytes: 512 * 1024 * 1024, + pidsMax: 256, + memoryMib: 768, + timeoutMs: 30_000, + network: { + guestAddress: '192.168.127.2/30', + proxyAddress: '192.168.127.1:3128', + egressAllow: ['tcp:192.168.127.1:3128'], + hostLoopbackForwards: ['172.30.0.30:10002'], + }, + }, + }; +} + +function outcome(category: 'success' | 'guest-exit' | 'vmm-failure', statusCode: number) { + return { + schema_version: 1, + instance_id: 'a'.repeat(32), + backend: 'kvm', + outcome: { + operation: 'run', + category, + status_code: statusCode, + }, + network_policy: { + status: 'applied', + status_code: 0, + mode: 'rules', + allow_rule_count: 1, + deny_rule_count: 0, + host_loopback: 'allow', + }, + teardown: Object.fromEntries(NVX_TEARDOWN_STAGES.map((stage) => [stage, true])), + }; +} + +describe('NVX one-shot execution adapter', () => { + it('builds only a one-shot sandbox run command with fixed-deny network defaults', async () => { + const { root, request } = await fixture(); + try { + const args = buildNvxOneShotArguments( + request, + path.join(request.filesystem.runDirectory, 'outcome.json'), + ); + expect(args.slice(0, 3)).toEqual(['scripts/nvx.py', 'sandbox', 'run']); + expect(args).toEqual(expect.arrayContaining([ + '--layer', + `distro,${request.filesystem.layers[0].path},11111111-2222-5333-8444-555555555555`, + '--scratch', + request.filesystem.scratch.path, + '--network-profile', + 'portable', + '--network-egress', + 'deny', + '--network-ingress', + 'deny', + '--host-loopback', + 'allow', + '--outcome-report', + path.join(request.filesystem.runDirectory, 'outcome.json'), + ])); + expect(args).not.toEqual(expect.arrayContaining([ + 'provision', + 'start', + 'exec', + 'stop', + 'deprovision', + '--state-dir', + ])); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('preserves guest exit 125, filters streamed output, and keeps bounded raw tails', async () => { + const { root, request } = await fixture(); + const stdout = new PassThrough(); + const presented: Buffer[] = []; + stdout.on('data', (chunk: Buffer) => presented.push(chunk)); + let hostEnvironment: NodeJS.ProcessEnv | undefined; + const dependencies: NvxOneShotAdapterDependencies = { + pythonBinary: '/usr/bin/python3', + runProcess: jest.fn(async (processRequest) => { + hostEnvironment = processRequest.env; + await processRequest.onStdout(Buffer.from(':')); + await processRequest.onStdout(Buffer.from(':notice::unsafe\nsafe\n')); + await processRequest.onStderr(Buffer.from('stderr-tail')); + const outcomePath = processRequest.args[processRequest.args.length - 1]; + await fs.writeFile( + outcomePath, + `${JSON.stringify(outcome('guest-exit', 125))}\n`, + { mode: 0o600 }, + ); + await fs.chmod(outcomePath, 0o600); + return { exitCode: 125, signal: null, timedOut: false, cancelled: false }; + }), + }; + const previousToken = process.env.GITHUB_TOKEN; + process.env.GITHUB_TOKEN = 'must-not-leak'; + try { + const result = await new NvxOneShotAdapter(dependencies).execute({ + ...request, + stdout, + rawTailBytes: 8, + }); + + expect(result).toMatchObject({ + exitCode: 125, + category: 'guest-exit', + signal: null, + timedOut: false, + }); + expect(Buffer.concat(presented).toString()).toContain( + '[awf blocked workflow command] : :notice::unsafe', + ); + expect(result.rawStdoutTail.length).toBe(8); + expect(result.rawStderrTail.toString()).toBe('err-tail'); + expect(hostEnvironment?.GITHUB_TOKEN).toBeUndefined(); + expect(hostEnvironment).toMatchObject({ + PYTHONUNBUFFERED: '1', + TMPDIR: request.filesystem.runDirectory, + }); + } finally { + if (previousToken === undefined) delete process.env.GITHUB_TOKEN; + else process.env.GITHUB_TOKEN = previousToken; + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('returns 124 on a host timeout without requiring a completed outcome', async () => { + const { root, request } = await fixture(); + const dependencies: NvxOneShotAdapterDependencies = { + pythonBinary: '/usr/bin/python3', + runProcess: jest.fn(async () => ({ + exitCode: null, + signal: 'SIGKILL' as NodeJS.Signals, + timedOut: true, + cancelled: false, + })), + }; + try { + await expect(new NvxOneShotAdapter(dependencies).execute(request)) + .resolves.toMatchObject({ + exitCode: 124, + category: 'timeout', + signal: 'SIGKILL', + timedOut: true, + }); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('requires the workload identity to own the scratch filesystem', async () => { + const { root, request } = await fixture(); + const runProcess = jest.fn(); + try { + await expect(new NvxOneShotAdapter({ + pythonBinary: '/usr/bin/python3', + runProcess, + }).execute({ + ...request, + workloadUid: 1000, + workloadGid: 1000, + })).rejects.toThrow(/scratch owner 65534:65534 must match workload identity 1000:1000/); + expect(runProcess).not.toHaveBeenCalled(); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects missing outcomes, wrapper mismatches, and modified images', async () => { + const missing = await fixture(); + const noOutcomeDependencies: NvxOneShotAdapterDependencies = { + pythonBinary: '/usr/bin/python3', + runProcess: jest.fn(async () => ({ + exitCode: 1, + signal: null, + timedOut: false, + cancelled: false, + })), + }; + try { + await expect(new NvxOneShotAdapter(noOutcomeDependencies).execute(missing.request)) + .rejects.toThrow(/did not produce a structured outcome/); + } finally { + await fs.rm(missing.root, { recursive: true, force: true }); + } + + const mismatch = await fixture(); + const mismatchDependencies: NvxOneShotAdapterDependencies = { + pythonBinary: '/usr/bin/python3', + runProcess: jest.fn(async (processRequest) => { + const outcomePath = processRequest.args[processRequest.args.length - 1]; + await fs.writeFile( + outcomePath, + `${JSON.stringify(outcome('vmm-failure', 1))}\n`, + { mode: 0o600 }, + ); + await fs.chmod(outcomePath, 0o600); + return { exitCode: 125, signal: null, timedOut: false, cancelled: false }; + }), + }; + try { + await expect(new NvxOneShotAdapter(mismatchDependencies).execute(mismatch.request)) + .rejects.toThrow(/wrapper exit 125 does not match.*vmm-failure:1/); + } finally { + await fs.rm(mismatch.root, { recursive: true, force: true }); + } + + const modified = await fixture(); + const runProcess = jest.fn(); + await fs.chmod(modified.bundle.layers[0].path, 0o600); + await fs.appendFile(modified.bundle.layers[0].path, 'tampered'); + await fs.chmod(modified.bundle.layers[0].path, 0o400); + try { + await expect(new NvxOneShotAdapter({ + pythonBinary: '/usr/bin/python3', + runProcess, + }).execute(modified.request)).rejects.toThrow(/digest changed/); + expect(runProcess).not.toHaveBeenCalled(); + } finally { + await fs.rm(modified.root, { recursive: true, force: true }); + } + }); + + it('rejects a successful workload when the structured network policy differs', async () => { + const { root, request } = await fixture(); + const dependencies: NvxOneShotAdapterDependencies = { + pythonBinary: '/usr/bin/python3', + runProcess: jest.fn(async (processRequest) => { + const report = outcome('success', 0); + report.network_policy.allow_rule_count = 0; + const outcomePath = processRequest.args[processRequest.args.length - 1]; + await fs.writeFile(outcomePath, `${JSON.stringify(report)}\n`, { mode: 0o600 }); + await fs.chmod(outcomePath, 0o600); + return { exitCode: 0, signal: null, timedOut: false, cancelled: false }; + }), + }; + try { + await expect(new NvxOneShotAdapter(dependencies).execute(request)) + .rejects.toMatchObject({ + category: 'outcome-mismatch', + message: expect.stringMatching(/network policy does not match/), + }); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('preserves bounded output evidence when the outcome is missing', async () => { + const { root, request } = await fixture(); + const dependencies: NvxOneShotAdapterDependencies = { + pythonBinary: '/usr/bin/python3', + runProcess: jest.fn(async (processRequest) => { + await processRequest.onStderr(Buffer.from('diagnostic')); + return { exitCode: 1, signal: null, timedOut: false, cancelled: false }; + }), + }; + try { + await expect(new NvxOneShotAdapter(dependencies).execute(request)) + .rejects.toMatchObject({ + category: 'missing-outcome', + rawStderrTail: Buffer.from('diagnostic'), + }); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('rejects a symbolic-link outcome report', async () => { + const { root, request } = await fixture(); + const externalOutcome = path.join(root, 'external-outcome.json'); + await fs.writeFile( + externalOutcome, + `${JSON.stringify(outcome('success', 0))}\n`, + { mode: 0o600 }, + ); + const dependencies: NvxOneShotAdapterDependencies = { + pythonBinary: '/usr/bin/python3', + runProcess: jest.fn(async (processRequest) => { + const outcomePath = processRequest.args[processRequest.args.length - 1]; + await fs.symlink(externalOutcome, outcomePath); + return { exitCode: 0, signal: null, timedOut: false, cancelled: false }; + }), + }; + try { + await expect(new NvxOneShotAdapter(dependencies).execute(request)) + .rejects.toMatchObject({ + category: 'invalid-outcome', + message: expect.stringMatching(/must be a regular file/), + }); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('returns a distinct cancellation result', async () => { + const { root, request } = await fixture(); + const dependencies: NvxOneShotAdapterDependencies = { + pythonBinary: '/usr/bin/python3', + runProcess: jest.fn(async () => ({ + exitCode: null, + signal: 'SIGTERM' as NodeJS.Signals, + timedOut: false, + cancelled: true, + })), + }; + try { + await expect(new NvxOneShotAdapter(dependencies).execute(request)) + .resolves.toMatchObject({ + exitCode: 130, + category: 'cancelled', + signal: 'SIGTERM', + timedOut: false, + }); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('escalates process-group termination after the launcher exits', async () => { + const childPidPath = path.join( + await fs.mkdtemp(path.join(os.tmpdir(), 'awf-nvx-process-')), + 'child.pid', + ); + const childScript = [ + "const fs = require('fs');", + "process.on('SIGTERM', () => {});", + `fs.writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid));`, + 'setInterval(() => {}, 1_000);', + ].join(''); + const launcherScript = [ + "const { spawn } = require('child_process');", + `spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`, + 'setInterval(() => {}, 1_000);', + ].join(''); + try { + const started = Date.now(); + const controller = new AbortController(); + const run = testHelpers.runNvxProcess({ + command: process.execPath, + args: ['-e', launcherScript], + cwd: process.cwd(), + env: process.env, + abortSignal: controller.signal, + onStdout: async () => {}, + onStderr: async () => {}, + }); + const childPid = Number(await waitForFile(childPidPath)); + controller.abort(); + const result = await run; + expect(result.cancelled).toBe(true); + expect(Date.now() - started).toBeGreaterThanOrEqual(2_000); + expect(await processHasExited(childPid)).toBe(true); + } finally { + await fs.rm(path.dirname(childPidPath), { recursive: true, force: true }); + } + }, 10_000); +}); + +async function waitForFile(filePath: string): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + try { + return await fs.readFile(filePath, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + } + throw new Error(`Timed out waiting for ${filePath}`); +} + +async function processHasExited(pid: number): Promise { + if (process.platform !== 'linux') { + try { + process.kill(pid, 0); + return false; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') return true; + throw error; + } + } + try { + const stat = await fs.readFile(`/proc/${pid}/stat`, 'utf8'); + return stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0] === 'Z'; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return true; + throw error; + } +} diff --git a/src/nvx/one-shot-adapter.ts b/src/nvx/one-shot-adapter.ts new file mode 100644 index 000000000..941d65f82 --- /dev/null +++ b/src/nvx/one-shot-adapter.ts @@ -0,0 +1,848 @@ +import { constants, createReadStream, promises as fs } from 'fs'; +import { createHash } from 'crypto'; +import * as path from 'path'; +import { spawn } from 'child_process'; +import type { Readable, Writable } from 'stream'; +import { WorkflowCommandFilter } from '../microvm/workflow-command-filter'; +import type { + NvxFilesystemBundle, + NvxLayerArtifact, +} from './filesystem-builder'; +import { + parseNvxOneShotOutcome, + type NvxOneShotOutcome, +} from './outcome'; + +const NVX_MAX_OUTCOME_BYTES = 64 * 1024; +const NVX_DEFAULT_RAW_TAIL_BYTES = 64 * 1024; +const NVX_TERMINATION_GRACE_MS = 2_000; +const NVX_DEFAULT_WORKLOAD_ID = 65534; + +export interface NvxOneShotNetworkPlan { + readonly guestAddress: string; + readonly proxyAddress: string; + readonly egressAllow?: readonly string[]; + readonly egressDeny?: readonly string[]; + readonly hostLoopbackForwards?: readonly string[]; +} + +export interface NvxOneShotExecutionRequest { + readonly nvxRoot: string; + readonly filesystem: NvxFilesystemBundle; + readonly entrypoint: string; + readonly args?: readonly string[]; + readonly hostname?: string; + readonly workloadUid?: number; + readonly workloadGid?: number; + readonly memoryMaxBytes?: number; + readonly pidsMax?: number; + readonly memoryMib?: number; + readonly timeoutMs?: number; + readonly abortSignal?: AbortSignal; + readonly network: NvxOneShotNetworkPlan; + readonly stdout?: Writable; + readonly stderr?: Writable; + readonly rawTailBytes?: number; +} + +export interface NvxOneShotExecutionResult { + readonly exitCode: number; + readonly category: + | NvxOneShotOutcome['outcome']['category'] + | 'timeout' + | 'cancelled' + | 'signal'; + readonly signal: NodeJS.Signals | null; + readonly timedOut: boolean; + readonly outcome?: NvxOneShotOutcome; + readonly rawStdoutTail: Buffer; + readonly rawStderrTail: Buffer; +} + +interface NvxProcessRequest { + readonly command: string; + readonly args: readonly string[]; + readonly cwd: string; + readonly env: NodeJS.ProcessEnv; + readonly timeoutMs?: number; + readonly abortSignal?: AbortSignal; + onStdout(chunk: Buffer): Promise; + onStderr(chunk: Buffer): Promise; +} + +interface NvxProcessResult { + readonly exitCode: number | null; + readonly signal: NodeJS.Signals | null; + readonly timedOut: boolean; + readonly cancelled: boolean; +} + +export interface NvxOneShotAdapterDependencies { + runProcess(request: NvxProcessRequest): Promise; + pythonBinary: string; +} + +export class NvxOneShotExecutionError extends Error { + readonly cause?: unknown; + + constructor( + message: string, + readonly category: 'missing-outcome' | 'invalid-outcome' | 'outcome-mismatch', + readonly rawStdoutTail: Buffer, + readonly rawStderrTail: Buffer, + cause?: unknown, + ) { + super(message); + this.name = 'NvxOneShotExecutionError'; + this.cause = cause; + } +} + +const defaultDependencies: NvxOneShotAdapterDependencies = { + runProcess: runNvxProcess, + pythonBinary: '/usr/bin/python3', +}; + +/** + * Runs one complete workload through `nvx.py sandbox run`. + * + * The adapter deliberately exposes no provision/start/exec/stop lifecycle. + * It is not registered as an AWF runtime until host confinement and network + * verification are implemented in Phase 3. + */ +export class NvxOneShotAdapter { + constructor( + private readonly dependencies: NvxOneShotAdapterDependencies = defaultDependencies, + ) {} + + async execute(request: NvxOneShotExecutionRequest): Promise { + validateRequest(request); + await verifyFilesystemBundle(request.filesystem); + const nvxScript = path.join(request.nvxRoot, 'scripts', 'nvx.py'); + await assertRegularFile(nvxScript, 'NVX launcher'); + const outcomePath = path.join(request.filesystem.runDirectory, 'outcome.json'); + await assertNewOutcomePath(outcomePath, request.filesystem.runDirectory); + + const rawTailBytes = request.rawTailBytes ?? NVX_DEFAULT_RAW_TAIL_BYTES; + if (!Number.isSafeInteger(rawTailBytes) || rawTailBytes < 0) { + throw new Error(`NVX raw output tail limit must be a non-negative integer: ${rawTailBytes}`); + } + const stdoutTail = new BoundedByteTail(rawTailBytes); + const stderrTail = new BoundedByteTail(rawTailBytes); + const stdoutFilter = new WorkflowCommandFilter(); + const stderrFilter = new WorkflowCommandFilter(); + const processResult = await this.dependencies.runProcess({ + command: this.dependencies.pythonBinary, + args: buildNvxOneShotArguments(request, outcomePath), + cwd: request.nvxRoot, + env: buildNvxHostEnvironment(request.filesystem.runDirectory), + timeoutMs: request.timeoutMs, + abortSignal: request.abortSignal, + onStdout: async (chunk) => { + stdoutTail.push(chunk); + if (request.stdout) { + await writeWithBackpressure(request.stdout, stdoutFilter.push(chunk)); + } + }, + onStderr: async (chunk) => { + stderrTail.push(chunk); + if (request.stderr) { + await writeWithBackpressure(request.stderr, stderrFilter.push(chunk)); + } + }, + }); + if (request.stdout) { + await writeWithBackpressure(request.stdout, stdoutFilter.finish()); + } + if (request.stderr) { + await writeWithBackpressure(request.stderr, stderrFilter.finish()); + } + + if (processResult.timedOut) { + return { + exitCode: 124, + category: 'timeout', + signal: processResult.signal, + timedOut: true, + rawStdoutTail: stdoutTail.value(), + rawStderrTail: stderrTail.value(), + }; + } + if (processResult.cancelled) { + return { + exitCode: 130, + category: 'cancelled', + signal: processResult.signal, + timedOut: false, + rawStdoutTail: stdoutTail.value(), + rawStderrTail: stderrTail.value(), + }; + } + if (processResult.exitCode === null) { + return { + exitCode: signalExitCode(processResult.signal), + category: 'signal', + signal: processResult.signal, + timedOut: false, + rawStdoutTail: stdoutTail.value(), + rawStderrTail: stderrTail.value(), + }; + } + + let outcome: NvxOneShotOutcome; + try { + outcome = await readOutcome(outcomePath); + } catch (error) { + const category = error instanceof Error && + error.message.includes('did not produce') + ? 'missing-outcome' + : 'invalid-outcome'; + throw new NvxOneShotExecutionError( + formatError(error), + category, + stdoutTail.value(), + stderrTail.value(), + error, + ); + } + try { + assertMatchingExitCode(processResult.exitCode, outcome); + if (outcome.outcome.category !== 'vmm-failure') { + assertNetworkPolicy(request.network, outcome); + } + } catch (error) { + throw new NvxOneShotExecutionError( + formatError(error), + 'outcome-mismatch', + stdoutTail.value(), + stderrTail.value(), + error, + ); + } + return { + exitCode: outcome.outcome.statusCode, + category: outcome.outcome.category, + signal: processResult.signal, + timedOut: false, + outcome, + rawStdoutTail: stdoutTail.value(), + rawStderrTail: stderrTail.value(), + }; + } +} + +export function buildNvxOneShotArguments( + request: NvxOneShotExecutionRequest, + outcomePath: string, +): readonly string[] { + const uid = request.workloadUid ?? NVX_DEFAULT_WORKLOAD_ID; + const gid = request.workloadGid ?? NVX_DEFAULT_WORKLOAD_ID; + const args = [ + path.join('scripts', 'nvx.py'), + 'sandbox', + 'run', + ]; + for (const layer of orderedLayers(request.filesystem.layers)) { + args.push('--layer', `${layer.role},${layer.path},${layer.uuid}`); + } + args.push( + '--scratch', request.filesystem.scratch.path, + '--entrypoint', request.entrypoint, + ); + for (const argument of request.args ?? []) args.push(`--arg=${argument}`); + args.push( + '--hostname', request.hostname ?? 'awf-nvx', + '--workload-user', `${uid}:${gid}`, + ); + if (request.memoryMaxBytes !== undefined) { + args.push('--memory-max', String(request.memoryMaxBytes)); + } + if (request.pidsMax !== undefined) args.push('--pids-max', String(request.pidsMax)); + args.push( + '--memory-mib', String(request.memoryMib ?? 512), + '--hypervisor', 'kvm', + '--net', request.network.guestAddress, + '--network-profile', 'portable', + '--network-egress', 'deny', + '--network-ingress', 'deny', + ); + for (const rule of request.network.egressAllow ?? []) { + args.push('--network-egress-allow', rule); + } + for (const rule of request.network.egressDeny ?? []) { + args.push('--network-egress-deny', rule); + } + const forwards = request.network.hostLoopbackForwards ?? []; + args.push('--host-loopback', forwards.length > 0 ? 'allow' : 'deny'); + args.push('--network-proxy', request.network.proxyAddress); + for (const forward of forwards) args.push('--host-loopback-forward', forward); + args.push('--outcome-report', outcomePath); + return args; +} + +async function verifyFilesystemBundle(bundle: NvxFilesystemBundle): Promise { + const runDirectory = path.resolve(bundle.runDirectory); + const runDirectoryStat = await fs.lstat(runDirectory); + if (!runDirectoryStat.isDirectory() || runDirectoryStat.isSymbolicLink()) { + throw new Error(`NVX run directory must be a real directory: ${runDirectory}`); + } + if ((runDirectoryStat.mode & 0o077) !== 0) { + throw new Error('NVX run directory must not be accessible to group or other users'); + } + if (bundle.layers.length < 1 || bundle.layers.length > 3) { + throw new Error('NVX filesystem bundle must contain one to three layers'); + } + const roles = new Set(); + for (const layer of orderedLayers(bundle.layers)) { + if (layer.role !== 'distro' && layer.role !== 'runtime' && layer.role !== 'custom') { + throw new Error(`Unsupported NVX layer role: ${String(layer.role)}`); + } + if (roles.has(layer.role)) throw new Error(`Duplicate NVX layer role: ${layer.role}`); + roles.add(layer.role); + assertContained(runDirectory, layer.path, `NVX ${layer.role} layer`); + if (path.resolve(layer.path) !== path.join(runDirectory, `${layer.role}.erofs`)) { + throw new Error(`NVX ${layer.role} layer has an unexpected artifact path`); + } + assertSafeDiskPath(layer.path); + const stat = await assertRegularFile(layer.path, `NVX ${layer.role} layer`); + if ((stat.mode & 0o777) !== 0o400) { + throw new Error(`NVX ${layer.role} layer image must have mode 0400`); + } + assertUuid(layer.uuid, `NVX ${layer.role} layer UUID`); + assertSha256(layer.sha256, `NVX ${layer.role} layer digest`); + assertSha256( + layer.sourceManifestSha256, + `NVX ${layer.role} source manifest digest`, + ); + const digest = await sha256File(layer.path); + if (digest !== layer.sha256) { + throw new Error(`NVX ${layer.role} layer digest changed after preparation`); + } + } + assertContained(runDirectory, bundle.scratch.path, 'NVX scratch image'); + if (path.resolve(bundle.scratch.path) !== path.join(runDirectory, 'scratch.ext4')) { + throw new Error('NVX scratch image has an unexpected artifact path'); + } + assertSafeDiskPath(bundle.scratch.path); + assertUuid(bundle.scratch.uuid, 'NVX scratch UUID'); + const scratch = await assertRegularFile(bundle.scratch.path, 'NVX scratch image'); + if (scratch.size !== bundle.scratch.sizeBytes) { + throw new Error('NVX scratch image size changed after preparation'); + } + if ((scratch.mode & 0o077) !== 0) { + throw new Error('NVX scratch image must not be accessible to group or other users'); + } + if (path.resolve(bundle.manifestPath) !== path.join(runDirectory, 'manifest.json')) { + throw new Error('NVX filesystem manifest has an unexpected artifact path'); + } + const manifestStat = await assertRegularFile( + bundle.manifestPath, + 'NVX filesystem manifest', + ); + if ((manifestStat.mode & 0o777) !== 0o400) { + throw new Error('NVX filesystem manifest must have mode 0400'); + } + if (manifestStat.size < 1 || manifestStat.size > NVX_MAX_OUTCOME_BYTES) { + throw new Error('NVX filesystem manifest has an invalid size'); + } + let manifest: unknown; + try { + manifest = JSON.parse(await fs.readFile(bundle.manifestPath, 'utf8')); + } catch (error) { + throw new Error(`NVX filesystem manifest is invalid: ${formatError(error)}`); + } + const expectedManifest = { + schemaVersion: 1, + sourceDateEpoch: bundle.sourceDateEpoch, + layers: orderedLayers(bundle.layers).map((layer) => ({ + role: layer.role, + file: path.basename(layer.path), + uuid: layer.uuid, + sha256: layer.sha256, + sourceManifestSha256: layer.sourceManifestSha256, + sourceEntries: layer.sourceEntries, + excludedPaths: layer.excludedPaths, + })), + scratch: { + file: path.basename(bundle.scratch.path), + uuid: bundle.scratch.uuid, + sizeBytes: bundle.scratch.sizeBytes, + uid: bundle.scratch.uid, + gid: bundle.scratch.gid, + }, + }; + if (canonicalJson(manifest) !== canonicalJson(expectedManifest)) { + throw new Error('NVX filesystem manifest does not match the prepared artifacts'); + } +} + +function validateRequest(request: NvxOneShotExecutionRequest): void { + if (request.entrypoint.includes('\0') || !request.entrypoint.startsWith('/')) { + throw new Error('NVX entrypoint must be an absolute path without NUL bytes'); + } + if (/\s/.test(request.entrypoint)) { + throw new Error('NVX entrypoint must not contain whitespace'); + } + for (const argument of request.args ?? []) { + if (!argument || /\s/.test(argument) || argument.includes('\0')) { + throw new Error('NVX one-shot arguments must be nonempty and contain no whitespace'); + } + } + const hostname = request.hostname ?? 'awf-nvx'; + if ( + hostname.length < 1 || + hostname.length > 63 || + hostname.startsWith('-') || + hostname.endsWith('-') || + !/^[a-z0-9-]+$/.test(hostname) + ) { + throw new Error(`Invalid NVX hostname: ${hostname}`); + } + const workloadUid = request.workloadUid ?? NVX_DEFAULT_WORKLOAD_ID; + const workloadGid = request.workloadGid ?? NVX_DEFAULT_WORKLOAD_ID; + assertPositiveInteger(workloadUid, 'NVX workload UID'); + assertPositiveInteger(workloadGid, 'NVX workload GID'); + if ( + request.filesystem.scratch.uid !== workloadUid || + request.filesystem.scratch.gid !== workloadGid + ) { + throw new Error( + `NVX scratch owner ${request.filesystem.scratch.uid}:${request.filesystem.scratch.gid} ` + + `must match workload identity ${workloadUid}:${workloadGid}`, + ); + } + assertOptionalPositiveInteger(request.memoryMaxBytes, 'NVX memory limit'); + assertOptionalPositiveInteger(request.pidsMax, 'NVX process limit'); + assertOptionalPositiveInteger(request.memoryMib, 'NVX guest memory'); + assertOptionalPositiveInteger(request.timeoutMs, 'NVX timeout'); + assertIpv4Cidr(request.network.guestAddress, 'NVX guest network address'); + assertIpv4Endpoint(request.network.proxyAddress, 'NVX network proxy address'); + for (const value of [ + ...(request.network.egressAllow ?? []), + ...(request.network.egressDeny ?? []), + ...(request.network.hostLoopbackForwards ?? []), + ]) { + if (!value || value.includes('\0') || /\s/.test(value)) { + throw new Error(`Invalid NVX network rule: ${value}`); + } + } +} + +function orderedLayers(layers: readonly NvxLayerArtifact[]): readonly NvxLayerArtifact[] { + const order = new Map([ + ['distro', 0], + ['runtime', 1], + ['custom', 2], + ]); + return [...layers].sort((left, right) => + (order.get(left.role) ?? Number.MAX_SAFE_INTEGER) - + (order.get(right.role) ?? Number.MAX_SAFE_INTEGER) + ); +} + +async function readOutcome(outcomePath: string): Promise { + let handle; + try { + handle = await fs.open( + outcomePath, + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0), + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + throw new Error('NVX one-shot execution did not produce a structured outcome report'); + } + if ((error as NodeJS.ErrnoException).code === 'ELOOP') { + throw new Error(`NVX outcome report must be a regular file: ${outcomePath}`); + } + throw error; + } + + try { + const stat = await handle.stat(); + if (!stat.isFile()) { + throw new Error(`NVX outcome report must be a regular file: ${outcomePath}`); + } + if (stat.size < 1 || stat.size > NVX_MAX_OUTCOME_BYTES) { + throw new Error(`NVX outcome report size must be 1-${NVX_MAX_OUTCOME_BYTES} bytes`); + } + if ((stat.mode & 0o077) !== 0) { + throw new Error('NVX outcome report must not be accessible to group or other users'); + } + const contents = await handle.readFile(); + if (contents.length < 1 || contents.length > NVX_MAX_OUTCOME_BYTES) { + throw new Error(`NVX outcome report size must be 1-${NVX_MAX_OUTCOME_BYTES} bytes`); + } + return parseNvxOneShotOutcome(contents.toString('utf8')); + } finally { + await handle.close(); + } +} + +async function assertNewOutcomePath(outcomePath: string, runDirectory: string): Promise { + assertContained(runDirectory, outcomePath, 'NVX outcome report'); + const parent = await fs.lstat(path.dirname(outcomePath)); + if (!parent.isDirectory() || parent.isSymbolicLink()) { + throw new Error('NVX outcome report parent must be a real directory'); + } + try { + await fs.lstat(outcomePath); + throw new Error(`NVX outcome report already exists: ${outcomePath}`); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== 'ENOENT') throw error; + } +} + +function assertMatchingExitCode( + wrapperExitCode: number, + outcome: NvxOneShotOutcome, +): void { + if (wrapperExitCode !== outcome.outcome.statusCode) { + throw new Error( + `NVX wrapper exit ${wrapperExitCode} does not match structured outcome ` + + `${outcome.outcome.category}:${outcome.outcome.statusCode}`, + ); + } +} + +function buildNvxHostEnvironment(runDirectory: string): NodeJS.ProcessEnv { + const environment: NodeJS.ProcessEnv = { + PATH: '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + LANG: 'C.UTF-8', + LC_ALL: 'C.UTF-8', + PYTHONUNBUFFERED: '1', + TMPDIR: runDirectory, + }; + return environment; +} + +async function runNvxProcess(request: NvxProcessRequest): Promise { + const child = spawn(request.command, [...request.args], { + cwd: request.cwd, + env: request.env, + detached: process.platform !== 'win32', + stdio: ['ignore', 'pipe', 'pipe'], + }); + const stdoutPump = pump(child.stdout, request.onStdout); + const stderrPump = pump(child.stderr, request.onStderr); + let timedOut = false; + let cancelled = false; + let forceKillTimer: NodeJS.Timeout | undefined; + let pendingExit: { exitCode: number | null; signal: NodeJS.Signals | null } | undefined; + let resolveCompletion!: (result: NvxProcessResult) => void; + let rejectCompletion!: (error: Error) => void; + const completion = new Promise((resolve, reject) => { + resolveCompletion = resolve; + rejectCompletion = reject; + child.once('error', reject); + child.once('exit', (exitCode, signal) => { + clearTimeout(timeout); + pendingExit = { exitCode, signal }; + if (forceKillTimer === undefined) { + resolveCompletion({ exitCode, signal, timedOut, cancelled }); + } + }); + }); + const terminate = (reason: 'timeout' | 'cancelled'): void => { + if (timedOut || cancelled) return; + if (reason === 'timeout') timedOut = true; + else cancelled = true; + const terminationError = killProcessTree( + child.pid, + 'SIGTERM', + () => child.kill('SIGTERM'), + ); + if (terminationError) { + rejectCompletion(terminationError); + return; + } + forceKillTimer = setTimeout(() => { + forceKillTimer = undefined; + void forceKill(); + }, NVX_TERMINATION_GRACE_MS); + }; + const forceKill = async (): Promise => { + const forceKillError = pendingExit + ? await killExitedProcessGroup(child.pid, 'SIGKILL') + : killProcessTree(child.pid, 'SIGKILL', () => child.kill('SIGKILL')); + if (forceKillError) { + rejectCompletion(forceKillError); + } else if (pendingExit) { + // Keep escalation active after the launcher exits so descendants cannot survive. + resolveCompletion({ + exitCode: pendingExit.exitCode, + signal: pendingExit.signal, + timedOut, + cancelled, + }); + } + }; + const timeout = request.timeoutMs === undefined + ? undefined + : setTimeout(() => terminate('timeout'), request.timeoutMs); + const onAbort = (): void => terminate('cancelled'); + request.abortSignal?.addEventListener('abort', onAbort, { once: true }); + if (request.abortSignal?.aborted) onAbort(); + try { + const result = await completion; + await Promise.all([stdoutPump, stderrPump]); + return result; + } finally { + clearTimeout(timeout); + clearTimeout(forceKillTimer); + request.abortSignal?.removeEventListener('abort', onAbort); + } +} + +/** @internal Test-only access to process-group lifecycle behavior. */ +// ts-prune-ignore-next +export const testHelpers = { runNvxProcess }; + +async function pump( + source: Readable | null, + destination: (chunk: Buffer) => Promise, +): Promise { + if (!source) return; + for await (const chunk of source) { + await destination(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } +} + +function killProcessTree( + pid: number | undefined, + signal: NodeJS.Signals, + fallback: () => boolean, +): Error | undefined { + if (pid === undefined) return new Error('NVX process has no PID'); + try { + if (process.platform === 'win32') { + if (!fallback()) return new Error(`Failed to signal NVX process with ${signal}`); + } else { + process.kill(-pid, signal); + } + + return undefined; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'ESRCH') return undefined; + try { + if (fallback()) return undefined; + } catch (fallbackError) { + return new Error( + `Failed to signal NVX process group with ${signal}: ${formatError(fallbackError)}`, + ); + } + return new Error( + `Failed to signal NVX process group with ${signal}: ${formatError(error)}`, + ); + } +} + +async function killExitedProcessGroup( + pgid: number | undefined, + signal: NodeJS.Signals, +): Promise { + if (pgid === undefined) return new Error('NVX process has no PID'); + try { + for (const entry of await fs.readdir('/proc')) { + if (!/^\d+$/.test(entry)) continue; + try { + const stat = await fs.readFile(`/proc/${entry}/stat`, 'utf8'); + const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + if (fields[2] === String(pgid)) process.kill(Number(entry), signal); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== 'ENOENT' && code !== 'ESRCH') throw error; + } + } + return undefined; + } catch (error) { + return new Error( + `Failed to signal exited NVX process group with ${signal}: ${formatError(error)}`, + ); + } +} + +async function writeWithBackpressure(destination: Writable, data: Buffer): Promise { + if (data.length === 0) return; + await new Promise((resolve, reject) => { + const cleanup = (): void => { + destination.off('drain', onDrain); + destination.off('error', onError); + }; + const onDrain = (): void => { + cleanup(); + resolve(); + }; + const onError = (error: Error): void => { + cleanup(); + reject(error); + }; + destination.once('drain', onDrain); + destination.once('error', onError); + if (destination.write(data)) { + cleanup(); + resolve(); + } + }); +} + +class BoundedByteTail { + private bytes = Buffer.alloc(0); + + constructor(private readonly limit: number) {} + + push(chunk: Buffer): void { + if (this.limit === 0) return; + const combined = this.bytes.length === 0 ? chunk : Buffer.concat([this.bytes, chunk]); + this.bytes = combined.length <= this.limit + ? Buffer.from(combined) + : Buffer.from(combined.subarray(combined.length - this.limit)); + } + + value(): Buffer { + return Buffer.from(this.bytes); + } +} + +async function assertRegularFile(filePath: string, label: string) { + const stat = await fs.lstat(filePath); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error(`${label} must be a regular file: ${filePath}`); + } + return stat; +} + +function assertSafeDiskPath(filePath: string): void { + if (filePath.includes(',') || filePath.includes(';') || filePath.includes('\0')) { + throw new Error(`NVX disk path contains an unsupported delimiter: ${filePath}`); + } +} + +function assertContained(root: string, candidate: string, label: string): void { + const relative = path.relative(path.resolve(root), path.resolve(candidate)); + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new Error(`${label} escapes ${root}: ${candidate}`); + } +} + +function assertPositiveInteger(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error(`${label} must be a positive integer`); + } +} + +function assertOptionalPositiveInteger(value: number | undefined, label: string): void { + if (value !== undefined) assertPositiveInteger(value, label); +} + +function assertUuid(value: string, label: string): void { + if ( + !/^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test( + value, + ) + ) { + throw new Error(`${label} must be a canonical lowercase UUID`); + } +} + +function assertSha256(value: string, label: string): void { + if (!/^[a-f0-9]{64}$/.test(value)) { + throw new Error(`${label} must be a lowercase SHA-256 digest`); + } +} + +function signalExitCode(signal: NodeJS.Signals | null): number { + switch (signal) { + case 'SIGHUP': return 129; + case 'SIGINT': return 130; + case 'SIGQUIT': return 131; + case 'SIGKILL': return 137; + case 'SIGTERM': return 143; + default: return 128; + } +} + +async function sha256File(filePath: string): Promise { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(filePath)) hash.update(chunk as Buffer); + return hash.digest('hex'); +} + +function assertNetworkPolicy( + requested: NvxOneShotNetworkPlan, + outcome: NvxOneShotOutcome, +): void { + const expectedHostLoopback = (requested.hostLoopbackForwards?.length ?? 0) > 0 + ? 'allow' + : 'deny'; + const expectedAllowRules = requested.egressAllow?.length ?? 0; + const expectedDenyRules = requested.egressDeny?.length ?? 0; + if ( + outcome.networkPolicy.status !== 'applied' || + outcome.networkPolicy.statusCode !== 0 || + outcome.networkPolicy.mode !== 'rules' || + outcome.networkPolicy.allowRuleCount !== expectedAllowRules || + outcome.networkPolicy.denyRuleCount !== expectedDenyRules || + outcome.networkPolicy.hostLoopback !== expectedHostLoopback + ) { + throw new Error( + 'NVX structured network policy does not match the requested fail-closed policy', + ); + } +} + +function formatError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(',')}]`; + } + if (value && typeof value === 'object') { + return `{${Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`) + .join(',')}}`; + } + return JSON.stringify(value); +} + +function assertIpv4Cidr(value: string, label: string): void { + const [address, prefix, ...extra] = value.split('/'); + if ( + extra.length > 0 || + !isIpv4Address(address) || + prefix === undefined || + !Number.isInteger(Number(prefix)) || + Number(prefix) < 0 || + Number(prefix) > 32 + ) { + throw new Error(`Invalid ${label}: ${value}`); + } +} + +function assertIpv4Endpoint(value: string, label: string): void { + const separator = value.lastIndexOf(':'); + const address = value.slice(0, separator); + const port = Number(value.slice(separator + 1)); + if ( + separator < 1 || + !isIpv4Address(address) || + !Number.isInteger(port) || + port < 1 || + port > 65_535 + ) { + throw new Error(`Invalid ${label}: ${value}`); + } +} + +function isIpv4Address(value: string): boolean { + const octets = value.split('.'); + return octets.length === 4 && octets.every((octet) => { + if (!/^\d{1,3}$/.test(octet)) return false; + const numeric = Number(octet); + return numeric >= 0 && numeric <= 255 && String(numeric) === octet; + }); +} diff --git a/src/nvx/outcome.test.ts b/src/nvx/outcome.test.ts new file mode 100644 index 000000000..85ed6ffb4 --- /dev/null +++ b/src/nvx/outcome.test.ts @@ -0,0 +1,68 @@ +import { + NVX_TEARDOWN_STAGES, + parseNvxOneShotOutcome, +} from './outcome'; + +function report( + category: 'success' | 'guest-exit' | 'vmm-failure' = 'success', + statusCode = category === 'success' ? 0 : 1, +) { + return { + schema_version: 1, + instance_id: 'a'.repeat(32), + backend: 'kvm', + outcome: { + operation: 'run', + category, + status_code: statusCode, + }, + network_policy: { + status: 'applied', + status_code: 0, + mode: 'rules', + allow_rule_count: 2, + deny_rule_count: 1, + host_loopback: 'allow', + }, + teardown: Object.fromEntries(NVX_TEARDOWN_STAGES.map((stage) => [stage, true])), + }; +} + +describe('NVX one-shot outcome parser', () => { + it('accepts complete success and guest-exit outcomes', () => { + expect(parseNvxOneShotOutcome(JSON.stringify(report())).outcome) + .toEqual({ operation: 'run', category: 'success', statusCode: 0 }); + expect(parseNvxOneShotOutcome(JSON.stringify(report('guest-exit', 125))).outcome) + .toEqual({ operation: 'run', category: 'guest-exit', statusCode: 125 }); + }); + + it('rejects partial, false, or additional teardown stages', () => { + const partial = report(); + delete (partial.teardown as Record).vm_stopped; + expect(() => parseNvxOneShotOutcome(JSON.stringify(partial))) + .toThrow(/teardown keys must be exactly/); + + const failed = report(); + failed.teardown.vm_stopped = false; + expect(() => parseNvxOneShotOutcome(JSON.stringify(failed))) + .toThrow(/teardown stage did not complete: vm_stopped/); + + const additional = report(); + (additional.teardown as Record).unexpected = true; + expect(() => parseNvxOneShotOutcome(JSON.stringify(additional))) + .toThrow(/teardown keys must be exactly/); + }); + + it('rejects inconsistent categories, malformed reports, and extra fields', () => { + expect(() => parseNvxOneShotOutcome(JSON.stringify(report('success', 125)))) + .toThrow(/success outcome must have status_code 0/); + expect(() => parseNvxOneShotOutcome(JSON.stringify(report('guest-exit', 0)))) + .toThrow(/guest-exit outcome must have a nonzero status_code/); + expect(() => parseNvxOneShotOutcome('{')) + .toThrow(/not valid JSON/); + expect(() => parseNvxOneShotOutcome(JSON.stringify({ + ...report(), + unexpected: true, + }))).toThrow(/outcome report keys must be exactly/); + }); +}); diff --git a/src/nvx/outcome.ts b/src/nvx/outcome.ts new file mode 100644 index 000000000..1d603c555 --- /dev/null +++ b/src/nvx/outcome.ts @@ -0,0 +1,186 @@ +export const NVX_OUTCOME_SCHEMA_VERSION = 1; +export const NVX_TEARDOWN_STAGES = [ + 'control_channels_closed', + 'guest_workload_stopped', + 'network_released', + 'openvmm_process_terminated', + 'temporary_storage_removed', + 'virtiofs_released', + 'vm_stopped', +] as const; + +export type NvxOutcomeCategory = 'success' | 'guest-exit' | 'vmm-failure'; + +export interface NvxOneShotOutcome { + readonly schemaVersion: 1; + readonly instanceId: string; + readonly backend: 'kvm'; + readonly outcome: { + readonly operation: 'run'; + readonly category: NvxOutcomeCategory; + readonly statusCode: number; + }; + readonly networkPolicy: { + readonly status: string; + readonly statusCode: number; + readonly mode: string; + readonly allowRuleCount: number; + readonly denyRuleCount: number; + readonly hostLoopback: string; + }; + readonly teardown: Readonly>; +} + +export function parseNvxOneShotOutcome(contents: string): NvxOneShotOutcome { + let parsed: unknown; + try { + parsed = JSON.parse(contents); + } catch (error) { + throw new Error(`NVX outcome report is not valid JSON: ${formatError(error)}`); + } + const report = requireObject(parsed, 'NVX outcome report'); + assertExactKeys(report, [ + 'schema_version', + 'instance_id', + 'backend', + 'outcome', + 'network_policy', + 'teardown', + ], 'NVX outcome report'); + if (report.schema_version !== NVX_OUTCOME_SCHEMA_VERSION) { + throw new Error('NVX outcome report schema_version must be 1'); + } + const instanceId = requireString(report.instance_id, 'instance_id'); + if (!/^[a-f0-9]{32}$/.test(instanceId)) { + throw new Error('NVX outcome instance_id must be 32 lowercase hexadecimal characters'); + } + if (report.backend !== 'kvm') { + throw new Error('NVX outcome backend must be "kvm"'); + } + + const outcome = requireObject(report.outcome, 'outcome'); + assertExactKeys(outcome, ['operation', 'category', 'status_code'], 'outcome'); + if (outcome.operation !== 'run') { + throw new Error('NVX one-shot outcome operation must be "run"'); + } + if ( + outcome.category !== 'success' && + outcome.category !== 'guest-exit' && + outcome.category !== 'vmm-failure' + ) { + throw new Error(`Unsupported NVX one-shot outcome category: ${String(outcome.category)}`); + } + const statusCode = requireInteger(outcome.status_code, 'outcome.status_code', 0, 255); + if (outcome.category === 'success' && statusCode !== 0) { + throw new Error('NVX success outcome must have status_code 0'); + } + if (outcome.category !== 'success' && statusCode === 0) { + throw new Error(`NVX ${outcome.category} outcome must have a nonzero status_code`); + } + + const network = requireObject(report.network_policy, 'network_policy'); + assertExactKeys(network, [ + 'status', + 'status_code', + 'mode', + 'allow_rule_count', + 'deny_rule_count', + 'host_loopback', + ], 'network_policy'); + const teardown = requireObject(report.teardown, 'teardown'); + assertExactKeys(teardown, [...NVX_TEARDOWN_STAGES], 'teardown'); + const incompleteStage = NVX_TEARDOWN_STAGES.find((stage) => + Object.entries(teardown).some(([name, value]) => name === stage && value !== true) + ); + if (incompleteStage) { + throw new Error(`NVX teardown stage did not complete: ${incompleteStage}`); + } + + return { + schemaVersion: 1, + instanceId, + backend: 'kvm', + outcome: { + operation: 'run', + category: outcome.category, + statusCode, + }, + networkPolicy: { + status: requireString(network.status, 'network_policy.status'), + statusCode: requireInteger( + network.status_code, + 'network_policy.status_code', + 0, + 255, + ), + mode: requireString(network.mode, 'network_policy.mode'), + allowRuleCount: requireInteger( + network.allow_rule_count, + 'network_policy.allow_rule_count', + 0, + Number.MAX_SAFE_INTEGER, + ), + denyRuleCount: requireInteger( + network.deny_rule_count, + 'network_policy.deny_rule_count', + 0, + Number.MAX_SAFE_INTEGER, + ), + hostLoopback: requireString( + network.host_loopback, + 'network_policy.host_loopback', + ), + }, + teardown: Object.fromEntries( + NVX_TEARDOWN_STAGES.map((stage) => [stage, true]), + ) as Record, + }; +} + +function requireObject(value: unknown, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be a JSON object`); + } + return value as Record; +} + +function requireString(value: unknown, label: string): string { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`${label} must be a non-empty string`); + } + return value; +} + +function requireInteger( + value: unknown, + label: string, + minimum: number, + maximum: number, +): number { + if ( + !Number.isSafeInteger(value) || + (value as number) < minimum || + (value as number) > maximum + ) { + throw new Error(`${label} must be an integer in ${minimum}-${maximum}`); + } + return value as number; +} + +function assertExactKeys( + value: Record, + expected: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const normalizedExpected = [...expected].sort(); + if (JSON.stringify(actual) !== JSON.stringify(normalizedExpected)) { + throw new Error( + `${label} keys must be exactly: ${normalizedExpected.join(', ')}`, + ); + } +} + +function formatError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +}