From a6d440f0dbac5b086b9bd0744b3b60066f3c8f58 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Sat, 31 Oct 2026 03:23:11 -0400 Subject: [PATCH 1/6] fix(start): prove an unbound non-loopback address absent by binding it (#8045) A timed-out probe of a configured non-loopback bind (a Tailscale IP) with nothing listening was reported unknown, so the #7851 ownership check refused startup forever. Bind the address once: success proves no owner holds it, any bind error keeps the refusal. --- src/server/proxy-liveness.ts | 42 ++++++++++++++++- ...xy-liveness-nonloopback-bind-proof.test.ts | 45 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 tests/server/proxy-liveness-nonloopback-bind-proof.test.ts diff --git a/src/server/proxy-liveness.ts b/src/server/proxy-liveness.ts index 78ec8c74e9c..56314f0d3b2 100644 --- a/src/server/proxy-liveness.ts +++ b/src/server/proxy-liveness.ts @@ -13,6 +13,7 @@ import { loadConfig } from "../config"; import { isLoopbackHostname, isWildcardHostname } from "../codex/loopback-target"; import { normalizeBindHostname } from "./bind-host"; import { isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort, verifyPidIdentity } from "../config/process-state"; +import { createServer } from "node:net"; import { loopbackFamilyAvailability } from "../lib/loopback-family"; import { LOCAL_ATTESTATION_CHALLENGE_HEADER, @@ -46,6 +47,33 @@ export interface HealthzIdentity { export type EndpointLiveness = "live" | "dead" | "unknown"; +/** + * Whether this process can bind `hostname:port` right now. A successful bind proves no process + * owns that address, which no probe of it can: connecting to this machine's own non-loopback + * address (a Tailscale IP, a LAN IP) with nothing listening TIMES OUT on some hosts instead of + * being refused, and a timeout is indistinguishable from a slow owner. The listener is closed + * immediately. Any bind error (EADDRINUSE, EADDRNOTAVAIL, EACCES) answers false, so an address + * we cannot bind stays "unknown" and startup keeps refusing. + */ +export function canBindAddress(hostname: string, port: number, timeoutMs = 1_500): Promise { + if (!Number.isInteger(port) || port <= 0 || port > 65_535) return Promise.resolve(false); + return new Promise(resolve => { + const server = createServer(); + let settled = false; + const finish = (bound: boolean): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + try { server.close(); } catch { /* never bound */ } + resolve(bound); + }; + const timer = setTimeout(() => finish(false), timeoutMs); + server.once("error", () => finish(false)); + server.once("listening", () => finish(true)); + try { server.listen({ host: hostname, port, exclusive: true }); } catch { finish(false); } + }); +} + export interface LivenessIo { fetchFn?: typeof fetch; readPidFn?: () => number | null; @@ -57,6 +85,11 @@ export interface LivenessIo { /** Fresh command-line evidence: undefined/empty is unreadable, never confirmed foreign. */ readCommandLineFn?: typeof readProcessCommandLine; loopbackAvailabilityFn?: typeof loopbackFamilyAvailability; + /** + * Test seam for the bind proof (see `canBindAddress`): whether `hostname:port` can be bound by + * this process right now, which proves no process owns that address. + */ + bindProofFn?: (hostname: string, port: number) => Promise; readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string } | null; configFn?: () => { port?: number; hostname?: string }; timeoutMs?: number; @@ -525,7 +558,14 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { const bindHostname = opts.hostname?.trim() || "127.0.0.1"; const hostname = normalizeBindHostname(opts.hostname); if (!observer || (!isLoopbackHostname(hostname) && !isWildcardHostname(hostname))) { - const identity = await proxyIdentityAtObserved(port, opts, probeIo, observer?.endpoint); + // Recovery only (an observer is present): a transport miss on a specific non-loopback bind + // address is "unknown", and a bind proof is the one thing that turns it into "dead". + let verdict: EndpointLiveness | undefined; + const identity = await proxyIdentityAtObserved(port, opts, probeIo, observer ? (r => { verdict = r; }) : undefined); + if (observer && !identity && verdict !== undefined) { + if (verdict === "unknown" && await (io.bindProofFn ?? canBindAddress)(probeHostname(opts.hostname), port)) verdict = "dead"; + observer.endpoint(verdict); + } return identity ? { ...identity, hostname: bindHostname, ...(observer ? { probeHostname: probeHostname(opts.hostname) } : {}) } : null; } diff --git a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts new file mode 100644 index 00000000000..87e5b86d52b --- /dev/null +++ b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts @@ -0,0 +1,45 @@ +import { expect, test } from "bun:test"; +import { createServer } from "node:net"; +import { canBindAddress, observeProxyLiveness, type LivenessIo } from "../../src/server/proxy-liveness"; + +// Connecting to this machine's own non-loopback address (a Tailscale IP) with nothing listening can +// time out instead of being refused. A timeout is "unknown", which startup refuses forever; only a +// bind proof may turn it into "absent". +function ioFor(bindProofFn: LivenessIo["bindProofFn"]): LivenessIo { + return { + readPidFn: () => null, + readRuntimeFn: () => null, + verifyPidFn: pid => pid, + configFn: () => ({ port: 10100, hostname: "100.88.46.13" }), + loopbackAvailabilityFn: () => "available", + attempts: 1, + timeoutMs: 50, + fetchFn: (async () => { throw Object.assign(new Error("timed out"), { name: "TimeoutError" }); }) as typeof fetch, + ...(bindProofFn ? { bindProofFn } : {}), + }; +} + +test("a timed-out probe of a non-loopback bind is absent when the address can be bound", async () => { + let asked: [string, number] | undefined; + const observation = await observeProxyLiveness(ioFor(async (host, port) => { asked = [host, port]; return true; })); + expect(observation.kind).toBe("absent"); + expect(asked).toEqual(["100.88.46.13", 10100]); +}); + +test("a timed-out probe of a non-loopback bind stays unknown when the address cannot be bound", async () => { + const observation = await observeProxyLiveness(ioFor(async () => false)); + expect(observation.kind).toBe("unknown"); +}); + +test("canBindAddress is true for a free port and false while another listener holds it", async () => { + const holder = createServer(); + await new Promise(resolve => holder.listen({ host: "127.0.0.1", port: 0 }, resolve)); + const { port } = holder.address() as { port: number }; + try { + expect(await canBindAddress("127.0.0.1", port)).toBe(false); + } finally { + await new Promise(resolve => holder.close(() => resolve())); + } + expect(await canBindAddress("127.0.0.1", port)).toBe(true); + expect(await canBindAddress("127.0.0.1", 0)).toBe(false); +}); From 5abe1850dfe8c8180ede7c2569e8ecdade2e275b Mon Sep 17 00:00:00 2001 From: "Andrew (agent)" Date: Tue, 3 Nov 2026 10:44:51 -0500 Subject: [PATCH 2/6] fix(start): bind-prove the raw IPv6 address within the discovery deadline (#8045) Address review on #8046: the socket proof received probeHostname's bracketed URL spelling, so an IPv6 non-loopback bind stayed unknown; the proof could also outlast io.deadlineAt and flip unknown to absent. Strip brackets for the socket, cap the proof by the remaining budget and discard a late result. Register the test in the layout inventory and record the recovery-only bind proof in structure/runtime.md. --- .../test-layout/layout-start-ownership.json | 1 + src/server/proxy-liveness.ts | 22 +++++++++++++--- structure/runtime.md | 2 +- tests/fixtures/test-layout-expected.json | 1 + ...xy-liveness-nonloopback-bind-proof.test.ts | 26 +++++++++++++++++-- 5 files changed, 45 insertions(+), 7 deletions(-) diff --git a/scripts/test-layout/layout-start-ownership.json b/scripts/test-layout/layout-start-ownership.json index b112773799a..7caaff72af5 100644 --- a/scripts/test-layout/layout-start-ownership.json +++ b/scripts/test-layout/layout-start-ownership.json @@ -4,5 +4,6 @@ "proxy-liveness-owner-evidence.test.ts": "server", "proxy-liveness-recovery.test.ts": "server", "proxy-liveness-bind-scope.test.ts": "server", + "proxy-liveness-nonloopback-bind-proof.test.ts": "server", "system-restart-ownership.test.ts": "server" } diff --git a/src/server/proxy-liveness.ts b/src/server/proxy-liveness.ts index 56314f0d3b2..f2da8623354 100644 --- a/src/server/proxy-liveness.ts +++ b/src/server/proxy-liveness.ts @@ -55,7 +55,9 @@ export type EndpointLiveness = "live" | "dead" | "unknown"; * immediately. Any bind error (EADDRINUSE, EADDRNOTAVAIL, EACCES) answers false, so an address * we cannot bind stays "unknown" and startup keeps refusing. */ -export function canBindAddress(hostname: string, port: number, timeoutMs = 1_500): Promise { +const BIND_PROOF_TIMEOUT_MS = 1_500; + +export function canBindAddress(hostname: string, port: number, timeoutMs = BIND_PROOF_TIMEOUT_MS): Promise { if (!Number.isInteger(port) || port <= 0 || port > 65_535) return Promise.resolve(false); return new Promise(resolve => { const server = createServer(); @@ -87,9 +89,10 @@ export interface LivenessIo { loopbackAvailabilityFn?: typeof loopbackFamilyAvailability; /** * Test seam for the bind proof (see `canBindAddress`): whether `hostname:port` can be bound by - * this process right now, which proves no process owns that address. + * this process right now, which proves no process owns that address. `hostname` is the raw + * socket address (IPv6 unbracketed) and `timeoutMs` the remaining discovery budget. */ - bindProofFn?: (hostname: string, port: number) => Promise; + bindProofFn?: (hostname: string, port: number, timeoutMs: number) => Promise; readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string } | null; configFn?: () => { port?: number; hostname?: string }; timeoutMs?: number; @@ -553,6 +556,17 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { const budgetExhausted = (): boolean => deadlineAt !== undefined && nowFn() >= deadlineAt; const expired = (): null => { observer?.endpoint("unknown"); return null; }; + // The proof stays inside the discovery deadline: it is skipped once the budget is spent, its + // own timeout is capped by what remains, and a proof that finishes past the deadline is + // discarded, so an exhausted discovery still answers "unknown". A socket takes the raw + // address, never the bracketed URL spelling `probeHostname` composes for IPv6. + const bindProof = async (hostname: string, port: number): Promise => { + const remainingMs = deadlineAt === undefined ? BIND_PROOF_TIMEOUT_MS : Math.min(BIND_PROOF_TIMEOUT_MS, deadlineAt - nowFn()); + if (remainingMs <= 0) return false; + const socketHost = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; + const bound = await (io.bindProofFn ?? canBindAddress)(socketHost, port, remainingMs); + return bound && !budgetExhausted(); + }; const probe = async (port: number, opts: { hostname?: string; expectedPid?: number }) => { // Missing/blank means an actual loopback bind, not a later config's bind scope. const bindHostname = opts.hostname?.trim() || "127.0.0.1"; @@ -563,7 +577,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { let verdict: EndpointLiveness | undefined; const identity = await proxyIdentityAtObserved(port, opts, probeIo, observer ? (r => { verdict = r; }) : undefined); if (observer && !identity && verdict !== undefined) { - if (verdict === "unknown" && await (io.bindProofFn ?? canBindAddress)(probeHostname(opts.hostname), port)) verdict = "dead"; + if (verdict === "unknown" && await bindProof(hostname, port)) verdict = "dead"; observer.endpoint(verdict); } return identity ? { ...identity, hostname: bindHostname, diff --git a/structure/runtime.md b/structure/runtime.md index 24fd3ad5035..c7e00ca7448 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -181,7 +181,7 @@ uninstall still restore. `src/service/cli.ts` removes the service token on unins `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. `ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Only definitive absence permits PID deletion and journal reconciliation: no alive recorded PID unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. Unreadable/empty PID command lines and any OpenCodex invocation remain unknown; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Only definitive absence permits PID deletion and journal reconciliation: no alive recorded PID unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback bind address (a Tailscale or LAN IP, never loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers bound, unbindable, IPv6 and deadline cases. Unreadable/empty PID command lines and any OpenCodex invocation remain unknown; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index b97dd7e3e9c..9727c76c1ef 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1170,6 +1170,7 @@ "proxy-liveness-recovery.test.ts": "server", "proxy-liveness-owner-evidence.test.ts": "server", "proxy-liveness-bind-scope.test.ts": "server", + "proxy-liveness-nonloopback-bind-proof.test.ts": "server", "system-restart-ownership.test.ts": "server", "proxy-liveness.test.ts": "server", "readyz-features.test.ts": "server", "served-by-header.test.ts": "server", "qoder-adapter.test.ts": "providers", "qoder-live-models.test.ts": "providers", "qoder-mcp-server.test.ts": "providers", "qoder-scaffold-guard.test.ts": "providers", diff --git a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts index 87e5b86d52b..f359a0ebe50 100644 --- a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts +++ b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts @@ -5,17 +5,18 @@ import { canBindAddress, observeProxyLiveness, type LivenessIo } from "../../src // Connecting to this machine's own non-loopback address (a Tailscale IP) with nothing listening can // time out instead of being refused. A timeout is "unknown", which startup refuses forever; only a // bind proof may turn it into "absent". -function ioFor(bindProofFn: LivenessIo["bindProofFn"]): LivenessIo { +function ioFor(bindProofFn: LivenessIo["bindProofFn"], hostname = "100.88.46.13", extra: Partial = {}): LivenessIo { return { readPidFn: () => null, readRuntimeFn: () => null, verifyPidFn: pid => pid, - configFn: () => ({ port: 10100, hostname: "100.88.46.13" }), + configFn: () => ({ port: 10100, hostname }), loopbackAvailabilityFn: () => "available", attempts: 1, timeoutMs: 50, fetchFn: (async () => { throw Object.assign(new Error("timed out"), { name: "TimeoutError" }); }) as typeof fetch, ...(bindProofFn ? { bindProofFn } : {}), + ...extra, }; } @@ -31,6 +32,27 @@ test("a timed-out probe of a non-loopback bind stays unknown when the address ca expect(observation.kind).toBe("unknown"); }); +test("a non-loopback IPv6 bind is proven with the unbracketed socket address", async () => { + for (const hostname of ["fd7a:115c:a1e0::1234", "[fd7a:115c:a1e0::1234]"]) { + let asked: string | undefined; + const observation = await observeProxyLiveness(ioFor(async host => { asked = host; return true; }, hostname)); + expect(observation.kind).toBe("absent"); + expect(asked).toBe("fd7a:115c:a1e0::1234"); + } +}); + +test("the bind proof stays within the discovery deadline", async () => { + let clock = 0; + let budget: number | undefined; + const overran = await observeProxyLiveness(ioFor(async (_host, _port, timeoutMs) => { + budget = timeoutMs; + clock = 2_000; + return true; + }, undefined, { nowFn: () => clock, deadlineAt: 1_000 })); + expect(budget).toBeLessThanOrEqual(1_000); + expect(overran.kind).toBe("unknown"); +}); + test("canBindAddress is true for a free port and false while another listener holds it", async () => { const holder = createServer(); await new Promise(resolve => holder.listen({ host: "127.0.0.1", port: 0 }, resolve)); From 217a92d621c743ee7d945e776b71d0dc28c913c5 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 8 Nov 2026 18:27:47 +0900 Subject: [PATCH 3/6] Preserve live orphan runtime ownership during recovery --- src/server/proxy-liveness.ts | 23 +++++- structure/runtime.md | 2 +- ...xy-liveness-nonloopback-bind-proof.test.ts | 81 ++++++++++++++++++- 3 files changed, 100 insertions(+), 6 deletions(-) diff --git a/src/server/proxy-liveness.ts b/src/server/proxy-liveness.ts index f2da8623354..5068b184fde 100644 --- a/src/server/proxy-liveness.ts +++ b/src/server/proxy-liveness.ts @@ -515,19 +515,34 @@ export type ProxyLivenessObservation = /** Recovery additionally covers both available loopback families and fresh PID identity. */ export async function observeProxyLiveness(io: LivenessIo = {}): Promise { - let alivePid: number | null = null, unknown = false; + let alivePid: number | null = null, runtimePid: number | undefined, unknown = false; const live = await discoverLiveProxy(io, { pid: pid => { alivePid = pid; }, + runtimePid: pid => { runtimePid = pid; }, endpoint: result => { if (result === "unknown") unknown = true; }, }); if (live) return { kind: "live", live }; - if (alivePid !== null && !unknown) { + const candidates = new Set(); + if (alivePid !== null) candidates.add(alivePid); + if (runtimePid !== undefined && Number.isSafeInteger(runtimePid) && runtimePid > 0) candidates.add(runtimePid); + for (const pid of candidates) { + if (unknown) break; + if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) { unknown = true; break; } + if (pid !== alivePid) { + try { process.kill(pid, 0); } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") continue; + // Permission or an indeterminate probe is not proof that the process is dead. + unknown = true; + break; + } + } // A failed verifier alone conflates foreign with unreadable. Only fresh positive // evidence of a foreign command line releases a reused PID; never signal it. - const commandLine = (io.readCommandLineFn ?? readProcessCommandLine)(alivePid); + const commandLine = (io.readCommandLineFn ?? readProcessCommandLine)(pid); unknown = !commandLine?.trim() || isOcxCommandLine(commandLine); if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) unknown = true; } + if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) unknown = true; return { kind: unknown ? "unknown" : "absent", live: null }; } @@ -542,6 +557,7 @@ export async function findLiveProxy(io: LivenessIo = {}): Promise void; + runtimePid: (pid: number | undefined) => void; endpoint: (result: EndpointLiveness) => void; }): Promise { // Prefer the cheap alive-pid check: the Windows cmdline probe (WMIC/PowerShell) is too @@ -641,6 +657,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { // sibling command, manual deletion). The runtime record still says where it listens — // identity-probe it so ensure/update/stop see the live proxy instead of shadowing it. const record = readRuntimeFn(); + observer?.runtimePid(record?.pid); if (record?.port && record.port !== probedPort) { if (budgetExhausted()) return expired(); const expectedPid = typeof record.pid === "number" ? record.pid : undefined; diff --git a/structure/runtime.md b/structure/runtime.md index 6939dc22258..258c8f15ccc 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -181,7 +181,7 @@ uninstall still restore. Native client signal cleanup follows the [served-connec `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. `ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. The Remote Link Child listener checks its startup owner and link against durable client state once per `/readyz`, after readiness probing, and for authenticated machine/link status only on those routes. If they differ, `/readyz` and those status routes answer 503 with `client_state_changed`; the dashboard shows that reason with its recovery guidance. This reports divergence; it does not disconnect the relay or reconstruct a connection. `tests/clients/client-machine-listener.test.ts` covers missing and replaced records. Explicit `ocx disconnect --orphaned-token --yes` in `src/client/orphan-token-recovery.ts` preserves unowned token files only after the start lease, lifecycle lock, liveness proof and config-lock revalidation; Hub, admission-token use and installed or unverified services refuse. Windows closes source handles before rename while private hard links pin identity; a refusal removes a pin only while another name still holds the token, and an extra hard link left by an interrupted run is refused with a message naming the `.identity` pin to remove from the `orphaned-token-*` folder. Missing ownership is never reconstructed. `ocx config import` refuses to replace a connected client record under the config lock. `tests/clients/client-link-connect.test.ts` covers these refusals and local liveness. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Only definitive absence permits PID deletion and journal reconciliation: no alive recorded PID unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback bind address (a Tailscale or LAN IP, never loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers bound, unbindable, IPv6 and deadline cases. Unreadable/empty PID command lines and any OpenCodex invocation remain unknown; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback bind address (a Tailscale or LAN IP, never loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers bound, unbindable, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts index f359a0ebe50..5ac13ec3875 100644 --- a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts +++ b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts @@ -1,6 +1,6 @@ -import { expect, test } from "bun:test"; +import { expect, spyOn, test } from "bun:test"; import { createServer } from "node:net"; -import { canBindAddress, observeProxyLiveness, type LivenessIo } from "../../src/server/proxy-liveness"; +import { canBindAddress, findLiveProxy, observeProxyLiveness, type LivenessIo } from "../../src/server/proxy-liveness"; // Connecting to this machine's own non-loopback address (a Tailscale IP) with nothing listening can // time out instead of being refused. A timeout is "unknown", which startup refuses forever; only a @@ -32,6 +32,83 @@ test("a timed-out probe of a non-loopback bind stays unknown when the address ca expect(observation.kind).toBe("unknown"); }); +for (const commandLine of ["ocx start", undefined, ""]) { + test(`an orphan runtime PID with ${JSON.stringify(commandLine)} identity blocks bind-proof recovery`, async () => { + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "100.88.46.13" }), + readCommandLineFn: () => commandLine, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); +} + +test("dead and confirmed foreign orphan runtime PIDs permit bind-proof recovery", async () => { + const exited = Bun.spawn([process.execPath, "-e", ""], { stdout: "ignore", stderr: "ignore" }); + await exited.exited; + for (const [pid, commandLine] of [[exited.pid, undefined], [process.pid, "foreign-server"]] as const) { + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid, port: 10100, hostname: "100.88.46.13" }), + readCommandLineFn: () => commandLine, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); + } +}); + +test("a foreign PID file does not erase a different live runtime owner", async () => { + const io = ioFor(async () => true, undefined, { + readPidFn: () => 4242, + readRuntimeFn: pid => pid === undefined ? { pid: process.pid, port: 10100, hostname: "100.88.46.13" } : null, + readCommandLineFn: pid => pid === 4242 ? "foreign-server" : "ocx start", + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); +}); + +test("an indeterminate orphan process probe cannot authorize recovery", async () => { + const kill = spyOn(process, "kill").mockImplementation(() => { throw Object.assign(new Error("denied"), { code: "EPERM" }); }); + try { + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "100.88.46.13" }), + readCommandLineFn: () => "foreign-server", + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + expect(kill).toHaveBeenCalledWith(process.pid, 0); + } finally { kill.mockRestore(); } +}); + +test("runtime PID identity inspection cannot outlive the recovery deadline", async () => { + let clock = 0; + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "100.88.46.13" }), + readCommandLineFn: () => { clock = 1000; return "foreign-server"; }, + nowFn: () => clock, deadlineAt: 1000, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); +}); + +test("a dead PID probe finishing after the deadline remains unknown", async () => { + let clock = 0; + const kill = spyOn(process, "kill").mockImplementation(() => { + clock = 1000; + throw Object.assign(new Error("no process"), { code: "ESRCH" }); + }); + try { + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "100.88.46.13" }), + nowFn: () => clock, deadlineAt: 1000, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + } finally { kill.mockRestore(); } +}); + +test("an orphan PID is never returned for a pidless responding proxy", async () => { + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "100.88.46.13" }), + fetchFn: (async () => Response.json({ service: "opencodex" })) as typeof fetch, + }); + expect((await findLiveProxy(io))?.pid).toBeNull(); + expect(await observeProxyLiveness(io)).toMatchObject({ kind: "live", live: { pid: null, port: 10100 } }); +}); + test("a non-loopback IPv6 bind is proven with the unbracketed socket address", async () => { for (const hostname of ["fd7a:115c:a1e0::1234", "[fd7a:115c:a1e0::1234]"]) { let asked: string | undefined; From 3dde52b7f198a644e14407870b96eae4dd263297 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 8 Nov 2026 18:40:04 +0900 Subject: [PATCH 4/6] Restrict startup bind proof to literal IP addresses --- src/server/proxy-liveness.ts | 4 +++- structure/runtime.md | 2 +- ...xy-liveness-nonloopback-bind-proof.test.ts | 22 +++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/server/proxy-liveness.ts b/src/server/proxy-liveness.ts index 5068b184fde..09ea6ab549d 100644 --- a/src/server/proxy-liveness.ts +++ b/src/server/proxy-liveness.ts @@ -13,7 +13,7 @@ import { loadConfig } from "../config"; import { isLoopbackHostname, isWildcardHostname } from "../codex/loopback-target"; import { normalizeBindHostname } from "./bind-host"; import { isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort, verifyPidIdentity } from "../config/process-state"; -import { createServer } from "node:net"; +import { createServer, isIP } from "node:net"; import { loopbackFamilyAvailability } from "../lib/loopback-family"; import { LOCAL_ATTESTATION_CHALLENGE_HEADER, @@ -580,6 +580,8 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { const remainingMs = deadlineAt === undefined ? BIND_PROOF_TIMEOUT_MS : Math.min(BIND_PROOF_TIMEOUT_MS, deadlineAt - nowFn()); if (remainingMs <= 0) return false; const socketHost = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; + // A hostname can resolve to different addresses for the probe and the bind. + if (isIP(socketHost) === 0) return false; const bound = await (io.bindProofFn ?? canBindAddress)(socketHost, port, remainingMs); return bound && !budgetExhausted(); }; diff --git a/structure/runtime.md b/structure/runtime.md index 258c8f15ccc..6ad921e4ac1 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -181,7 +181,7 @@ uninstall still restore. Native client signal cleanup follows the [served-connec `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. `ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. The Remote Link Child listener checks its startup owner and link against durable client state once per `/readyz`, after readiness probing, and for authenticated machine/link status only on those routes. If they differ, `/readyz` and those status routes answer 503 with `client_state_changed`; the dashboard shows that reason with its recovery guidance. This reports divergence; it does not disconnect the relay or reconstruct a connection. `tests/clients/client-machine-listener.test.ts` covers missing and replaced records. Explicit `ocx disconnect --orphaned-token --yes` in `src/client/orphan-token-recovery.ts` preserves unowned token files only after the start lease, lifecycle lock, liveness proof and config-lock revalidation; Hub, admission-token use and installed or unverified services refuse. Windows closes source handles before rename while private hard links pin identity; a refusal removes a pin only while another name still holds the token, and an extra hard link left by an interrupted run is refused with a message naming the `.identity` pin to remove from the `orphaned-token-*` folder. Missing ownership is never reconstructed. `ocx config import` refuses to replace a connected client record under the config lock. `tests/clients/client-link-connect.test.ts` covers these refusals and local liveness. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback bind address (a Tailscale or LAN IP, never loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers bound, unbindable, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts index 5ac13ec3875..8d3775da0ab 100644 --- a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts +++ b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts @@ -32,6 +32,28 @@ test("a timed-out probe of a non-loopback bind stays unknown when the address ca expect(observation.kind).toBe("unknown"); }); +for (const source of ["config", "runtime"] as const) { + test(`an inconclusive ${source} hostname stays unknown without a bind proof`, async () => { + let proofCalls = 0; + const hostname = "multi-address.fixture.invalid"; + const extra: Partial = source === "runtime" + ? { readRuntimeFn: () => ({ port: 10100, hostname }) } + : {}; + const observation = await observeProxyLiveness(ioFor(async () => { proofCalls++; return true; }, hostname, extra)); + expect(observation.kind).toBe("unknown"); + expect(proofCalls).toBe(0); + }); +} + +test("a hostname listener that identifies itself remains live", async () => { + let proofCalls = 0; + const observation = await observeProxyLiveness(ioFor(async () => { proofCalls++; return true; }, "multi-address.fixture.invalid", { + fetchFn: (async () => Response.json({ service: "opencodex" })) as typeof fetch, + })); + expect(observation.kind).toBe("live"); + expect(proofCalls).toBe(0); +}); + for (const commandLine of ["ocx start", undefined, ""]) { test(`an orphan runtime PID with ${JSON.stringify(commandLine)} identity blocks bind-proof recovery`, async () => { const io = ioFor(async () => true, undefined, { From 98eaa5c4d934822968bb541ba0efee8c321ef478 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 8 Nov 2026 19:34:54 +0900 Subject: [PATCH 5/6] Fix recovery classification of loopback literals --- src/server/proxy-liveness.ts | 21 +++++- structure/runtime.md | 2 +- ...xy-liveness-nonloopback-bind-proof.test.ts | 73 +++++++++++++++++++ 3 files changed, 91 insertions(+), 5 deletions(-) diff --git a/src/server/proxy-liveness.ts b/src/server/proxy-liveness.ts index 09ea6ab549d..0fc50ffdee2 100644 --- a/src/server/proxy-liveness.ts +++ b/src/server/proxy-liveness.ts @@ -589,7 +589,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { // Missing/blank means an actual loopback bind, not a later config's bind scope. const bindHostname = opts.hostname?.trim() || "127.0.0.1"; const hostname = normalizeBindHostname(opts.hostname); - if (!observer || (!isLoopbackHostname(hostname) && !isWildcardHostname(hostname))) { + if (!observer || (!recoveryLoopbackFamily(hostname) && !isWildcardHostname(hostname))) { // Recovery only (an observer is present): a transport miss on a specific non-loopback bind // address is "unknown", and a bind proof is the one thing that turns it into "dead". let verdict: EndpointLiveness | undefined; @@ -702,6 +702,19 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { return null; } +// Recovery classification is intentionally local: it must not broaden admission or bind policy. +function recoveryLoopbackFamily(hostname: string): 4 | 6 | null { + if (isLoopbackHostname(hostname)) return hostname.includes(":") ? 6 : 4; + const address = hostname.replace(/^\[(.*)\]$/, "$1"); + const family = isIP(address); + if (family === 4) return address.startsWith("127.") ? 4 : null; + if (family !== 6) return null; + // URL serializes validated IPv6 literals canonically, including dotted IPv4-mapped forms. + const canonical = new URL(`http://[${address.split("%")[0]}]/`).hostname.slice(1, -1); + if (canonical === "::1") return 6; + return /^::ffff:7f[0-9a-f]{2}:[0-9a-f]{1,4}$/.test(canonical) ? 4 : null; +} + /** * Loopback addresses to ask about a port whose holder must be identified before a * decision that is destructive when it answers "nobody". @@ -722,9 +735,9 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { export function loopbackProbeHosts(hostname: string | undefined): string[] { const normalized = normalizeBindHostname(hostname); const primary = probeHostname(normalized); - if (isLoopbackHostname(normalized) || isWildcardHostname(normalized)) { - return primary === "[::1]" ? ["[::1]", "127.0.0.1"] : ["127.0.0.1", "[::1]"]; - } + if (isWildcardHostname(normalized)) return ["127.0.0.1", "[::1]"]; + const family = recoveryLoopbackFamily(normalized); + if (family) return [primary, family === 6 ? "127.0.0.1" : "[::1]"]; return [primary]; } diff --git a/structure/runtime.md b/structure/runtime.md index 6ad921e4ac1..ac3c9e57c48 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -181,7 +181,7 @@ uninstall still restore. Native client signal cleanup follows the [served-connec `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. `ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. The Remote Link Child listener checks its startup owner and link against durable client state once per `/readyz`, after readiness probing, and for authenticated machine/link status only on those routes. If they differ, `/readyz` and those status routes answer 503 with `client_state_changed`; the dashboard shows that reason with its recovery guidance. This reports divergence; it does not disconnect the relay or reconstruct a connection. `tests/clients/client-machine-listener.test.ts` covers missing and replaced records. Explicit `ocx disconnect --orphaned-token --yes` in `src/client/orphan-token-recovery.ts` preserves unowned token files only after the start lease, lifecycle lock, liveness proof and config-lock revalidation; Hub, admission-token use and installed or unverified services refuse. Windows closes source handles before rename while private hard links pin identity; a refusal removes a pin only while another name still holds the token, and an extra hard link left by an interrupted run is refused with a message naming the `.identity` pin to remove from the `orphaned-token-*` folder. Missing ownership is never reconstructed. `ocx config import` refuses to replace a connected client record under the config lock. `tests/clients/client-link-connect.test.ts` covers these refusals and local liveness. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts index 8d3775da0ab..09aaee51941 100644 --- a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts +++ b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts @@ -20,6 +20,79 @@ function ioFor(bindProofFn: LivenessIo["bindProofFn"], hostname = "100.88.46.13" }; } +for (const [hostname, primary, other] of [ + ["127.0.0.2", "127.0.0.2", "[::1]"], + ["127.255.255.254", "127.255.255.254", "[::1]"], + ["::0:1", "[::1]", "127.0.0.1"], + ["[0:0:0:0:0:0:0:1]", "[::1]", "127.0.0.1"], + ["::ffff:127.0.0.1", "[::ffff:7f00:1]", "[::1]"], + ["[0:0:0:0:0:ffff:7fff:fffe]", "[::ffff:7fff:fffe]", "[::1]"], +] as const) { + for (const source of ["config", "runtime"] as const) { + test(`loopback ${hostname} ${source} probes both families without bind proof`, async () => { + const calls: string[] = []; + let proofs = 0; + const extra: Partial = source === "runtime" + ? { readRuntimeFn: () => ({ port: 10100, hostname }) } : {}; + const io = ioFor(async () => { proofs++; return true; }, hostname, { + ...extra, fetchFn: (async (url: string) => { + calls.push(new URL(url).hostname); + throw Object.assign(new Error("timeout"), { code: "ETIMEDOUT" }); + }) as typeof fetch, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + expect(calls).toEqual(source === "runtime" ? [primary, other, primary, other] : [primary, other]); + expect(proofs).toBe(0); + for (const uncertain of [primary, other]) { + const result = await observeProxyLiveness({ ...io, fetchFn: (async (url: string) => { + throw Object.assign(new Error("probe failed"), { + code: new URL(url).hostname === uncertain ? "ETIMEDOUT" : "ECONNREFUSED", + }); + }) as typeof fetch }); + expect(result).toEqual({ kind: "unknown", live: null }); + } + expect(proofs).toBe(0); + }); + } + test(`loopback ${hostname} retains runtime owner evidence on its concrete address`, async () => { + const observation = await observeProxyLiveness(ioFor(async () => true, hostname, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname }), + fetchFn: (async (url: string) => new URL(url).hostname === primary + ? Response.json({ service: "opencodex", pid: process.pid }) + : Promise.reject(Object.assign(new Error("refused"), { code: "ECONNREFUSED" }))) as typeof fetch, + })); + expect(observation.kind).toBe("live"); + expect(observation.live).toMatchObject({ pid: process.pid, port: 10100, hostname, source: "runtime" }); + expect(new URL(`http://${observation.live?.probeHostname}:10100`).hostname).toBe(primary); + }); + test(`loopback ${hostname} finds an owner on the other family`, async () => { + const observation = await observeProxyLiveness(ioFor(async () => true, hostname, { + fetchFn: (async (url: string) => new URL(url).hostname === other + ? Response.json({ service: "opencodex", pid: process.pid }) + : Promise.reject(Object.assign(new Error("refused"), { code: "ECONNREFUSED" }))) as typeof fetch, + })); + expect(observation.kind).toBe("live"); + expect(observation.live).toMatchObject({ pid: process.pid, hostname, probeHostname: other, source: "config" }); + }); + test(`loopback ${hostname} requires both negative probes and checks the runtime PID`, async () => { + const io = ioFor(async () => { throw new Error("loopback bind proof called"); }, hostname, { + fetchFn: (async () => { throw Object.assign(new Error("refused"), { code: "ECONNREFUSED" }); }) as typeof fetch, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); + expect(await observeProxyLiveness({ ...io, + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname }), + readCommandLineFn: () => "ocx start", + })).toEqual({ kind: "unknown", live: null }); + }); +} + +test("mapped non-loopback IPv4 remains eligible for the bind proof", async () => { + let socketHost = ""; + const observation = await observeProxyLiveness(ioFor(async host => { socketHost = host; return true; }, "[::ffff:192.0.2.1]")); + expect(observation.kind).toBe("absent"); + expect(socketHost).toBe("::ffff:192.0.2.1"); +}); + test("a timed-out probe of a non-loopback bind is absent when the address can be bound", async () => { let asked: [string, number] | undefined; const observation = await observeProxyLiveness(ioFor(async (host, port) => { asked = [host, port]; return true; })); From 32b238dfa86158095363477f88cafb839c149c3b Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 8 Nov 2026 19:51:51 +0900 Subject: [PATCH 6/6] Keep expanded loopback probes within recovery --- src/server/proxy-liveness.ts | 15 ++++++--- structure/runtime.md | 2 +- ...xy-liveness-nonloopback-bind-proof.test.ts | 33 ++++++++++++++++++- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/server/proxy-liveness.ts b/src/server/proxy-liveness.ts index 0fc50ffdee2..72947685992 100644 --- a/src/server/proxy-liveness.ts +++ b/src/server/proxy-liveness.ts @@ -603,7 +603,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { } // Parallel families share the existing per-probe attempts/timeout and absolute // deadline. A slow family cannot multiply the serial discovery budget. - const hosts = loopbackProbeHosts(hostname).filter(host => + const hosts = recoveryProbeHosts(hostname).filter(host => (io.loopbackAvailabilityFn ?? loopbackFamilyAvailability)(host) !== "unavailable"); const identities = await Promise.all(hosts.map(async hostname => { const identity = await proxyIdentityAtObserved(port, { ...opts, hostname }, probeIo, observer.endpoint); @@ -715,6 +715,13 @@ function recoveryLoopbackFamily(hostname: string): 4 | 6 | null { return /^::ffff:7f[0-9a-f]{2}:[0-9a-f]{1,4}$/.test(canonical) ? 4 : null; } +function recoveryProbeHosts(hostname: string): string[] { + if (isWildcardHostname(hostname)) return ["127.0.0.1", "[::1]"]; + const family = recoveryLoopbackFamily(hostname); + if (family) return [probeHostname(hostname), family === 6 ? "127.0.0.1" : "[::1]"]; + return [probeHostname(hostname)]; +} + /** * Loopback addresses to ask about a port whose holder must be identified before a * decision that is destructive when it answers "nobody". @@ -735,9 +742,9 @@ function recoveryLoopbackFamily(hostname: string): 4 | 6 | null { export function loopbackProbeHosts(hostname: string | undefined): string[] { const normalized = normalizeBindHostname(hostname); const primary = probeHostname(normalized); - if (isWildcardHostname(normalized)) return ["127.0.0.1", "[::1]"]; - const family = recoveryLoopbackFamily(normalized); - if (family) return [primary, family === 6 ? "127.0.0.1" : "[::1]"]; + if (isLoopbackHostname(normalized) || isWildcardHostname(normalized)) { + return primary === "[::1]" ? ["[::1]", "127.0.0.1"] : ["127.0.0.1", "[::1]"]; + } return [primary]; } diff --git a/structure/runtime.md b/structure/runtime.md index ac3c9e57c48..dad717f1d1b 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -181,7 +181,7 @@ uninstall still restore. Native client signal cleanup follows the [served-connec `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. `ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. The Remote Link Child listener checks its startup owner and link against durable client state once per `/readyz`, after readiness probing, and for authenticated machine/link status only on those routes. If they differ, `/readyz` and those status routes answer 503 with `client_state_changed`; the dashboard shows that reason with its recovery guidance. This reports divergence; it does not disconnect the relay or reconstruct a connection. `tests/clients/client-machine-listener.test.ts` covers missing and replaced records. Explicit `ocx disconnect --orphaned-token --yes` in `src/client/orphan-token-recovery.ts` preserves unowned token files only after the start lease, lifecycle lock, liveness proof and config-lock revalidation; Hub, admission-token use and installed or unverified services refuse. Windows closes source handles before rename while private hard links pin identity; a refusal removes a pin only while another name still holds the token, and an extra hard link left by an interrupted run is refused with a message naming the `.identity` pin to remove from the `orphaned-token-*` folder. Missing ownership is never reconstructed. `ocx config import` refuses to replace a connected client record under the config lock. `tests/clients/client-link-connect.test.ts` covers these refusals and local liveness. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts index 09aaee51941..86363fa5f8a 100644 --- a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts +++ b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts @@ -1,6 +1,6 @@ import { expect, spyOn, test } from "bun:test"; import { createServer } from "node:net"; -import { canBindAddress, findLiveProxy, observeProxyLiveness, type LivenessIo } from "../../src/server/proxy-liveness"; +import { canBindAddress, findLiveProxy, observeProxyLiveness, probeEndpointLiveness, probePortOwner, type LivenessIo } from "../../src/server/proxy-liveness"; // Connecting to this machine's own non-loopback address (a Tailscale IP) with nothing listening can // time out instead of being refused. A timeout is "unknown", which startup refuses forever; only a @@ -20,6 +20,37 @@ function ioFor(bindProofFn: LivenessIo["bindProofFn"], hostname = "100.88.46.13" }; } +for (const [hostname, primary] of [ + ["127.0.0.2", "127.0.0.2"], + ["::0:1", "[::1]"], + ["::ffff:127.0.0.1", "[::ffff:7f00:1]"], +] as const) { + for (const caller of ["endpoint", "port-owner"] as const) test(`${caller} does not adopt another family's owner for ${hostname}`, async () => { + const calls: string[] = []; + const fetchFn = (async (url: string) => { + const host = new URL(url).hostname; + calls.push(host); + if (host !== primary) return Response.json({ service: "opencodex", pid: 4242 }); + throw Object.assign(new Error("refused"), { code: "ECONNREFUSED" }); + }) as typeof fetch; + if (caller === "endpoint") expect(await probeEndpointLiveness({ port: 10100, hostname }, { fetchFn })).toBe("dead"); + else expect(await probePortOwner(10100, { hostname }, { fetchFn })).toBeNull(); + expect(calls).toEqual([primary]); + }); + test(`recovery still checks both families for ${hostname}`, async () => { + const calls: string[] = []; + const io = ioFor(undefined, hostname, { fetchFn: (async (url: string) => { + const host = new URL(url).hostname; + calls.push(host); + if (host !== primary) return Response.json({ service: "opencodex", pid: 4242 }); + throw Object.assign(new Error("refused"), { code: "ECONNREFUSED" }); + }) as typeof fetch }); + expect(await observeProxyLiveness(io)).toMatchObject({ kind: "live", live: { pid: 4242, hostname } }); + expect(calls).toHaveLength(2); + expect(calls).toContain(primary); + }); +} + for (const [hostname, primary, other] of [ ["127.0.0.2", "127.0.0.2", "[::1]"], ["127.255.255.254", "127.255.255.254", "[::1]"],