diff --git a/scripts/test-layout/layout-start-ownership.json b/scripts/test-layout/layout-start-ownership.json index b112773799a..7caaff72af5 100644 --- a/scripts/test-layout/layout-start-ownership.json +++ b/scripts/test-layout/layout-start-ownership.json @@ -4,5 +4,6 @@ "proxy-liveness-owner-evidence.test.ts": "server", "proxy-liveness-recovery.test.ts": "server", "proxy-liveness-bind-scope.test.ts": "server", + "proxy-liveness-nonloopback-bind-proof.test.ts": "server", "system-restart-ownership.test.ts": "server" } diff --git a/src/server/proxy-liveness.ts b/src/server/proxy-liveness.ts index 78ec8c74e9c..72947685992 100644 --- a/src/server/proxy-liveness.ts +++ b/src/server/proxy-liveness.ts @@ -13,6 +13,7 @@ import { loadConfig } from "../config"; import { isLoopbackHostname, isWildcardHostname } from "../codex/loopback-target"; import { normalizeBindHostname } from "./bind-host"; import { isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort, verifyPidIdentity } from "../config/process-state"; +import { createServer, isIP } from "node:net"; import { loopbackFamilyAvailability } from "../lib/loopback-family"; import { LOCAL_ATTESTATION_CHALLENGE_HEADER, @@ -46,6 +47,35 @@ export interface HealthzIdentity { export type EndpointLiveness = "live" | "dead" | "unknown"; +/** + * Whether this process can bind `hostname:port` right now. A successful bind proves no process + * owns that address, which no probe of it can: connecting to this machine's own non-loopback + * address (a Tailscale IP, a LAN IP) with nothing listening TIMES OUT on some hosts instead of + * being refused, and a timeout is indistinguishable from a slow owner. The listener is closed + * immediately. Any bind error (EADDRINUSE, EADDRNOTAVAIL, EACCES) answers false, so an address + * we cannot bind stays "unknown" and startup keeps refusing. + */ +const BIND_PROOF_TIMEOUT_MS = 1_500; + +export function canBindAddress(hostname: string, port: number, timeoutMs = BIND_PROOF_TIMEOUT_MS): Promise { + if (!Number.isInteger(port) || port <= 0 || port > 65_535) return Promise.resolve(false); + return new Promise(resolve => { + const server = createServer(); + let settled = false; + const finish = (bound: boolean): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + try { server.close(); } catch { /* never bound */ } + resolve(bound); + }; + const timer = setTimeout(() => finish(false), timeoutMs); + server.once("error", () => finish(false)); + server.once("listening", () => finish(true)); + try { server.listen({ host: hostname, port, exclusive: true }); } catch { finish(false); } + }); +} + export interface LivenessIo { fetchFn?: typeof fetch; readPidFn?: () => number | null; @@ -57,6 +87,12 @@ export interface LivenessIo { /** Fresh command-line evidence: undefined/empty is unreadable, never confirmed foreign. */ readCommandLineFn?: typeof readProcessCommandLine; loopbackAvailabilityFn?: typeof loopbackFamilyAvailability; + /** + * Test seam for the bind proof (see `canBindAddress`): whether `hostname:port` can be bound by + * this process right now, which proves no process owns that address. `hostname` is the raw + * socket address (IPv6 unbracketed) and `timeoutMs` the remaining discovery budget. + */ + bindProofFn?: (hostname: string, port: number, timeoutMs: number) => Promise; readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string } | null; configFn?: () => { port?: number; hostname?: string }; timeoutMs?: number; @@ -479,19 +515,34 @@ export type ProxyLivenessObservation = /** Recovery additionally covers both available loopback families and fresh PID identity. */ export async function observeProxyLiveness(io: LivenessIo = {}): Promise { - let alivePid: number | null = null, unknown = false; + let alivePid: number | null = null, runtimePid: number | undefined, unknown = false; const live = await discoverLiveProxy(io, { pid: pid => { alivePid = pid; }, + runtimePid: pid => { runtimePid = pid; }, endpoint: result => { if (result === "unknown") unknown = true; }, }); if (live) return { kind: "live", live }; - if (alivePid !== null && !unknown) { + const candidates = new Set(); + if (alivePid !== null) candidates.add(alivePid); + if (runtimePid !== undefined && Number.isSafeInteger(runtimePid) && runtimePid > 0) candidates.add(runtimePid); + for (const pid of candidates) { + if (unknown) break; + if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) { unknown = true; break; } + if (pid !== alivePid) { + try { process.kill(pid, 0); } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") continue; + // Permission or an indeterminate probe is not proof that the process is dead. + unknown = true; + break; + } + } // A failed verifier alone conflates foreign with unreadable. Only fresh positive // evidence of a foreign command line releases a reused PID; never signal it. - const commandLine = (io.readCommandLineFn ?? readProcessCommandLine)(alivePid); + const commandLine = (io.readCommandLineFn ?? readProcessCommandLine)(pid); unknown = !commandLine?.trim() || isOcxCommandLine(commandLine); if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) unknown = true; } + if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) unknown = true; return { kind: unknown ? "unknown" : "absent", live: null }; } @@ -506,6 +557,7 @@ export async function findLiveProxy(io: LivenessIo = {}): Promise void; + runtimePid: (pid: number | undefined) => void; endpoint: (result: EndpointLiveness) => void; }): Promise { // Prefer the cheap alive-pid check: the Windows cmdline probe (WMIC/PowerShell) is too @@ -520,18 +572,38 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { const budgetExhausted = (): boolean => deadlineAt !== undefined && nowFn() >= deadlineAt; const expired = (): null => { observer?.endpoint("unknown"); return null; }; + // The proof stays inside the discovery deadline: it is skipped once the budget is spent, its + // own timeout is capped by what remains, and a proof that finishes past the deadline is + // discarded, so an exhausted discovery still answers "unknown". A socket takes the raw + // address, never the bracketed URL spelling `probeHostname` composes for IPv6. + const bindProof = async (hostname: string, port: number): Promise => { + const remainingMs = deadlineAt === undefined ? BIND_PROOF_TIMEOUT_MS : Math.min(BIND_PROOF_TIMEOUT_MS, deadlineAt - nowFn()); + if (remainingMs <= 0) return false; + const socketHost = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; + // A hostname can resolve to different addresses for the probe and the bind. + if (isIP(socketHost) === 0) return false; + const bound = await (io.bindProofFn ?? canBindAddress)(socketHost, port, remainingMs); + return bound && !budgetExhausted(); + }; const probe = async (port: number, opts: { hostname?: string; expectedPid?: number }) => { // Missing/blank means an actual loopback bind, not a later config's bind scope. const bindHostname = opts.hostname?.trim() || "127.0.0.1"; const hostname = normalizeBindHostname(opts.hostname); - if (!observer || (!isLoopbackHostname(hostname) && !isWildcardHostname(hostname))) { - const identity = await proxyIdentityAtObserved(port, opts, probeIo, observer?.endpoint); + if (!observer || (!recoveryLoopbackFamily(hostname) && !isWildcardHostname(hostname))) { + // Recovery only (an observer is present): a transport miss on a specific non-loopback bind + // address is "unknown", and a bind proof is the one thing that turns it into "dead". + let verdict: EndpointLiveness | undefined; + const identity = await proxyIdentityAtObserved(port, opts, probeIo, observer ? (r => { verdict = r; }) : undefined); + if (observer && !identity && verdict !== undefined) { + if (verdict === "unknown" && await bindProof(hostname, port)) verdict = "dead"; + observer.endpoint(verdict); + } return identity ? { ...identity, hostname: bindHostname, ...(observer ? { probeHostname: probeHostname(opts.hostname) } : {}) } : null; } // Parallel families share the existing per-probe attempts/timeout and absolute // deadline. A slow family cannot multiply the serial discovery budget. - const hosts = loopbackProbeHosts(hostname).filter(host => + const hosts = recoveryProbeHosts(hostname).filter(host => (io.loopbackAvailabilityFn ?? loopbackFamilyAvailability)(host) !== "unavailable"); const identities = await Promise.all(hosts.map(async hostname => { const identity = await proxyIdentityAtObserved(port, { ...opts, hostname }, probeIo, observer.endpoint); @@ -587,6 +659,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { // sibling command, manual deletion). The runtime record still says where it listens — // identity-probe it so ensure/update/stop see the live proxy instead of shadowing it. const record = readRuntimeFn(); + observer?.runtimePid(record?.pid); if (record?.port && record.port !== probedPort) { if (budgetExhausted()) return expired(); const expectedPid = typeof record.pid === "number" ? record.pid : undefined; @@ -629,6 +702,26 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { return null; } +// Recovery classification is intentionally local: it must not broaden admission or bind policy. +function recoveryLoopbackFamily(hostname: string): 4 | 6 | null { + if (isLoopbackHostname(hostname)) return hostname.includes(":") ? 6 : 4; + const address = hostname.replace(/^\[(.*)\]$/, "$1"); + const family = isIP(address); + if (family === 4) return address.startsWith("127.") ? 4 : null; + if (family !== 6) return null; + // URL serializes validated IPv6 literals canonically, including dotted IPv4-mapped forms. + const canonical = new URL(`http://[${address.split("%")[0]}]/`).hostname.slice(1, -1); + if (canonical === "::1") return 6; + return /^::ffff:7f[0-9a-f]{2}:[0-9a-f]{1,4}$/.test(canonical) ? 4 : null; +} + +function recoveryProbeHosts(hostname: string): string[] { + if (isWildcardHostname(hostname)) return ["127.0.0.1", "[::1]"]; + const family = recoveryLoopbackFamily(hostname); + if (family) return [probeHostname(hostname), family === 6 ? "127.0.0.1" : "[::1]"]; + return [probeHostname(hostname)]; +} + /** * Loopback addresses to ask about a port whose holder must be identified before a * decision that is destructive when it answers "nobody". diff --git a/structure/runtime.md b/structure/runtime.md index c641fd7de9b..dad717f1d1b 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -181,7 +181,7 @@ uninstall still restore. Native client signal cleanup follows the [served-connec `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. `ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. The Remote Link Child listener checks its startup owner and link against durable client state once per `/readyz`, after readiness probing, and for authenticated machine/link status only on those routes. If they differ, `/readyz` and those status routes answer 503 with `client_state_changed`; the dashboard shows that reason with its recovery guidance. This reports divergence; it does not disconnect the relay or reconstruct a connection. `tests/clients/client-machine-listener.test.ts` covers missing and replaced records. Explicit `ocx disconnect --orphaned-token --yes` in `src/client/orphan-token-recovery.ts` preserves unowned token files only after the start lease, lifecycle lock, liveness proof and config-lock revalidation; Hub, admission-token use and installed or unverified services refuse. Windows closes source handles before rename while private hard links pin identity; a refusal removes a pin only while another name still holds the token, and an extra hard link left by an interrupted run is refused with a message naming the `.identity` pin to remove from the `orphaned-token-*` folder. Missing ownership is never reconstructed. `ocx config import` refuses to replace a connected client record under the config lock. `tests/clients/client-link-connect.test.ts` covers these refusals and local liveness. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Only definitive absence permits PID deletion and journal reconciliation: no alive recorded PID unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. Unreadable/empty PID command lines and any OpenCodex invocation remain unknown; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index dc905cc69bd..0f44a3ecd58 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1204,6 +1204,7 @@ "proxy-liveness-recovery.test.ts": "server", "proxy-liveness-owner-evidence.test.ts": "server", "proxy-liveness-bind-scope.test.ts": "server", + "proxy-liveness-nonloopback-bind-proof.test.ts": "server", "system-restart-ownership.test.ts": "server", "proxy-liveness.test.ts": "server", "readyz-features.test.ts": "server", "served-by-header.test.ts": "server", "qoder-adapter.test.ts": "providers", "qoder-live-models.test.ts": "providers", "qoder-discovery-timeout.test.ts": "providers", "qoder-mcp-server.test.ts": "providers", "qoder-scaffold-guard.test.ts": "providers", diff --git a/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts new file mode 100644 index 00000000000..86363fa5f8a --- /dev/null +++ b/tests/server/proxy-liveness-nonloopback-bind-proof.test.ts @@ -0,0 +1,270 @@ +import { expect, spyOn, test } from "bun:test"; +import { createServer } from "node:net"; +import { canBindAddress, findLiveProxy, observeProxyLiveness, probeEndpointLiveness, probePortOwner, type LivenessIo } from "../../src/server/proxy-liveness"; + +// Connecting to this machine's own non-loopback address (a Tailscale IP) with nothing listening can +// time out instead of being refused. A timeout is "unknown", which startup refuses forever; only a +// bind proof may turn it into "absent". +function ioFor(bindProofFn: LivenessIo["bindProofFn"], hostname = "100.88.46.13", extra: Partial = {}): LivenessIo { + return { + readPidFn: () => null, + readRuntimeFn: () => null, + verifyPidFn: pid => pid, + configFn: () => ({ port: 10100, hostname }), + loopbackAvailabilityFn: () => "available", + attempts: 1, + timeoutMs: 50, + fetchFn: (async () => { throw Object.assign(new Error("timed out"), { name: "TimeoutError" }); }) as typeof fetch, + ...(bindProofFn ? { bindProofFn } : {}), + ...extra, + }; +} + +for (const [hostname, primary] of [ + ["127.0.0.2", "127.0.0.2"], + ["::0:1", "[::1]"], + ["::ffff:127.0.0.1", "[::ffff:7f00:1]"], +] as const) { + for (const caller of ["endpoint", "port-owner"] as const) test(`${caller} does not adopt another family's owner for ${hostname}`, async () => { + const calls: string[] = []; + const fetchFn = (async (url: string) => { + const host = new URL(url).hostname; + calls.push(host); + if (host !== primary) return Response.json({ service: "opencodex", pid: 4242 }); + throw Object.assign(new Error("refused"), { code: "ECONNREFUSED" }); + }) as typeof fetch; + if (caller === "endpoint") expect(await probeEndpointLiveness({ port: 10100, hostname }, { fetchFn })).toBe("dead"); + else expect(await probePortOwner(10100, { hostname }, { fetchFn })).toBeNull(); + expect(calls).toEqual([primary]); + }); + test(`recovery still checks both families for ${hostname}`, async () => { + const calls: string[] = []; + const io = ioFor(undefined, hostname, { fetchFn: (async (url: string) => { + const host = new URL(url).hostname; + calls.push(host); + if (host !== primary) return Response.json({ service: "opencodex", pid: 4242 }); + throw Object.assign(new Error("refused"), { code: "ECONNREFUSED" }); + }) as typeof fetch }); + expect(await observeProxyLiveness(io)).toMatchObject({ kind: "live", live: { pid: 4242, hostname } }); + expect(calls).toHaveLength(2); + expect(calls).toContain(primary); + }); +} + +for (const [hostname, primary, other] of [ + ["127.0.0.2", "127.0.0.2", "[::1]"], + ["127.255.255.254", "127.255.255.254", "[::1]"], + ["::0:1", "[::1]", "127.0.0.1"], + ["[0:0:0:0:0:0:0:1]", "[::1]", "127.0.0.1"], + ["::ffff:127.0.0.1", "[::ffff:7f00:1]", "[::1]"], + ["[0:0:0:0:0:ffff:7fff:fffe]", "[::ffff:7fff:fffe]", "[::1]"], +] as const) { + for (const source of ["config", "runtime"] as const) { + test(`loopback ${hostname} ${source} probes both families without bind proof`, async () => { + const calls: string[] = []; + let proofs = 0; + const extra: Partial = source === "runtime" + ? { readRuntimeFn: () => ({ port: 10100, hostname }) } : {}; + const io = ioFor(async () => { proofs++; return true; }, hostname, { + ...extra, fetchFn: (async (url: string) => { + calls.push(new URL(url).hostname); + throw Object.assign(new Error("timeout"), { code: "ETIMEDOUT" }); + }) as typeof fetch, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + expect(calls).toEqual(source === "runtime" ? [primary, other, primary, other] : [primary, other]); + expect(proofs).toBe(0); + for (const uncertain of [primary, other]) { + const result = await observeProxyLiveness({ ...io, fetchFn: (async (url: string) => { + throw Object.assign(new Error("probe failed"), { + code: new URL(url).hostname === uncertain ? "ETIMEDOUT" : "ECONNREFUSED", + }); + }) as typeof fetch }); + expect(result).toEqual({ kind: "unknown", live: null }); + } + expect(proofs).toBe(0); + }); + } + test(`loopback ${hostname} retains runtime owner evidence on its concrete address`, async () => { + const observation = await observeProxyLiveness(ioFor(async () => true, hostname, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname }), + fetchFn: (async (url: string) => new URL(url).hostname === primary + ? Response.json({ service: "opencodex", pid: process.pid }) + : Promise.reject(Object.assign(new Error("refused"), { code: "ECONNREFUSED" }))) as typeof fetch, + })); + expect(observation.kind).toBe("live"); + expect(observation.live).toMatchObject({ pid: process.pid, port: 10100, hostname, source: "runtime" }); + expect(new URL(`http://${observation.live?.probeHostname}:10100`).hostname).toBe(primary); + }); + test(`loopback ${hostname} finds an owner on the other family`, async () => { + const observation = await observeProxyLiveness(ioFor(async () => true, hostname, { + fetchFn: (async (url: string) => new URL(url).hostname === other + ? Response.json({ service: "opencodex", pid: process.pid }) + : Promise.reject(Object.assign(new Error("refused"), { code: "ECONNREFUSED" }))) as typeof fetch, + })); + expect(observation.kind).toBe("live"); + expect(observation.live).toMatchObject({ pid: process.pid, hostname, probeHostname: other, source: "config" }); + }); + test(`loopback ${hostname} requires both negative probes and checks the runtime PID`, async () => { + const io = ioFor(async () => { throw new Error("loopback bind proof called"); }, hostname, { + fetchFn: (async () => { throw Object.assign(new Error("refused"), { code: "ECONNREFUSED" }); }) as typeof fetch, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); + expect(await observeProxyLiveness({ ...io, + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname }), + readCommandLineFn: () => "ocx start", + })).toEqual({ kind: "unknown", live: null }); + }); +} + +test("mapped non-loopback IPv4 remains eligible for the bind proof", async () => { + let socketHost = ""; + const observation = await observeProxyLiveness(ioFor(async host => { socketHost = host; return true; }, "[::ffff:192.0.2.1]")); + expect(observation.kind).toBe("absent"); + expect(socketHost).toBe("::ffff:192.0.2.1"); +}); + +test("a timed-out probe of a non-loopback bind is absent when the address can be bound", async () => { + let asked: [string, number] | undefined; + const observation = await observeProxyLiveness(ioFor(async (host, port) => { asked = [host, port]; return true; })); + expect(observation.kind).toBe("absent"); + expect(asked).toEqual(["100.88.46.13", 10100]); +}); + +test("a timed-out probe of a non-loopback bind stays unknown when the address cannot be bound", async () => { + const observation = await observeProxyLiveness(ioFor(async () => false)); + expect(observation.kind).toBe("unknown"); +}); + +for (const source of ["config", "runtime"] as const) { + test(`an inconclusive ${source} hostname stays unknown without a bind proof`, async () => { + let proofCalls = 0; + const hostname = "multi-address.fixture.invalid"; + const extra: Partial = source === "runtime" + ? { readRuntimeFn: () => ({ port: 10100, hostname }) } + : {}; + const observation = await observeProxyLiveness(ioFor(async () => { proofCalls++; return true; }, hostname, extra)); + expect(observation.kind).toBe("unknown"); + expect(proofCalls).toBe(0); + }); +} + +test("a hostname listener that identifies itself remains live", async () => { + let proofCalls = 0; + const observation = await observeProxyLiveness(ioFor(async () => { proofCalls++; return true; }, "multi-address.fixture.invalid", { + fetchFn: (async () => Response.json({ service: "opencodex" })) as typeof fetch, + })); + expect(observation.kind).toBe("live"); + expect(proofCalls).toBe(0); +}); + +for (const commandLine of ["ocx start", undefined, ""]) { + test(`an orphan runtime PID with ${JSON.stringify(commandLine)} identity blocks bind-proof recovery`, async () => { + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "100.88.46.13" }), + readCommandLineFn: () => commandLine, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); +} + +test("dead and confirmed foreign orphan runtime PIDs permit bind-proof recovery", async () => { + const exited = Bun.spawn([process.execPath, "-e", ""], { stdout: "ignore", stderr: "ignore" }); + await exited.exited; + for (const [pid, commandLine] of [[exited.pid, undefined], [process.pid, "foreign-server"]] as const) { + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid, port: 10100, hostname: "100.88.46.13" }), + readCommandLineFn: () => commandLine, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); + } +}); + +test("a foreign PID file does not erase a different live runtime owner", async () => { + const io = ioFor(async () => true, undefined, { + readPidFn: () => 4242, + readRuntimeFn: pid => pid === undefined ? { pid: process.pid, port: 10100, hostname: "100.88.46.13" } : null, + readCommandLineFn: pid => pid === 4242 ? "foreign-server" : "ocx start", + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); +}); + +test("an indeterminate orphan process probe cannot authorize recovery", async () => { + const kill = spyOn(process, "kill").mockImplementation(() => { throw Object.assign(new Error("denied"), { code: "EPERM" }); }); + try { + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "100.88.46.13" }), + readCommandLineFn: () => "foreign-server", + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + expect(kill).toHaveBeenCalledWith(process.pid, 0); + } finally { kill.mockRestore(); } +}); + +test("runtime PID identity inspection cannot outlive the recovery deadline", async () => { + let clock = 0; + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "100.88.46.13" }), + readCommandLineFn: () => { clock = 1000; return "foreign-server"; }, + nowFn: () => clock, deadlineAt: 1000, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); +}); + +test("a dead PID probe finishing after the deadline remains unknown", async () => { + let clock = 0; + const kill = spyOn(process, "kill").mockImplementation(() => { + clock = 1000; + throw Object.assign(new Error("no process"), { code: "ESRCH" }); + }); + try { + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "100.88.46.13" }), + nowFn: () => clock, deadlineAt: 1000, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + } finally { kill.mockRestore(); } +}); + +test("an orphan PID is never returned for a pidless responding proxy", async () => { + const io = ioFor(async () => true, undefined, { + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "100.88.46.13" }), + fetchFn: (async () => Response.json({ service: "opencodex" })) as typeof fetch, + }); + expect((await findLiveProxy(io))?.pid).toBeNull(); + expect(await observeProxyLiveness(io)).toMatchObject({ kind: "live", live: { pid: null, port: 10100 } }); +}); + +test("a non-loopback IPv6 bind is proven with the unbracketed socket address", async () => { + for (const hostname of ["fd7a:115c:a1e0::1234", "[fd7a:115c:a1e0::1234]"]) { + let asked: string | undefined; + const observation = await observeProxyLiveness(ioFor(async host => { asked = host; return true; }, hostname)); + expect(observation.kind).toBe("absent"); + expect(asked).toBe("fd7a:115c:a1e0::1234"); + } +}); + +test("the bind proof stays within the discovery deadline", async () => { + let clock = 0; + let budget: number | undefined; + const overran = await observeProxyLiveness(ioFor(async (_host, _port, timeoutMs) => { + budget = timeoutMs; + clock = 2_000; + return true; + }, undefined, { nowFn: () => clock, deadlineAt: 1_000 })); + expect(budget).toBeLessThanOrEqual(1_000); + expect(overran.kind).toBe("unknown"); +}); + +test("canBindAddress is true for a free port and false while another listener holds it", async () => { + const holder = createServer(); + await new Promise(resolve => holder.listen({ host: "127.0.0.1", port: 0 }, resolve)); + const { port } = holder.address() as { port: number }; + try { + expect(await canBindAddress("127.0.0.1", port)).toBe(false); + } finally { + await new Promise(resolve => holder.close(() => resolve())); + } + expect(await canBindAddress("127.0.0.1", port)).toBe(true); + expect(await canBindAddress("127.0.0.1", 0)).toBe(false); +});