fix(windows): writable fsync handles, hermetic ACL test lanes, teardown codemod

devlog/_plan/260902_windows_ci_release/011_wp2_evidence.md

@@ -0,0 +1,39 @@
+# 011 — wp2/wp3 evidence: dispatch rounds and reviewer verdicts
+
+## Dispatch 1 — 33595585136 on e1acb7f7a (ACL lifecycle + fsync + fixtures, before review fixes)
+
+| shard | before (33590540220) | after |
+|---|---|---|
+| windows 1/4 | ~120 fails, account-store/auth-api cascades | 8 fails: service.test ×7 (real icacls given a synthetic SID → EICACLS), issue-702 ×1 (unlinked an unpublished async spill) |
+| windows 2/4 | ~100 fails | 2 fails: responses-state budget case (gate swallowed the snapshot harden → 30 s ACL deadline → 60 s ceiling), write-failure case (copy fallback keyed on process.platform) |
+| windows 3/4 | ~40 fails | 1 fail: native-profile-manager first-child boot > private 5 s wait |
+| windows 4/4 | cancelled | cancelled by the shard-1 gate |
+
+Linux test 3/4 failed once on `late async spill completion` (ETIMEDOUT from the shared ACL budget);
+passes 6/6 locally and on the dev push run — treated as the same gate-swallowed-snapshot-harden
+mechanism, fixed by `isSpillAclTarget`.
+
+## Dispatch 2 — 33597649234 on 079bec4e0 (all of the above fixed)
+
+windows 1/4 SUCCESS, 2/4 SUCCESS, 3/4 SUCCESS — first Windows-green shards on any branch since
+33290817128 (2026-08-30). windows 4/4 cancelled at the 25-minute job ceiling with ZERO test
+failures: 199 files done at 06:27:35 (started 06:10:46), then the log ends inside
+`responses-native-main-refresh.test.ts` — 8 minutes with no output. The same file passes in
+~1.5 s in isolation on the Windows desktop (`desktop-c795oh4`, bun 1.3.14). Shard 4 also carries
+`codex-composed-acceptance` (299 s) and `native-profile-startup` (80 s). Investigation: full
+shard 4 run on the desktop, log at `C:\Temp\ocx-shard4b.log`.
+
+## Reviewer rounds (read-only, sol/high)
+
+- Lovelace on d0feec0ad..ae6212463: FAIL — P2 flush skipped when an earlier finalizer rejects;
+  P3 time-based pending oracle. Fixed in 1c41988ad (finally + rejected-release regression, driven red).
+- Hooke on origin/dev..079bec4e0 (8 commits): FAIL — P2 admission suite pinned to the sync lane
+  lost Windows coverage of runPendingResponseSpill; P3 assert the propagated rejection message.
+  Fixed in 384090052 (two win32 admission variants after queue settle; message asserted).
+  Confirmed safe: "r+" callers, isSpillAclTarget equivalence, windowsSecretAclApplies() ==
+  process.platform in production, service.test stub masks nothing, no AGENTS/privacy/Lab violation.
+
+## fuck-powershell
+
+56e1801 async-child-holds-dir-after-stop (87→88), 2f2107d fsync-readonly-handle-eperm +
+file-url-pathname-drive-slash (88→90). Graph 313 nodes / 643 edges, validate OK.

src/lib/service-secrets.ts

@@ -66,7 +66,9 @@ export function writeServiceApiTokenFile(token: string): PersistedServiceApiToke
 }
 
 function fsyncRegularFile(path: string): void {
-  const fd = openSync(path, "r");
+  // "r+", not "r": Windows rejects fsync on a read-only handle with EPERM, so a read-only
+  // open turned every token backup/replace/restore into a hard failure there.
+  const fd = openSync(path, "r+");
   try { fsyncSync(fd); } finally { closeSync(fd); }
 }
 

src/responses/spill-store.ts

@@ -215,7 +215,9 @@ function isErrno(error: unknown, code: string): boolean {
 }
 
 function canUseExclusiveCopyFallback(error: unknown): boolean {
-  return process.platform === "win32" || ["EPERM", "EACCES", "ENOSYS", "ENOTSUP", "EOPNOTSUPP", "EXDEV"]
+  // Same platform seam as harden(): a fixture pinned to the POSIX lane on a Windows host must
+  // see a link failure as a failure, not as a cue to copy.
+  return windowsSecretAclApplies() || ["EPERM", "EACCES", "ENOSYS", "ENOTSUP", "EOPNOTSUPP", "EXDEV"]
     .some(code => isErrno(error, code));
 }
 
@@ -235,7 +237,9 @@ function nextSpillHardenDeadlineMs(budget: SpillAclBudget | undefined): number |
 }
 
 function harden(path: string, mode: number, budget?: SpillAclBudget): void {
-  const aclApplies = budget ? windowsSecretAclApplies() : process.platform === "win32";
+  // One predicate for both lanes: the test seam that forces a platform must reach the
+  // sync harden too, or a fixture pinned to "linux" on a Windows host still spawns icacls.
+  const aclApplies = windowsSecretAclApplies();
   try {
     chmodSync(path, mode);
   } catch {
@@ -384,7 +388,8 @@ function publishNoReplace(
       else copyFileSync(tempPath, destinationPath, constants.COPYFILE_EXCL);
       copied = true;
       harden(destinationPath, 0o600, budget);
-      const copyFd = openSync(destinationPath, "r");
+      // "r+": a read-only handle cannot be fsynced on Windows (EPERM).
+      const copyFd = openSync(destinationPath, "r+");
       try {
         if (spillIoForTest?.fsync) spillIoForTest.fsync(copyFd);
         else fsyncSync(copyFd);
@@ -422,7 +427,7 @@ async function publishNoReplaceAsync(
       copied = true;
       await hardenAsync(destinationPath, 0o600, budget, retryTimedOutOnce);
       throwIfPublicationSuperseded(publicationControl);
-      const copyFd = openSync(destinationPath, "r");
+      const copyFd = openSync(destinationPath, "r+");
       try {
         if (spillIoForTest?.fsync) spillIoForTest.fsync(copyFd);
         else fsyncSync(copyFd);

tests/account-import.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, describe, expect, test } from "bun:test";
-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { createAntigravityAccountImportAdapter } from "../src/oauth/account-import/google-antigravity-adapter";
@@ -14,6 +14,7 @@ import {
   type ValidatedAntigravityCredential,
 } from "../src/oauth/account-import/types";
 import { getAccountSet, upsertCredentialByIdentity } from "../src/oauth/store";
+import { removeTreeWithRetry } from "./helpers/remove-tree";
 
 const CANARY = "cockpit-canary-refresh-token-DO-NOT-LEAK";
 const originalHome = process.env.OPENCODEX_HOME;
@@ -22,7 +23,7 @@ let testHome = "";
 afterEach(() => {
   if (originalHome === undefined) delete process.env.OPENCODEX_HOME;
   else process.env.OPENCODEX_HOME = originalHome;
-  if (testHome) rmSync(testHome, { recursive: true, force: true });
+  if (testHome) removeTreeWithRetry(testHome);
   testHome = "";
 });
 

tests/account-pool-management-api.test.ts

@@ -1,6 +1,6 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
 import { managementFetch as fetch } from "./helpers/management-auth";
-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { handleCodexAuthAPI } from "../src/codex/auth-api";
@@ -32,7 +32,7 @@ describe("Codex account pool strategy management API", () => {
   afterEach(() => {
     if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME;
     else process.env.OPENCODEX_HOME = previousOpencodexHome;
-    rmSync(TEST_DIR, { recursive: true, force: true });
+    removeTreeWithRetry(TEST_DIR);
   });
 
   test("GET /api/codex-auth/active surfaces strategy defaults", async () => {
@@ -454,7 +454,7 @@ describe("generic OAuth pool-settings contract (#695)", () => {
   afterEach(() => {
     if (previousHome === undefined) delete process.env.OPENCODEX_HOME;
     else process.env.OPENCODEX_HOME = previousHome;
-    if (testDir) rmSync(testDir, { recursive: true, force: true });
+    if (testDir) removeTreeWithRetry(testDir);
   });
 
   test("GET/PUT round-trip for a generic OAuth provider; api-key providers and bad values get 400", async () => {

tests/active-registry-admission.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync} from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { MAX_ACTIVE_TURNS, abortAndReleaseAllTurns, activeRegistryMetrics, trackStreamLifetime, tryAdmitTurn, unregisterTurn } from "../src/server/lifecycle";
@@ -24,6 +24,7 @@ import {
 import { saveConfig } from "../src/config";
 import { startServer } from "../src/server";
 import type { OcxConfig } from "../src/types";
+import { removeTreeWithRetry } from "./helpers/remove-tree";
 
 describe("active registry admission", () => {
   test("active turn 257 returns structured server_busy before handler work", async () => {
@@ -54,7 +55,7 @@ describe("active registry admission", () => {
       await server.stop(true);
       if (previousHome === undefined) delete process.env.OPENCODEX_HOME;
       else process.env.OPENCODEX_HOME = previousHome;
-      rmSync(home, { recursive: true, force: true });
+      removeTreeWithRetry(home);
     }
   });
 
@@ -86,7 +87,7 @@ describe("active registry admission", () => {
       await server.stop(true);
       if (previousHome === undefined) delete process.env.OPENCODEX_HOME;
       else process.env.OPENCODEX_HOME = previousHome;
-      rmSync(home, { recursive: true, force: true });
+      removeTreeWithRetry(home);
     }
   });
 
@@ -138,7 +139,7 @@ describe("active registry admission", () => {
       upstream.stop(true);
       if (previousHome === undefined) delete process.env.OPENCODEX_HOME;
       else process.env.OPENCODEX_HOME = previousHome;
-      rmSync(home, { recursive: true, force: true });
+      removeTreeWithRetry(home);
     }
   });
 

tests/adapter-event-oauth-failover.test.ts

@@ -1,11 +1,12 @@
 import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync} from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import type { ProviderAdapter } from "../src/adapters/base";
 import { clearGenericFailoverHealth } from "../src/oauth/generic-account-failover";
 import { saveCredential } from "../src/oauth/store";
 import type { AdapterEvent, OcxConfig, OcxProviderConfig } from "../src/types";
+import { removeTreeWithRetry } from "./helpers/remove-tree";
 
 const actualResolver = await import("../src/server/adapter-resolve");
 const actualResolveAdapter = actualResolver.resolveAdapter;
@@ -94,7 +95,7 @@ afterEach(() => {
   clearGenericFailoverHealth();
   if (originalHome === undefined) delete process.env.OPENCODEX_HOME;
   else process.env.OPENCODEX_HOME = originalHome;
-  rmSync(home, { recursive: true, force: true });
+  removeTreeWithRetry(home);
 });
 
 describe("#2568 adapter-event OAuth failover", () => {

tests/agent-task-recovery-combo.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { existsSync, mkdtempSync, readFileSync} from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
@@ -20,6 +20,7 @@ import {
   recoverySse,
   routedConfig,
 } from "./helpers/agent-task-recovery";
+import { removeTreeWithRetry } from "./helpers/remove-tree";
 
 function providerCompletion(): Response {
   return Response.json({
@@ -60,7 +61,7 @@ describe("combo path encrypted agent task recovery", () => {
     globalThis.fetch = originalFetch;
     resetAgentTaskRecoveryState();
     clearResponseStateForTests();
-    rmSync(home, { recursive: true, force: true });
+    removeTreeWithRetry(home);
     if (priorHome === undefined) delete process.env["OPENCODEX_HOME"];
     else process.env["OPENCODEX_HOME"] = priorHome;
   });

tests/alibaba-region-backup.test.ts

@@ -6,12 +6,13 @@ import {
   AlibabaBackupIntegrityError,
   backupConfigBeforeAlibabaRegionMigration,
 } from "../src/providers/alibaba-region-backup";
+import { removeTreeWithRetry } from "./helpers/remove-tree";
 
 test("absent source produces no backup", () => {
   const dir = mkdtempSync(join(tmpdir(), "ocx-bak-"));
   try {
     expect(backupConfigBeforeAlibabaRegionMigration(join(dir, "config.json"))).toBe("absent");
-  } finally { rmSync(dir, { recursive: true, force: true }); }
+  } finally { removeTreeWithRetry(dir); }
 });
 
 test("creates a snapshot, then never replaces it", () => {
@@ -24,7 +25,7 @@ test("creates a snapshot, then never replaces it", () => {
     expect(readFileSync(backupPath, "utf8")).toBe('{"before":true}');
     expect(backupConfigBeforeAlibabaRegionMigration(configPath)).toBe("reused");
     expect(readFileSync(backupPath, "utf8")).toBe('{"before":true}');
-  } finally { rmSync(dir, { recursive: true, force: true }); }
+  } finally { removeTreeWithRetry(dir); }
 });
 
 test("an existing snapshot is kept even after the config legitimately changes", () => {
@@ -40,7 +41,7 @@ test("an existing snapshot is kept even after the config legitimately changes",
     expect(backupConfigBeforeAlibabaRegionMigration(configPath)).toBe("reused");
     // The earliest snapshot survives: it predates every migration.
     expect(readFileSync(backupPath, "utf8")).toBe('{"before":true}');
-  } finally { rmSync(dir, { recursive: true, force: true }); }
+  } finally { removeTreeWithRetry(dir); }
 });
 
 test("a short copy is never published", () => {
@@ -56,7 +57,7 @@ test("a short copy is never published", () => {
       remove: path => rmSync(path, { force: true }),
     })).toThrow(AlibabaBackupIntegrityError);
     expect(existsSync(`${configPath}.pre-alibaba-region-v1.bak`)).toBe(false);
-  } finally { rmSync(dir, { recursive: true, force: true }); }
+  } finally { removeTreeWithRetry(dir); }
 });
 
 test("a failed copy leaves no snapshot and no temp file", () => {
@@ -74,5 +75,5 @@ test("a failed copy leaves no snapshot and no temp file", () => {
     })).toThrow("disk full");
     expect(existsSync(`${configPath}.pre-alibaba-region-v1.bak`)).toBe(false);
     expect(removed).toHaveLength(1);
-  } finally { rmSync(dir, { recursive: true, force: true }); }
+  } finally { removeTreeWithRetry(dir); }
 });

tests/alibaba-region-migration.test.ts

@@ -1,11 +1,12 @@
 import { expect, test } from "bun:test";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync} from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { loadConfig, saveConfig } from "../src/config";
 import { projectAlibabaRegionMigration } from "../src/providers/alibaba-region-migration";
 import { routeModel } from "../src/router";
 import type { OcxConfig } from "../src/types";
+import { removeTreeWithRetry } from "./helpers/remove-tree";
 
 const INTL_URL = "https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1";
 
@@ -70,7 +71,7 @@ test("the migrated config survives a reload", () => {
   } finally {
     if (prev === undefined) delete process.env.OPENCODEX_HOME;
     else process.env.OPENCODEX_HOME = prev;
-    rmSync(home, { recursive: true, force: true });
+    removeTreeWithRetry(home);
   }
 });
 
@@ -142,7 +143,7 @@ test("a namespace-blocked migration remains valid across reload", () => {
   } finally {
     if (prev === undefined) delete process.env.OPENCODEX_HOME;
     else process.env.OPENCODEX_HOME = prev;
-    rmSync(home, { recursive: true, force: true });
+    removeTreeWithRetry(home);
   }
 });
 

tests/anthropic-account-pool.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync} from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { clearPoolRotationState, notePoolRotationFailure, POOL_KEY_ANTHROPIC } from "../src/codex/pool-rotation";
@@ -20,6 +20,7 @@ import {
 import { getAccountSet, saveCredential, setActiveAccount } from "../src/oauth/store";
 import { clearAccountQuotaCache, setCachedProviderAccountQuotaForTests } from "../src/providers/quota";
 import type { OcxAccountPoolQuotaWindow, OcxAccountPoolRotationStrategy, OcxConfig } from "../src/types";
+import { removeTreeWithRetry } from "./helpers/remove-tree";
 
 const originalHome = process.env.OPENCODEX_HOME;
 let home: string;
@@ -38,7 +39,7 @@ afterEach(() => {
   clearAccountQuotaCache("anthropic");
   if (originalHome === undefined) delete process.env.OPENCODEX_HOME;
   else process.env.OPENCODEX_HOME = originalHome;
-  rmSync(home, { recursive: true, force: true });
+  removeTreeWithRetry(home);
 });
 
 async function seedTwoAccounts() {

tests/anthropic-image-retry-e2e.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync} from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { saveConfig } from "../src/config";
@@ -9,6 +9,7 @@ import { resetNormalizeStateForTests } from "../src/adapters/anthropic-image-nor
 import { sniffImageDimensions } from "../src/adapters/anthropic-image-guard";
 import type { OcxConfig } from "../src/types";
 import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home";
+import { removeTreeWithRetry } from "./helpers/remove-tree";
 
 let testDir = "";
 let previousHome: string | undefined;
@@ -31,7 +32,7 @@ afterEach(() => {
   else process.env.OPENCODEX_HOME = previousHome;
   isolatedCodexHome?.restore();
   isolatedCodexHome = null;
-  if (testDir) rmSync(testDir, { recursive: true, force: true });
+  if (testDir) removeTreeWithRetry(testDir);
   clearKeyCooldowns();
 });
 

tests/api-catalog-route.test.ts

@@ -1,11 +1,12 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { handleManagementAPI } from "../src/server/management-api";
 import { loadConfig, saveConfig } from "../src/config";
 import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home";
 import { ManagementRequest, managementHeaders } from "./helpers/management-auth";
 import type { OcxConfig } from "../src/types";
+import { removeTreeWithRetry } from "./helpers/remove-tree";
 
 const TEST_DIR = join(import.meta.dir, `.tmp-api-catalog-route-${process.pid}`);
 const previousOpencodexHome = process.env.OPENCODEX_HOME;
@@ -31,7 +32,7 @@ afterEach(() => {
   isolatedCodexHome = null;
   if (previousOpencodexHome === undefined) {
     delete process.env.OPENCODEX_HOME;
-    rmSync(TEST_DIR, { recursive: true, force: true });
+    removeTreeWithRetry(TEST_DIR);
   } else {
     process.env.OPENCODEX_HOME = previousOpencodexHome;
   }