diff --git a/docs/architecture/packaging.md b/docs/architecture/packaging.md index 973812e00d..3c24a4484d 100644 --- a/docs/architecture/packaging.md +++ b/docs/architecture/packaging.md @@ -73,7 +73,7 @@ The release workflow runs the three scripts directly. Without the parser pack, D - `scripts/opencode/stage.mjs`, which `build:opencode` runs, stages opencode `1.18.34` and ripgrep `15.1.0` into `electron/opencode/`, each archive checked against its SHA-256 in `scripts/opencode/pins.mjs`. On x64 it takes opencode's `-baseline` build, which opencode compiles without AVX2. The release archives carry only the executable, so opencode's licence comes from its tag, also pinned; ripgrep's three licence files ship in `opencode/ripgrep/`. It runs `opencode --version` and `rg --version` from the staged folder before it swaps the folder into place, and restages when the staged opencode reports another version. The Linux executable needs glibc 2.17 and no libstdc++. It stages only when `scripts/opencode/enabled.mjs` says opencode is on, which is off until a model passes the agent test and `SURFSENSE_LOCAL_OPENCODE_ENABLED=1` overrides; off, it removes an earlier stage, so neither `pnpm dev` nor an installer carries one. -`pnpm dist` in `electron/` runs every staging step before `electron-builder`, the native runtimes first, so a missing toolchain stops it before the frontend and the Python binaries are built. It stages sd.cpp and audio.cpp with `--strict`, so an installer never ships without them; `predev` does not, so dev runs without a toolchain. +`pnpm dist` in `electron/` runs every staging step before `electron-builder`, the native runtimes first, so a missing toolchain stops it before the frontend and the Python binaries are built. It stages sd.cpp and audio.cpp with `--strict`, so an installer never ships without them; `predev` does not, so dev runs without a toolchain. On Windows, the rename that swaps each stage into place waits up to a minute while another process still has a file in it open, such as a Windows service reading an executable the script just ran. `frontend/` and `electron/` each pin their pnpm in `packageManager` (`pnpm@11.27.1`), which pnpm switches to locally and the desktop workflows read through `package_json_file`. The repository root's `packageManager` is the web app's, `pnpm@10.26.0`, and does not apply here. diff --git a/surfsense_local/electron/scripts/audiocpp/stage.mjs b/surfsense_local/electron/scripts/audiocpp/stage.mjs index 40090d7b54..770e71218e 100644 --- a/surfsense_local/electron/scripts/audiocpp/stage.mjs +++ b/surfsense_local/electron/scripts/audiocpp/stage.mjs @@ -14,6 +14,7 @@ import { stageEspeak } from "./espeak.mjs" import { copyMsvcRuntime } from "../msvc-runtime.mjs" import { NotStaged, notStaged } from "../not-staged/message.mjs" import { packageManager } from "../not-staged/package-manager.mjs" +import { renameWhenUnlocked } from "../windows-locks/rename-when-unlocked.mjs" import { TAG } from "./pins.mjs" import { copyServerFiles, SERVER } from "./server-files.mjs" import { unpackUpstream } from "./upstream-archive.mjs" @@ -85,7 +86,7 @@ async function main() { rmSync(backup, { recursive: true, force: true }) if (existsSync(OUT)) renameSync(OUT, backup) try { - renameSync(stage, OUT) + await renameWhenUnlocked(stage, OUT) } catch (error) { if (existsSync(backup)) renameSync(backup, OUT) throw error diff --git a/surfsense_local/electron/scripts/fetch-llamacpp.mjs b/surfsense_local/electron/scripts/fetch-llamacpp.mjs index 73f2d5d732..7611b84cc5 100644 --- a/surfsense_local/electron/scripts/fetch-llamacpp.mjs +++ b/surfsense_local/electron/scripts/fetch-llamacpp.mjs @@ -33,6 +33,8 @@ import { tmpdir } from "node:os" import { basename, dirname, join } from "node:path" import { fileURLToPath } from "node:url" +import { renameWhenUnlocked } from "./windows-locks/rename-when-unlocked.mjs" + export const BUILD = "b11050" const BASE = `https://github.com/ggml-org/llama.cpp/releases/download/${BUILD}` const HERE = fileURLToPath(new URL(".", import.meta.url)) @@ -189,7 +191,7 @@ async function main() { rmSync(backup, { recursive: true, force: true }) if (existsSync(OUT)) renameSync(OUT, backup) try { - renameSync(stage, OUT) + await renameWhenUnlocked(stage, OUT) } catch (error) { if (existsSync(backup)) renameSync(backup, OUT) throw error diff --git a/surfsense_local/electron/scripts/opencode/stage.mjs b/surfsense_local/electron/scripts/opencode/stage.mjs index 18d908777a..23f233dc46 100644 --- a/surfsense_local/electron/scripts/opencode/stage.mjs +++ b/surfsense_local/electron/scripts/opencode/stage.mjs @@ -8,6 +8,7 @@ import { basename, join } from "node:path" import { fileURLToPath } from "node:url" import { download, unpack } from "../pinned-download.mjs" +import { renameWhenUnlocked } from "../windows-locks/rename-when-unlocked.mjs" import { opencodeEnabled } from "./enabled.mjs" import { HOSTS, OPENCODE_LICENCE, RIPGREP_VERSION, VERSION } from "./pins.mjs" @@ -103,7 +104,7 @@ async function main() { rmSync(backup, { recursive: true, force: true }) if (existsSync(OUT)) renameSync(OUT, backup) try { - renameSync(stage, OUT) + await renameWhenUnlocked(stage, OUT) } catch (error) { if (existsSync(backup)) renameSync(backup, OUT) throw error diff --git a/surfsense_local/electron/scripts/sdcpp/stage.mjs b/surfsense_local/electron/scripts/sdcpp/stage.mjs index e5b864cabe..db0f6236eb 100644 --- a/surfsense_local/electron/scripts/sdcpp/stage.mjs +++ b/surfsense_local/electron/scripts/sdcpp/stage.mjs @@ -13,6 +13,7 @@ import { copyMsvcRuntime, visualStudio } from "../msvc-runtime.mjs" import { NotStaged, notStaged } from "../not-staged/message.mjs" import { packageManager } from "../not-staged/package-manager.mjs" import { VISUAL_STUDIO } from "../not-staged/tools.mjs" +import { renameWhenUnlocked } from "../windows-locks/rename-when-unlocked.mjs" import { compile, missingToolchain } from "./compile.mjs" import { TAG } from "./pins.mjs" import { copyServerFiles, SERVER } from "./server-files.mjs" @@ -87,7 +88,7 @@ async function main() { rmSync(backup, { recursive: true, force: true }) if (existsSync(OUT)) renameSync(OUT, backup) try { - renameSync(stage, OUT) + await renameWhenUnlocked(stage, OUT) } catch (error) { if (existsSync(backup)) renameSync(backup, OUT) throw error diff --git a/surfsense_local/electron/scripts/windows-locks/rename-when-unlocked.mjs b/surfsense_local/electron/scripts/windows-locks/rename-when-unlocked.mjs new file mode 100644 index 0000000000..9ee602fa21 --- /dev/null +++ b/surfsense_local/electron/scripts/windows-locks/rename-when-unlocked.mjs @@ -0,0 +1,22 @@ +// A rename that waits out Windows holding a file open inside the folder being renamed. +import { renameSync } from "node:fs" +import { setTimeout as sleep } from "node:timers/promises" + +// Only Windows answers this way for a file another process holds open; elsewhere they are real permission errors. +const LOCKED = new Set(["EPERM", "EACCES", "EBUSY"]) + +/** Rename `from` to `to`, retrying while another process holds a file in it, such as a Windows + * service reading a binary we just ran. 60 s by default: the window graceful-fs waits for the same lock. */ +export async function renameWhenUnlocked(from, to, { rename = renameSync, timeoutMs = 60_000, platform = process.platform } = {}) { + const start = Date.now() + for (let attempt = 1; ; attempt++) { + try { + return rename(from, to) + } catch (error) { + if (platform !== "win32" || !LOCKED.has(error.code) || Date.now() - start >= timeoutMs) throw error + if (attempt === 1) console.warn(`rename of ${from} blocked (${error.code}), retrying for up to ${timeoutMs / 1000} s`) + await sleep(Math.min(10 * attempt, 100)) + } + } +} + diff --git a/surfsense_local/electron/scripts/windows-locks/rename-when-unlocked.test.mjs b/surfsense_local/electron/scripts/windows-locks/rename-when-unlocked.test.mjs new file mode 100644 index 0000000000..0cc92232b4 --- /dev/null +++ b/surfsense_local/electron/scripts/windows-locks/rename-when-unlocked.test.mjs @@ -0,0 +1,38 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import { renameWhenUnlocked } from "./rename-when-unlocked.mjs" + +/** A stand-in for renameSync that fails with each code in turn, then succeeds. */ +function renameFailingWith(...codes) { + return () => { + const code = codes.shift() + if (code) throw Object.assign(new Error(`rename failed: ${code}`), { code }) + } +} + +test("a rename that fails for another reason is rethrown without a retry", async () => { + await assert.rejects(renameWhenUnlocked("stage", "out", { rename: renameFailingWith("ENOENT"), platform: "win32" }), { + code: "ENOENT", + }) +}) + +test("a rename Windows refuses twice goes through on the third try", async () => { + await assert.doesNotReject( + renameWhenUnlocked("stage", "out", { rename: renameFailingWith("EPERM", "EPERM"), platform: "win32" }), + ) +}) + +test("a rename locked for longer than the time limit gives up and rethrows EPERM", async () => { + const lockedFor24Tries = renameFailingWith(...Array(24).fill("EPERM")) + await assert.rejects(renameWhenUnlocked("stage", "out", { rename: lockedFor24Tries, timeoutMs: 50, platform: "win32" }), { + code: "EPERM", + }) +}) + +test("off Windows, a refused rename is rethrown without a retry", async () => { + await assert.rejects( + renameWhenUnlocked("stage", "out", { rename: renameFailingWith("EPERM"), platform: "linux" }), + { code: "EPERM" }, + ) +})