Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions apps/server/src/storageCleanup.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1241,6 +1241,59 @@ describe("storage cleanup reports and local file policies", () => {
}),
),
);
it.live.each(["same-device", "other-device", "no-manifest"] as const)(
"counts the pnpm virtual store only when its store forces copies (%s)",
(store) =>
runCleanupTest(
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const storeDir = yield* fs.makeTempDirectoryScoped({ prefix: "pnpm-store-" });
const { service, worktree, setPolicy } = yield* cleanupFixture().pipe(
Effect.provideService(FileSystem.FileSystem, {
...fs,
stat: (target) =>
fs
.stat(target)
.pipe(
Effect.map((info) =>
store === "other-device" && target === storeDir
? { ...info, dev: info.dev + 1n }
: info,
),
),
}),
);
setPolicy("tracked-changes");
const nodeModules = `${worktree}/node_modules`;
yield* fs.makeDirectory(`${nodeModules}/.pnpm/pkg@1.0.0`, { recursive: true });
yield* fs.makeDirectory(`${nodeModules}/.vite`);
yield* fs.writeFileString(
`${nodeModules}/.pnpm/pkg@1.0.0/index.js`,
"clone ".repeat(10_000),
);
yield* fs.writeFileString(`${nodeModules}/.vite/deps.js`, "cache");
if (store !== "no-manifest")
yield* fs.writeFileString(
`${nodeModules}/.modules.yaml`,
JSON.stringify({ storeDir, virtualStoreDir: ".pnpm" }),
);
const counted = [
...(yield* fs.readDirectory(worktree)).flatMap((name) =>
name === "node_modules" ? [] : [`${worktree}/${name}`],
),
`${nodeModules}/.vite/deps.js`,
...(store === "no-manifest" ? [] : [`${nodeModules}/.modules.yaml`]),
...(store === "same-device" ? [] : [`${nodeModules}/.pnpm/pkg@1.0.0/index.js`]),
];
const bytes = (yield* Effect.forEach(counted, (file) => allocatedBytes(fs, file))).reduce(
(sum, size) => sum + size,
0,
);
const report = yield* service.runNow;
expect(report.entries[0]).toMatchObject({ outcome: "removed", bytes });
}),
),
);
it.live("still removes a worktree when measuring its size fails", () =>
runCleanupTest(
Effect.gen(function* () {
Expand Down
29 changes: 29 additions & 0 deletions apps/server/src/storageCleanup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import type { PlatformError } from "effect/PlatformError";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import * as SubscriptionRef from "effect/SubscriptionRef";
import { parse as parseYaml } from "yaml";

import * as ServerConfig from "./config.ts";
import * as GitManager from "./git/GitManager.ts";
Expand All @@ -55,6 +56,13 @@ const decodeCleanupThread = Schema.decodeUnknownEffect(
const decodeCleanupSession = Schema.decodeUnknownEffect(
Schema.fromJsonString(OrchestrationV2ProviderSessionJson),
);
// pnpm 11 writes this file as JSON, older versions as YAML.
const decodePnpmModulesManifest = Schema.decodeUnknownEffect(
Schema.Struct({
storeDir: Schema.String,
virtualStoreDir: Schema.optional(Schema.String),
}),
);

const DAY_MS = 86_400_000;
// After a settle, the project's settle script needs time to open its
Expand Down Expand Up @@ -247,14 +255,35 @@ const make = Effect.gen(function* () {
);
});

// pnpm imports packages by clone or reflink when its store shares a device
// with the worktree. Those files have one link but share blocks with the
// store, so removing them frees almost nothing. A store on another device
// forces real copies, which still count.
const sharedPnpmVirtualStore = Effect.fnUntraced(
function* (root: string) {
const nodeModules = path.join(root, "node_modules");
const text = yield* fs.readFileString(path.join(nodeModules, ".modules.yaml"));
const manifest = yield* decodePnpmModulesManifest(yield* Effect.try(() => parseYaml(text)));
const virtualStore = path.resolve(nodeModules, manifest.virtualStoreDir ?? ".pnpm");
const [store, virtual] = yield* Effect.all([
fs.stat(manifest.storeDir),
fs.stat(virtualStore),
]);
return store.dev === virtual.dev ? virtualStore : null;
},
Effect.orElseSucceed(() => null),
);

const measureWorktree = Effect.fn("storageCleanup.measureWorktree")(
function* (worktreePath: string) {
const root = yield* fs.realPath(worktreePath);
const skipped = yield* sharedPnpmVirtualStore(root);
const pending = [root];
let entries = 1;
let bytes = 0;
while (pending.length > 0) {
const target = pending.pop()!;
if (target === skipped) continue;
// Effect's stat follows links. Probe with readLink first to get lstat
// semantics, including skipping dangling links and directory links.
const isLink = yield* fs.readLink(target).pipe(
Expand Down
Loading