fix: drop unmapped alchemy-effect vendor gitlink

.github/workflows/ci.yml

@@ -26,12 +26,10 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      # Do not set persist-credentials: false. Checkout then runs
-      # `git submodule foreach`, which exits 128 on the vendored gitlink
-      # .repos/alchemy-effect/.vendor/alchemy. This job does not push;
-      # contents: read is what keeps the token from writing.
       - name: Checkout
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
 
       - name: Setup Vite+
         uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1

.github/workflows/mobile-eas-preview.yml

@@ -39,6 +39,7 @@ jobs:
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - name: Setup Vite+
         if: steps.expo-token.outputs.present == 'true'

.github/workflows/mobile-fingerprint-check.yml

@@ -49,6 +49,7 @@ jobs:
           # top of base), so the "head" fingerprint is the state main would
           # actually be in after merging — stale branches compare cleanly.
           fetch-depth: 0
+          persist-credentials: false
 
       - name: Setup Vite+
         uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1

.github/workflows/mobile-showcase-screenshots.yml

@@ -38,6 +38,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v6
         with:
+          persist-credentials: false
           sparse-checkout: |
             /*
             !/.repos/
@@ -90,6 +91,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
+          persist-credentials: false
           sparse-checkout: |
             /*
             !/.repos/

scripts/sync-reference-repos.test.ts

@@ -1,3 +1,4 @@
+import * as NodeURL from "node:url";
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import { assert, it } from "@effect/vitest";
 import * as Effect from "effect/Effect";
@@ -6,15 +7,47 @@ import * as Layer from "effect/Layer";
 import * as Path from "effect/Path";
 import * as Sink from "effect/Sink";
 import * as Stream from "effect/Stream";
-import { ChildProcessSpawner } from "effect/unstable/process";
+import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
 
 import { referenceRepos } from "./lib/reference-repos.ts";
 import {
+  dropUnmappedGitlinks,
+  gitlinkPathsFromLsFiles,
   planReferenceRepoSync,
   resolveReferenceRepoRef,
   syncReferenceRepos,
+  type ReferenceRepoSyncPlan,
 } from "./sync-reference-repos.ts";
 
+const workspaceRoot = NodeURL.fileURLToPath(new URL("..", import.meta.url));
+
+const collectStreamAsString = <E>(stream: Stream.Stream<Uint8Array, E>) =>
+  stream.pipe(
+    Stream.decodeText(),
+    Stream.runFold(
+      () => "",
+      (acc, chunk) => acc + chunk,
+    ),
+  );
+
+const runGit = (cwd: string, args: ReadonlyArray<string>) =>
+  Effect.gen(function* () {
+    const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+    const child = yield* spawner.spawn(ChildProcess.make("git", args, { cwd }));
+    const [stdout, stderr, exitCode] = yield* Effect.all(
+      [
+        collectStreamAsString(child.stdout),
+        collectStreamAsString(child.stderr),
+        child.exitCode.pipe(Effect.map(Number)),
+      ],
+      { concurrency: "unbounded" },
+    );
+    if (exitCode !== 0) {
+      assert.fail(`git ${args.join(" ")} exited ${exitCode}: ${stderr}`);
+    }
+    return stdout;
+  }).pipe(Effect.scoped);
+
 const encoder = new TextEncoder();
 const effectSmol = referenceRepos[0]!;
 const alchemyEffect = referenceRepos[1]!;
@@ -236,6 +269,10 @@ it.layer(NodeServices.layer)("sync-reference-repos", (it) => {
             "--squash",
           ],
         },
+        {
+          command: "git",
+          args: ["ls-files", "-s", "-z", "--", ".repos/effect-smol"],
+        },
       ]);
     });
   });
@@ -296,7 +333,79 @@ it.layer(NodeServices.layer)("sync-reference-repos", (it) => {
       assert.notProperty(error, "args");
       assert.notProperty(error, "stderr");
       assert.notInclude(error.message, "secret-token-value");
+      assert.equal(error.step, "subtree");
       assert.ok(!("cause" in error));
     });
   });
+
+  it.effect("reads only mode-160000 paths from ls-files output", () =>
+    Effect.sync(() => {
+      const output = [
+        "100644 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0\tREADME.md",
+        "160000 c9f5e549cf023632c3df948c207a58336192b3c7 0\t.repos/alchemy-effect/.vendor/alchemy",
+        "160000 bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0\t.repos/alchemy-effect/nested path",
+        "",
+      ].join("\0");
+
+      assert.deepStrictEqual(gitlinkPathsFromLsFiles(output), [
+        ".repos/alchemy-effect/.vendor/alchemy",
+        ".repos/alchemy-effect/nested path",
+      ]);
+      assert.deepStrictEqual(gitlinkPathsFromLsFiles(""), []);
+    }),
+  );
+
+  it.effect("commits removal of an unmapped gitlink without a .gitmodules entry", () =>
+    Effect.gen(function* () {
+      const fs = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+      const rootDir = yield* fs.makeTempDirectoryScoped({
+        prefix: "sync-reference-repos-gitlink-",
+      });
+      const gitlink = `${alchemyEffect.prefix}/.vendor/alchemy`;
+      const readme = path.join(rootDir, alchemyEffect.prefix, "README.md");
+      yield* fs.makeDirectory(path.dirname(readme), { recursive: true });
+      yield* fs.writeFileString(readme, "keep\n");
+
+      const git = (args: ReadonlyArray<string>) => runGit(rootDir, args);
+      yield* git(["init"]);
+      yield* git(["config", "user.email", "papercut@example.com"]);
+      yield* git(["config", "user.name", "Papercut"]);
+      yield* git(["config", "commit.gpgsign", "false"]);
+      yield* git(["add", "--", `${alchemyEffect.prefix}/README.md`]);
+      yield* git(["commit", "-m", "init"]);
+      yield* git([
+        "update-index",
+        "--add",
+        "--cacheinfo",
+        `160000,c9f5e549cf023632c3df948c207a58336192b3c7,${gitlink}`,
+      ]);
+      yield* git(["commit", "-m", "add gitlink"]);
+
+      const plan = {
+        repo: alchemyEffect,
+        action: "add",
+        ref: "vtest",
+        args: ["subtree", "add"],
+      } satisfies ReferenceRepoSyncPlan;
+      yield* dropUnmappedGitlinks(rootDir, plan).pipe(Effect.scoped);
+
+      const listed = yield* git(["ls-files", "-s", "-z"]);
+      assert.deepStrictEqual(gitlinkPathsFromLsFiles(listed), []);
+      yield* git(["submodule", "status"]);
+      yield* git(["submodule", "foreach", "--recursive", "git status --short"]);
+      const subject = (yield* git(["log", "-1", "--format=%s"])).trim();
+      assert.equal(subject, `chore: drop unmapped gitlinks under ${alchemyEffect.prefix}`);
+      assert.equal(yield* git(["show", `HEAD:${alchemyEffect.prefix}/README.md`]), "keep\n");
+    }),
+  );
+
+  it.effect("this checkout has no unmapped gitlinks", () =>
+    Effect.gen(function* () {
+      const listed = yield* runGit(workspaceRoot, ["ls-files", "-s", "-z"]);
+      assert.deepStrictEqual(gitlinkPathsFromLsFiles(listed), []);
+      yield* runGit(workspaceRoot, ["submodule", "status"]);
+      yield* runGit(workspaceRoot, ["submodule", "foreach", "--recursive", "git status --short"]);
+    }),
+  );
 });

scripts/sync-reference-repos.ts

@@ -70,10 +70,14 @@ export class ReferenceRepoVersionResolutionError extends Schema.TaggedErrorClass
   }
 }
 
+const gitSyncSteps = ["subtree", "ls-files", "rm", "commit"] as const;
+type GitSyncStep = (typeof gitSyncSteps)[number];
+
 export class ReferenceRepoGitSubtreeError extends Schema.TaggedErrorClass<ReferenceRepoGitSubtreeError>()(
   "ReferenceRepoGitSubtreeError",
   {
     operation: Schema.Literals(["spawn", "communicate", "exit"]),
+    step: Schema.Literals(gitSyncSteps),
     repoId: Schema.String,
     action: Schema.Literals(["add", "pull"]),
     repository: Schema.String,
@@ -87,7 +91,7 @@ export class ReferenceRepoGitSubtreeError extends Schema.TaggedErrorClass<Refere
   },
 ) {
   override get message(): string {
-    return `Git subtree ${this.action} for reference repo "${this.repoId}" failed during "${this.operation}".`;
+    return `Git ${this.step} for reference repo "${this.repoId}" failed during "${this.operation}" while syncing with subtree ${this.action}.`;
   }
 }
 
@@ -220,17 +224,50 @@ export const planReferenceRepoSync = Effect.fn("planReferenceRepoSync")(function
   } satisfies ReferenceRepoSyncPlan;
 });
 
-const runGit = Effect.fn("runGit")(function* (rootDir: string, plan: ReferenceRepoSyncPlan) {
+/**
+ * Index records from `git ls-files -s -z`. Mode 160000 is a gitlink.
+ * A gitlink with no root `.gitmodules` entry makes `git submodule foreach`
+ * exit 128, which is what checkout uses to clean a persistent worktree.
+ */
+export function gitlinkPathsFromLsFiles(output: string): ReadonlyArray<string> {
+  const paths: Array<string> = [];
+  for (const record of output.split("\0")) {
+    if (!record.startsWith("160000 ")) {
+      continue;
+    }
+    const tab = record.indexOf("\t");
+    if (tab < 0) {
+      continue;
+    }
+    const filePath = record.slice(tab + 1);
+    if (filePath.length > 0) {
+      paths.push(filePath);
+    }
+  }
+  return paths;
+}
+
+function isPathUnderPrefix(filePath: string, prefix: string): boolean {
+  return filePath === prefix || filePath.startsWith(`${prefix}/`);
+}
+
+const spawnGit = Effect.fn("spawnGit")(function* (
+  rootDir: string,
+  args: ReadonlyArray<string>,
+  plan: ReferenceRepoSyncPlan,
+  step: GitSyncStep,
+) {
   const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
   const errorContext = {
     repoId: plan.repo.id,
     action: plan.action,
     repository: plan.repo.repository,
     ref: plan.ref,
     rootDir,
-    argumentCount: plan.args.length,
+    argumentCount: args.length,
+    step,
   } as const;
-  const child = yield* spawner.spawn(ChildProcess.make("git", plan.args, { cwd: rootDir })).pipe(
+  const child = yield* spawner.spawn(ChildProcess.make("git", args, { cwd: rootDir })).pipe(
     Effect.mapError(
       (cause) =>
         new ReferenceRepoGitSubtreeError({
@@ -268,11 +305,57 @@ const runGit = Effect.fn("runGit")(function* (rootDir: string, plan: ReferenceRe
     });
   }
 
+  return stdout;
+});
+
+const runGit = Effect.fn("runGit")(function* (rootDir: string, plan: ReferenceRepoSyncPlan) {
+  const stdout = yield* spawnGit(rootDir, plan.args, plan, "subtree");
   if (stdout.trim().length > 0) {
     yield* Console.log(stdout.trim());
   }
 });
 
+export const dropUnmappedGitlinks = Effect.fn("dropUnmappedGitlinks")(function* (
+  rootDir: string,
+  plan: ReferenceRepoSyncPlan,
+) {
+  const listed = yield* spawnGit(
+    rootDir,
+    ["ls-files", "-s", "-z", "--", plan.repo.prefix],
+    plan,
+    "ls-files",
+  );
+  const gitlinks = gitlinkPathsFromLsFiles(listed).filter((filePath) =>
+    isPathUnderPrefix(filePath, plan.repo.prefix),
+  );
+  if (gitlinks.length === 0) {
+    return;
+  }
+
+  const removed = yield* spawnGit(rootDir, ["rm", "-f", "--cached", "--", ...gitlinks], plan, "rm");
+  if (removed.trim().length > 0) {
+    yield* Console.log(removed.trim());
+  }
+
+  const committed = yield* spawnGit(
+    rootDir,
+    [
+      "commit",
+      "-m",
+      `chore: drop unmapped gitlinks under ${plan.repo.prefix}`,
+      "-m",
+      "Vendored subtree sync copies nested submodule gitlinks. They have no root .gitmodules entry, so git submodule foreach fails during checkout clean.",
+      "--",
+      ...gitlinks,
+    ],
+    plan,
+    "commit",
+  );
+  if (committed.trim().length > 0) {
+    yield* Console.log(committed.trim());
+  }
+});
+
 export const syncReferenceRepos = Effect.fn("syncReferenceRepos")(function* (
   options: ReferenceRepoSyncOptions = {},
 ) {
@@ -287,6 +370,7 @@ export const syncReferenceRepos = Effect.fn("syncReferenceRepos")(function* (
     yield* Console.log(`Syncing ${repo.id} from ${plan.ref} with git subtree ${plan.action}.`);
     if (!(options.dryRun ?? false)) {
       yield* runGit(rootDir, plan).pipe(Effect.scoped);
+      yield* dropUnmappedGitlinks(rootDir, plan).pipe(Effect.scoped);
     }
   }