diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f80226bfcf2..c9b8afe1a7c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,12 +26,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - # Do not set persist-credentials: false. Checkout then runs - # `git submodule foreach`, which exits 128 on the vendored gitlink - # .repos/alchemy-effect/.vendor/alchemy. This job does not push; - # contents: read is what keeps the token from writing. - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false - name: Setup Vite+ uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1 diff --git a/.github/workflows/mobile-eas-preview.yml b/.github/workflows/mobile-eas-preview.yml index c7c81abd2f77..d664e766a010 100644 --- a/.github/workflows/mobile-eas-preview.yml +++ b/.github/workflows/mobile-eas-preview.yml @@ -39,6 +39,7 @@ jobs: uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: fetch-depth: 0 + persist-credentials: false - name: Setup Vite+ if: steps.expo-token.outputs.present == 'true' diff --git a/.github/workflows/mobile-fingerprint-check.yml b/.github/workflows/mobile-fingerprint-check.yml index fcb418b336d7..f4a26887912e 100644 --- a/.github/workflows/mobile-fingerprint-check.yml +++ b/.github/workflows/mobile-fingerprint-check.yml @@ -49,6 +49,7 @@ jobs: # top of base), so the "head" fingerprint is the state main would # actually be in after merging — stale branches compare cleanly. fetch-depth: 0 + persist-credentials: false - name: Setup Vite+ uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1 diff --git a/.github/workflows/mobile-showcase-screenshots.yml b/.github/workflows/mobile-showcase-screenshots.yml index 1e302905d40f..0e8da49de0c8 100644 --- a/.github/workflows/mobile-showcase-screenshots.yml +++ b/.github/workflows/mobile-showcase-screenshots.yml @@ -38,6 +38,7 @@ jobs: - name: Checkout uses: actions/checkout@v6 with: + persist-credentials: false sparse-checkout: | /* !/.repos/ @@ -90,6 +91,7 @@ jobs: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: + persist-credentials: false sparse-checkout: | /* !/.repos/ diff --git a/.repos/alchemy-effect/.vendor/alchemy b/.repos/alchemy-effect/.vendor/alchemy deleted file mode 160000 index c9f5e549cf02..000000000000 --- a/.repos/alchemy-effect/.vendor/alchemy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c9f5e549cf023632c3df948c207a58336192b3c7 diff --git a/scripts/sync-reference-repos.test.ts b/scripts/sync-reference-repos.test.ts index e3189936352c..961e769eec46 100644 --- a/scripts/sync-reference-repos.test.ts +++ b/scripts/sync-reference-repos.test.ts @@ -1,3 +1,4 @@ +import * as NodeURL from "node:url"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; @@ -6,15 +7,47 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; -import { ChildProcessSpawner } from "effect/unstable/process"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { referenceRepos } from "./lib/reference-repos.ts"; import { + dropUnmappedGitlinks, + gitlinkPathsFromLsFiles, planReferenceRepoSync, resolveReferenceRepoRef, syncReferenceRepos, + type ReferenceRepoSyncPlan, } from "./sync-reference-repos.ts"; +const workspaceRoot = NodeURL.fileURLToPath(new URL("..", import.meta.url)); + +const collectStreamAsString = (stream: Stream.Stream) => + stream.pipe( + Stream.decodeText(), + Stream.runFold( + () => "", + (acc, chunk) => acc + chunk, + ), + ); + +const runGit = (cwd: string, args: ReadonlyArray) => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn(ChildProcess.make("git", args, { cwd })); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + collectStreamAsString(child.stdout), + collectStreamAsString(child.stderr), + child.exitCode.pipe(Effect.map(Number)), + ], + { concurrency: "unbounded" }, + ); + if (exitCode !== 0) { + assert.fail(`git ${args.join(" ")} exited ${exitCode}: ${stderr}`); + } + return stdout; + }).pipe(Effect.scoped); + const encoder = new TextEncoder(); const effectSmol = referenceRepos[0]!; const alchemyEffect = referenceRepos[1]!; @@ -236,6 +269,10 @@ it.layer(NodeServices.layer)("sync-reference-repos", (it) => { "--squash", ], }, + { + command: "git", + args: ["ls-files", "-s", "-z", "--", ".repos/effect-smol"], + }, ]); }); }); @@ -296,7 +333,79 @@ it.layer(NodeServices.layer)("sync-reference-repos", (it) => { assert.notProperty(error, "args"); assert.notProperty(error, "stderr"); assert.notInclude(error.message, "secret-token-value"); + assert.equal(error.step, "subtree"); assert.ok(!("cause" in error)); }); }); + + it.effect("reads only mode-160000 paths from ls-files output", () => + Effect.sync(() => { + const output = [ + "100644 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0\tREADME.md", + "160000 c9f5e549cf023632c3df948c207a58336192b3c7 0\t.repos/alchemy-effect/.vendor/alchemy", + "160000 bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0\t.repos/alchemy-effect/nested path", + "", + ].join("\0"); + + assert.deepStrictEqual(gitlinkPathsFromLsFiles(output), [ + ".repos/alchemy-effect/.vendor/alchemy", + ".repos/alchemy-effect/nested path", + ]); + assert.deepStrictEqual(gitlinkPathsFromLsFiles(""), []); + }), + ); + + it.effect("commits removal of an unmapped gitlink without a .gitmodules entry", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const rootDir = yield* fs.makeTempDirectoryScoped({ + prefix: "sync-reference-repos-gitlink-", + }); + const gitlink = `${alchemyEffect.prefix}/.vendor/alchemy`; + const readme = path.join(rootDir, alchemyEffect.prefix, "README.md"); + yield* fs.makeDirectory(path.dirname(readme), { recursive: true }); + yield* fs.writeFileString(readme, "keep\n"); + + const git = (args: ReadonlyArray) => runGit(rootDir, args); + yield* git(["init"]); + yield* git(["config", "user.email", "papercut@example.com"]); + yield* git(["config", "user.name", "Papercut"]); + yield* git(["config", "commit.gpgsign", "false"]); + yield* git(["add", "--", `${alchemyEffect.prefix}/README.md`]); + yield* git(["commit", "-m", "init"]); + yield* git([ + "update-index", + "--add", + "--cacheinfo", + `160000,c9f5e549cf023632c3df948c207a58336192b3c7,${gitlink}`, + ]); + yield* git(["commit", "-m", "add gitlink"]); + + const plan = { + repo: alchemyEffect, + action: "add", + ref: "vtest", + args: ["subtree", "add"], + } satisfies ReferenceRepoSyncPlan; + yield* dropUnmappedGitlinks(rootDir, plan).pipe(Effect.scoped); + + const listed = yield* git(["ls-files", "-s", "-z"]); + assert.deepStrictEqual(gitlinkPathsFromLsFiles(listed), []); + yield* git(["submodule", "status"]); + yield* git(["submodule", "foreach", "--recursive", "git status --short"]); + const subject = (yield* git(["log", "-1", "--format=%s"])).trim(); + assert.equal(subject, `chore: drop unmapped gitlinks under ${alchemyEffect.prefix}`); + assert.equal(yield* git(["show", `HEAD:${alchemyEffect.prefix}/README.md`]), "keep\n"); + }), + ); + + it.effect("this checkout has no unmapped gitlinks", () => + Effect.gen(function* () { + const listed = yield* runGit(workspaceRoot, ["ls-files", "-s", "-z"]); + assert.deepStrictEqual(gitlinkPathsFromLsFiles(listed), []); + yield* runGit(workspaceRoot, ["submodule", "status"]); + yield* runGit(workspaceRoot, ["submodule", "foreach", "--recursive", "git status --short"]); + }), + ); }); diff --git a/scripts/sync-reference-repos.ts b/scripts/sync-reference-repos.ts index 91508d8d5279..95461c71b0a4 100644 --- a/scripts/sync-reference-repos.ts +++ b/scripts/sync-reference-repos.ts @@ -70,10 +70,14 @@ export class ReferenceRepoVersionResolutionError extends Schema.TaggedErrorClass } } +const gitSyncSteps = ["subtree", "ls-files", "rm", "commit"] as const; +type GitSyncStep = (typeof gitSyncSteps)[number]; + export class ReferenceRepoGitSubtreeError extends Schema.TaggedErrorClass()( "ReferenceRepoGitSubtreeError", { operation: Schema.Literals(["spawn", "communicate", "exit"]), + step: Schema.Literals(gitSyncSteps), repoId: Schema.String, action: Schema.Literals(["add", "pull"]), repository: Schema.String, @@ -87,7 +91,7 @@ export class ReferenceRepoGitSubtreeError extends Schema.TaggedErrorClass { + const paths: Array = []; + for (const record of output.split("\0")) { + if (!record.startsWith("160000 ")) { + continue; + } + const tab = record.indexOf("\t"); + if (tab < 0) { + continue; + } + const filePath = record.slice(tab + 1); + if (filePath.length > 0) { + paths.push(filePath); + } + } + return paths; +} + +function isPathUnderPrefix(filePath: string, prefix: string): boolean { + return filePath === prefix || filePath.startsWith(`${prefix}/`); +} + +const spawnGit = Effect.fn("spawnGit")(function* ( + rootDir: string, + args: ReadonlyArray, + plan: ReferenceRepoSyncPlan, + step: GitSyncStep, +) { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const errorContext = { repoId: plan.repo.id, @@ -228,9 +264,10 @@ const runGit = Effect.fn("runGit")(function* (rootDir: string, plan: ReferenceRe repository: plan.repo.repository, ref: plan.ref, rootDir, - argumentCount: plan.args.length, + argumentCount: args.length, + step, } as const; - const child = yield* spawner.spawn(ChildProcess.make("git", plan.args, { cwd: rootDir })).pipe( + const child = yield* spawner.spawn(ChildProcess.make("git", args, { cwd: rootDir })).pipe( Effect.mapError( (cause) => new ReferenceRepoGitSubtreeError({ @@ -268,11 +305,57 @@ const runGit = Effect.fn("runGit")(function* (rootDir: string, plan: ReferenceRe }); } + return stdout; +}); + +const runGit = Effect.fn("runGit")(function* (rootDir: string, plan: ReferenceRepoSyncPlan) { + const stdout = yield* spawnGit(rootDir, plan.args, plan, "subtree"); if (stdout.trim().length > 0) { yield* Console.log(stdout.trim()); } }); +export const dropUnmappedGitlinks = Effect.fn("dropUnmappedGitlinks")(function* ( + rootDir: string, + plan: ReferenceRepoSyncPlan, +) { + const listed = yield* spawnGit( + rootDir, + ["ls-files", "-s", "-z", "--", plan.repo.prefix], + plan, + "ls-files", + ); + const gitlinks = gitlinkPathsFromLsFiles(listed).filter((filePath) => + isPathUnderPrefix(filePath, plan.repo.prefix), + ); + if (gitlinks.length === 0) { + return; + } + + const removed = yield* spawnGit(rootDir, ["rm", "-f", "--cached", "--", ...gitlinks], plan, "rm"); + if (removed.trim().length > 0) { + yield* Console.log(removed.trim()); + } + + const committed = yield* spawnGit( + rootDir, + [ + "commit", + "-m", + `chore: drop unmapped gitlinks under ${plan.repo.prefix}`, + "-m", + "Vendored subtree sync copies nested submodule gitlinks. They have no root .gitmodules entry, so git submodule foreach fails during checkout clean.", + "--", + ...gitlinks, + ], + plan, + "commit", + ); + if (committed.trim().length > 0) { + yield* Console.log(committed.trim()); + } +}); + export const syncReferenceRepos = Effect.fn("syncReferenceRepos")(function* ( options: ReferenceRepoSyncOptions = {}, ) { @@ -287,6 +370,7 @@ export const syncReferenceRepos = Effect.fn("syncReferenceRepos")(function* ( yield* Console.log(`Syncing ${repo.id} from ${plan.ref} with git subtree ${plan.action}.`); if (!(options.dryRun ?? false)) { yield* runGit(rootDir, plan).pipe(Effect.scoped); + yield* dropUnmappedGitlinks(rootDir, plan).pipe(Effect.scoped); } }