Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions apps/desktop/electron.vite.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,9 @@ export default defineConfig({
// pty-daemon - long-lived per-org Unix-socket server that owns PTYs.
// Spawned by PtyDaemonCoordinator; survives host-service restarts.
"pty-daemon": resolve("src/main/pty-daemon/index.ts"),
// host-service worker thread — emitted side-by-side with
// host-service.js so the pool's script resolution finds it.
"host-worker": resolve("src/main/host-worker/index.ts"),
},
output: {
dir: resolve(devPath, "main"),
Expand Down
4 changes: 4 additions & 0 deletions apps/desktop/src/main/host-worker/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
// Desktop bundle entry for the host-service worker thread. Emitted as
// dist/main/host-worker.js, side-by-side with host-service.js so the pool's
// script resolution finds it (see host-worker-pool.ts).
import "@superset/host-service/host-worker";
7 changes: 7 additions & 0 deletions packages/cli/scripts/build-dist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,13 @@ async function main(): Promise<void> {
const hostServiceBundle = await buildHostService();
cpSync(hostServiceBundle, join(stagingRoot, "lib", "host-service.js"));

// host-worker.js ships side-by-side too: the worker pool resolves it
// next to host-service.js (falling back to inline execution if absent).
cpSync(
join(dirname(hostServiceBundle), "host-worker.js"),
join(stagingRoot, "lib", "host-worker.js"),
);

// pty-daemon ships side-by-side with host-service.js. The host-service
// resolves the script path via `resolveSupervisorScriptPath()` which
// looks for `pty-daemon.js` next to itself first; without this copy the
Expand Down
25 changes: 24 additions & 1 deletion packages/host-service/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,4 +51,27 @@ if (!result.success) {
process.exit(1);
}

console.log(`[host-service] bundled to ${outdir}/host-service.js`);
// Worker-thread bundle, emitted side-by-side so the pool's script
// resolution finds it next to host-service.js (see host-worker-pool.ts).
const workerResult = await Bun.build({
entrypoints: ["src/workers/host-worker.ts"],
target: "node",
outdir,
naming: "host-worker.js",
format: "esm",
define: {
"process.env.NODE_ENV": JSON.stringify("production"),
},
});

if (!workerResult.success) {
console.error("[host-service] host-worker build failed:");
for (const log of workerResult.logs) {
console.error(log);
}
process.exit(1);
}

console.log(
`[host-service] bundled to ${outdir}/host-service.js + ${outdir}/host-worker.js`,
);
4 changes: 4 additions & 0 deletions packages/host-service/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@
"types": "./src/trpc/index.ts",
"default": "./src/trpc/index.ts"
},
"./host-worker": {
"types": "./src/workers/host-worker.ts",
"default": "./src/workers/host-worker.ts"
},
"./terminal-env": {
"types": "./src/terminal/env.ts",
"default": "./src/terminal/env.ts"
Expand Down
19 changes: 15 additions & 4 deletions packages/host-service/src/runtime/git/git.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,28 @@ import { createUserSimpleGit } from "./simple-git";
import type { GitCredentialProvider, GitFactory } from "./types";
import { getRemoteUrl } from "./utils";

export function createGitFactory(provider: GitCredentialProvider): GitFactory {
return async (repoPath: string) => {
/**
* Resolve the env a git invocation for `repoPath` needs (credentials for the
* repo's remote + lock hygiene). Split out from the factory so worker tasks
* can receive the env as plain data and build their own SimpleGit off-thread.
*/
export function createGitEnvResolver(provider: GitCredentialProvider) {
return async (repoPath: string): Promise<Record<string, string>> => {
const initialCredentials = await provider.getCredentials(null);
const git = createUserSimpleGit(repoPath).env(initialCredentials.env);
const remoteUrl = await getRemoteUrl(git);
const credentials = await provider.getCredentials(remoteUrl);

return git.env({
return {
...initialCredentials.env,
...credentials.env,
GIT_OPTIONAL_LOCKS: "0",
});
};
};
}

export function createGitFactory(provider: GitCredentialProvider): GitFactory {
const resolveEnv = createGitEnvResolver(provider);
return async (repoPath: string) =>
createUserSimpleGit(repoPath).env(await resolveEnv(repoPath));
}
2 changes: 1 addition & 1 deletion packages/host-service/src/runtime/git/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
export { createGitFactory } from "./git";
export { createGitEnvResolver, createGitFactory } from "./git";
export type { ResolvedRef, ResolveRefOptions } from "./refs";
export {
asLocalRef,
Expand Down
89 changes: 77 additions & 12 deletions packages/host-service/src/trpc/router/git/git.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,17 @@ import { TRPCError } from "@trpc/server";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { pullRequests, workspaces } from "../../../db/schema";
import { createGitEnvResolver } from "../../../runtime/git";
import type { HostServiceContext } from "../../../types";
import { getHostWorkerPool } from "../../../workers/host-worker-pool";
import {
gitCommitFilesTask,
gitStatusSnapshotTask,
} from "../../../workers/tasks/git";
import { protectedProcedure, queryProcedure, router } from "../../index";
import { resolveGithubRepo } from "../workspace-creation/shared/project-helpers";
import type {
ChangedFile,
CheckConclusionState,
CheckRun,
CheckStatusState,
Expand All @@ -19,11 +27,9 @@ import type {
} from "./types";
import { gitConfigWrite } from "./utils/config-write";
import {
getChangedFilesForDiff,
getDefaultBranchName,
resolveBaseComparison,
} from "./utils/git-helpers";
import { getGitStatusSnapshot } from "./utils/git-status";
import { gitStatusRefreshLimiter } from "./utils/git-status-refresh-limiter";
import {
type GraphQLThreadsResult,
Expand All @@ -32,6 +38,51 @@ import {
} from "./utils/graphql";
import { resolveWorktreePath } from "./utils/resolve-worktree";

// Front-door cap for commit-file diffs. Statuses are admitted by
// gitStatusRefreshLimiter; without a cap here, a burst of distinct-commit
// diffs could occupy every pool worker ahead of limiter-admitted statuses.
const MAX_CONCURRENT_COMMIT_FILE_TASKS = 2;
let activeCommitFileTasks = 0;
const commitFileWaiters: (() => void)[] = [];
async function withCommitFilesSlot<T>(fn: () => Promise<T>): Promise<T> {
if (activeCommitFileTasks >= MAX_CONCURRENT_COMMIT_FILE_TASKS) {
await new Promise<void>((resolve) => commitFileWaiters.push(resolve));
}
activeCommitFileTasks++;
try {
return await fn();
} finally {
activeCommitFileTasks--;
commitFileWaiters.shift()?.();
}
}

// Identical requests share one slot AND one task — deduping outside the
// semaphore keeps same-commit bursts from consuming both cap slots or
// re-running a task that finished while they waited for a slot.
const inFlightCommitFiles = new Map<string, Promise<ChangedFile[]>>();
function runCommitFilesDeduped(
key: string,
fn: () => Promise<ChangedFile[]>,
): Promise<ChangedFile[]> {
const existing = inFlightCommitFiles.get(key);
if (existing) return existing;
const task = withCommitFilesSlot(fn).finally(() => {
inFlightCommitFiles.delete(key);
});
inFlightCommitFiles.set(key, task);
return task;
}

/** Credential env for a worker git task, resolved in-process (the provider
* can't cross the thread boundary) and passed to the worker as plain data. */
function resolveGitTaskEnv(
ctx: Pick<HostServiceContext, "credentials">,
worktreePath: string,
): Promise<Record<string, string>> {
return createGitEnvResolver(ctx.credentials)(worktreePath);
}

function assertSafeRelativePath(filePath: string): void {
if (isAbsolute(filePath)) {
throw new TRPCError({
Expand Down Expand Up @@ -109,12 +160,12 @@ export const gitRouter = router({
priority: input.priority,
run: async () => {
const worktreePath = resolveWorktreePath(ctx, input.workspaceId);
const git = await ctx.git(worktreePath);
return getGitStatusSnapshot({
git,
worktreePath,
baseBranch: input.baseBranch,
});
const gitEnv = await resolveGitTaskEnv(ctx, worktreePath);
return getHostWorkerPool().run(
gitStatusSnapshotTask,
{ worktreePath, baseBranch: input.baseBranch, gitEnv },
{ timeoutMs: 15_000 },
);
},
});
}),
Expand Down Expand Up @@ -170,10 +221,24 @@ export const gitRouter = router({
)
.query(async ({ ctx, input }) => {
const worktreePath = resolveWorktreePath(ctx, input.workspaceId);
const git = await ctx.git(worktreePath);

const from = input.fromHash ? input.fromHash : `${input.commitHash}^`;
const files = await getChangedFilesForDiff(git, [from, input.commitHash]);
const gitEnv = await resolveGitTaskEnv(ctx, worktreePath);
const dedupeKey = `${input.workspaceId}:commit-files:${input.fromHash ?? ""}:${input.commitHash}`;
const files = await runCommitFilesDeduped(dedupeKey, () =>
getHostWorkerPool().run(
gitCommitFilesTask,
{
worktreePath,
commitHash: input.commitHash,
fromHash: input.fromHash,
gitEnv,
},
{
timeoutMs: 15_000,
strategy: "coalesce",
dedupeKey,
},
),
);

return { files };
}),
Expand Down
Loading
Loading