diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/ChatSession/components/Composer/hooks/useUploadAttachments/useUploadAttachments.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/ChatSession/components/Composer/hooks/useUploadAttachments/useUploadAttachments.ts index 60d8a3b05a3..3a46d51ac8d 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/ChatSession/components/Composer/hooks/useUploadAttachments/useUploadAttachments.ts +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/ChatSession/components/Composer/hooks/useUploadAttachments/useUploadAttachments.ts @@ -1,51 +1,33 @@ import { useLingui } from "@lingui/react/macro"; import { errorMessage } from "@superset/i18n/errors"; import { toast } from "@superset/ui/sonner"; -import { workspaceTrpc } from "@superset/workspace-client"; import { useCallback } from "react"; -import { - PasteUploadLimitError, - uploadPastedFiles, -} from "../../../../../../utils/uploadPastedFiles"; +import { useSendAttachments } from "../../../../../../hooks/useSendAttachments"; +import { AttachmentLimitError } from "../../../../../../utils/sendAttachmentsToWorkspace"; import { formatAttachmentTag } from "../../../../utils/attachmentTags"; export function useUploadAttachments(workspaceId: string) { const { t } = useLingui(); - const { data: workspace } = workspaceTrpc.workspace.get.useQuery({ - id: workspaceId, - }); - const createDirectory = - workspaceTrpc.filesystem.createDirectory.useMutation(); - const writeFile = workspaceTrpc.filesystem.writeFile.useMutation(); - const worktreePath = workspace?.worktreePath; + const sendAttachments = useSendAttachments(workspaceId); return useCallback( async (files: File[]): Promise => { if (files.length === 0) return []; try { - if (!worktreePath) throw new Error("Workspace is not ready"); - const paths = await uploadPastedFiles({ - deps: { - createDirectory: (input) => createDirectory.mutateAsync(input), - writeFile: (input) => writeFile.mutateAsync(input), - }, - workspaceId, - worktreePath, - files, - }); + const paths = await sendAttachments(files); return paths.map((path, index) => formatAttachmentTag({ path, type: files[index]?.type ?? "" }), ); } catch (error) { toast.error(t({ message: "Couldn't attach files" }), { description: - error instanceof PasteUploadLimitError + error instanceof AttachmentLimitError ? error.message : errorMessage(error, t({ message: "Unknown error" })), }); return null; } }, - [worktreePath, workspaceId, createDirectory, writeFile, t], + [sendAttachments, t], ); } diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/TerminalPane/TerminalPane.tsx b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/TerminalPane/TerminalPane.tsx index 03ac1677111..343f281d668 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/TerminalPane/TerminalPane.tsx +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/TerminalPane/TerminalPane.tsx @@ -46,6 +46,7 @@ import { useTheme } from "renderer/stores/theme"; import { resolveTerminalThemeType } from "renderer/stores/theme/utils"; import { isWithinWorkspacePath } from "shared/absolute-paths"; import { useLinkClickHint } from "../../hooks/useLinkClickHint"; +import { useSendAttachments } from "../../hooks/useSendAttachments"; import { runFileLinkAction, runFolderLinkAction, @@ -53,10 +54,7 @@ import { type TerminalLinkActionDeps, terminalPullRequestLinkAction, } from "../../utils/runTerminalLinkAction"; -import { - PasteUploadLimitError, - uploadPastedFiles, -} from "../../utils/uploadPastedFiles"; +import { AttachmentLimitError } from "../../utils/sendAttachmentsToWorkspace"; import { TerminalAgentAutoResume } from "./components/TerminalAgentAutoResume"; import { TerminalCopiedIndicator } from "./components/TerminalCopiedIndicator"; import { TerminalNarrowedBanner } from "./components/TerminalNarrowedBanner"; @@ -408,9 +406,9 @@ export function TerminalPane({ // The default paste path forwards Ctrl+V and lets the TUI read the OS // clipboard — which only exists on the machine the PTY runs on. When the // workspace lives on another host (relay-reached machine, cloud sandbox), - // ship the clipboard bytes there via filesystem.writeFile and paste the - // resulting paths instead. Local workspaces keep the Ctrl+V forward, which - // lets TUIs attach the image natively. + // send the clipboard files to the workspace and paste the resulting paths + // instead. Local workspaces keep the Ctrl+V forward, which lets TUIs + // attach the image natively. const { machineId } = useLocalHostService(); const hostWorkspaces = useHostWorkspaces(); const workspaceHostId = hostWorkspaces.workspaces.find( @@ -424,28 +422,14 @@ export function TerminalPane({ Boolean(machineId) && workspaceHostId !== machineId; - const writeFileMutation = workspaceTrpc.filesystem.writeFile.useMutation(); - const createDirectoryMutation = - workspaceTrpc.filesystem.createDirectory.useMutation(); - const writeFileRef = useRef(writeFileMutation.mutateAsync); - writeFileRef.current = writeFileMutation.mutateAsync; - const createDirectoryRef = useRef(createDirectoryMutation.mutateAsync); - createDirectoryRef.current = createDirectoryMutation.mutateAsync; + const sendAttachments = useSendAttachments(workspaceId); // Fire-and-forget: ship files to the workspace, then paste the paths. const uploadAndPasteFiles = useCallback( - (files: File[], worktree: string) => { + (files: File[]) => { void (async () => { try { - const paths = await uploadPastedFiles({ - deps: { - createDirectory: (input) => createDirectoryRef.current(input), - writeFile: (input) => writeFileRef.current(input), - }, - workspaceId, - worktreePath: worktree, - files, - }); + const paths = await sendAttachments(files); terminalRuntimeRegistry.paste( terminalId, shellEscapePaths(paths), @@ -454,7 +438,7 @@ export function TerminalPane({ } catch (error) { console.error("[v2 Terminal] remote file upload failed", error); toast.error( - error instanceof PasteUploadLimitError + error instanceof AttachmentLimitError ? error.message : files.length === 1 ? t({ @@ -467,15 +451,15 @@ export function TerminalPane({ } })(); }, - [terminalId, terminalInstanceId, workspaceId, t], + [terminalId, terminalInstanceId, sendAttachments, t], ); useEffect(() => { - if (!isRemoteHost || !worktreePath) return; + if (!isRemoteHost) return; terminalRuntimeRegistry.setImagePasteOverride( terminalId, - (files) => uploadAndPasteFiles(files, worktreePath), + uploadAndPasteFiles, terminalInstanceId, ); return () => { @@ -485,13 +469,7 @@ export function TerminalPane({ terminalInstanceId, ); }; - }, [ - terminalId, - terminalInstanceId, - worktreePath, - isRemoteHost, - uploadAndPasteFiles, - ]); + }, [terminalId, terminalInstanceId, isRemoteHost, uploadAndPasteFiles]); useTerminalInterruptClear({ terminalId, @@ -595,7 +573,7 @@ export function TerminalPane({ // remote workspace's PTY runs on. When every dropped entry is a plain // file, ship the bytes instead. Folders keep the path flow (their File // entries carry no content), which at least preserves today's behavior. - if (isRemoteHost && worktreePath) { + if (isRemoteHost) { const items = Array.from(event.dataTransfer.items); const allPlainFiles = items.length > 0 && @@ -610,7 +588,7 @@ export function TerminalPane({ terminalRuntimeRegistry .getTerminal(terminalId, terminalInstanceId) ?.focus(); - uploadAndPasteFiles(files, worktreePath); + uploadAndPasteFiles(files); return; } } diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/TerminalPane/components/TerminalRichInput/TerminalRichInput.tsx b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/TerminalPane/components/TerminalRichInput/TerminalRichInput.tsx index d43131d5b51..b0a71f92c0f 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/TerminalPane/components/TerminalRichInput/TerminalRichInput.tsx +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/components/TerminalPane/components/TerminalRichInput/TerminalRichInput.tsx @@ -21,10 +21,8 @@ import { usePluginMentionOptions } from "renderer/hooks/usePluginMentionOptions" import { useHotkeyDisplay } from "renderer/hotkeys"; import { track } from "renderer/lib/analytics"; import { terminalRuntimeRegistry } from "renderer/lib/terminal/terminal-runtime-registry"; -import { - PasteUploadLimitError, - uploadPastedFiles, -} from "../../../../utils/uploadPastedFiles"; +import { useSendAttachments } from "../../../../hooks/useSendAttachments"; +import { AttachmentLimitError } from "../../../../utils/sendAttachmentsToWorkspace"; import { TerminalPaneIcon } from "../TerminalPaneIcon"; import { prepareTerminalSubmission } from "./prepareTerminalSubmission"; @@ -125,13 +123,7 @@ function TerminalRichInputInner({ [trpcUtils, workspaceId], ); - const writeFileMutation = workspaceTrpc.filesystem.writeFile.useMutation(); - const createDirectoryMutation = - workspaceTrpc.filesystem.createDirectory.useMutation(); - const writeFileRef = useRef(writeFileMutation.mutateAsync); - writeFileRef.current = writeFileMutation.mutateAsync; - const createDirectoryRef = useRef(createDirectoryMutation.mutateAsync); - createDirectoryRef.current = createDirectoryMutation.mutateAsync; + const sendAttachments = useSendAttachments(workspaceId); const handleSubmit = useCallback( async (message: PromptInputMessage) => { @@ -141,27 +133,15 @@ function TerminalRichInputInner({ // async onSubmit), so the user does not lose the draft or the files. let attachmentPaths: string[] = []; if (message.files.length > 0) { - if (!cwd) { - toast.error(t({ message: "Workspace path is not available yet" })); - throw new Error("no worktree path"); - } try { - attachmentPaths = await uploadPastedFiles({ - deps: { - createDirectory: (input) => createDirectoryRef.current(input), - writeFile: (input) => writeFileRef.current(input), - }, - workspaceId, - worktreePath: cwd, - files: await toFiles(message.files), - }); + attachmentPaths = await sendAttachments(await toFiles(message.files)); } catch (error) { console.error( "[v2 TerminalRichInput] attachment upload failed", error, ); toast.error( - error instanceof PasteUploadLimitError + error instanceof AttachmentLimitError ? error.message : message.files.length === 1 ? "Failed to attach the file" @@ -193,8 +173,7 @@ function TerminalRichInputInner({ terminalInstanceId, controller, workspaceId, - cwd, - t, + sendAttachments, sendToTerminalAgent, ], ); diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/hooks/useSendAttachments/index.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/hooks/useSendAttachments/index.ts new file mode 100644 index 00000000000..f52d0716aa7 --- /dev/null +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/hooks/useSendAttachments/index.ts @@ -0,0 +1 @@ +export { useSendAttachments } from "./useSendAttachments"; diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/hooks/useSendAttachments/useSendAttachments.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/hooks/useSendAttachments/useSendAttachments.ts new file mode 100644 index 00000000000..5741acaa914 --- /dev/null +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/hooks/useSendAttachments/useSendAttachments.ts @@ -0,0 +1,54 @@ +import { workspaceTrpc } from "@superset/workspace-client"; +import { useCallback, useRef } from "react"; +import { useWorkspaceHostTarget } from "renderer/hooks/host-service/useWorkspaceHostUrl"; +import { apiTrpcClient } from "renderer/lib/api-trpc-client"; +import { sendAttachmentsToWorkspace } from "../../utils/sendAttachmentsToWorkspace"; + +async function putObject( + target: { url: string; headers: Record }, + file: File, +): Promise { + return await fetch(target.url, { + method: "PUT", + headers: target.headers, + body: file, + }); +} + +/** + * A stable function that gets files into the workspace's worktree and + * returns their worktree-relative paths. Stable so a terminal's paste + * override, registered once, always sees the current host target. + */ +export function useSendAttachments( + workspaceId: string, +): (files: File[]) => Promise { + const target = useWorkspaceHostTarget(workspaceId); + const isCloudWorkspace = + target.status === "ready" && target.kind === "sandbox"; + const isCloudRef = useRef(isCloudWorkspace); + isCloudRef.current = isCloudWorkspace; + + const materialize = + workspaceTrpc.attachments.materializeIntoWorkspace.useMutation(); + const materializeRef = useRef(materialize.mutateAsync); + materializeRef.current = materialize.mutateAsync; + + return useCallback( + (files: File[]) => + sendAttachmentsToWorkspace({ + deps: { + createUpload: (input) => + apiTrpcClient.attachment.createUpload.mutate(input), + putObject, + attachFiles: (input) => + apiTrpcClient.cloudWorkspace.attachFiles.mutate(input), + materialize: (input) => materializeRef.current(input), + }, + workspaceId, + isCloudWorkspace: isCloudRef.current, + files, + }), + [workspaceId], + ); +} diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/sendAttachmentsToWorkspace/index.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/sendAttachmentsToWorkspace/index.ts new file mode 100644 index 00000000000..c3094082875 --- /dev/null +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/sendAttachmentsToWorkspace/index.ts @@ -0,0 +1,5 @@ +export { + AttachmentLimitError, + type SendAttachmentsDeps, + sendAttachmentsToWorkspace, +} from "./sendAttachmentsToWorkspace"; diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/sendAttachmentsToWorkspace/sendAttachmentsToWorkspace.test.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/sendAttachmentsToWorkspace/sendAttachmentsToWorkspace.test.ts new file mode 100644 index 00000000000..e8ddd2758b8 --- /dev/null +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/sendAttachmentsToWorkspace/sendAttachmentsToWorkspace.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it } from "bun:test"; +import { + MAX_ATTACHMENT_BYTES, + MAX_ATTACHMENTS, +} from "@superset/shared/attachment-limits"; +import { + AttachmentLimitError, + type SendAttachmentsDeps, + sendAttachmentsToWorkspace, +} from "./sendAttachmentsToWorkspace"; + +interface Recorded { + uploads: Array<{ name: string; contentType: string; sizeBytes: number }>; + puts: Array<{ url: string; file: File }>; + attached: Array<{ id: string; fileIds: string[] }>; + materialized: Array<{ workspaceId: string; fileIds: string[] }>; +} + +function makeDeps(options: { putStatus?: number } = {}) { + const recorded: Recorded = { + uploads: [], + puts: [], + attached: [], + materialized: [], + }; + let nextId = 0; + const deps: SendAttachmentsDeps = { + async createUpload(input) { + recorded.uploads.push(input); + nextId += 1; + return { + fileId: `file-${nextId}`, + upload: { url: `https://r2.test/${nextId}`, headers: {} }, + }; + }, + async putObject(target, file) { + recorded.puts.push({ url: target.url, file }); + return new Response(null, { status: options.putStatus ?? 200 }); + }, + async attachFiles(input) { + recorded.attached.push(input); + }, + async materialize(input) { + recorded.materialized.push(input); + return { + paths: input.fileIds.map((id) => `.superset/attachments/${id}.png`), + }; + }, + }; + return { deps, recorded }; +} + +const png = (name: string, size = 4) => + new File([new Uint8Array(size)], name, { type: "image/png" }); + +describe("sendAttachmentsToWorkspace", () => { + it("uploads each file to storage, then materializes them by id in order", async () => { + const { deps, recorded } = makeDeps(); + const paths = await sendAttachmentsToWorkspace({ + deps, + workspaceId: "ws-1", + isCloudWorkspace: false, + files: [png("a.png"), png("b.png", 9)], + }); + + expect(recorded.uploads).toEqual([ + { name: "a.png", contentType: "image/png", sizeBytes: 4 }, + { name: "b.png", contentType: "image/png", sizeBytes: 9 }, + ]); + expect(recorded.puts.map((p) => p.url)).toEqual([ + "https://r2.test/1", + "https://r2.test/2", + ]); + expect(recorded.materialized).toEqual([ + { workspaceId: "ws-1", fileIds: ["file-1", "file-2"] }, + ]); + expect(recorded.attached).toEqual([]); + expect(paths).toEqual([ + ".superset/attachments/file-1.png", + ".superset/attachments/file-2.png", + ]); + }); + + it("attaches the files to a cloud workspace before the box fetches them", async () => { + const { deps, recorded } = makeDeps(); + await sendAttachmentsToWorkspace({ + deps, + workspaceId: "cloud-ws", + isCloudWorkspace: true, + files: [png("shot.png")], + }); + expect(recorded.attached).toEqual([ + { id: "cloud-ws", fileIds: ["file-1"] }, + ]); + expect(recorded.materialized).toEqual([ + { workspaceId: "cloud-ws", fileIds: ["file-1"] }, + ]); + }); + + it("names a nameless blob from its mime type and defaults the content type", async () => { + const { deps, recorded } = makeDeps(); + await sendAttachmentsToWorkspace({ + deps, + workspaceId: "ws-1", + isCloudWorkspace: false, + files: [ + new File([new Uint8Array(2)], "", { type: "image/jpeg" }), + new File([new Uint8Array(2)], "", { type: "" }), + ], + }); + expect(recorded.uploads).toEqual([ + { name: "attachment_1.jpg", contentType: "image/jpeg", sizeBytes: 2 }, + { + name: "attachment_2", + contentType: "application/octet-stream", + sizeBytes: 2, + }, + ]); + }); + + it("fails before uploading anything when a file is over the limit", async () => { + const { deps, recorded } = makeDeps(); + const big = { + name: "huge.png", + type: "image/png", + size: MAX_ATTACHMENT_BYTES + 1, + } as File; + await expect( + sendAttachmentsToWorkspace({ + deps, + workspaceId: "ws-1", + isCloudWorkspace: false, + files: [png("ok.png"), big], + }), + ).rejects.toBeInstanceOf(AttachmentLimitError); + expect(recorded.uploads).toEqual([]); + }); + + it("refuses more files than one send may carry", async () => { + const { deps, recorded } = makeDeps(); + const files = Array.from({ length: MAX_ATTACHMENTS + 1 }, (_, i) => + png(`${i}.png`), + ); + await expect( + sendAttachmentsToWorkspace({ + deps, + workspaceId: "ws-1", + isCloudWorkspace: false, + files, + }), + ).rejects.toBeInstanceOf(AttachmentLimitError); + expect(recorded.uploads).toEqual([]); + }); + + it("surfaces a rejected PUT and never materializes", async () => { + const { deps, recorded } = makeDeps({ putStatus: 403 }); + await expect( + sendAttachmentsToWorkspace({ + deps, + workspaceId: "ws-1", + isCloudWorkspace: false, + files: [png("a.png")], + }), + ).rejects.toThrow("Upload failed for a.png (403)"); + expect(recorded.materialized).toEqual([]); + }); + + it("does nothing for an empty list", async () => { + const { deps, recorded } = makeDeps(); + expect( + await sendAttachmentsToWorkspace({ + deps, + workspaceId: "ws-1", + isCloudWorkspace: true, + files: [], + }), + ).toEqual([]); + expect(recorded.uploads).toEqual([]); + expect(recorded.attached).toEqual([]); + }); +}); diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/sendAttachmentsToWorkspace/sendAttachmentsToWorkspace.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/sendAttachmentsToWorkspace/sendAttachmentsToWorkspace.ts new file mode 100644 index 00000000000..75b029eb8af --- /dev/null +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/sendAttachmentsToWorkspace/sendAttachmentsToWorkspace.ts @@ -0,0 +1,122 @@ +import { + MAX_ATTACHMENT_BYTES, + MAX_ATTACHMENTS, +} from "@superset/shared/attachment-limits"; +import { attachmentFallbackName } from "@superset/shared/workspace-attachments"; + +/** Thrown for limits the user can act on; the message is toast-ready. */ +export class AttachmentLimitError extends Error {} + +function formatMb(bytes: number): string { + return `${Math.round(bytes / 1024 / 1024)}MB`; +} + +export interface AttachmentUploadTarget { + url: string; + headers: Record; +} + +export interface SendAttachmentsDeps { + createUpload(input: { + name: string; + contentType: string; + sizeBytes: number; + }): Promise<{ fileId: string; upload: AttachmentUploadTarget }>; + putObject(target: AttachmentUploadTarget, file: File): Promise; + /** Cloud only: a box can fetch only the files attached to it. */ + attachFiles(input: { id: string; fileIds: string[] }): Promise; + materialize(input: { + workspaceId: string; + fileIds: string[]; + }): Promise<{ paths: string[] }>; +} + +const MIME_EXTENSIONS: Record = { + "image/png": "png", + "image/jpeg": "jpg", + "image/gif": "gif", + "image/webp": "webp", + "image/svg+xml": "svg", + "image/tiff": "tiff", + "image/bmp": "bmp", +}; + +const FALLBACK_CONTENT_TYPE = "application/octet-stream"; + +/** + * Clipboard files keep their name when they have one (Chromium synthesizes + * "image.png" for a screenshot; Finder copies keep the real name); a blob + * with none takes its extension from the mime type, the only signal it + * carries. The host sanitizes and deduplicates whatever arrives. + */ +function nameOf(file: File, index: number): string { + if (file.name) return file.name; + const extension = MIME_EXTENSIONS[file.type]; + return attachmentFallbackName(index, extension ? `.${extension}` : ""); +} + +function assertWithinLimits(files: File[]): void { + if (files.length > MAX_ATTACHMENTS) { + throw new AttachmentLimitError( + `${files.length} files attached — a message carries at most ${MAX_ATTACHMENTS}`, + ); + } + for (const file of files) { + if (file.size > MAX_ATTACHMENT_BYTES) { + throw new AttachmentLimitError( + `"${file.name || "attachment"}" is ${formatMb(file.size)} — files over ${formatMb(MAX_ATTACHMENT_BYTES)} can't be attached`, + ); + } + } +} + +async function uploadOne( + deps: SendAttachmentsDeps, + file: File, + index: number, +): Promise { + const name = nameOf(file, index); + const { fileId, upload } = await deps.createUpload({ + name, + contentType: file.type || FALLBACK_CONTENT_TYPE, + sizeBytes: file.size, + }); + const response = await deps.putObject(upload, file); + if (!response.ok) { + throw new Error(`Upload failed for ${name} (${response.status})`); + } + return fileId; +} + +/** + * Gets composer files into a workspace's worktree and returns the + * worktree-relative paths an agent is handed — the same route the mobile + * composer and the new-workspace modal take: bytes go client → cloud storage + * over a presigned PUT, and the host pulls them from there. The host-service + * request carries only ids, so neither a relay's body cap nor a sandbox + * ingress that cuts an upload the host answers early ever sees the bytes. + * + * The caller owns the error surface; a thrown `AttachmentLimitError` is + * already worded for a toast. + */ +export async function sendAttachmentsToWorkspace(options: { + deps: SendAttachmentsDeps; + workspaceId: string; + isCloudWorkspace: boolean; + files: File[]; +}): Promise { + const { deps, workspaceId, isCloudWorkspace, files } = options; + if (files.length === 0) return []; + assertWithinLimits(files); + + // Order is kept: the host names attachments positionally and the caller + // pairs each path back with the file it came from. + const fileIds = await Promise.all( + files.map((file, index) => uploadOne(deps, file, index)), + ); + if (isCloudWorkspace) { + await deps.attachFiles({ id: workspaceId, fileIds }); + } + const { paths } = await deps.materialize({ workspaceId, fileIds }); + return paths; +} diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/uploadPastedFiles/index.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/uploadPastedFiles/index.ts deleted file mode 100644 index c1582bc0276..00000000000 --- a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/uploadPastedFiles/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export { - PasteUploadLimitError, - uploadPastedFiles, -} from "./uploadPastedFiles"; diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/uploadPastedFiles/uploadPastedFiles.test.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/uploadPastedFiles/uploadPastedFiles.test.ts deleted file mode 100644 index afd97401f77..00000000000 --- a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/uploadPastedFiles/uploadPastedFiles.test.ts +++ /dev/null @@ -1,237 +0,0 @@ -import { afterAll, beforeAll, describe, expect, it } from "bun:test"; -import { WORKSPACE_ATTACHMENTS_DIR } from "@superset/shared/workspace-attachments"; -import { - PasteUploadLimitError, - type UploadPastedFilesDeps, - uploadPastedFiles, -} from "./uploadPastedFiles"; - -// fileToBase64 goes through FileReader.readAsDataURL, which the renderer has -// but Bun's test runtime may not — pin a minimal implementation over -// Blob.arrayBuffer so the tests exercise the real base64 pipeline. -let fileReads = 0; - -class TestFileReader { - result: string | null = null; - error: Error | null = null; - onload: (() => void) | null = null; - onerror: (() => void) | null = null; - - readAsDataURL(file: File) { - fileReads += 1; - file - .arrayBuffer() - .then((buffer) => { - const base64 = Buffer.from(buffer).toString("base64"); - this.result = `data:${file.type};base64,${base64}`; - this.onload?.(); - }) - .catch((error) => { - this.error = error; - this.onerror?.(); - }); - } -} - -const originalFileReader = (globalThis as { FileReader?: unknown }).FileReader; - -beforeAll(() => { - (globalThis as { FileReader?: unknown }).FileReader = TestFileReader; -}); - -afterAll(() => { - (globalThis as { FileReader?: unknown }).FileReader = originalFileReader; -}); - -interface RecordedWrite { - absolutePath: string; - content: string | { kind: "base64"; data: string }; - options: { create: boolean; overwrite: boolean }; -} - -/** - * Fake deps backed by a real "directory": create-only writes report "exists" - * on a name collision exactly like the host does, so the collision-probing - * naming is exercised for real. - */ -function makeDeps(options: { existing?: string[]; failWrites?: boolean } = {}) { - const createdDirs: string[] = []; - const writes: RecordedWrite[] = []; - const existing = new Set(options.existing ?? []); - const deps: UploadPastedFilesDeps = { - async createDirectory(input) { - createdDirs.push(input.absolutePath); - }, - async writeFile(input) { - const write: RecordedWrite = { - absolutePath: input.absolutePath, - content: input.content, - options: input.options, - }; - writes.push(write); - if (options.failWrites && !input.absolutePath.endsWith(".gitignore")) { - return { ok: false, reason: "conflict" }; - } - if (existing.has(input.absolutePath) && !input.options.overwrite) { - return { ok: false, reason: "exists" }; - } - existing.add(input.absolutePath); - return { ok: true }; - }, - }; - return { deps, createdDirs, writes }; -} - -const WORKTREE = "/home/user/.superset/worktrees/demo"; -const DIR_ABS = `${WORKTREE}/${WORKSPACE_ATTACHMENTS_DIR}`; - -describe("uploadPastedFiles", () => { - it("creates the attachments dir, self-ignores it, writes each file, returns relative paths", async () => { - const { deps, createdDirs, writes } = makeDeps(); - const files = [ - new File([new Uint8Array([1, 2, 3])], "image.png", { - type: "image/png", - }), - new File([new Uint8Array([4, 5])], "notes.pdf", { - type: "application/pdf", - }), - ]; - - const paths = await uploadPastedFiles({ - deps, - workspaceId: "ws-1", - worktreePath: WORKTREE, - files, - }); - - expect(createdDirs).toEqual([DIR_ABS]); - expect(writes[0]).toEqual({ - absolutePath: `${DIR_ABS}/.gitignore`, - content: "*\n", - // create-only: a user's own .gitignore edit is never clobbered. - options: { create: true, overwrite: false }, - }); - // Worktree-relative, matching the agent-launch and mobile convention. - expect(paths).toEqual([ - `${WORKSPACE_ATTACHMENTS_DIR}/image.png`, - `${WORKSPACE_ATTACHMENTS_DIR}/notes.pdf`, - ]); - expect(writes[1]?.options).toEqual({ create: true, overwrite: false }); - expect(writes[1]?.content).toEqual({ - kind: "base64", - data: Buffer.from([1, 2, 3]).toString("base64"), - }); - }); - - it("never overwrites an earlier paste — collisions probe to a fresh _N name", async () => { - const { deps, writes } = makeDeps({ - existing: [`${DIR_ABS}/image.png`, `${DIR_ABS}/image_1.png`], - }); - - const paths = await uploadPastedFiles({ - deps, - workspaceId: "ws-1", - worktreePath: WORKTREE, - files: [new File(["x"], "image.png", { type: "image/png" })], - }); - - expect(paths).toEqual([`${WORKSPACE_ATTACHMENTS_DIR}/image_2.png`]); - // Every probe was create-only; nothing was written over. - expect(writes.every((w) => w.options.overwrite === false)).toBe(true); - }); - - it("tolerates a .gitignore that already exists from an earlier paste", async () => { - const { deps } = makeDeps({ existing: [`${DIR_ABS}/.gitignore`] }); - - const paths = await uploadPastedFiles({ - deps, - workspaceId: "ws-1", - worktreePath: WORKTREE, - files: [new File(["x"], "image.png", { type: "image/png" })], - }); - - expect(paths).toEqual([`${WORKSPACE_ATTACHMENTS_DIR}/image.png`]); - }); - - it("throws when a file write is refused", async () => { - const { deps } = makeDeps({ failWrites: true }); - - await expect( - uploadPastedFiles({ - deps, - workspaceId: "ws-1", - worktreePath: WORKTREE, - files: [new File(["x"], "image.png", { type: "image/png" })], - }), - ).rejects.toThrow("conflict"); - }); - - it("refuses oversized files before reading any bytes", async () => { - const { deps, createdDirs, writes } = makeDeps(); - fileReads = 0; - const big = new File([new Uint8Array(1)], "video.mov", { - type: "video/quicktime", - }); - // A 2GB Blob in the test would be real memory; fake the size instead — - // the check reads file.size only. - Object.defineProperty(big, "size", { value: 2 * 1024 * 1024 * 1024 }); - - await expect( - uploadPastedFiles({ - deps, - workspaceId: "ws-1", - worktreePath: WORKTREE, - files: [big], - }), - ).rejects.toThrow(PasteUploadLimitError); - // Failed fast: nothing read, no directory created, no bytes shipped. - expect(fileReads).toBe(0); - expect(createdDirs).toHaveLength(0); - expect(writes).toHaveLength(0); - }); - - it("refuses batches over the total budget", async () => { - const { deps, createdDirs, writes } = makeDeps(); - fileReads = 0; - const files = Array.from({ length: 5 }, (_, i) => { - const f = new File([new Uint8Array(1)], `part${i}.bin`, { - type: "application/octet-stream", - }); - Object.defineProperty(f, "size", { value: 45 * 1024 * 1024 }); - return f; - }); - - await expect( - uploadPastedFiles({ - deps, - workspaceId: "ws-1", - worktreePath: WORKTREE, - files, - }), - ).rejects.toThrow(PasteUploadLimitError); - expect(fileReads).toBe(0); - expect(createdDirs).toHaveLength(0); - expect(writes).toHaveLength(0); - }); - - it("sanitizes path-hostile clipboard names and falls back to attachment_N", async () => { - const { deps } = makeDeps(); - - const paths = await uploadPastedFiles({ - deps, - workspaceId: "ws-1", - worktreePath: WORKTREE, - files: [ - new File(["x"], "../shot: 1/final.png", { type: "image/png" }), - new File(["y"], "", { type: "image/png" }), - new File(["z"], "", { type: "application/x-thing" }), - ], - }); - - expect(paths).toEqual([ - `${WORKSPACE_ATTACHMENTS_DIR}/.._shot__1_final.png`, - `${WORKSPACE_ATTACHMENTS_DIR}/attachment_2.png`, - `${WORKSPACE_ATTACHMENTS_DIR}/attachment_3`, - ]); - }); -}); diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/uploadPastedFiles/uploadPastedFiles.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/uploadPastedFiles/uploadPastedFiles.ts deleted file mode 100644 index 6d6e061c2cd..00000000000 --- a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/v2-workspace/$workspaceId/hooks/usePaneRegistry/utils/uploadPastedFiles/uploadPastedFiles.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { - attachmentFallbackName, - attachmentNameWithSuffix, - sanitizeAttachmentFileName, - WORKSPACE_ATTACHMENTS_DIR, -} from "@superset/shared/workspace-attachments"; -import { fileToBase64 } from "renderer/lib/file-to-base64"; - -// Pastes follow the shared workspace-attachments convention (same directory -// and naming as agent-launch and the mobile composer): worktree-relative -// paths, since `filesystem.writeFile` refuses paths outside the worktree. -// The directory self-ignores (see the `.gitignore` write below) so -// attachments never dirty git status. - -/** Collision-probe cap; matches the spirit of createUniqueEntry's bound. */ -const MAX_NAME_ATTEMPTS = 50; - -// Same caps as the agent-launch adapter's attachment writer. The bytes are -// FileReader-read and base64'd (~1.33x) in renderer memory, then travel as -// one JSON body — an accidentally pasted multi-GB Finder file must fail -// fast instead of stalling the renderer and the host. -const MAX_SINGLE_FILE_BYTES = 50 * 1024 * 1024; -const MAX_TOTAL_BYTES = 200 * 1024 * 1024; - -/** Thrown for limits the user can act on; the message is toast-ready. */ -export class PasteUploadLimitError extends Error {} - -function formatMb(bytes: number): string { - return `${Math.round(bytes / 1024 / 1024)}MB`; -} - -interface FsWriteOutcome { - ok: boolean; - reason?: string; -} - -export interface UploadPastedFilesDeps { - createDirectory(input: { - workspaceId: string; - absolutePath: string; - recursive: boolean; - }): Promise; - writeFile(input: { - workspaceId: string; - absolutePath: string; - content: string | { kind: "base64"; data: string }; - options: { create: boolean; overwrite: boolean }; - }): Promise; -} - -const MIME_EXTENSIONS: Record = { - "image/png": "png", - "image/jpeg": "jpg", - "image/gif": "gif", - "image/webp": "webp", - "image/svg+xml": "svg", - "image/tiff": "tiff", - "image/bmp": "bmp", -}; - -/** - * Clipboard names (Chromium synthesizes "image.png" for screenshots; Finder - * copies keep their real name) go through the shared sanitizer; generated - * fallbacks take their extension from the mime type, the only signal a - * clipboard file carries. - */ -function baseFileName(file: File, index: number): string { - const sanitized = sanitizeAttachmentFileName(file.name); - if (sanitized) return sanitized; - const extension = MIME_EXTENSIONS[file.type]; - return attachmentFallbackName(index, extension ? `.${extension}` : ""); -} - -/** - * Write one file under a collision-free name. The batch conventions dedupe - * against an in-memory set; pastes arrive one at a time across a session, so - * probe the real directory instead (create-only writes report "exists") — - * a second screenshot must never overwrite a path an earlier prompt already - * references. - */ -async function writeUnique( - deps: UploadPastedFilesDeps, - workspaceId: string, - dirAbs: string, - base: string, - data: string, -): Promise { - for (let attempt = 0; attempt < MAX_NAME_ATTEMPTS; attempt++) { - const fileName = attachmentNameWithSuffix(base, attempt); - const outcome = await deps.writeFile({ - workspaceId, - absolutePath: `${dirAbs}/${fileName}`, - content: { kind: "base64", data }, - options: { create: true, overwrite: false }, - }); - if (outcome.ok !== false) return fileName; - if (outcome.reason !== "exists") { - throw new Error(outcome.reason ?? "write failed"); - } - } - throw new Error(`Could not find a free name for ${base}`); -} - -/** - * Ship clipboard files to a workspace whose PTY runs on another machine and - * return the worktree-relative paths written, ready to paste into the - * terminal. Throws on the first failed write — the caller owns the error - * surface. - */ -export async function uploadPastedFiles(options: { - deps: UploadPastedFilesDeps; - workspaceId: string; - worktreePath: string; - files: File[]; -}): Promise { - const { deps, workspaceId, worktreePath, files } = options; - const dirAbs = `${worktreePath}/${WORKSPACE_ATTACHMENTS_DIR}`; - - let totalBytes = 0; - for (const file of files) { - if (file.size > MAX_SINGLE_FILE_BYTES) { - throw new PasteUploadLimitError( - `"${file.name || "pasted file"}" is ${formatMb(file.size)} — files over ${formatMb(MAX_SINGLE_FILE_BYTES)} can't be sent to a remote workspace`, - ); - } - totalBytes += file.size; - } - if (totalBytes > MAX_TOTAL_BYTES) { - throw new PasteUploadLimitError( - `Pasted files total ${formatMb(totalBytes)} — more than the ${formatMb(MAX_TOTAL_BYTES)} limit for a remote workspace`, - ); - } - - await deps.createDirectory({ - workspaceId, - absolutePath: dirAbs, - recursive: true, - }); - - // `*` ignores every attachment and the .gitignore itself, so agents never - // see attachments as untracked changes. create-only: a user's own edit wins. - const ignoreOutcome = await deps.writeFile({ - workspaceId, - absolutePath: `${dirAbs}/.gitignore`, - content: "*\n", - options: { create: true, overwrite: false }, - }); - if (ignoreOutcome.ok === false && ignoreOutcome.reason !== "exists") { - throw new Error(ignoreOutcome.reason ?? "write failed"); - } - - const paths: string[] = []; - for (const [index, file] of files.entries()) { - const data = await fileToBase64(file); - const fileName = await writeUnique( - deps, - workspaceId, - dirAbs, - baseFileName(file, index), - data, - ); - paths.push(`${WORKSPACE_ATTACHMENTS_DIR}/${fileName}`); - } - return paths; -} diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index 013bb5229ac..d50acae57d7 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -322,13 +322,28 @@ forwards Ctrl+V and lets the TUI (Claude Code, Codex) read the image from the OS clipboard — of the machine the PTY runs on. A sandbox (or any relay-reached host) never holds the user's local screenshot, so the paste silently did nothing or surfaced "Failed to paste image". Fixed renderer-side: -for non-local hosts the desktop ships the clipboard bytes over -`filesystem.writeFile` into the shared `.superset/attachments/` worktree dir -(the same convention the agent-launch terminal adapter and the mobile -composer use — mobile proved the pattern) and pastes the worktree-relative -path instead (`setImagePasteOverride` in the terminal runtime registry). -Chosen over a new host endpoint because deployed sandboxes never update -their baked host-service. +for non-local hosts the desktop sends the clipboard files the way every +composer sends attachments — `attachment.createUpload` on the API, a +presigned PUT to storage, `cloudWorkspace.attachFiles` for a box, then +`attachments.materializeIntoWorkspace` on the host — and pastes the +worktree-relative paths instead (`setImagePasteOverride` in the terminal +runtime registry, `useSendAttachments` in the pane registry). + +**The ingress drops an upload the host answers early.** tRPC's streamed batch +(`httpBatchStreamLink`, `trpc-accept: application/jsonl`) returns its response +head before it has read the request body. Behind a Vercel sandbox's ingress +(`sb-*.vercel.run`, which the gate forwards to) that early response ends the +exchange: the rest of the upload is dropped, the procedure never runs, and the +client sees a stream that ends after the head line — "value$1 is not iterable" +in tRPC's consumer. It is a race, not a size limit: a 500 KB body on a home +uplink loses it about half the time, a 1 MB body on a datacenter link wins +it. This is why shipping file bytes through `filesystem.writeFile` broke +(SUPER-2658) and why attachments go through storage instead. host-service now +buffers every `/trpc/*` body before handing it to tRPC, so no response starts +before the request is complete; a sandbox provisioned before that bundle still +has the race for any request with a sizeable body. Reproducing with curl: pass +`-H "Expect:"` — curl's automatic `Expect: 100-continue` on bodies ≥1 MB +masks the bug, and browsers never send it. **A URL launcher succeeds and reaches nobody.** The image ships `xdg-utils` (`packages/sandbox/bundle/rootfs/usr/local/share/superset/desktop.Aptfile`) diff --git a/packages/host-service/src/app.ts b/packages/host-service/src/app.ts index 5158acc6f72..3ced7eb7891 100644 --- a/packages/host-service/src/app.ts +++ b/packages/host-service/src/app.ts @@ -506,6 +506,19 @@ export function createApp(options: CreateAppOptions): CreateAppResult { }); registerChatV3Routes({ app, db, mount: chatV3, upgradeWebSocket }); + // tRPC answers a streamed batch before it has read the request body, and + // a sandbox's ingress treats that early response as the end of the + // exchange and drops the upload still in flight. Hand tRPC a body it + // already has, so no response starts before the request is complete. + app.use("/trpc/*", async (c, next) => { + if (c.req.raw.body !== null && !c.req.raw.bodyUsed) { + c.req.raw = new Request(c.req.raw, { + body: await c.req.raw.arrayBuffer(), + }); + } + await next(); + }); + app.use( "/trpc/*", trpcServer({ diff --git a/packages/host-service/src/chat-v3/agentImages.ts b/packages/host-service/src/chat-v3/agentImages.ts index 92c6e3cef32..8702c5b8fcf 100644 --- a/packages/host-service/src/chat-v3/agentImages.ts +++ b/packages/host-service/src/chat-v3/agentImages.ts @@ -22,6 +22,7 @@ import { WORKSPACE_ATTACHMENTS_DIR, } from "@superset/shared/workspace-attachments"; import mimeTypes from "mime-types"; +import { ensureWorkspaceAttachmentsDir } from "../runtime/workspace-attachments-dir"; const NAME_HASH_CHARS = 12; @@ -38,18 +39,6 @@ function stemFor(name: string | undefined): string { return sanitized ?? "image"; } -function attachmentsDir(cwd: string): string { - const dir = join(cwd, WORKSPACE_ATTACHMENTS_DIR); - mkdirSync(dir, { recursive: true }); - try { - closeSync(openSync(join(dir, ".gitignore"), "wx")); - writeFileSync(join(dir, ".gitignore"), "*\n"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - } - return dir; -} - function isUsableFile(path: string, maxBytes: number): boolean { try { const stat = statSync(path); @@ -100,7 +89,7 @@ function storeImage( .digest("hex") .slice(0, NAME_HASH_CHARS); const fileName = `${stemFor(name)}-${hash}.${extensionFor(source.mimeType, name)}`; - const target = join(attachmentsDir(cwd), fileName); + const target = join(ensureWorkspaceAttachmentsDir(cwd), fileName); if (!existsSync(target)) writeFileSync(target, bytes); return { path: `${WORKSPACE_ATTACHMENTS_DIR}/${fileName}` }; } @@ -147,7 +136,7 @@ export function storeAgentImage( } const stored = storeImage(source, cwd, maxBytes); if (stored && referencePath) { - attachmentsDir(cwd); + ensureWorkspaceAttachmentsDir(cwd); mkdirSync(referenceDir, { recursive: true }); writeFileSync(referencePath, JSON.stringify(basename(stored.path))); } diff --git a/packages/host-service/src/runtime/workspace-attachments-dir.ts b/packages/host-service/src/runtime/workspace-attachments-dir.ts new file mode 100644 index 00000000000..809fd73810b --- /dev/null +++ b/packages/host-service/src/runtime/workspace-attachments-dir.ts @@ -0,0 +1,21 @@ +import { closeSync, mkdirSync, openSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { WORKSPACE_ATTACHMENTS_DIR } from "@superset/shared/workspace-attachments"; + +/** + * The worktree directory attachments are written into, created with a + * `.gitignore` that hides everything in it so an agent never sees + * attachments as untracked changes. The ignore file is create-only: a + * user's own edit wins. + */ +export function ensureWorkspaceAttachmentsDir(worktreePath: string): string { + const dir = join(worktreePath, WORKSPACE_ATTACHMENTS_DIR); + mkdirSync(dir, { recursive: true }); + try { + closeSync(openSync(join(dir, ".gitignore"), "wx")); + writeFileSync(join(dir, ".gitignore"), "*\n"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } + return dir; +} diff --git a/packages/host-service/src/trpc/router/attachments/attachments.ts b/packages/host-service/src/trpc/router/attachments/attachments.ts index c13505ba6dc..9a675bbad08 100644 --- a/packages/host-service/src/trpc/router/attachments/attachments.ts +++ b/packages/host-service/src/trpc/router/attachments/attachments.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { closeSync, mkdirSync, openSync } from "node:fs"; +import { closeSync, openSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { @@ -10,6 +10,7 @@ import { import { TRPCError } from "@trpc/server"; import mimeTypes from "mime-types"; import { z } from "zod"; +import { ensureWorkspaceAttachmentsDir } from "../../../runtime/workspace-attachments-dir"; import { protectedProcedure, router } from "../../index"; import { resolveWorktreePath } from "../git/utils/resolve-worktree"; import { MAX_INLINE_ATTACHMENT_BYTES } from "./constants"; @@ -172,8 +173,7 @@ export const attachmentsRouter = router({ ) .mutation(async ({ ctx, input }) => { const worktreePath = resolveWorktreePath(ctx, input.workspaceId); - const directory = join(worktreePath, WORKSPACE_ATTACHMENTS_DIR); - mkdirSync(directory, { recursive: true }); + const directory = ensureWorkspaceAttachmentsDir(worktreePath); const resolved = await ctx.api.attachment.resolve.mutate({ fileIds: input.fileIds, diff --git a/packages/host-service/test/integration/trpc-request-body.integration.test.ts b/packages/host-service/test/integration/trpc-request-body.integration.test.ts new file mode 100644 index 00000000000..2baa91a704e --- /dev/null +++ b/packages/host-service/test/integration/trpc-request-body.integration.test.ts @@ -0,0 +1,66 @@ +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { createTestHost, type TestHost } from "../helpers/createTestHost"; + +let host: TestHost; + +beforeAll(async () => { + host = await createTestHost(); +}); + +afterAll(async () => { + await host.dispose(); +}); + +/** A request body that stays open until the test closes it. */ +function openBody(): { + stream: ReadableStream; + finish: (text: string) => void; +} { + let controller!: ReadableStreamDefaultController; + const stream = new ReadableStream({ + start(c) { + controller = c; + }, + }); + return { + stream, + finish(text) { + controller.enqueue(new TextEncoder().encode(text)); + controller.close(); + }, + }; +} + +describe("tRPC request bodies", () => { + // A sandbox's ingress drops the exchange when the host answers before the + // upload is complete, so a streamed batch must not start its response + // until the whole body has arrived. + it("does not answer a streamed batch before the body has fully arrived", async () => { + const body = openBody(); + let settled = false; + const response = host + .fetch("http://host-service.test/trpc/health.check?batch=1", { + method: "POST", + headers: { + authorization: `Bearer ${host.psk}`, + "content-type": "application/json", + "trpc-accept": "application/jsonl", + }, + body: body.stream, + // @ts-expect-error -- required by the Fetch spec for a stream body + duplex: "half", + }) + .then((res) => { + settled = true; + return res; + }); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(settled).toBe(false); + + body.finish(JSON.stringify({ "0": { json: null } })); + const res = await response; + expect(res.status).toBe(200); + expect(await res.text()).toContain('"status":"ok"'); + }); +}); diff --git a/packages/i18n/locales/cs/messages.po b/packages/i18n/locales/cs/messages.po index 37a3a7c27f0..bbc5e9a5c86 100644 --- a/packages/i18n/locales/cs/messages.po +++ b/packages/i18n/locales/cs/messages.po @@ -19714,9 +19714,6 @@ msgstr "Pracovní prostor je otevřený, ale terminál se nepodařilo spustit." msgid "Workspace path is not available" msgstr "Cesta k pracovnímu prostoru není dostupná" -msgid "Workspace path is not available yet" -msgstr "Cesta pracovního prostoru zatím není dostupná" - msgid "Workspace restored" msgstr "Pracovní prostor obnoven" diff --git a/packages/i18n/locales/de/messages.po b/packages/i18n/locales/de/messages.po index 389c9b5ba0c..35a5d26fbee 100644 --- a/packages/i18n/locales/de/messages.po +++ b/packages/i18n/locales/de/messages.po @@ -19714,9 +19714,6 @@ msgstr "Arbeitsbereich geöffnet, aber das Terminal konnte nicht starten." msgid "Workspace path is not available" msgstr "Pfad des Arbeitsbereichs ist nicht verfügbar" -msgid "Workspace path is not available yet" -msgstr "Der Workspace-Pfad ist noch nicht verfügbar" - msgid "Workspace restored" msgstr "Arbeitsbereich wiederhergestellt" diff --git a/packages/i18n/locales/en/messages.po b/packages/i18n/locales/en/messages.po index adc31fed96c..6164b502f78 100644 --- a/packages/i18n/locales/en/messages.po +++ b/packages/i18n/locales/en/messages.po @@ -19714,9 +19714,6 @@ msgstr "Workspace opened, but terminal failed to start." msgid "Workspace path is not available" msgstr "Workspace path is not available" -msgid "Workspace path is not available yet" -msgstr "Workspace path is not available yet" - msgid "Workspace restored" msgstr "Workspace restored" diff --git a/packages/i18n/locales/es/messages.po b/packages/i18n/locales/es/messages.po index 00a7e808bde..7f13299cb84 100644 --- a/packages/i18n/locales/es/messages.po +++ b/packages/i18n/locales/es/messages.po @@ -19714,9 +19714,6 @@ msgstr "El espacio de trabajo se abrió, pero el terminal no se pudo iniciar." msgid "Workspace path is not available" msgstr "La ruta del espacio de trabajo no está disponible" -msgid "Workspace path is not available yet" -msgstr "La ruta del espacio de trabajo aún no está disponible" - msgid "Workspace restored" msgstr "Espacio de trabajo restaurado" diff --git a/packages/i18n/locales/fr/messages.po b/packages/i18n/locales/fr/messages.po index 7569ede4369..f45921bd274 100644 --- a/packages/i18n/locales/fr/messages.po +++ b/packages/i18n/locales/fr/messages.po @@ -19714,9 +19714,6 @@ msgstr "Espace de travail ouvert, mais le terminal n'a pas démarré." msgid "Workspace path is not available" msgstr "Le chemin de l'espace de travail n'est pas disponible" -msgid "Workspace path is not available yet" -msgstr "Le chemin de l’espace de travail n’est pas encore disponible" - msgid "Workspace restored" msgstr "Espace de travail restauré" diff --git a/packages/i18n/locales/id/messages.po b/packages/i18n/locales/id/messages.po index ae32b61c684..7d4ac6ffcc7 100644 --- a/packages/i18n/locales/id/messages.po +++ b/packages/i18n/locales/id/messages.po @@ -19714,9 +19714,6 @@ msgstr "Workspace terbuka, tapi terminal gagal dijalankan." msgid "Workspace path is not available" msgstr "Path workspace tidak tersedia" -msgid "Workspace path is not available yet" -msgstr "Jalur ruang kerja belum tersedia" - msgid "Workspace restored" msgstr "Workspace dipulihkan" diff --git a/packages/i18n/locales/it/messages.po b/packages/i18n/locales/it/messages.po index 113d8fbdd87..04f108b55ae 100644 --- a/packages/i18n/locales/it/messages.po +++ b/packages/i18n/locales/it/messages.po @@ -19714,9 +19714,6 @@ msgstr "Workspace aperto, ma il terminale non è partito." msgid "Workspace path is not available" msgstr "Il percorso del workspace non è disponibile" -msgid "Workspace path is not available yet" -msgstr "Il percorso dello spazio di lavoro non è ancora disponibile" - msgid "Workspace restored" msgstr "Workspace ripristinato" diff --git a/packages/i18n/locales/ja/messages.po b/packages/i18n/locales/ja/messages.po index 664e86c8721..734ac5b5031 100644 --- a/packages/i18n/locales/ja/messages.po +++ b/packages/i18n/locales/ja/messages.po @@ -19714,9 +19714,6 @@ msgstr "ワークスペースを開きましたが、ターミナルを起動で msgid "Workspace path is not available" msgstr "ワークスペースのパスを利用できません" -msgid "Workspace path is not available yet" -msgstr "ワークスペースのパスはまだ利用できません" - msgid "Workspace restored" msgstr "ワークスペースを復元しました" diff --git a/packages/i18n/locales/ko/messages.po b/packages/i18n/locales/ko/messages.po index 4f58404ddd2..6a12be455bb 100644 --- a/packages/i18n/locales/ko/messages.po +++ b/packages/i18n/locales/ko/messages.po @@ -19714,9 +19714,6 @@ msgstr "워크스페이스를 열었지만 터미널을 시작하지 못했습 msgid "Workspace path is not available" msgstr "워크스페이스 경로를 사용할 수 없습니다" -msgid "Workspace path is not available yet" -msgstr "작업 공간 경로를 아직 사용할 수 없습니다" - msgid "Workspace restored" msgstr "워크스페이스를 복원했습니다" diff --git a/packages/i18n/locales/nl/messages.po b/packages/i18n/locales/nl/messages.po index bcc90d57bb7..dda056a7366 100644 --- a/packages/i18n/locales/nl/messages.po +++ b/packages/i18n/locales/nl/messages.po @@ -19714,9 +19714,6 @@ msgstr "Werkruimte geopend, maar de terminal kon niet starten." msgid "Workspace path is not available" msgstr "Pad van werkruimte is niet beschikbaar" -msgid "Workspace path is not available yet" -msgstr "Het werkruimtepad is nog niet beschikbaar" - msgid "Workspace restored" msgstr "Werkruimte hersteld" diff --git a/packages/i18n/locales/pl/messages.po b/packages/i18n/locales/pl/messages.po index e5a8667ed34..733f0ad9793 100644 --- a/packages/i18n/locales/pl/messages.po +++ b/packages/i18n/locales/pl/messages.po @@ -19714,9 +19714,6 @@ msgstr "Obszar roboczy otwarty, ale terminal się nie uruchomił." msgid "Workspace path is not available" msgstr "Ścieżka obszaru roboczego jest niedostępna" -msgid "Workspace path is not available yet" -msgstr "Ścieżka obszaru roboczego nie jest jeszcze dostępna" - msgid "Workspace restored" msgstr "Przywrócono obszar roboczy" diff --git a/packages/i18n/locales/pt-BR/messages.po b/packages/i18n/locales/pt-BR/messages.po index 858e012c3fe..1faaa7cc783 100644 --- a/packages/i18n/locales/pt-BR/messages.po +++ b/packages/i18n/locales/pt-BR/messages.po @@ -19714,9 +19714,6 @@ msgstr "Área de trabalho aberta, mas o terminal não iniciou." msgid "Workspace path is not available" msgstr "O caminho da área de trabalho não está disponível" -msgid "Workspace path is not available yet" -msgstr "O caminho do espaço de trabalho ainda não está disponível" - msgid "Workspace restored" msgstr "Área de trabalho restaurada" diff --git a/packages/i18n/locales/ru/messages.po b/packages/i18n/locales/ru/messages.po index 64bcc18d072..ddbe019acb3 100644 --- a/packages/i18n/locales/ru/messages.po +++ b/packages/i18n/locales/ru/messages.po @@ -19714,9 +19714,6 @@ msgstr "Рабочая область открыта, но терминал не msgid "Workspace path is not available" msgstr "Путь к рабочей области недоступен" -msgid "Workspace path is not available yet" -msgstr "Путь рабочего пространства пока недоступен" - msgid "Workspace restored" msgstr "Рабочая область восстановлена" diff --git a/packages/i18n/locales/tr/messages.po b/packages/i18n/locales/tr/messages.po index 9bfbefb7fc6..ee462b8a42f 100644 --- a/packages/i18n/locales/tr/messages.po +++ b/packages/i18n/locales/tr/messages.po @@ -19714,9 +19714,6 @@ msgstr "Çalışma alanı açıldı, ancak terminal başlatılamadı." msgid "Workspace path is not available" msgstr "Çalışma alanı yolu kullanılamıyor" -msgid "Workspace path is not available yet" -msgstr "Çalışma alanı yolu henüz kullanılamıyor" - msgid "Workspace restored" msgstr "Çalışma alanı geri yüklendi" diff --git a/packages/i18n/locales/vi/messages.po b/packages/i18n/locales/vi/messages.po index 89e1787a1d0..ba9c671c437 100644 --- a/packages/i18n/locales/vi/messages.po +++ b/packages/i18n/locales/vi/messages.po @@ -19714,9 +19714,6 @@ msgstr "Đã mở workspace, nhưng terminal không khởi động được." msgid "Workspace path is not available" msgstr "Không có đường dẫn workspace" -msgid "Workspace path is not available yet" -msgstr "Đường dẫn không gian làm việc chưa sẵn sàng" - msgid "Workspace restored" msgstr "Đã khôi phục workspace" diff --git a/packages/i18n/locales/zh-CN/messages.po b/packages/i18n/locales/zh-CN/messages.po index 9fe38014ea4..4b2310ea7e3 100644 --- a/packages/i18n/locales/zh-CN/messages.po +++ b/packages/i18n/locales/zh-CN/messages.po @@ -19714,9 +19714,6 @@ msgstr "工作区已打开,但终端启动失败。" msgid "Workspace path is not available" msgstr "工作区路径不可用" -msgid "Workspace path is not available yet" -msgstr "工作区路径尚不可用" - msgid "Workspace restored" msgstr "工作区已恢复" diff --git a/packages/i18n/locales/zh-TW/messages.po b/packages/i18n/locales/zh-TW/messages.po index 775a4daec2f..06757221612 100644 --- a/packages/i18n/locales/zh-TW/messages.po +++ b/packages/i18n/locales/zh-TW/messages.po @@ -19714,9 +19714,6 @@ msgstr "工作區已開啟,但終端啟動失敗。" msgid "Workspace path is not available" msgstr "工作區路徑不可用" -msgid "Workspace path is not available yet" -msgstr "工作區路徑尚不可用" - msgid "Workspace restored" msgstr "工作區已還原" diff --git a/packages/trpc/src/router/attachment/attachment.ts b/packages/trpc/src/router/attachment/attachment.ts index a0b96c3162a..47fadc75c44 100644 --- a/packages/trpc/src/router/attachment/attachment.ts +++ b/packages/trpc/src/router/attachment/attachment.ts @@ -1,25 +1,17 @@ import { db } from "@superset/db/client"; -import { attachments, files } from "@superset/db/schema"; +import { files, type SelectFile } from "@superset/db/schema"; import { fileOriginalKey } from "@superset/shared/usercontent"; import type { TRPCRouterRecord } from "@trpc/server"; -import { and, eq, inArray } from "drizzle-orm"; import { headObject, presignedGetUrl, presignedPutUrl } from "../../lib/r2"; +import type { SandboxCaller } from "../../lib/sandbox/api-credential"; import { protectedProcedure, userError } from "../../trpc"; import { requireActiveOrgMembership } from "../utils/active-org"; +import { resolvableFiles, TRANSFER_ONLY } from "./resolvable"; import { createAttachmentUploadSchema, resolveAttachmentsSchema, } from "./schema"; -/** - * What marks a `files` row as one of these. Every other kind of file is - * identified by its content — page assets deduplicate on the digest — and no - * real file hashes to the empty string, so this both keeps an attachment out - * of any reuse lookup and keeps `resolve` off files it has no business - * handing a URL for. - */ -const TRANSFER_ONLY = ""; - /** * Composer attachments: bytes on their way from a client to the machine an * agent is running on. @@ -29,11 +21,10 @@ const TRANSFER_ONLY = ""; * bodies in a Worker isolate and is refused outright by Cloudflare's edge * above 100 MB. What travels over the relay is a list of file ids. * - * Rows here stay `pending` for their whole life, which is the point: the - * object is a transfer buffer, dead as soon as the host has written it into - * the worktree, and `pending` is exactly what the file sweep reclaims a day - * later. Nothing references these bytes afterwards, so there is no `ready` - * state to move to and no parent to attach them to. + * A row stays `pending` while it is only a transfer buffer, which the file + * sweep reclaims a day later. A file sent to a cloud workspace is kept + * instead: `anchorAttachments` marks it `ready` on that workspace, and it is + * reaped with the workspace's box. */ export const attachmentRouter = { /** @@ -89,14 +80,8 @@ export const attachmentRouter = { * Presigned GETs for a host about to materialize these into a worktree, * with the metadata it needs to name the files. * - * Scoped to the caller's own organization and to attachments, which is - * what keeps a host to the bytes it asked for: `files.id` is the only - * thing the caller presents, and knowing one is not authority to read it. - * Without the second scope this would sign a URL for any file in the org, - * a page's private assets included. - * - * A box calling for itself gets only the files attached to its own - * workspace: access to the box is access to its files, nothing wider. + * Scoped by `resolvableFiles`: `files.id` is the only thing the caller + * presents, and knowing one is not authority to read it. * * A mutation despite reading nothing but rows: the URLs it mints are * short-lived credentials, and a query's response is cacheable. @@ -104,33 +89,7 @@ export const attachmentRouter = { resolve: protectedProcedure .input(resolveAttachmentsSchema) .mutation(async ({ ctx, input }) => { - const organizationId = await requireActiveOrgMembership(ctx); - - const rows = await db - .select() - .from(files) - .where( - and( - inArray(files.id, input.fileIds), - eq(files.organizationId, organizationId), - eq(files.sha256, TRANSFER_ONLY), - ctx.sandboxCaller - ? inArray( - files.id, - db - .select({ id: attachments.fileId }) - .from(attachments) - .where( - and( - eq(attachments.parentKind, "cloud_workspace"), - eq(attachments.parentId, ctx.sandboxCaller.workspaceId), - ), - ), - ) - : undefined, - ), - ); - const byId = new Map(rows.map((row) => [row.id, row])); + const byId = await loadResolvable(ctx, input.fileIds); // In the order asked for: the caller names attachments positionally // and a reordered answer would rename them. @@ -144,29 +103,69 @@ export const attachmentRouter = { i18nKey: "serverError.attachment.notFound", }); } - - // A presigned PUT lands without the API seeing it, so this is - // where an upload is held to what it declared. A short read - // that never finished is the case worth catching — it would - // otherwise reach the agent as a truncated file. - const key = fileOriginalKey(file.id); - const head = await headObject(key); - if (!head || head.sizeBytes !== file.sizeBytes) { + const resolved = await signUploaded(file); + if (!resolved) { throw userError({ code: "BAD_REQUEST", message: "Attachment was not uploaded — send the bytes first", i18nKey: "serverError.attachment.notUploaded", }); } + return resolved; + }), + ); + }), - return { - fileId: file.id, - name: file.name, - contentType: file.contentType, - sizeBytes: file.sizeBytes, - url: await presignedGetUrl(key), - }; + /** + * `resolve` for a reader that shows what it can: a transcript with one + * file gone still renders the rest. Every id gets an answer, in the order + * asked for, and one the caller may not see is reported the same as one + * that does not exist. + */ + resolveEach: protectedProcedure + .input(resolveAttachmentsSchema) + .mutation(async ({ ctx, input }) => { + const byId = await loadResolvable(ctx, input.fileIds); + return await Promise.all( + input.fileIds.map(async (fileId) => { + const file = byId.get(fileId); + const resolved = file ? await signUploaded(file) : null; + return resolved ?? { fileId, missing: true as const }; }), ); }), } satisfies TRPCRouterRecord; + +async function loadResolvable( + ctx: Parameters[0] & { + sandboxCaller: SandboxCaller | null; + }, + fileIds: string[], +) { + const rows = await resolvableFiles({ + organizationId: await requireActiveOrgMembership(ctx), + userId: ctx.session.user.id, + boxWorkspaceId: ctx.sandboxCaller?.workspaceId ?? null, + fileIds, + }); + return new Map(rows.map((row) => [row.id, row])); +} + +/** + * A presigned PUT lands without the API seeing it, so this is where an + * upload is held to what it declared. A short read that never finished is + * the case worth catching — it would otherwise reach the agent as a + * truncated file. Null when the bytes are not there. + */ +async function signUploaded(file: SelectFile) { + const key = fileOriginalKey(file.id); + const head = await headObject(key); + if (!head || head.sizeBytes !== file.sizeBytes) return null; + return { + fileId: file.id, + name: file.name, + contentType: file.contentType, + sizeBytes: file.sizeBytes, + url: await presignedGetUrl(key), + }; +} diff --git a/packages/trpc/src/router/attachment/resolvable.integration.ts b/packages/trpc/src/router/attachment/resolvable.integration.ts new file mode 100644 index 00000000000..d1dd4af50d0 --- /dev/null +++ b/packages/trpc/src/router/attachment/resolvable.integration.ts @@ -0,0 +1,148 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { db, dbWs } from "@superset/db/client"; +import { + attachments, + cloudWorkspaces, + environments, + files, + organizations, + users, +} from "@superset/db/schema"; +import { eq, inArray } from "drizzle-orm"; +import { resolvableFiles, TRANSFER_ONLY } from "./resolvable"; + +const ORG = crypto.randomUUID(); +const OWNER = crypto.randomUUID(); +const TEAMMATE = crypto.randomUUID(); +const now = Date.now(); + +const insertFile = async ( + status: "pending" | "ready", + sha256 = TRANSFER_ONLY, +) => { + const id = crypto.randomUUID(); + await db.insert(files).values({ + id, + organizationId: ORG, + name: "screenshot.png", + contentType: "image/png", + sizeBytes: 4, + sha256, + status, + createdByUserId: OWNER, + }); + return id; +}; + +const insertWorkspace = async (visibility: "org" | "just_me") => { + const [environment] = await db + .select({ id: environments.id }) + .from(environments) + .where(eq(environments.organizationId, ORG)); + const id = crypto.randomUUID(); + await db.insert(cloudWorkspaces).values({ + id, + organizationId: ORG, + name: "box", + branch: "main", + baseBranch: "main", + providerSandboxId: `box-${id}`, + environmentId: environment?.id as string, + visibility, + createdByUserId: OWNER, + }); + return id; +}; + +const anchor = (fileId: string, workspaceId: string) => + db.insert(attachments).values({ + fileId, + parentKind: "cloud_workspace", + parentId: workspaceId, + }); + +const resolvable = async ( + userId: string, + fileIds: string[], + boxWorkspaceId: string | null = null, +) => + ( + await resolvableFiles({ + organizationId: ORG, + userId, + boxWorkspaceId, + fileIds, + }) + ) + .map((row) => row.id) + .sort(); + +beforeAll(async () => { + await db + .insert(organizations) + .values({ id: ORG, name: "Resolve Org", slug: `resolve-org-${now}` }); + await db.insert(users).values([ + { id: OWNER, name: "Owner", email: `owner-${now}@example.com` }, + { id: TEAMMATE, name: "Teammate", email: `teammate-${now}@example.com` }, + ]); + await db.insert(environments).values({ + organizationId: ORG, + name: "default", + sourceKind: "image", + sourceRef: "image", + bundleSha: "sha", + }); +}); + +afterAll(async () => { + await db.delete(organizations).where(eq(organizations.id, ORG)); + await db.delete(users).where(inArray(users.id, [OWNER, TEAMMATE])); + await dbWs.$client.end?.().catch(() => {}); +}); + +describe("resolvableFiles", () => { + test("an upload in flight resolves for anyone in the org", async () => { + const pending = await insertFile("pending"); + expect(await resolvable(TEAMMATE, [pending])).toEqual([pending]); + }); + + test("a kept file resolves only through a workspace the caller can see", async () => { + const shared = await insertWorkspace("org"); + const hidden = await insertWorkspace("just_me"); + const inShared = await insertFile("ready"); + const inHidden = await insertFile("ready"); + await anchor(inShared, shared); + await anchor(inHidden, hidden); + + expect(await resolvable(OWNER, [inShared, inHidden])).toEqual( + [inShared, inHidden].sort(), + ); + expect(await resolvable(TEAMMATE, [inShared, inHidden])).toEqual([ + inShared, + ]); + }); + + test("a kept file on no cloud workspace does not resolve", async () => { + const loose = await insertFile("ready"); + expect(await resolvable(OWNER, [loose])).toEqual([]); + }); + + test("a box resolves only its own files, never one in flight", async () => { + const box = await insertWorkspace("org"); + const other = await insertWorkspace("org"); + const own = await insertFile("ready"); + const theirs = await insertFile("ready"); + const pending = await insertFile("pending"); + await anchor(own, box); + await anchor(theirs, other); + + expect(await resolvable(OWNER, [own, theirs, pending], box)).toEqual([own]); + }); + + test("a page asset never resolves", async () => { + const box = await insertWorkspace("org"); + const asset = await insertFile("ready", "ab".repeat(32)); + await anchor(asset, box); + expect(await resolvable(OWNER, [asset])).toEqual([]); + }); +}); diff --git a/packages/trpc/src/router/attachment/resolvable.ts b/packages/trpc/src/router/attachment/resolvable.ts new file mode 100644 index 00000000000..5e229aa2031 --- /dev/null +++ b/packages/trpc/src/router/attachment/resolvable.ts @@ -0,0 +1,69 @@ +import { db } from "@superset/db/client"; +import { attachments, cloudWorkspaces, files } from "@superset/db/schema"; +import { and, eq, inArray, or, type SQL } from "drizzle-orm"; +import { visibleTo } from "../cloud-workspace/access"; + +/** + * What marks a `files` row as a composer attachment. Every other kind of file is + * identified by its content — page assets deduplicate on the digest — and no + * real file hashes to the empty string, so this both keeps an attachment out + * of any reuse lookup and keeps `resolve` off files it has no business + * handing a URL for. + */ +export const TRANSFER_ONLY = ""; + +/** + * The attachments among `fileIds` a caller may read. + * + * A box gets only the files anchored to its own workspace. Anyone else gets + * an upload still in flight, and a kept file only through a cloud workspace + * they can see. In flight stays org-wide because a host resolves with its + * owner's credential, and the sender may be a teammate who shares that host. + */ +export function resolvableFiles({ + organizationId, + userId, + boxWorkspaceId, + fileIds, +}: { + organizationId: string; + userId: string; + boxWorkspaceId: string | null; + fileIds: string[]; +}) { + return db + .select() + .from(files) + .where( + and( + inArray(files.id, fileIds), + eq(files.organizationId, organizationId), + eq(files.sha256, TRANSFER_ONLY), + boxWorkspaceId + ? inArray( + files.id, + anchoredTo(eq(cloudWorkspaces.id, boxWorkspaceId)), + ) + : or( + eq(files.status, "pending"), + inArray( + files.id, + anchoredTo( + and( + eq(cloudWorkspaces.organizationId, organizationId), + visibleTo(userId), + ), + ), + ), + ), + ), + ); +} + +function anchoredTo(workspaces: SQL | undefined) { + return db + .select({ id: attachments.fileId }) + .from(attachments) + .innerJoin(cloudWorkspaces, eq(cloudWorkspaces.id, attachments.parentId)) + .where(and(eq(attachments.parentKind, "cloud_workspace"), workspaces)); +} diff --git a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts index 7bb23bd4ba8..bcdd69c977c 100644 --- a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts +++ b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts @@ -53,7 +53,7 @@ import { recordCloudWorkspaceActivity } from "./activity"; import { queueSettleWorkspace, settleWorkspaceNow } from "./meter"; import { nextSandboxNameFor } from "./provision"; import { listPullRequests } from "./pull-requests"; -import { queueReap } from "./reap"; +import { queueReap, reapWorkspaceFiles } from "./reap"; import { cloudWorkspaceRecordRouter } from "./record"; import { cloudWorkspaceSharingRouter } from "./sharing"; import { queueProvision, startCloudWorkspace } from "./start"; @@ -713,6 +713,7 @@ export const cloudWorkspaceRouter = { ), ); await deleteSandbox(row.providerSandboxId); + await reapWorkspaceFiles(row.id); }); } await recordCloudWorkspaceActivity( diff --git a/packages/trpc/src/router/cloud-workspace/reap.test.ts b/packages/trpc/src/router/cloud-workspace/reap.test.ts index 4247bd6c04f..9a54f220332 100644 --- a/packages/trpc/src/router/cloud-workspace/reap.test.ts +++ b/packages/trpc/src/router/cloud-workspace/reap.test.ts @@ -1,11 +1,13 @@ import { beforeEach, describe, expect, test } from "bun:test"; import { db } from "@superset/db/client"; import { stub } from "../../../test/stub"; +import * as files from "../../lib/files"; import * as sandbox from "../../lib/sandbox"; import * as jobs from "./jobs"; import * as meter from "./meter"; type Row = { + id: string; status: string; deletedAt: Date | null; provider: string; @@ -19,6 +21,7 @@ let queued: Array<{ path: string; body: unknown; delaySeconds?: number }> = []; let failingStage: Stage | null = null; let meterCalls: string[] = []; let meterFailure: Error | null = null; +let reapedFiles: string[] = []; stub(db.query.cloudWorkspaces, { findFirst: () => Promise.resolve(row) }); stub(sandbox, { @@ -49,6 +52,15 @@ stub(jobs, { }, }); +stub(files, { + detachAll: ({ parentIds }: { parentIds: string[] }) => + Promise.resolve(parentIds.map((id) => `file-of-${id}`)), + reapOrphanFiles: (fileIds: string[]) => { + reapedFiles.push(...fileIds); + return Promise.resolve(); + }, +}); + stub(meter, { settleWorkspaceNow: () => { meterCalls.push("settle"); @@ -112,6 +124,7 @@ describe("queueReap", () => { describe("reapArchivedCloudWorkspace", () => { beforeEach(() => { row = { + id: input.cloudWorkspaceId, status: "deleted", deletedAt: archivedAt, provider: "vercel", @@ -121,6 +134,7 @@ describe("reapArchivedCloudWorkspace", () => { queued = []; meterCalls = []; meterFailure = null; + reapedFiles = []; }); test("the stop stage stops the box and queues its final settle", async () => { @@ -130,6 +144,7 @@ describe("reapArchivedCloudWorkspace", () => { expect(sandboxCalls).toEqual(["stop:ws-box"]); expect(meterCalls).toEqual(["queue"]); expect(queued).toEqual([]); + expect(reapedFiles).toEqual([]); }); test("the delete stage settles the box's usage before deleting it", async () => { @@ -140,6 +155,11 @@ describe("reapArchivedCloudWorkspace", () => { expect(sandboxCalls).toEqual(["delete:ws-box"]); }); + test("the delete stage removes the files sent to the workspace", async () => { + await reapArchivedCloudWorkspace({ ...input, stage: "delete" }); + expect(reapedFiles).toEqual([`file-of-${input.cloudWorkspaceId}`]); + }); + test("a delete still deletes when the final settle fails: at most one lock window goes unbilled", async () => { meterFailure = new Error("autumn down"); expect( @@ -155,6 +175,7 @@ describe("reapArchivedCloudWorkspace", () => { "skipped", ); expect(sandboxCalls).toEqual([]); + expect(reapedFiles).toEqual([]); }); test(`the ${stage} stage leaves a later archive's box alone`, async () => { @@ -175,6 +196,7 @@ describe("reapArchivedCloudWorkspace", () => { await reapArchivedCloudWorkspace({ ...input, stage: "delete" }), ).toBe("skipped"); row = { + id: input.cloudWorkspaceId, status: "deleted", deletedAt: archivedAt, provider: "e2b", diff --git a/packages/trpc/src/router/cloud-workspace/reap.ts b/packages/trpc/src/router/cloud-workspace/reap.ts index 5f298a645f2..864d1d5d120 100644 --- a/packages/trpc/src/router/cloud-workspace/reap.ts +++ b/packages/trpc/src/router/cloud-workspace/reap.ts @@ -1,6 +1,7 @@ import { db } from "@superset/db/client"; import { cloudWorkspaces } from "@superset/db/schema"; import { eq } from "drizzle-orm"; +import { detachAll, reapOrphanFiles } from "../../lib/files"; import { deleteSandbox, stopSandbox } from "../../lib/sandbox"; import { publishCloudWorkspaceJob } from "./jobs"; import { queueSettleWorkspace, settleWorkspaceNow } from "./meter"; @@ -36,6 +37,7 @@ export async function reapArchivedCloudWorkspace( console.error(`[cloud-workspace] ${row.id} final settle failed`, error), ); await deleteSandbox(row.providerSandboxId); + await reapWorkspaceFiles(row.id); return "reaped"; } await stopSandbox(row.providerSandboxId); @@ -43,6 +45,21 @@ export async function reapArchivedCloudWorkspace( return "stopped"; } +/** Drops the files sent to a workspace whose box is gone for good. */ +export async function reapWorkspaceFiles(cloudWorkspaceId: string) { + await reapOrphanFiles( + await detachAll({ + parentKind: "cloud_workspace", + parentIds: [cloudWorkspaceId], + }), + ).catch((error) => + console.error( + `[cloud-workspace] ${cloudWorkspaceId} file reap failed`, + error, + ), + ); +} + function queueReapStage( input: ReapArchivedCloudWorkspaceInput, delaySeconds: number,