diff --git a/packages/cli/src/server/studioServer.ts b/packages/cli/src/server/studioServer.ts
index f48458a1f6..8de534fb2a 100644
--- a/packages/cli/src/server/studioServer.ts
+++ b/packages/cli/src/server/studioServer.ts
@@ -18,14 +18,17 @@ import {
import { VERSION as version } from "../version.js";
import { buildStudioHeadScripts, resolveCliTelemetryDistinctId } from "./telemetryIdentity.js";
import { emitStudioRenderComplete, emitStudioRenderError } from "./studioRenderTelemetry.js";
+import { isDevMode } from "../utils/env.js";
import {
createStudioManualEditsRenderBodyScript,
createStudioApi,
createProjectSignature,
+ createBackgroundRemovalJob,
getMimeType,
type StudioApiAdapter,
type ResolvedProject,
type RenderJobState,
+ type BackgroundRemovalRender,
} from "@hyperframes/studio-server";
import { getElementScreenshotClip } from "@hyperframes/studio-server/screenshot-clip";
import type { ScreenshotClip } from "@hyperframes/studio-server/screenshot-clip";
@@ -35,6 +38,12 @@ const STUDIO_MANUAL_EDITS_PATH = ".hyperframes/studio-manual-edits.json";
const REMOTE_GIF_IMG_SRC_RE =
/
]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+\.gif(?:[?#][^"']*)?)["'][^>]*>/gi;
+async function loadStudioProducer() {
+ return isDevMode()
+ ? await import("../../../producer/src/index.js")
+ : await import("@hyperframes/producer");
+}
+
// ── Path resolution ─────────────────────────────────────────────────────────
function resolveDistDir(): string {
@@ -302,7 +311,10 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
// we can point it at our hot-reloadable local runtime endpoint. Inlining
// ~150 KB of runtime body on every preview render would defeat browser
// caching across composition edits.
- let html = await bundleToSingleHtml(dir, { runtime: "placeholder" });
+ let html = await bundleToSingleHtml(dir, {
+ runtime: "placeholder",
+ inlineColorGradingLuts: false,
+ });
html = html.replace(
'data-hyperframes-preview-runtime="1" src=""',
'data-hyperframes-preview-runtime="1" src="/api/runtime.js"',
@@ -361,7 +373,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
(async () => {
let renderJob: RenderJob | undefined;
try {
- const { createRenderJob, executeRenderJob } = await import("@hyperframes/producer");
+ const { createRenderJob, executeRenderJob } = await loadStudioProducer();
const { ensureBrowser } = await import("../browser/manager.js");
try {
@@ -416,6 +428,16 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
return state;
},
+ startBackgroundRemoval(opts) {
+ return createBackgroundRemovalJob(opts, async (renderOpts) => {
+ const sourcePipelinePath = "../background-removal/pipeline.ts";
+ const pipeline = (await import("../background-removal/pipeline.js").catch(
+ () => import(sourcePipelinePath),
+ )) as { render: BackgroundRemovalRender };
+ return pipeline.render(renderOpts);
+ });
+ },
+
async generateThumbnail(opts): Promise {
const browser = await getThumbnailBrowser();
if (!browser) {
@@ -438,6 +460,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
{ timeout: 5000 },
)
.catch(() => {});
+ // fallow-ignore-next-line code-duplication
await page.evaluate((t: number) => {
const w = window as Window & {
__player?: { seek?: (time: number) => void };
diff --git a/packages/studio-server/src/createStudioApi.ts b/packages/studio-server/src/createStudioApi.ts
index 99de7631be..1b1fecb510 100644
--- a/packages/studio-server/src/createStudioApi.ts
+++ b/packages/studio-server/src/createStudioApi.ts
@@ -11,6 +11,7 @@ import { registerWaveformRoutes } from "./routes/waveform.js";
import { registerFontRoutes } from "./routes/fonts.js";
import { registerRegistryRoutes } from "./routes/registry.js";
import { registerSelectionRoutes } from "./routes/selection.js";
+import { registerMediaRoutes } from "./routes/media.js";
/**
* Create a Hono sub-app with all studio API routes.
@@ -29,6 +30,7 @@ export function createStudioApi(adapter: StudioApiAdapter): Hono {
registerRenderRoutes(api, adapter);
registerThumbnailRoutes(api, adapter);
registerSelectionRoutes(api, adapter);
+ registerMediaRoutes(api, adapter);
registerWaveformRoutes(api, adapter);
registerFontRoutes(api);
registerRegistryRoutes(api, adapter);
diff --git a/packages/studio-server/src/helpers/backgroundRemovalJob.ts b/packages/studio-server/src/helpers/backgroundRemovalJob.ts
new file mode 100644
index 0000000000..910d45b89b
--- /dev/null
+++ b/packages/studio-server/src/helpers/backgroundRemovalJob.ts
@@ -0,0 +1,90 @@
+import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js";
+
+export type BackgroundRemovalJobOptions = Parameters<
+ NonNullable
+>[0];
+
+export type BackgroundRemovalProgressEvent =
+ | { kind: "info"; message: string }
+ | { kind: "metadata"; width: number; height: number; fps: number; frameCount: number }
+ | { kind: "frame"; index: number; total: number; avgMsPerFrame: number };
+
+export type BackgroundRemovalRender = (options: {
+ inputPath: string;
+ outputPath: string;
+ backgroundOutputPath?: string;
+ device?: BackgroundRemovalJobOptions["device"];
+ quality?: BackgroundRemovalJobOptions["quality"];
+ onProgress?: (event: BackgroundRemovalProgressEvent) => void;
+}) => Promise<{
+ provider: string;
+ framesProcessed: number;
+ durationSeconds: number;
+ avgMsPerFrame: number;
+}>;
+
+export function createBackgroundRemovalJob(
+ opts: BackgroundRemovalJobOptions,
+ render: BackgroundRemovalRender,
+): MediaProcessingJobState {
+ const state: MediaProcessingJobState = {
+ id: opts.jobId,
+ status: "processing",
+ progress: 0,
+ stage: "Preparing background removal",
+ inputAssetPath: opts.inputAssetPath,
+ outputAssetPath: opts.outputAssetPath,
+ outputPath: opts.outputPath,
+ ...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}),
+ ...(opts.backgroundOutputAssetPath
+ ? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath }
+ : {}),
+ };
+
+ void (async () => {
+ try {
+ const result = await render({
+ inputPath: opts.inputPath,
+ outputPath: opts.outputPath,
+ backgroundOutputPath: opts.backgroundOutputPath,
+ device: opts.device,
+ quality: opts.quality,
+ onProgress: (event) => updateBackgroundRemovalProgress(state, event),
+ });
+ state.status = "complete";
+ state.progress = 100;
+ state.stage = "Complete";
+ state.provider = result.provider;
+ state.framesProcessed = result.framesProcessed;
+ state.durationSeconds = result.durationSeconds;
+ state.avgMsPerFrame = result.avgMsPerFrame;
+ } catch (err) {
+ state.status = "failed";
+ state.error = err instanceof Error ? err.message : String(err);
+ state.stage = "Failed";
+ }
+ })();
+
+ return state;
+}
+
+function updateBackgroundRemovalProgress(
+ state: MediaProcessingJobState,
+ event: BackgroundRemovalProgressEvent,
+): void {
+ if (event.kind === "info") {
+ state.stage = event.message;
+ return;
+ }
+ if (event.kind === "metadata") {
+ state.stage = `Source ${event.width}×${event.height}`;
+ state.progress = 2;
+ return;
+ }
+ state.progress = event.total ? Math.min(99, Math.floor((event.index / event.total) * 100)) : 0;
+ state.stage = event.total
+ ? `Removing background ${event.index}/${event.total}`
+ : `Removing background frame ${event.index}`;
+ state.framesProcessed = event.index;
+ state.avgMsPerFrame = event.avgMsPerFrame;
+}
diff --git a/packages/studio-server/src/helpers/mediaMetadata.test.ts b/packages/studio-server/src/helpers/mediaMetadata.test.ts
new file mode 100644
index 0000000000..8c6dd69e04
--- /dev/null
+++ b/packages/studio-server/src/helpers/mediaMetadata.test.ts
@@ -0,0 +1,97 @@
+import { describe, expect, it } from "vitest";
+import { classifyMediaColor, probeMediaMetadata } from "./mediaMetadata.js";
+
+describe("classifyMediaColor", () => {
+ it("detects HDR PQ from BT.2020 + smpte2084 metadata", () => {
+ expect(
+ classifyMediaColor({
+ codec_type: "video",
+ codec_name: "hevc",
+ profile: "Main 10",
+ pix_fmt: "yuv420p10le",
+ color_space: "bt2020nc",
+ color_transfer: "smpte2084",
+ color_primaries: "bt2020",
+ }),
+ ).toMatchObject({
+ dynamicRange: "hdr",
+ hdrTransfer: "pq",
+ label: "HDR PQ",
+ isHdr: true,
+ });
+ });
+
+ it("detects HDR HLG from arib-std-b67 metadata", () => {
+ expect(
+ classifyMediaColor({
+ codec_type: "video",
+ color_space: "bt2020nc",
+ color_transfer: "arib-std-b67",
+ color_primaries: "bt2020",
+ }),
+ ).toMatchObject({
+ dynamicRange: "hdr",
+ hdrTransfer: "hlg",
+ label: "HDR HLG",
+ isHdr: true,
+ });
+ });
+
+ it("labels BT.709 media as SDR Rec.709", () => {
+ expect(
+ classifyMediaColor({
+ codec_type: "video",
+ color_space: "bt709",
+ color_transfer: "bt709",
+ color_primaries: "bt709",
+ }),
+ ).toMatchObject({
+ dynamicRange: "sdr",
+ hdrTransfer: null,
+ label: "SDR Rec.709",
+ isHdr: false,
+ });
+ });
+});
+
+describe("probeMediaMetadata", () => {
+ it("reads the first video stream from ffprobe JSON", () => {
+ const metadata = probeMediaMetadata("/tmp/clip.mp4", () => ({
+ status: 0,
+ stdout: JSON.stringify({
+ streams: [
+ { codec_type: "audio", codec_name: "aac" },
+ {
+ codec_type: "video",
+ codec_name: "hevc",
+ pix_fmt: "yuv420p10le",
+ color_space: "bt2020nc",
+ color_transfer: "smpte2084",
+ color_primaries: "bt2020",
+ },
+ ],
+ }),
+ stderr: "",
+ }));
+
+ expect(metadata).toMatchObject({
+ kind: "video",
+ color: { isHdr: true, label: "HDR PQ" },
+ });
+ });
+
+ it("returns unknown metadata when ffprobe is unavailable", () => {
+ expect(
+ probeMediaMetadata("/tmp/clip.mp4", () => ({
+ status: null,
+ stdout: "",
+ stderr: "",
+ error: { code: "ENOENT" } as NodeJS.ErrnoException,
+ })),
+ ).toMatchObject({
+ kind: "video",
+ color: { dynamicRange: "unknown", isHdr: false },
+ probeError: "ffprobe unavailable",
+ });
+ });
+});
diff --git a/packages/studio-server/src/helpers/mediaMetadata.ts b/packages/studio-server/src/helpers/mediaMetadata.ts
new file mode 100644
index 0000000000..e932be08e0
--- /dev/null
+++ b/packages/studio-server/src/helpers/mediaMetadata.ts
@@ -0,0 +1,160 @@
+import { spawnSync, type SpawnSyncOptions } from "node:child_process";
+import { extname } from "node:path";
+
+type FfprobeRunner = (
+ command: string,
+ args: string[],
+ options?: SpawnSyncOptions,
+) => {
+ status: number | null;
+ stdout: string | Buffer;
+ stderr: string | Buffer;
+ error?: NodeJS.ErrnoException;
+};
+
+export type MediaDynamicRange = "hdr" | "sdr" | "unknown";
+export type MediaHdrTransfer = "pq" | "hlg" | "unknown";
+
+export interface MediaColorMetadata {
+ dynamicRange: MediaDynamicRange;
+ hdrTransfer: MediaHdrTransfer | null;
+ label: string;
+ isHdr: boolean;
+ codecName?: string;
+ profile?: string;
+ pixelFormat?: string;
+ colorSpace?: string;
+ colorTransfer?: string;
+ colorPrimaries?: string;
+ bitsPerRawSample?: string;
+}
+
+export interface MediaMetadata {
+ kind: "video" | "image" | "audio" | "unknown";
+ color: MediaColorMetadata;
+ probeError?: string;
+}
+
+interface FfprobeStream {
+ codec_type?: string;
+ codec_name?: string;
+ profile?: string;
+ pix_fmt?: string;
+ color_space?: string;
+ color_transfer?: string;
+ color_primaries?: string;
+ bits_per_raw_sample?: string;
+}
+
+const VIDEO_EXT = new Set([".mp4", ".mov", ".webm", ".mkv", ".avi", ".m4v"]);
+const IMAGE_EXT = new Set([".jpg", ".jpeg", ".png", ".webp", ".avif"]);
+const AUDIO_EXT = new Set([".mp3", ".wav", ".ogg", ".m4a", ".aac"]);
+
+function lower(value: string | undefined): string {
+ return value?.toLowerCase() ?? "";
+}
+
+function inferKindFromPath(path: string): MediaMetadata["kind"] {
+ const ext = extname(path).toLowerCase();
+ if (VIDEO_EXT.has(ext)) return "video";
+ if (IMAGE_EXT.has(ext)) return "image";
+ if (AUDIO_EXT.has(ext)) return "audio";
+ return "unknown";
+}
+
+function colorLabel(input: {
+ isHdr: boolean;
+ hdrTransfer: MediaHdrTransfer | null;
+ colorPrimaries: string;
+ colorSpace: string;
+ colorTransfer: string;
+}): string {
+ if (input.isHdr) {
+ if (input.hdrTransfer === "pq") return "HDR PQ";
+ if (input.hdrTransfer === "hlg") return "HDR HLG";
+ return "HDR";
+ }
+ if (
+ input.colorPrimaries.includes("bt709") ||
+ input.colorSpace.includes("bt709") ||
+ input.colorTransfer.includes("bt709")
+ ) {
+ return "SDR Rec.709";
+ }
+ return "SDR/unknown";
+}
+
+export function classifyMediaColor(stream: FfprobeStream | null | undefined): MediaColorMetadata {
+ const colorPrimaries = lower(stream?.color_primaries);
+ const colorSpace = lower(stream?.color_space);
+ const colorTransfer = lower(stream?.color_transfer);
+ const isHdr =
+ colorPrimaries.includes("bt2020") ||
+ colorSpace.includes("bt2020") ||
+ colorTransfer === "smpte2084" ||
+ colorTransfer === "arib-std-b67";
+ const hdrTransfer: MediaHdrTransfer | null = isHdr
+ ? colorTransfer === "smpte2084"
+ ? "pq"
+ : colorTransfer === "arib-std-b67"
+ ? "hlg"
+ : "unknown"
+ : null;
+
+ return {
+ dynamicRange: stream ? (isHdr ? "hdr" : "sdr") : "unknown",
+ hdrTransfer,
+ label: stream
+ ? colorLabel({ isHdr, hdrTransfer, colorPrimaries, colorSpace, colorTransfer })
+ : "Unknown",
+ isHdr,
+ codecName: stream?.codec_name,
+ profile: stream?.profile,
+ pixelFormat: stream?.pix_fmt,
+ colorSpace: stream?.color_space,
+ colorTransfer: stream?.color_transfer,
+ colorPrimaries: stream?.color_primaries,
+ bitsPerRawSample: stream?.bits_per_raw_sample,
+ };
+}
+
+export function probeMediaMetadata(
+ filePath: string,
+ runner: FfprobeRunner = spawnSync as unknown as FfprobeRunner,
+): MediaMetadata {
+ const kind = inferKindFromPath(filePath);
+ if (kind === "audio" || kind === "unknown") {
+ return { kind, color: classifyMediaColor(null) };
+ }
+
+ const result = runner(
+ "ffprobe",
+ [
+ "-v",
+ "error",
+ "-show_entries",
+ "stream=codec_type,codec_name,profile,pix_fmt,color_space,color_transfer,color_primaries,bits_per_raw_sample",
+ "-of",
+ "json",
+ filePath,
+ ],
+ { timeout: 15_000, maxBuffer: 1024 * 1024 },
+ );
+
+ if (result.error?.code === "ENOENT") {
+ return { kind, color: classifyMediaColor(null), probeError: "ffprobe unavailable" };
+ }
+ if (result.status !== 0) {
+ return { kind, color: classifyMediaColor(null), probeError: "ffprobe failed" };
+ }
+
+ try {
+ const parsed = JSON.parse(String(result.stdout || "{}")) as { streams?: FfprobeStream[] };
+ const stream = parsed.streams?.find((item) =>
+ kind === "image" ? item.codec_type === "video" : item.codec_type === kind,
+ );
+ return { kind, color: classifyMediaColor(stream) };
+ } catch {
+ return { kind, color: classifyMediaColor(null), probeError: "ffprobe returned invalid json" };
+ }
+}
diff --git a/packages/studio-server/src/index.ts b/packages/studio-server/src/index.ts
index 87cd61103c..4211df1db6 100644
--- a/packages/studio-server/src/index.ts
+++ b/packages/studio-server/src/index.ts
@@ -4,6 +4,7 @@ export type {
StudioApiAdapter,
ResolvedProject,
RenderJobState,
+ MediaProcessingJobState,
LintResult,
StudioSelectionResponse,
StudioSelectionSnapshot,
@@ -13,6 +14,10 @@ export { isSafePath, walkDir } from "./helpers/safePath.js";
export { getMimeType, MIME_TYPES } from "./helpers/mime.js";
export { buildSubCompositionHtml } from "./helpers/subComposition.js";
export { getElementScreenshotClip, type ScreenshotClip } from "./helpers/screenshotClip.js";
+export {
+ createBackgroundRemovalJob,
+ type BackgroundRemovalRender,
+} from "./helpers/backgroundRemovalJob.js";
export {
STUDIO_MANUAL_EDITS_PATH,
createStudioManualEditsRenderBodyScript,
diff --git a/packages/studio-server/src/routes/media.test.ts b/packages/studio-server/src/routes/media.test.ts
new file mode 100644
index 0000000000..f02e928f6f
--- /dev/null
+++ b/packages/studio-server/src/routes/media.test.ts
@@ -0,0 +1,231 @@
+// fallow-ignore-file code-duplication
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { Hono } from "hono";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { registerMediaRoutes } from "./media";
+import type { MediaProcessingJobState, StudioApiAdapter } from "../types";
+
+const tempProjectDirs: string[] = [];
+
+afterEach(() => {
+ for (const dir of tempProjectDirs.splice(0)) {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+function createAdapter(
+ startBackgroundRemoval?: StudioApiAdapter["startBackgroundRemoval"],
+ probeMediaMetadata?: NonNullable[2]>["probeMediaMetadata"],
+): {
+ app: Hono;
+ projectDir: string;
+ startBackgroundRemoval: ReturnType;
+} {
+ const projectDir = mkdtempSync(join(tmpdir(), "hf-media-route-test-"));
+ tempProjectDirs.push(projectDir);
+
+ mkdirSync(join(projectDir, "assets"), { recursive: true });
+ writeFileSync(join(projectDir, "assets", "clip.mp4"), "video");
+ writeFileSync(join(projectDir, "assets", "photo.jpg"), "image");
+
+ const spy = vi.fn(startBackgroundRemoval);
+ const adapter: StudioApiAdapter = {
+ listProjects: () => [],
+ resolveProject: async (id: string) => ({ id, dir: projectDir }),
+ bundle: async () => null,
+ lint: async () => ({ findings: [] }),
+ runtimeUrl: "/api/runtime.js",
+ rendersDir: () => "/tmp/renders",
+ startRender: () => ({
+ id: "job-1",
+ status: "rendering",
+ progress: 0,
+ outputPath: "/tmp/out.mp4",
+ }),
+ ...(startBackgroundRemoval ? { startBackgroundRemoval: spy } : {}),
+ };
+ const app = new Hono();
+ registerMediaRoutes(app, adapter, probeMediaMetadata ? { probeMediaMetadata } : undefined);
+ return { app, projectDir, startBackgroundRemoval: spy };
+}
+
+function completeJob(opts: Parameters>[0]) {
+ return {
+ id: opts.jobId,
+ status: "complete",
+ progress: 100,
+ inputAssetPath: opts.inputAssetPath,
+ outputAssetPath: opts.outputAssetPath,
+ outputPath: opts.outputPath,
+ } satisfies MediaProcessingJobState;
+}
+
+describe("registerMediaRoutes", () => {
+ it("returns metadata for a project-local media asset", async () => {
+ const probe = vi.fn(() => ({
+ kind: "video" as const,
+ color: {
+ dynamicRange: "hdr" as const,
+ hdrTransfer: "hlg" as const,
+ label: "HDR HLG",
+ isHdr: true,
+ },
+ }));
+ const { app, projectDir } = createAdapter(undefined, probe);
+
+ const response = await app.request(
+ "http://localhost/projects/demo/media/metadata?path=assets%2Fclip.mp4",
+ );
+ const data = (await response.json()) as { metadata: { color: { label: string } } };
+
+ expect(response.status).toBe(200);
+ expect(data.metadata.color.label).toBe("HDR HLG");
+ expect(probe).toHaveBeenCalledWith(join(projectDir, "assets", "clip.mp4"));
+ });
+
+ it("rejects media metadata paths outside the project", async () => {
+ const { app } = createAdapter();
+
+ const response = await app.request(
+ "http://localhost/projects/demo/media/metadata?path=..%2Fsecret.mp4",
+ );
+
+ expect(response.status).toBe(403);
+ });
+
+ it("rejects null bytes in media metadata paths", async () => {
+ const { app } = createAdapter();
+
+ const response = await app.request(
+ "http://localhost/projects/demo/media/metadata?path=assets%00clip.mp4",
+ );
+
+ expect(response.status).toBe(403);
+ });
+
+ it("returns 501 when background removal is not available", async () => {
+ const { app } = createAdapter();
+
+ const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ inputPath: "assets/clip.mp4" }),
+ });
+
+ expect(response.status).toBe(501);
+ });
+
+ it("rejects remote input paths", async () => {
+ const { app, startBackgroundRemoval } = createAdapter(completeJob);
+
+ const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ inputPath: "https://example.com/clip.mp4" }),
+ });
+
+ expect(response.status).toBe(400);
+ expect(startBackgroundRemoval).not.toHaveBeenCalled();
+ });
+
+ it("rejects null bytes in background-removal paths", async () => {
+ const { app, startBackgroundRemoval } = createAdapter(completeJob);
+
+ const inputResponse = await app.request(
+ "http://localhost/projects/demo/media/remove-background",
+ {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ inputPath: "assets/clip\0.mp4" }),
+ },
+ );
+ const outputResponse = await app.request(
+ "http://localhost/projects/demo/media/remove-background",
+ {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ inputPath: "assets/clip.mp4", outputPath: "assets/out\0.webm" }),
+ },
+ );
+
+ expect(inputResponse.status).toBe(403);
+ expect(outputResponse.status).toBe(403);
+ expect(startBackgroundRemoval).not.toHaveBeenCalled();
+ });
+
+ it("starts a video cutout job with safe default output paths", async () => {
+ const { app, projectDir, startBackgroundRemoval } = createAdapter(completeJob);
+
+ const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ inputPath: "assets/clip.mp4", createBackgroundPlate: true }),
+ });
+ const data = (await response.json()) as {
+ jobId: string;
+ outputPath: string;
+ backgroundOutputPath: string;
+ };
+
+ expect(response.status).toBe(200);
+ expect(data.outputPath).toBe("assets/cutouts/clip-cutout.webm");
+ expect(data.backgroundOutputPath).toBe("assets/cutouts/clip-plate.webm");
+ expect(startBackgroundRemoval).toHaveBeenCalledWith(
+ expect.objectContaining({
+ project: { id: "demo", dir: projectDir },
+ inputAssetPath: "assets/clip.mp4",
+ outputAssetPath: "assets/cutouts/clip-cutout.webm",
+ backgroundOutputAssetPath: "assets/cutouts/clip-plate.webm",
+ quality: "balanced",
+ device: "auto",
+ jobId: data.jobId,
+ }),
+ );
+ });
+
+ it("normalizes query strings from local media paths", async () => {
+ const { app, startBackgroundRemoval } = createAdapter(completeJob);
+
+ const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ inputPath: "./assets/clip.mp4?v=123#frame" }),
+ });
+
+ expect(response.status).toBe(200);
+ expect(startBackgroundRemoval).toHaveBeenCalledWith(
+ expect.objectContaining({
+ inputAssetPath: "assets/clip.mp4",
+ outputAssetPath: "assets/cutouts/clip-cutout.webm",
+ }),
+ );
+ });
+
+ it("requires png output for image cutouts", async () => {
+ const { app, startBackgroundRemoval } = createAdapter(completeJob);
+
+ const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ inputPath: "assets/photo.jpg", outputPath: "assets/photo.webm" }),
+ });
+
+ expect(response.status).toBe(400);
+ expect(startBackgroundRemoval).not.toHaveBeenCalled();
+ });
+
+ it("keeps output paths inside the project", async () => {
+ const { app, startBackgroundRemoval } = createAdapter(completeJob);
+
+ const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ inputPath: "assets/clip.mp4", outputPath: "../escape.webm" }),
+ });
+
+ expect(response.status).toBe(403);
+ expect(startBackgroundRemoval).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/studio-server/src/routes/media.ts b/packages/studio-server/src/routes/media.ts
new file mode 100644
index 0000000000..76c52f5e62
--- /dev/null
+++ b/packages/studio-server/src/routes/media.ts
@@ -0,0 +1,255 @@
+import type { Hono } from "hono";
+import { streamSSE } from "hono/streaming";
+import { existsSync, mkdirSync } from "node:fs";
+import { basename, dirname, extname, join } from "node:path";
+import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js";
+import { resolveWithinProject } from "../helpers/safePath.js";
+import { probeMediaMetadata } from "../helpers/mediaMetadata.js";
+
+const VIDEO_EXTENSIONS = new Set([".mp4", ".mov", ".webm", ".mkv", ".avi"]);
+const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".webp"]);
+const VIDEO_OUTPUT_EXTENSIONS = new Set([".webm", ".mov"]);
+const QUALITIES = new Set(["fast", "balanced", "best"]);
+const DEVICES = new Set(["auto", "cpu", "coreml", "cuda"]);
+
+type BackgroundRemovalQuality = "fast" | "balanced" | "best";
+type BackgroundRemovalDevice = "auto" | "cpu" | "coreml" | "cuda";
+
+interface BackgroundRemovalBody {
+ inputPath?: string;
+ outputPath?: string;
+ createBackgroundPlate?: boolean;
+ quality?: string;
+ device?: string;
+}
+
+type JobWithCreatedAt = MediaProcessingJobState & { createdAt: number };
+type ProbeMediaMetadata = typeof probeMediaMetadata;
+
+function isVideoPath(path: string): boolean {
+ return VIDEO_EXTENSIONS.has(extname(path).toLowerCase());
+}
+
+function isImagePath(path: string): boolean {
+ return IMAGE_EXTENSIONS.has(extname(path).toLowerCase());
+}
+
+function normalizeProjectAssetPath(path: string): string {
+ return path
+ .trim()
+ .replace(/^[.]\//, "")
+ .replace(/[?#].*$/, "");
+}
+
+function containsNullByte(path: string): boolean {
+ return path.includes("\0");
+}
+
+function slugFileBase(path: string): string {
+ const name = basename(path, extname(path))
+ .replace(/[^a-zA-Z0-9._-]+/g, "-")
+ .replace(/^-+|-+$/g, "");
+ return name || "media";
+}
+
+function uniqueAssetPath(projectDir: string, assetPath: string): string {
+ const ext = extname(assetPath);
+ const withoutExt = assetPath.slice(0, -ext.length);
+ let candidate = assetPath;
+ for (let index = 2; existsSync(join(projectDir, candidate)); index++) {
+ candidate = `${withoutExt}-${index}${ext}`;
+ }
+ return candidate;
+}
+
+function defaultOutputPath(projectDir: string, inputPath: string): string {
+ const ext = isImagePath(inputPath) ? ".png" : ".webm";
+ return uniqueAssetPath(projectDir, `assets/cutouts/${slugFileBase(inputPath)}-cutout${ext}`);
+}
+
+function defaultPlatePath(projectDir: string, inputPath: string): string {
+ return uniqueAssetPath(projectDir, `assets/cutouts/${slugFileBase(inputPath)}-plate.webm`);
+}
+
+function makeJobId(projectId: string, mediaJobs: Map): string {
+ const stamp = new Date()
+ .toISOString()
+ .replace(/[-:.TZ]/g, "")
+ .slice(0, 14);
+ const safeProject = projectId.replace(/[^a-zA-Z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
+ const base = `${safeProject || "project"}_remove-bg_${stamp}`;
+ if (!mediaJobs.has(base)) return base;
+ for (let index = 2; ; index++) {
+ const candidate = `${base}-${index}`;
+ if (!mediaJobs.has(candidate)) return candidate;
+ }
+}
+
+function normalizeQuality(value: string | undefined): BackgroundRemovalQuality {
+ return QUALITIES.has(value ?? "") ? (value as BackgroundRemovalQuality) : "balanced";
+}
+
+function normalizeDevice(value: string | undefined): BackgroundRemovalDevice {
+ return DEVICES.has(value ?? "") ? (value as BackgroundRemovalDevice) : "auto";
+}
+
+export function registerMediaRoutes(
+ api: Hono,
+ adapter: StudioApiAdapter,
+ options: { probeMediaMetadata?: ProbeMediaMetadata } = {},
+): void {
+ const mediaJobs = new Map();
+ const TTL_MS = 300_000;
+ const readMediaMetadata = options.probeMediaMetadata ?? probeMediaMetadata;
+
+ function cleanupFinishedJobs(): void {
+ const now = Date.now();
+ for (const [id, job] of mediaJobs) {
+ if ((job.status === "complete" || job.status === "failed") && now - job.createdAt > TTL_MS) {
+ mediaJobs.delete(id);
+ }
+ }
+ }
+
+ api.get("/projects/:id/media/metadata", async (c) => {
+ const project = await adapter.resolveProject(c.req.param("id"));
+ if (!project) return c.json({ error: "not found" }, 404);
+
+ const assetPath = normalizeProjectAssetPath(c.req.query("path") ?? "");
+ if (!assetPath) return c.json({ error: "path required" }, 400);
+ if (containsNullByte(assetPath)) return c.json({ error: "forbidden" }, 403);
+ if (/^(?:https?:|data:|blob:)/i.test(assetPath)) {
+ return c.json({ error: "media metadata requires a project-local asset" }, 400);
+ }
+
+ const filePath = resolveWithinProject(project.dir, assetPath);
+ if (!filePath) return c.json({ error: "forbidden" }, 403);
+ if (!existsSync(filePath)) return c.json({ error: "media not found" }, 404);
+
+ return c.json({ path: assetPath, metadata: readMediaMetadata(filePath) });
+ });
+
+ api.post(
+ "/projects/:id/media/remove-background",
+ // fallow-ignore-next-line complexity
+ async (c) => {
+ cleanupFinishedJobs();
+ if (!adapter.startBackgroundRemoval) {
+ return c.json({ error: "background removal is not available in this Studio server" }, 501);
+ }
+
+ // fallow-ignore-next-line code-duplication
+ const project = await adapter.resolveProject(c.req.param("id"));
+ if (!project) return c.json({ error: "not found" }, 404);
+
+ const body = (await c.req.json().catch(() => ({}))) as BackgroundRemovalBody;
+ const inputAssetPath = body.inputPath ? normalizeProjectAssetPath(body.inputPath) : "";
+ if (!inputAssetPath) return c.json({ error: "inputPath required" }, 400);
+ if (containsNullByte(inputAssetPath)) return c.json({ error: "forbidden" }, 403);
+ if (/^(?:https?:|data:|blob:)/i.test(inputAssetPath)) {
+ return c.json({ error: "background removal requires a project-local media asset" }, 400);
+ }
+
+ const inputPath = resolveWithinProject(project.dir, inputAssetPath);
+ if (!inputPath) return c.json({ error: "forbidden" }, 403);
+ if (!existsSync(inputPath)) return c.json({ error: "input media not found" }, 404);
+
+ const inputIsVideo = isVideoPath(inputAssetPath);
+ const inputIsImage = isImagePath(inputAssetPath);
+ if (!inputIsVideo && !inputIsImage) {
+ return c.json({ error: "background removal supports video or image assets only" }, 400);
+ }
+
+ const requestedOutput = body.outputPath ? normalizeProjectAssetPath(body.outputPath) : "";
+ if (requestedOutput && containsNullByte(requestedOutput)) {
+ return c.json({ error: "forbidden" }, 403);
+ }
+ if (requestedOutput && !resolveWithinProject(project.dir, requestedOutput)) {
+ return c.json({ error: "forbidden" }, 403);
+ }
+ const outputAssetPath = requestedOutput
+ ? uniqueAssetPath(project.dir, requestedOutput)
+ : defaultOutputPath(project.dir, inputAssetPath);
+ const outputPath = resolveWithinProject(project.dir, outputAssetPath);
+ if (!outputPath) return c.json({ error: "forbidden" }, 403);
+ if (inputIsVideo && !VIDEO_OUTPUT_EXTENSIONS.has(extname(outputAssetPath).toLowerCase())) {
+ return c.json({ error: "video background removal output must be .webm or .mov" }, 400);
+ }
+ if (inputIsImage && extname(outputAssetPath).toLowerCase() !== ".png") {
+ return c.json({ error: "image background removal output must be .png" }, 400);
+ }
+
+ let backgroundOutputAssetPath: string | undefined;
+ let backgroundOutputPath: string | undefined;
+ if (body.createBackgroundPlate) {
+ if (!inputIsVideo) {
+ return c.json({ error: "background plates are only supported for video inputs" }, 400);
+ }
+ backgroundOutputAssetPath = defaultPlatePath(project.dir, inputAssetPath);
+ backgroundOutputPath =
+ resolveWithinProject(project.dir, backgroundOutputAssetPath) ?? undefined;
+ if (!backgroundOutputPath) {
+ return c.json({ error: "forbidden" }, 403);
+ }
+ }
+
+ mkdirSync(dirname(outputPath), { recursive: true });
+ if (backgroundOutputPath) mkdirSync(dirname(backgroundOutputPath), { recursive: true });
+
+ const jobId = makeJobId(project.id, mediaJobs);
+ const state = adapter.startBackgroundRemoval({
+ project,
+ inputPath,
+ inputAssetPath,
+ outputPath,
+ outputAssetPath,
+ backgroundOutputPath,
+ backgroundOutputAssetPath,
+ quality: normalizeQuality(body.quality),
+ device: normalizeDevice(body.device),
+ jobId,
+ }) as JobWithCreatedAt;
+ state.createdAt = Date.now();
+ mediaJobs.set(jobId, state);
+
+ return c.json({
+ jobId,
+ status: state.status,
+ outputPath: outputAssetPath,
+ backgroundOutputPath: backgroundOutputAssetPath,
+ });
+ },
+ );
+
+ api.get("/media-jobs/:jobId/progress", (c) => {
+ cleanupFinishedJobs();
+ const { jobId } = c.req.param();
+ const job = mediaJobs.get(jobId);
+ if (!job) return c.json({ error: "not found" }, 404);
+
+ return streamSSE(c, async (stream) => {
+ while (true) {
+ const current = mediaJobs.get(jobId);
+ if (!current) break;
+ await stream.writeSSE({
+ event: "progress",
+ data: JSON.stringify({
+ id: current.id,
+ status: current.status,
+ progress: current.progress,
+ stage: current.stage,
+ outputPath: current.outputAssetPath,
+ backgroundOutputPath: current.backgroundOutputAssetPath,
+ error: current.error,
+ provider: current.provider,
+ framesProcessed: current.framesProcessed,
+ durationSeconds: current.durationSeconds,
+ avgMsPerFrame: current.avgMsPerFrame,
+ }),
+ });
+ if (current.status === "complete" || current.status === "failed") break;
+ await stream.sleep(500);
+ }
+ });
+ });
+}
diff --git a/packages/studio-server/src/types.ts b/packages/studio-server/src/types.ts
index 97e5aca9fa..41fb5d5cd4 100644
--- a/packages/studio-server/src/types.ts
+++ b/packages/studio-server/src/types.ts
@@ -19,6 +19,23 @@ export interface RenderJobState {
error?: string;
}
+export interface MediaProcessingJobState {
+ id: string;
+ status: "processing" | "complete" | "failed";
+ progress: number;
+ stage?: string;
+ inputAssetPath: string;
+ outputAssetPath: string;
+ outputPath: string;
+ backgroundOutputAssetPath?: string;
+ backgroundOutputPath?: string;
+ error?: string;
+ provider?: string;
+ framesProcessed?: number;
+ durationSeconds?: number;
+ avgMsPerFrame?: number;
+}
+
/** Lint result from the core linter. */
export interface LintResult {
findings: Array<{
@@ -137,6 +154,19 @@ export interface StudioApiAdapter {
distinctId?: string;
}): RenderJobState;
+ startBackgroundRemoval?: (opts: {
+ project: ResolvedProject;
+ inputPath: string;
+ inputAssetPath: string;
+ outputPath: string;
+ outputAssetPath: string;
+ backgroundOutputPath?: string;
+ backgroundOutputAssetPath?: string;
+ quality: "fast" | "balanced" | "best";
+ device?: "auto" | "cpu" | "coreml" | "cuda";
+ jobId: string;
+ }) => MediaProcessingJobState;
+
/** Optional: generate a JPEG thumbnail via Puppeteer or similar. */
generateThumbnail?: (opts: {
project: ResolvedProject;
diff --git a/packages/studio/vite.adapter.ts b/packages/studio/vite.adapter.ts
index 3a5c68647e..23cc7c4c04 100644
--- a/packages/studio/vite.adapter.ts
+++ b/packages/studio/vite.adapter.ts
@@ -15,6 +15,8 @@ import {
type ResolvedProject,
type RenderJobState,
type StudioApiAdapter,
+ type BackgroundRemovalRender,
+ createBackgroundRemovalJob,
createProjectSignature,
} from "@hyperframes/studio-server";
import type { RegistryItem } from "@hyperframes/core/registry";
@@ -32,7 +34,10 @@ export function isPathWithin(parentDir: string, childPath: string): boolean {
export function createViteAdapter(dataDir: string, server: ViteDevServer): StudioApiAdapter {
let _bundler:
- | ((dir: string, options?: { runtime?: "inline" | "placeholder" }) => Promise)
+ | ((
+ dir: string,
+ options?: { runtime?: "inline" | "placeholder"; inlineColorGradingLuts?: boolean },
+ ) => Promise)
| null = null;
let _producerModuleLoader:
| (() => Promise<{
@@ -162,7 +167,7 @@ export function createViteAdapter(dataDir: string, server: ViteDevServer): Studi
async bundle(dir: string) {
const bundler = await getBundler();
if (!bundler) return null;
- let html = await bundler(dir, { runtime: "placeholder" });
+ let html = await bundler(dir, { runtime: "placeholder", inlineColorGradingLuts: false });
html = html.replace(
'data-hyperframes-preview-runtime="1" src=""',
`data-hyperframes-preview-runtime="1" src="${this.runtimeUrl}"`,
@@ -251,6 +256,16 @@ export function createViteAdapter(dataDir: string, server: ViteDevServer): Studi
return state;
},
+ startBackgroundRemoval(opts) {
+ return createBackgroundRemovalJob(opts, async (renderOpts) => {
+ const mod = await server.ssrLoadModule(
+ resolve(__dirname, "../cli/src/background-removal/pipeline.ts"),
+ );
+ const render = mod.render as BackgroundRemovalRender;
+ return render(renderOpts);
+ });
+ },
+
async generateThumbnail(opts) {
return generateThumbnail(opts);
},