diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index b3a949e65e59..e8593ff407f0 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -4,7 +4,10 @@ import type { ScheduledTask, ScheduledTaskUpsertInput, } from "@t3tools/contracts"; -import { resolveEnvironmentMachineKind } from "@t3tools/contracts"; +import { + MAX_WEBHOOK_DELIVERY_AGE_MINUTES, + resolveEnvironmentMachineKind, +} from "@t3tools/contracts"; import type { MenuAction } from "@react-native-menu/menu"; import { DateTimePicker } from "@expo/ui/community/datetime-picker"; import { @@ -12,6 +15,10 @@ import { squashAtomCommandFailure, type AtomCommandResult, } from "@t3tools/client-runtime/state/runtime"; +import { + DEFAULT_WEBHOOK_PROMPT, + parseMaxDeliveryAge, +} from "@t3tools/client-runtime/scheduled-task-webhook"; import { useCallback, useEffect, @@ -59,7 +66,6 @@ import { SettingsSection } from "./components/SettingsSection"; import { useSettingsEnvironmentFilter, type SettingsTarget } from "./settings-environment-filter"; import { editDraft, - DEFAULT_WEBHOOK_PROMPT, scheduledTaskDefaultModel, scheduleFromDraft, type ScheduledTaskDraft as Draft, @@ -132,7 +138,7 @@ function FormField(props: { readonly label: string; readonly value: string; readonly onChange: (value: string) => void; - readonly keyboardType?: "decimal-pad"; + readonly keyboardType?: "decimal-pad" | "number-pad"; readonly disabled?: boolean; readonly placeholder?: string; readonly borderTop?: boolean; @@ -611,6 +617,16 @@ function TaskForm({ ? { ...draft.schedule, signature: liveTask.schedule.signature } : draft.schedule, ); + if ( + draft.schedule.mode === "webhook" && + parseMaxDeliveryAge(draft.schedule.maxDeliveryAgeMinutes) === undefined + ) { + Alert.alert( + "Invalid age limit", + `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`, + ); + return; + } if ( !draft.title.trim() || !draft.prompt.trim() || @@ -823,7 +839,7 @@ function TaskForm({ options={[ { value: "fixed_time", label: "At a time" }, { value: "interval", label: "Interval" }, - { value: "webhook", label: "Webhook" }, + { value: "webhook", label: "On webhook" }, ]} selected={draft.schedule.mode} onSelect={(mode) => { @@ -910,13 +926,31 @@ function TaskForm({ /> ) : draft.schedule.mode === "webhook" ? ( - task.id === draft.task?.id) ?? draft.task ?? null - } - signatureConfigured={draft.schedule.signature !== null} - /> + <> + task.id === draft.task?.id) ?? null) + : draft.task + } + signatureConfigured={draft.schedule.signature !== null} + disabled={saving || environmentUnavailable} + /> + + setDraft({ ...draft, schedule: { ...draft.schedule, maxDeliveryAgeMinutes } }) + } + /> + ) : ( <> Alert.alert("Rotate URL?", "The current URL stops working immediately.", [ { text: "Cancel", style: "cancel" }, { text: "Rotate", style: "destructive", - onPress: () => + onPress: () => { + setRotating(true); void rotate({ environmentId, input: { id: task.id } }).then((result) => { + setRotating(false); if (result._tag === "Failure" && !isAtomCommandInterrupted(result)) { Alert.alert( "Could not rotate URL", String(squashAtomCommandFailure(result)), ); } - }), + }); + }, }, ]) } - className="min-h-11 justify-center active:opacity-70" + className="min-h-11 justify-center active:opacity-70 disabled:opacity-50" > - Rotate URL + + {rotating ? "Rotating…" : "Rotate URL"} + )} diff --git a/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts b/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts index 022a357404ed..9f83e0c8fa32 100644 --- a/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts +++ b/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts @@ -32,7 +32,25 @@ describe("scheduleDraftForTask", () => { it("round-trips a webhook schedule without a signature", () => { const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null } }); expect(draft.mode).toBe("webhook"); - expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature: null }); + expect(scheduleFromDraft(draft)).toEqual({ + type: "webhook", + signature: null, + maxDeliveryAgeMinutes: null, + }); + }); + + it("round-trips a webhook max age and treats blank input as no limit", () => { + const draft = scheduleDraftForTask({ + schedule: { type: "webhook", signature: null, maxDeliveryAgeMinutes: 45 }, + }); + expect(draft.maxDeliveryAgeMinutes).toBe("45"); + expect(scheduleFromDraft(draft)).toMatchObject({ maxDeliveryAgeMinutes: 45 }); + // An invalid limit is an invalid schedule, never a silently removed one. + expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "1.5" })).toBeNull(); + expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "0" })).toBeNull(); + expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "" })).toMatchObject({ + maxDeliveryAgeMinutes: null, + }); }); it("keeps a webhook signature on save without sending a secret", () => { @@ -44,7 +62,7 @@ describe("scheduleDraftForTask", () => { const saved = scheduleFromDraft( scheduleDraftForTask({ schedule: { type: "webhook", signature } }), ); - expect(saved).toEqual({ type: "webhook", signature }); + expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null }); expect(saved?.type === "webhook" && saved.signature && "secret" in saved.signature).toBe(false); }); }); diff --git a/apps/mobile/src/features/settings/scheduledTaskDraft.ts b/apps/mobile/src/features/settings/scheduledTaskDraft.ts index 2843700620e8..0ba9a437cdb6 100644 --- a/apps/mobile/src/features/settings/scheduledTaskDraft.ts +++ b/apps/mobile/src/features/settings/scheduledTaskDraft.ts @@ -9,6 +9,7 @@ import type { } from "@t3tools/contracts"; import { DEFAULT_SERVER_SETTINGS } from "@t3tools/contracts"; +import { parseMaxDeliveryAge } from "@t3tools/client-runtime/scheduled-task-webhook"; import { resolveProjectSettings, type LegacyProjectSettingsFields, @@ -44,6 +45,8 @@ export type ScheduleDraft = { readonly intervalMinutes: string; /** A webhook signature check configured elsewhere; mobile keeps it but does not edit it. */ readonly signature: ScheduledTaskWebhookSignature | null; + /** Minutes as typed; empty runs every held request regardless of age. */ + readonly maxDeliveryAgeMinutes: string; }; export const DEFAULT_SCHEDULE: ScheduleDraft = { @@ -52,11 +55,9 @@ export const DEFAULT_SCHEDULE: ScheduleDraft = { weekdays: [1, 2, 3, 4, 5], intervalMinutes: "15", signature: null, + maxDeliveryAgeMinutes: "", }; -/** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */ -export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}"; - export function scheduleDraftForTask(task: Pick): ScheduleDraft { switch (task.schedule.type) { case "fixed_time": @@ -74,12 +75,22 @@ export function scheduleDraftForTask(task: Pick): Sch intervalMinutes: String(Math.max(1, task.schedule.everyMs / 60_000)), }; case "webhook": - return { ...DEFAULT_SCHEDULE, mode: "webhook", signature: task.schedule.signature }; + return { + ...DEFAULT_SCHEDULE, + mode: "webhook", + signature: task.schedule.signature, + maxDeliveryAgeMinutes: + task.schedule.maxDeliveryAgeMinutes == null + ? "" + : String(task.schedule.maxDeliveryAgeMinutes), + }; } } export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSchedule | null { if (draft.mode === "webhook") { + const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes); + if (maxDeliveryAgeMinutes === undefined) return null; // No secret is sent, so the server keeps the stored one. return { type: "webhook", @@ -91,6 +102,7 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche encoding: draft.signature.encoding, prefix: draft.signature.prefix, }, + maxDeliveryAgeMinutes, }; } if (draft.mode === "interval") { @@ -145,6 +157,7 @@ function draftSignature(draft: ScheduledTaskDraft): string { draft.schedule.timeOfDay, [...draft.schedule.weekdays].sort((a, b) => a - b), draft.schedule.intervalMinutes, + draft.schedule.maxDeliveryAgeMinutes, draft.workspace, draft.baseRef, draft.checkoutPath, diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index aebefed787c1..a0490e77446f 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -209,6 +209,36 @@ describe("CloudManagedEndpointRuntime", () => { }), ); + it.effect("signals each registered tunnel connection", () => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const handle = makeHandle({ + pid: 700, + onKill: () => {}, + output: Stream.fromQueue(output), + }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const runtime = yield* buildCloudManagedEndpointRuntime(spawner); + yield* runtime.applyConfig({ + providerKind: "cloudflare_tunnel", + connectorToken: "token", + tunnelId: "tunnel-1", + }); + yield* Queue.offer( + output, + new TextEncoder().encode( + "2026-10-04T06:30:43Z INF Registered tunnel connection connIndex=0 event=0\n", + ), + ); + expect(Option.isSome(yield* Stream.runHead(runtime.tunnelConnected))).toBe(true); + }), + ); + it.effect("recovers a rejected tunnel without waiting for the connector to exit", () => Effect.gen(function* () { const output = yield* Queue.unbounded(); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 25426fcba4ce..6b54b7ee3bdf 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -47,6 +47,8 @@ export class CloudManagedEndpointRuntime extends Context.Service< ) => Effect.Effect; readonly recoveryRequests: Stream.Stream; readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect; + /** Emits when the connector registers a tunnel connection, i.e. the relay can reach us again. */ + readonly tunnelConnected: Stream.Stream; readonly withLinkStateLock: (effect: Effect.Effect) => Effect.Effect; } >()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {} @@ -134,6 +136,7 @@ export const make = Effect.gen(function* () { const activeRef = yield* Ref.make(null); const desiredConfigRef = yield* Ref.make(null); const recoveryRequests = yield* Queue.sliding(1); + const tunnelConnections = yield* Queue.sliding(1); const reconcileSemaphore = yield* Semaphore.make(1); const restartDelayRef = yield* Ref.make(0); const linkStateSemaphore = yield* Semaphore.make(1); @@ -236,7 +239,10 @@ export const make = Effect.gen(function* () { switch (classifyRelayClientOutput(line)) { case "connected": rejectedRegistrations = 0; - return Effect.logInfo("Relay client tunnel connection registered", attributes); + return Effect.logInfo("Relay client tunnel connection registered", attributes).pipe( + Effect.andThen(Queue.offer(tunnelConnections, undefined)), + Effect.asVoid, + ); case "warning": if (isRejectedRelayClientTunnelOutput(line)) { rejectedRegistrations += 1; @@ -412,6 +418,7 @@ export const make = Effect.gen(function* () { applyConfig, recoveryRequests: Stream.fromQueue(recoveryRequests), requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid), + tunnelConnected: Stream.fromQueue(tunnelConnections), withLinkStateLock: linkStateSemaphore.withPermits(1), }); diff --git a/apps/server/src/cloud/config.ts b/apps/server/src/cloud/config.ts index 9b1b281ba2da..92bd9c856ca6 100644 --- a/apps/server/src/cloud/config.ts +++ b/apps/server/src/cloud/config.ts @@ -16,6 +16,7 @@ export const RELAY_URL_SECRET = "cloud-relay-url"; export const RELAY_ISSUER_SECRET = "cloud-relay-issuer"; export const RELAY_ENVIRONMENT_CREDENTIAL_SECRET = "cloud-relay-environment-credential"; export const PUBLISH_AGENT_ACTIVITY_SECRET = "cloud-publish-agent-activity"; +export const HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET = "cloud-hold-webhooks-while-offline"; export const encodeEndpointRuntimeConfigJson = Schema.encodeEffect( Schema.fromJsonString(RelayManagedEndpointRuntimeConfig), @@ -73,3 +74,33 @@ export const readAgentActivityPublishingActive = ( environmentCredential !== "" ); }).pipe(Effect.orElseSucceed(() => false)); + +const readSecretString = ( + secrets: ServerSecretStore.ServerSecretStore["Service"], + name: string, +): Effect.Effect => + secrets.get(name).pipe( + Effect.map((bytes) => + Option.isSome(bytes) && bytes.value.length > 0 ? new TextDecoder().decode(bytes.value) : null, + ), + Effect.orElseSucceed(() => null), + ); + +/** The relay URL and environment credential, or null when not linked to T3 Connect. */ +export const readRelayConnection = (secrets: ServerSecretStore.ServerSecretStore["Service"]) => + Effect.all([ + readSecretString(secrets, RELAY_URL_SECRET), + readSecretString(secrets, RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + ]).pipe( + Effect.map(([url, environmentCredential]) => + url && environmentCredential ? { url, environmentCredential } : null, + ), + ); + +/** Whether this environment opted in to T3 Connect holding webhooks while it is offline. */ +export const readHoldWebhooksWhileOffline = ( + secrets: ServerSecretStore.ServerSecretStore["Service"], +) => + readSecretString(secrets, HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET).pipe( + Effect.map((value) => value === "true"), + ); diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 77d432308477..085cac39184d 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -249,6 +249,7 @@ describe("reconcileDesiredCloudLink", () => { applyConfig: unusedSecretStoreOperation, recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, + tunnelConnected: Stream.empty, withLinkStateLock: (effect) => effect, } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]), ), @@ -410,6 +411,7 @@ describe("releaseManagedTunnelOnShutdown", () => { }), recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, + tunnelConnected: Stream.empty, withLinkStateLock: (effect) => effect, }), ), diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 82337df5b6a7..4cb29c37b67f 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -8,6 +8,7 @@ import { EnvironmentCloudRelayConfigResult, EnvironmentHttpApi, EnvironmentHttpBadRequestError, + type EnvironmentCloudPreferencesRequest, EnvironmentHttpConflictError, EnvironmentHttpInternalServerError, EnvironmentHttpUnauthorizedError, @@ -69,6 +70,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { requireEnvironmentScope } from "../auth/http.ts"; import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import { makeRelayEnvironmentClient } from "../relay/relayEnvironmentClient.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { @@ -86,6 +88,9 @@ import { encodeEndpointRuntimeConfigJson, encodeConfirmedOriginJson, PUBLISH_AGENT_ACTIVITY_SECRET, + HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, + readHoldWebhooksWhileOffline, + readRelayConnection, RELAY_ENVIRONMENT_CREDENTIAL_SECRET, RELAY_ISSUER_SECRET, RELAY_URL_SECRET, @@ -1277,17 +1282,24 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* ( dependencies: CloudHttpDependencies, ) { - const [cloudUserId, relayUrl, relayIssuer, endpointRuntimeConfig, publishAgentActivity] = - yield* Effect.all( - [ - dependencies.secrets.get(CLOUD_LINKED_USER_ID), - dependencies.secrets.get(RELAY_URL_SECRET), - dependencies.secrets.get(RELAY_ISSUER_SECRET), - dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), - dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET), - ], - { concurrency: 5 }, - ); + const [ + cloudUserId, + relayUrl, + relayIssuer, + endpointRuntimeConfig, + publishAgentActivity, + holdWebhooks, + ] = yield* Effect.all( + [ + dependencies.secrets.get(CLOUD_LINKED_USER_ID), + dependencies.secrets.get(RELAY_URL_SECRET), + dependencies.secrets.get(RELAY_ISSUER_SECRET), + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET), + dependencies.secrets.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET), + ], + { concurrency: 6 }, + ); return { linked: Option.isSome(cloudUserId), cloudUserId: Option.isSome(cloudUserId) ? bytesToString(cloudUserId.value) : null, @@ -1299,6 +1311,8 @@ const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function publishAgentActivity: Option.isSome(publishAgentActivity) ? bytesToString(publishAgentActivity.value) === "true" : false, + holdWebhooksWhileOffline: + Option.isSome(holdWebhooks) && bytesToString(holdWebhooks.value) === "true", } satisfies EnvironmentCloudLinkStateResult; }); @@ -1329,8 +1343,9 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")( dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN), dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), + dependencies.secrets.remove(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET), ], - { concurrency: 8 }, + { concurrency: 9 }, ); yield* setCliDesiredCloudLink(false); return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; @@ -1343,12 +1358,52 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")( ), ); +const pushHoldWebhooksWhileOffline = Effect.fn("environment.cloud.pushHoldWebhooksWhileOffline")( + function* (dependencies: CloudHttpDependencies, holdWebhooksWhileOffline: boolean) { + const connection = yield* readRelayConnection(dependencies.secrets); + if (connection === null) { + return yield* new EnvironmentHttpBadRequestError({ + message: "Link this environment to T3 Connect first.", + }); + } + const environmentId = yield* dependencies.environment.getEnvironmentId; + const client = yield* makeRelayEnvironmentClient(connection); + yield* client.server + .updateLinkPreferences({ + params: { environmentId }, + payload: { holdWebhooksWhileOffline }, + }) + .pipe( + Effect.timeout("10 seconds"), + Effect.catch( + failEnvironmentCloudInternalError("Could not update T3 Connect webhook settings."), + ), + ); + }, +); + const cloudPreferencesHandler = Effect.fn("environment.cloud.preferences")( - function* ( - dependencies: CloudHttpDependencies, - payload: { readonly publishAgentActivity: boolean }, - ) { + function* (dependencies: CloudHttpDependencies, payload: EnvironmentCloudPreferencesRequest) { yield* requireEnvironmentScope(AuthRelayWriteScope); + if (payload.holdWebhooksWhileOffline !== undefined) { + // The relay decides whether to hold a request, so it is told first; the + // local copy is only saved once the relay has the same value, and the + // relay is put back if that save fails. + const previous = yield* readHoldWebhooksWhileOffline(dependencies.secrets); + yield* pushHoldWebhooksWhileOffline(dependencies, payload.holdWebhooksWhileOffline); + yield* dependencies.secrets + .set( + HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, + stringToBytes(String(payload.holdWebhooksWhileOffline)), + ) + .pipe( + Effect.tapError(() => + previous === payload.holdWebhooksWhileOffline + ? Effect.void + : pushHoldWebhooksWhileOffline(dependencies, previous).pipe(Effect.ignore), + ), + ); + } yield* dependencies.secrets.set( PUBLISH_AGENT_ACTIVITY_SECRET, stringToBytes(String(payload.publishAgentActivity)), diff --git a/apps/server/src/observability/Metrics.ts b/apps/server/src/observability/Metrics.ts index b75ace399ccc..f133eda3f8b0 100644 --- a/apps/server/src/observability/Metrics.ts +++ b/apps/server/src/observability/Metrics.ts @@ -70,6 +70,30 @@ export const terminalRestartsTotal = Metric.counter("t3_terminal_restarts_total" description: "Total terminal restart requests handled.", }); +/** + * One per webhook request that reached a task, by `outcome` (accepted, + * not_found, rejected_signature, disabled, rate_limited, queue_full, expired, + * prompt_too_long, error) and `source` (relay or direct). + */ +export const webhookDeliveriesTotal = Metric.counter("t3_webhook_deliveries_total", { + description: "Webhook requests handled, by outcome and source.", +}); + +export const webhookDeliveryDuration = Metric.timer("t3_webhook_delivery_duration", { + description: "Time to verify, log, and enqueue one webhook request.", +}); + +/** How long a relay-held request waited before this environment got it. */ +export const webhookHeldDelay = Metric.timer("t3_webhook_held_delay", { + description: + "Time between the relay receiving a webhook request and the environment handling it.", +}); + +/** Runs started by webhook deliveries, by `outcome` (started, skipped, failed). */ +export const webhookRunsTotal = Metric.counter("t3_webhook_runs_total", { + description: "Runs started from webhook deliveries, by outcome.", +}); + export const metricAttributes = ( attributes: Readonly>, ): ReadonlyArray<[string, string]> => Object.entries(compactMetricAttributes(attributes)); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 1067c2416085..ccfe7f50055a 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -71,6 +71,7 @@ import Migration0054 from "./Migrations/054_ProjectionThreadsAutoSettleDisabledA import Migration0055 from "./Migrations/055_OrchestrationV2.ts"; import Migration0056 from "./Migrations/056_RemoveRedundantProjectionIndexes.ts"; import Migration0057 from "./Migrations/057_ScheduledTaskWebhooks.ts"; +import Migration0058 from "./Migrations/058_WebhookRelayDeliveries.ts"; /** * Migration loader with all migrations defined inline. @@ -142,6 +143,7 @@ export const migrationEntries = [ [55, "OrchestrationV2", Migration0055], [56, "RemoveRedundantProjectionIndexes", Migration0056], [57, "ScheduledTaskWebhooks", Migration0057], + [58, "WebhookRelayDeliveries", Migration0058], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts index e3839e3f0d17..6ee7c94cf1ba 100644 --- a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts +++ b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts @@ -13,7 +13,7 @@ layer("055_OrchestrationV2", (it) => { Effect.sync(() => { assert.deepStrictEqual( migrationEntries.map(([id]) => id), - Array.from({ length: 57 }, (_, index) => index + 1), + Array.from({ length: 58 }, (_, index) => index + 1), ); }), ); @@ -29,6 +29,7 @@ layer("055_OrchestrationV2", (it) => { [55, "OrchestrationV2"], [56, "RemoveRedundantProjectionIndexes"], [57, "ScheduledTaskWebhooks"], + [58, "WebhookRelayDeliveries"], ]); assert.deepStrictEqual(yield* runMigrations(), []); @@ -52,6 +53,7 @@ layer("055_OrchestrationV2", (it) => { { migration_id: 55, name: "OrchestrationV2" }, { migration_id: 56, name: "RemoveRedundantProjectionIndexes" }, { migration_id: 57, name: "ScheduledTaskWebhooks" }, + { migration_id: 58, name: "WebhookRelayDeliveries" }, ]); const tables = yield* sql<{ readonly name: string }>` diff --git a/apps/server/src/persistence/Migrations/058_WebhookRelayDeliveries.ts b/apps/server/src/persistence/Migrations/058_WebhookRelayDeliveries.ts new file mode 100644 index 000000000000..6e1c264c1647 --- /dev/null +++ b/apps/server/src/persistence/Migrations/058_WebhookRelayDeliveries.ts @@ -0,0 +1,22 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + // Relay delivery ids already handled, kept longer than the relay holds a + // request so a replay can never run twice. The delivery log keeps only the + // newest 50 rows per task, which is too short for that. + yield* sql` + CREATE TABLE IF NOT EXISTS scheduled_task_webhook_relay_deliveries ( + relay_delivery_id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + seen_at TEXT NOT NULL + ) + `; + + yield* sql` + CREATE INDEX IF NOT EXISTS idx_scheduled_task_webhook_relay_deliveries_seen + ON scheduled_task_webhook_relay_deliveries(seen_at) + `; +}); diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index e872cb553bfa..fef7bd808aa1 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -38,6 +38,7 @@ describe("V2 preview upgrade", () => { [54, "ProjectionThreadsAutoSettleDisabledAt"], [56, "RemoveRedundantProjectionIndexes"], [57, "ScheduledTaskWebhooks"], + [58, "WebhookRelayDeliveries"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -118,6 +119,7 @@ describe("V2 preview upgrade", () => { [54, "ProjectionThreadsAutoSettleDisabledAt"], [56, "RemoveRedundantProjectionIndexes"], [57, "ScheduledTaskWebhooks"], + [58, "WebhookRelayDeliveries"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/relay/HeldHooksWaker.ts b/apps/server/src/relay/HeldHooksWaker.ts new file mode 100644 index 000000000000..9e0dae4ce624 --- /dev/null +++ b/apps/server/src/relay/HeldHooksWaker.ts @@ -0,0 +1,52 @@ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schedule from "effect/Schedule"; +import * as Stream from "effect/Stream"; + +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as CloudManagedEndpointRuntime from "../cloud/ManagedEndpointRuntime.ts"; +import { readHoldWebhooksWhileOffline, readRelayConnection } from "../cloud/config.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import { makeRelayEnvironmentClient } from "./relayEnvironmentClient.ts"; + +/** + * Tells T3 Connect this environment is reachable again, so the relay delivers + * the webhook requests it held while we were offline now rather than at its + * next backoff step. Nothing happens unless the environment opted in. + */ +const wakeHeldHooks = Effect.fn("HeldHooksWaker.wake")(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + if (!(yield* readHoldWebhooksWhileOffline(secrets))) return false; + const connection = yield* readRelayConnection(secrets); + if (connection === null) return false; + const environmentId = yield* (yield* ServerEnvironment.ServerEnvironment).getEnvironmentId; + const client = yield* makeRelayEnvironmentClient(connection); + const { pending } = yield* client.server.wakeHeldHooks({ params: { environmentId } }); + yield* Effect.annotateCurrentSpan({ "relay.inbox.pending": pending }); + return pending; +}); + +/** Wakes held webhooks each time the managed tunnel connects. */ +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const runtime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; + const wake = wakeHeldHooks().pipe( + Effect.timeout("10 seconds"), + // The relay retries on its own schedule too, so a few tries are enough. + Effect.retry({ schedule: Schedule.exponential("2 seconds"), times: 3 }), + Effect.tap((pending) => + pending ? Effect.logInfo("T3 Connect is delivering held webhook requests") : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning("Could not ask T3 Connect to deliver held webhook requests", { cause }), + ), + ); + yield* runtime.tunnelConnected.pipe( + // cloudflared registers several connections per (re)connect within a + // few seconds; they are one wake. + Stream.debounce("3 seconds"), + Stream.runForEach(() => wake), + Effect.forkScoped, + ); + }), +); diff --git a/apps/server/src/relay/relayEnvironmentClient.ts b/apps/server/src/relay/relayEnvironmentClient.ts new file mode 100644 index 000000000000..a63741c240dc --- /dev/null +++ b/apps/server/src/relay/relayEnvironmentClient.ts @@ -0,0 +1,21 @@ +import { RelayApi } from "@t3tools/contracts/relay"; +import * as Effect from "effect/Effect"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientRequest from "effect/http/HttpClientRequest"; +import * as HttpApiClient from "effect/http-api/HttpApiClient"; + +/** + * A typed RelayApi client that authenticates as this environment, for the + * environment-credential endpoints (link preferences, held webhooks). + */ +export const makeRelayEnvironmentClient = (connection: { + readonly url: string; + readonly environmentCredential: string; +}) => + HttpApiClient.make(RelayApi, { + baseUrl: connection.url, + transformClient: HttpClient.mapRequest( + HttpClientRequest.setHeader("authorization", `Bearer ${connection.environmentCredential}`), + ), + }).pipe(Effect.provide(FetchHttpClient.layer)); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index dca44f1e4112..f784484ad19b 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -1,6 +1,7 @@ import { CommandId, MessageId, + PROVIDER_SEND_TURN_MAX_INPUT_CHARS, ScheduledTask, ScheduledTaskError, ScheduledTaskId, @@ -27,6 +28,8 @@ import * as Crypto from "effect/Crypto"; import * as Data from "effect/Data"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Duration from "effect/Duration"; +import * as Metric from "effect/Metric"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; @@ -38,10 +41,16 @@ import * as Stream from "effect/Stream"; import * as SqlClient from "effect/sql/SqlClient"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; +import * as Metrics from "../observability/Metrics.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts"; -import { renderWebhookPrompt, type WebhookRequest } from "./webhookTemplate.ts"; +import { + redactHeaders, + redactQuery, + renderWebhookPrompt, + type WebhookRequest, +} from "./webhookTemplate.ts"; import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts"; /** Path prefix of the environment route that receives webhook requests. */ @@ -50,21 +59,44 @@ export const WEBHOOK_ROUTE_PREFIX = "/api/hooks"; const WEBHOOK_DELIVERY_RETENTION = 50; /** Body text kept in the delivery log. Larger bodies are cut and flagged. */ const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024; +/** Rendered prompt text kept in the delivery log. */ +const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024; /** Deliveries one task may hold at once, running or waiting their turn. */ const WEBHOOK_MAX_QUEUED_PER_TASK = 20; /** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */ const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60; -/** Where a webhook task's public URL points. `relayUrl` is null when the environment is not linked to T3 Connect. */ +/** + * Where a webhook task's public URL points: `${relayHookBaseUrl}/${taskId}/${token}`. + * Null when the environment has no managed tunnel on T3 Connect; clients then show the path. + */ interface WebhookOrigin { - readonly environmentId: string; - readonly relayUrl: string | null; + readonly relayHookBaseUrl: string | null; +} + +const ENDPOINT_KEY = /^[0-9a-f]{16}$/; + +/** + * The relay's hook URL prefix for this environment. The relay finds the + * environment by its managed tunnel's key (the tunnel name's last segment), + * so the URL never reveals the environment id. + */ +export function relayHookBaseUrl(input: { + readonly relayUrl: string; + readonly tunnelName: string | undefined; +}): string | null { + const endpointKey = input.tunnelName?.split("-").at(-1); + const relayUrl = input.relayUrl.replace(/\/+$/, ""); + if (relayUrl === "" || endpointKey === undefined || !ENDPOINT_KEY.test(endpointKey)) { + return null; + } + return `${relayUrl}/v1/hooks/${endpointKey}`; } export class ScheduledTaskWebhookOrigin extends Context.Reference>( "t3/scheduledTasks/ScheduledTaskWebhookOrigin", { - defaultValue: () => Effect.succeed({ environmentId: "local", relayUrl: null }), + defaultValue: () => Effect.succeed({ relayHookBaseUrl: null }), }, ) {} @@ -83,15 +115,45 @@ export interface WebhookTriggerRequest extends WebhookRequest { readonly hookId: string; readonly token: string; readonly body: Uint8Array; + /** Set by T3 Connect; the same id is never dispatched twice. */ + readonly relayDeliveryId?: string; + /** When the relay received a held request; defaults to now. */ + readonly receivedAt?: string; } /** What the HTTP route should answer. `not_found` covers unknown hooks and wrong tokens alike. */ +/** + * What happened to one webhook request, as recorded in metrics and spans. + * Sent back to the relay as `x-t3-hook-outcome`; never request contents. + */ +export type WebhookDeliveryOutcome = + | "accepted" + | "duplicate" + | "prompt_too_long" + | "queue_full" + | "rate_limited" + | "disabled" + | "rejected_signature" + | "expired" + | "not_found" + | "error"; + export type WebhookTriggerResult = - | { readonly _tag: "accepted"; readonly deliveryId: ScheduledTaskWebhookDeliveryId } + | { + readonly _tag: "accepted"; + readonly deliveryId: ScheduledTaskWebhookDeliveryId; + /** A 202 covers more than a started run; this says which. */ + readonly outcome: "accepted" | "duplicate" | "prompt_too_long"; + } | { readonly _tag: "not_found" } | { readonly _tag: "rejected_signature" } | { readonly _tag: "disabled" } - | { readonly _tag: "rate_limited" }; + | { + readonly _tag: "rate_limited"; + /** Too many requests to this hook, or too many runs already waiting. */ + readonly outcome: "rate_limited" | "queue_full"; + } + | { readonly _tag: "expired" }; const decodeTask = Schema.decodeUnknownEffect(ScheduledTask); const decodeTaskId = Schema.decodeUnknownOption(ScheduledTaskId); @@ -228,19 +290,6 @@ function errorMessage(error: unknown): string { return String(error); } -/** Headers kept out of the delivery log because they commonly carry credentials. */ -const REDACTED_HEADER = - /^(authorization|proxy-authorization|cookie|set-cookie)$|token|secret|signature|key|password|auth/i; - -function redactHeaders(headers: Readonly>): Record { - return Object.fromEntries( - Object.entries(headers).map(([name, value]) => [ - name, - REDACTED_HEADER.test(name) ? "[redacted]" : value, - ]), - ); -} - function webhookPath(taskId: string, token: string): string { return `${WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(taskId)}/${token}`; } @@ -250,13 +299,10 @@ function webhookEndpoint( origin: WebhookOrigin | null, ): ScheduledTask["webhook"] { if (row.webhook_token === null) return undefined; - const relayUrl = origin?.relayUrl?.replace(/\/+$/, "") ?? null; + const base = origin?.relayHookBaseUrl ?? null; return { path: webhookPath(row.task_id, row.webhook_token), - url: - relayUrl === null || origin === null - ? null - : `${relayUrl}/v1/hooks/${encodeURIComponent(origin.environmentId)}/${encodeURIComponent(row.task_id)}/${row.webhook_token}`, + url: base === null ? null : `${base}/${encodeURIComponent(row.task_id)}/${row.webhook_token}`, hasSecret: row.webhook_secret !== null, }; } @@ -985,6 +1031,7 @@ export const layer = Layer.effect( encoding: input.schedule.signature.encoding, prefix: input.schedule.signature.prefix, }, + maxDeliveryAgeMinutes: input.schedule.maxDeliveryAgeMinutes ?? null, } : input.schedule; const webhook = @@ -1197,6 +1244,7 @@ export const layer = Layer.effect( readonly signatureVerified: boolean; readonly missing: ReadonlyArray; readonly renderedPrompt: string | null; + readonly error?: string; }) => { const truncated = input.request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT; const loggedBody = truncated @@ -1215,12 +1263,22 @@ export const layer = Layer.effect( ) SELECT ${input.id}, ${input.taskId}, ${input.receivedAt}, ${input.request.method}, - ${input.request.query}, ${encodeHeadersJson(redactHeaders(input.request.headers))}, + ${redactQuery(input.request.query)}, + ${encodeHeadersJson(redactHeaders(input.request.headers))}, ${loggedBody}, ${input.request.body.byteLength}, ${truncated ? 1 : 0}, ${input.outcome}, ${input.signatureVerified ? 1 : 0}, ${encodeMissingFieldsJson(input.missing)}, - ${input.renderedPrompt}, NULL + ${input.renderedPrompt?.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT) ?? null}, + ${input.error ?? null} WHERE EXISTS (SELECT 1 FROM scheduled_tasks WHERE task_id = ${input.taskId}) + -- A held request retried after a rate limit reuses its relay + -- delivery id; the newer attempt replaces the logged one. + ON CONFLICT (delivery_id) DO UPDATE SET + outcome = excluded.outcome, + signature_verified = excluded.signature_verified, + missing_fields_json = excluded.missing_fields_json, + rendered_prompt = excluded.rendered_prompt, + error = excluded.error `; yield* sql` DELETE FROM scheduled_task_webhook_deliveries @@ -1295,10 +1353,55 @@ export const layer = Layer.effect( // run; scoped to the service so shutdown interrupts it. const serviceScope = yield* Effect.scope; + /** + * Records one handled request on the span and in metrics. `outcome` is + * finer than the result tag: it separates a full queue and an oversized + * prompt from the rest, which is what an operator needs to act on. + */ + const observeDelivery = ( + request: WebhookTriggerRequest, + outcome: WebhookDeliveryOutcome, + receivedAt: DateTime.DateTime | undefined, + now: DateTime.DateTime | undefined, + ) => + Effect.gen(function* () { + const source = request.relayDeliveryId === undefined ? "direct" : "relay"; + yield* Effect.annotateCurrentSpan({ + "scheduled_task.webhook.outcome": outcome, + "scheduled_task.webhook.source": source, + }); + yield* Metrics.increment(Metrics.webhookDeliveriesTotal, { outcome, source }); + if (request.receivedAt !== undefined && receivedAt !== undefined && now !== undefined) { + const heldMs = Math.max( + 0, + DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt), + ); + yield* Effect.annotateCurrentSpan({ "scheduled_task.webhook.held_ms": heldMs }); + yield* Metric.update(Metrics.webhookHeldDelay, Duration.millis(heldMs)); + } + }); + const triggerWebhook: ScheduledTaskService["Service"]["triggerWebhook"] = (request) => + triggerWebhookUnobserved(request).pipe( + Effect.tapError(() => observeDelivery(request, "error", undefined, undefined)), + Metrics.withMetrics({ timer: Metrics.webhookDeliveryDuration }), + Effect.withSpan("ScheduledTaskService.triggerWebhook", { + attributes: { + "scheduled_task.webhook.method": request.method, + "scheduled_task.webhook.body_bytes": request.body.byteLength, + "scheduled_task.webhook.source": + request.relayDeliveryId === undefined ? "direct" : "relay", + }, + }), + ); + + const triggerWebhookUnobserved = (request: WebhookTriggerRequest) => Effect.gen(function* () { const taskId = decodeTaskId(request.hookId); - if (Option.isNone(taskId)) return { _tag: "not_found" as const }; + const notFound = observeDelivery(request, "not_found", undefined, undefined).pipe( + Effect.as({ _tag: "not_found" as const }), + ); + if (Option.isNone(taskId)) return yield* notFound; const rows = yield* getRows(taskId.value).pipe( Effect.mapError((cause) => taskError("Could not load schedule task.", { taskId: taskId.value, cause }), @@ -1312,110 +1415,227 @@ export const layer = Layer.effect( row.webhook_token === null || !constantTimeEquals(request.token, row.webhook_token) ) { - return { _tag: "not_found" as const }; + return yield* notFound; } const task = yield* decodeRow(row); - if (task.schedule.type !== "webhook") return { _tag: "not_found" as const }; + yield* Effect.annotateCurrentSpan({ "scheduled_task.id": task.id }); + const schedule = task.schedule; + if (schedule.type !== "webhook") return yield* notFound; - const receivedAt = yield* localNow; - const deliveryUuid = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => taskError("Could not generate delivery id.", { cause })), + const now = yield* localNow; + // Anyone can reach the tunnel directly and set the relay's header, so + // a receive time is never later than now: a future one would pin the + // delivery in the log and slip past the task's max age. + const receivedAt = + request.receivedAt === undefined + ? now + : DateTime.min( + Option.getOrElse(DateTime.make(request.receivedAt), () => now), + now, + ); + const observe = (outcome: WebhookDeliveryOutcome) => + observeDelivery(request, outcome, receivedAt, now); + const deliveryId = ScheduledTaskWebhookDeliveryId.make( + request.relayDeliveryId === undefined + ? `delivery:${yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => taskError("Could not generate delivery id.", { cause })), + )}` + : `delivery:relay:${request.relayDeliveryId}`, ); - const deliveryId = ScheduledTaskWebhookDeliveryId.make(`delivery:${deliveryUuid}`); - const log = ( - outcome: ScheduledTaskWebhookDeliveryOutcome, - details: { - readonly signatureVerified?: boolean; - readonly missing?: ReadonlyArray; - readonly renderedPrompt?: string; - } = {}, - ) => - recordDelivery({ - id: deliveryId, - taskId: task.id, - receivedAt: iso(receivedAt), - request, - outcome, - signatureVerified: details.signatureVerified ?? false, - missing: details.missing ?? [], - renderedPrompt: details.renderedPrompt ?? null, - }); + // A held request may already have reached this environment directly + // before a timeout; it runs once. Claimed in its own table, kept longer + // than the relay holds a request, because the delivery log is trimmed. + // A rate-limited delivery stays held on the relay, so it must give up + // its claim or the next pass would treat it as already delivered. + const releaseClaim = (result: A) => + request.relayDeliveryId === undefined + ? Effect.succeed(result) + : sql` + DELETE FROM scheduled_task_webhook_relay_deliveries + WHERE relay_delivery_id = ${request.relayDeliveryId} + `.pipe(Effect.ignore, Effect.as(result)); + // From the claim until the run is forked nothing may interrupt: a + // request dropped in between (the relay hangs up after its timeout) + // would leave a claimed delivery that never runs, or a queue slot + // that is never released. Everything in here is local and quick. + return yield* Effect.uninterruptible( + Effect.gen(function* () { + if (request.relayDeliveryId !== undefined) { + const claimed = yield* sql<{ relay_delivery_id: string }>` + INSERT INTO scheduled_task_webhook_relay_deliveries + (relay_delivery_id, task_id, seen_at) + VALUES (${request.relayDeliveryId}, ${task.id}, ${iso(now)}) + ON CONFLICT (relay_delivery_id) DO NOTHING + RETURNING relay_delivery_id + `.pipe( + Effect.mapError((cause) => + taskError("Could not record webhook delivery.", { taskId: task.id, cause }), + ), + ); + if (claimed.length === 0) { + // A held request this environment already ran: accepted, not run twice. + yield* observe("duplicate"); + return { _tag: "accepted" as const, deliveryId, outcome: "duplicate" as const }; + } + // Older claims can no longer be replayed by the relay. + yield* sql` + DELETE FROM scheduled_task_webhook_relay_deliveries + WHERE seen_at < ${iso(DateTime.subtract(now, { hours: 48 }))} + `.pipe(Effect.ignore); + } + const log = ( + outcome: ScheduledTaskWebhookDeliveryOutcome, + details: { + readonly signatureVerified?: boolean; + readonly missing?: ReadonlyArray; + readonly renderedPrompt?: string; + readonly error?: string; + } = {}, + ) => + recordDelivery({ + id: deliveryId, + taskId: task.id, + receivedAt: iso(receivedAt), + request, + outcome, + signatureVerified: details.signatureVerified ?? false, + missing: details.missing ?? [], + renderedPrompt: details.renderedPrompt ?? null, + ...(details.error === undefined ? {} : { error: details.error }), + }); + + // Only the first rejected request in a window is logged, so a flood + // cannot write rows or push the real deliveries out of the log. + const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(now)); + if (slot !== "allowed") { + if (slot === "first_rejected") yield* log("rate_limited"); + yield* observe("rate_limited"); + return yield* releaseClaim({ + _tag: "rate_limited" as const, + outcome: "rate_limited" as const, + }); + } + if (!task.enabled) { + yield* log("disabled"); + yield* observe("disabled"); + return { _tag: "disabled" as const }; + } + const signature = schedule.signature; + if (signature !== null) { + const verified = + row.webhook_secret !== null && + verifyWebhookSignature({ + signature, + secret: row.webhook_secret, + headers: request.headers, + body: request.body, + }); + if (!verified) { + yield* log("rejected_signature"); + yield* observe("rejected_signature"); + return { _tag: "rejected_signature" as const }; + } + } + const maxAgeMinutes = schedule.maxDeliveryAgeMinutes ?? null; + if ( + maxAgeMinutes !== null && + DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt) > + maxAgeMinutes * 60_000 + ) { + yield* log("expired", { signatureVerified: signature !== null }); + yield* observe("expired"); + return { _tag: "expired" as const }; + } - // Only the first rejected request in a window is logged, so a flood - // cannot write rows or push the real deliveries out of the log. - const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(receivedAt)); - if (slot !== "allowed") { - if (slot === "first_rejected") yield* log("rate_limited"); - return { _tag: "rate_limited" as const }; - } - if (!task.enabled) { - yield* log("disabled"); - return { _tag: "disabled" as const }; - } - const signature = task.schedule.signature; - if (signature !== null) { - const verified = - row.webhook_secret !== null && - verifyWebhookSignature({ - signature, - secret: row.webhook_secret, - headers: request.headers, - body: request.body, + const rendered = renderWebhookPrompt(task.prompt, request); + // A provider refuses a turn this long, so it is not started. The + // delivery is not retryable, so the claim is kept. Providers trim + // the prompt before checking, so whitespace around it is free. + if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) { + yield* log("dispatch_failed", { + signatureVerified: signature !== null, + missing: rendered.missing, + renderedPrompt: rendered.prompt, + error: "The filled-in prompt is too long.", + }); + yield* observe("prompt_too_long"); + return { _tag: "accepted" as const, deliveryId, outcome: "prompt_too_long" as const }; + } + // Bound the deliveries one task holds, so steady traffic to a stuck + // task cannot pile up parked fibers. A refused request is not logged, + // so it cannot push real deliveries out of the log. + const queueKey = `${task.id}\u0000${task.createdAt}`; + const queued = yield* Ref.modify(webhookQueued, (counts) => { + const count = counts.get(queueKey) ?? 0; + return count >= WEBHOOK_MAX_QUEUED_PER_TASK + ? ([false, counts] as const) + : ([true, new Map(counts).set(queueKey, count + 1)] as const); }); - if (!verified) { - yield* log("rejected_signature"); - return { _tag: "rejected_signature" as const }; - } - } - - const rendered = renderWebhookPrompt(task.prompt, request); - // Bound the deliveries one task holds, so steady traffic to a stuck - // task cannot pile up parked fibers. A refused request is not logged, - // so it cannot push real deliveries out of the log. - const queueKey = `${task.id}\u0000${task.createdAt}`; - const queued = yield* Ref.modify(webhookQueued, (counts) => { - const count = counts.get(queueKey) ?? 0; - return count >= WEBHOOK_MAX_QUEUED_PER_TASK - ? ([false, counts] as const) - : ([true, new Map(counts).set(queueKey, count + 1)] as const); - }); - if (!queued) return { _tag: "rate_limited" as const }; - // Entries leave the map when their count reaches zero, so a deleted - // task's key does not linger once its last delivery finishes. - const release = Ref.update(webhookQueued, (counts) => { - const next = new Map(counts); - const count = (next.get(queueKey) ?? 1) - 1; - if (count <= 0) next.delete(queueKey); - else next.set(queueKey, count); - return next; - }); - yield* log("accepted", { - signatureVerified: signature !== null, - missing: rendered.missing, - renderedPrompt: rendered.prompt, - }).pipe(Effect.onError(() => release)); - const permit = yield* webhookPermit(task.id); - yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe( - Effect.flatMap((completed) => - completed.lastRunStatus === "failed" - ? markDeliveryFailed(deliveryId, "The run failed to start.") - : Effect.void, - ), - Effect.catchTag("WebhookDeliverySkipped", (skipped) => - markDeliveryFailed(deliveryId, skipped.reason), - ), - // The log is readable over RPC, so it gets a fixed reason; the - // cause, which can carry request data, stays in the server log. - Effect.catchCause((cause) => - Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe( - Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), - ), - ), - permit.withPermits(1), - Effect.ensuring(release), - Effect.forkIn(serviceScope), + if (!queued) { + // The task is busy with WEBHOOK_MAX_QUEUED_PER_TASK deliveries already. + yield* observe("queue_full"); + return yield* releaseClaim({ + _tag: "rate_limited" as const, + outcome: "queue_full" as const, + }); + } + // Entries leave the map when their count reaches zero, so a deleted + // task's key does not linger once its last delivery finishes. + const release = Ref.update(webhookQueued, (counts) => { + const next = new Map(counts); + const count = (next.get(queueKey) ?? 1) - 1; + if (count <= 0) next.delete(queueKey); + else next.set(queueKey, count); + return next; + }); + yield* log("accepted", { + signatureVerified: signature !== null, + missing: rendered.missing, + renderedPrompt: rendered.prompt, + }).pipe(Effect.onError(() => release)); + yield* observe("accepted"); + const permit = yield* webhookPermit(task.id); + const runOutcome = (outcome: "started" | "skipped" | "failed") => + Effect.all([ + Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }), + Metrics.increment(Metrics.webhookRunsTotal, { outcome }), + ]); + yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe( + Effect.flatMap((completed) => + completed.lastRunStatus === "failed" + ? runOutcome("failed").pipe( + Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), + ) + : runOutcome("started"), + ), + Effect.catchTag("WebhookDeliverySkipped", (skipped) => + runOutcome("skipped").pipe( + Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)), + ), + ), + // The log is readable over RPC, so it gets a fixed reason; the + // cause, which can carry request data, stays in the server log. + Effect.catchCause((cause) => + Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe( + Effect.andThen(runOutcome("failed")), + Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), + ), + ), + permit.withPermits(1), + // Its own trace: the request that triggered it has already been answered. + Effect.withSpan("ScheduledTaskService.runWebhookDelivery", { + root: true, + attributes: { + "scheduled_task.id": task.id, + "scheduled_task.webhook.delivery_id": deliveryId, + }, + }), + Effect.ensuring(release), + Effect.forkIn(serviceScope), + ); + return { _tag: "accepted" as const, deliveryId, outcome: "accepted" as const }; + }), ); - return { _tag: "accepted" as const, deliveryId }; }); return ScheduledTaskService.of({ diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index 7108d7b804f5..5fdaf45dcf3c 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -3,10 +3,14 @@ import * as NodeCrypto from "node:crypto"; import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto"; import { assert, it } from "@effect/vitest"; import { ScheduledTaskUpsertInput } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Metric from "effect/Metric"; import * as Queue from "effect/Queue"; +import * as EffectScheduler from "effect/Scheduler"; import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; @@ -63,7 +67,7 @@ const withService = ( readonly service: ScheduledTaskService.ScheduledTaskService["Service"]; readonly launches: Queue.Queue; }) => Effect.Effect, - options: { readonly gate?: Deferred.Deferred } = {}, + options: { readonly gate?: Deferred.Deferred; readonly relayHookBaseUrl?: string } = {}, ) => Effect.gen(function* () { const launches = yield* Queue.unbounded(); @@ -78,6 +82,10 @@ const withService = ( ), }), Layer.mock(ThreadManagementService.ThreadManagementService)({}), + Layer.succeed( + ScheduledTaskService.ScheduledTaskWebhookOrigin, + Effect.succeed({ relayHookBaseUrl: options.relayHookBaseUrl ?? null }), + ), ); return yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; @@ -120,6 +128,35 @@ it.effect("dispatches exactly the rendered prompt and logs the delivery", () => ), ); +it("builds the relay hook URL from the managed tunnel's key, never the environment id", () => { + const relayUrl = "https://relay.example.com/"; + assert.equal( + ScheduledTaskService.relayHookBaseUrl({ + relayUrl, + tunnelName: "t3coderelay-managedendpoint-dev-julius-0123456789abcdef", + }), + "https://relay.example.com/v1/hooks/0123456789abcdef", + ); + for (const tunnelName of [undefined, "t3coderelay-managedendpoint", "x-0123456789ABCDEF"]) { + assert.isNull(ScheduledTaskService.relayHookBaseUrl({ relayUrl, tunnelName })); + } +}); + +it.effect("gives webhook tasks a relay URL when the environment has a managed tunnel", () => + withService( + ({ service }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + const token = task.webhook!.path.split("/").at(-1); + assert.equal( + task.webhook?.url, + `https://relay.example.com/v1/hooks/0123456789abcdef/scheduled-task%3Ahook/${token}`, + ); + }), + { relayHookBaseUrl: "https://relay.example.com/v1/hooks/0123456789abcdef" }, + ), +); + it.effect("answers not found for a wrong token or unknown hook without logging", () => withService(({ service }) => Effect.gen(function* () { @@ -168,6 +205,7 @@ it.effect("checks the configured signature and keeps the secret write-only", () assert.deepEqual(task.schedule, { type: "webhook", signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" }, + maxDeliveryAgeMinutes: null, }); assert.isTrue(task.webhook?.hasSecret); @@ -298,12 +336,13 @@ it.effect("keeps the newest 50 deliveries when they share a timestamp", () => ), ); -it.effect("keeps credential headers out of the delivery log", () => +it.effect("keeps credential headers and query values out of the delivery log", () => withService(({ service }) => Effect.gen(function* () { const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); yield* service.triggerWebhook( requestFor(task, { + query: "page=2&api_key=k", headers: { "content-type": "application/json", authorization: "Bearer sender-token", @@ -317,6 +356,7 @@ it.effect("keeps credential headers out of the delivery log", () => id: task.id, deliveryId: summary!.id, }); + assert.equal(delivery.query, "page=2&api_key=[redacted]"); assert.equal(delivery.headers.authorization, "[redacted]"); assert.equal(delivery.headers["x-webhook-key"], "[redacted]"); assert.equal(delivery.headers["x-github-event"], "push"); @@ -454,6 +494,209 @@ it.effect("logs a body's first 64 KiB by bytes, not characters", () => ), ); +it.effect("counts the prompt as the provider does, without surrounding whitespace", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput({ prompt: "{{body.text}}" })); + // Over the limit as sent, within it once the padding is trimmed. + const text = `{"text":"${" ".repeat(1_000)}${"x".repeat(119_990)}${"\\n".repeat(1_000)}"}`; + const result = yield* service.triggerWebhook( + requestFor(task, { body: new TextEncoder().encode(text), bodyText: text }), + ); + assert.equal(result._tag, "accepted"); + yield* Queue.take(launches); + }), + ), +); + +it.effect("does not start a run when the filled-in prompt is too long", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput({ prompt: "{{body}}" })); + const text = "x".repeat(200_000); + const body = new TextEncoder().encode(text); + const result = yield* service.triggerWebhook(requestFor(task, { body, bodyText: text })); + assert.equal(result._tag, "accepted"); + assert.equal(yield* Queue.size(launches), 0); + const { delivery } = yield* service.getWebhookDelivery({ + id: task.id, + deliveryId: result._tag === "accepted" ? result.deliveryId : ("" as never), + }); + assert.equal(delivery.outcome, "dispatch_failed"); + assert.equal(delivery.error, "The filled-in prompt is too long."); + assert.equal(delivery.renderedPrompt?.length, 64 * 1024); + // The queue slot was never taken: a normal delivery still runs. + const ok = yield* service.triggerWebhook(requestFor(task)); + assert.equal(ok._tag, "accepted"); + yield* Queue.take(launches); + }), + ), +); + +it.effect("a held request already delivered directly runs only once", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + const direct = yield* service.triggerWebhook( + requestFor(task, { relayDeliveryId: "relay-1" }), + ); + const replayed = yield* service.triggerWebhook( + requestFor(task, { relayDeliveryId: "relay-1", receivedAt: "2026-10-04T10:00:00.000Z" }), + ); + assert.equal(direct._tag, "accepted"); + // Same delivery, answered the same way, but recorded as a duplicate. + assert.deepEqual(replayed, { ...direct, outcome: "duplicate" } as typeof replayed); + yield* Queue.take(launches); + const logged = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + assert.equal(logged.length, 1); + }), + ), +); + +it.effect("a held request whose sender hung up mid-request still runs", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + // The relay times out and the request fiber is interrupted. A small + // operation budget makes the request yield often, so stepping the + // interrupt one yield later each time lands it at every point in the + // request (one takes about 40 yields), then the relay retries. + for (let step = 0; step < 60; step++) { + const request = requestFor(task, { relayDeliveryId: `hung-up-${step}` }); + const fiber = yield* service + .triggerWebhook(request) + .pipe(Effect.provideService(EffectScheduler.MaxOpsBeforeYield, 8), Effect.forkChild); + for (let yields = 0; yields < step; yields++) yield* Effect.yieldNow; + yield* Fiber.interrupt(fiber); + const retried = yield* service.triggerWebhook(request); + assert.equal(retried._tag, "accepted"); + const deliveryId = retried._tag === "accepted" ? retried.deliveryId : undefined; + // Logged and run exactly once, whether or not the first attempt got through. + yield* service.getWebhookDelivery({ id: task.id, deliveryId: deliveryId! }); + const launch = yield* Queue.take(launches); + assert.include(launch.commandId, `hung-up-${step}`); + // Keep each step in a fresh rate-limit window. + yield* TestClock.adjust("61 seconds"); + } + assert.equal(yield* Queue.size(launches), 0); + }), + ), +); + +it.effect("a held request runs once even after the log has trimmed it", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + const first = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" })); + assert.equal(first._tag, "accepted"); + yield* Queue.take(launches); + // Push the original row out of the 50-row delivery log. + const paused = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); + yield* Effect.forEach(Array.from({ length: 55 }), () => + service.triggerWebhook(requestFor(paused.task)), + ); + yield* service.upsert(yield* webhookTaskInput()); + const replay = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" })); + assert.equal(replay._tag, "accepted"); + assert.equal(yield* Queue.size(launches), 0); + }), + ), +); + +it.effect("a rate-limited held request can run on a later pass", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); + // Spend the task's 60-a-minute budget. + yield* Effect.forEach(Array.from({ length: 60 }), () => + service.triggerWebhook(requestFor(task)), + ); + const limited = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "later" })); + assert.equal(limited._tag, "rate_limited"); + yield* service.upsert(yield* webhookTaskInput()); + yield* TestClock.adjust("61 seconds"); + const retried = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "later" })); + assert.equal(retried._tag, "accepted"); + yield* Queue.take(launches); + }), + ), +); + +it.effect("logs a held request at the time the relay received it", () => + withService(({ service }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); + const receivedAt = DateTime.formatIso(DateTime.subtract(yield* DateTime.now, { minutes: 5 })); + yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "relay-2", receivedAt })); + const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + assert.equal(delivery?.receivedAt, receivedAt); + }), + ), +); + +it.effect("a receive time in the future counts as now", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ schedule: { type: "webhook", maxDeliveryAgeMinutes: 30 } }), + ); + const now = yield* DateTime.now; + const result = yield* service.triggerWebhook( + requestFor(task, { + relayDeliveryId: "future", + receivedAt: DateTime.formatIso(DateTime.add(now, { days: 365 })), + }), + ); + assert.equal(result._tag, "accepted"); + yield* Queue.take(launches); + const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + assert.equal(delivery?.receivedAt, DateTime.formatIso(now)); + }), + ), +); + +it.effect("skips a held request older than the task's max age", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ schedule: { type: "webhook", maxDeliveryAgeMinutes: 30 } }), + ); + const now = yield* DateTime.now; + const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 })); + const fresh = DateTime.formatIso(DateTime.subtract(now, { minutes: 5 })); + const tooOld = yield* service.triggerWebhook( + requestFor(task, { relayDeliveryId: "old", receivedAt: old }), + ); + assert.equal(tooOld._tag, "expired"); + assert.equal(yield* Queue.size(launches), 0); + const ok = yield* service.triggerWebhook( + requestFor(task, { relayDeliveryId: "fresh", receivedAt: fresh }), + ); + assert.equal(ok._tag, "accepted"); + const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map( + (delivery) => delivery.outcome, + ); + assert.includeMembers(outcomes, ["expired", "accepted"]); + }), + ), +); + +it.effect("runs a held request of any age when no max age is set", () => + withService(({ service }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + const now = yield* DateTime.now; + const result = yield* service.triggerWebhook( + requestFor(task, { + relayDeliveryId: "ancient", + receivedAt: DateTime.formatIso(DateTime.subtract(now, { hours: 23 })), + }), + ); + assert.equal(result._tag, "accepted"); + }), + ), +); + it.effect("deleting a task removes its delivery log", () => withService(({ service }) => Effect.gen(function* () { @@ -464,3 +707,66 @@ it.effect("deleting a task removes its delivery log", () => }), ), ); + +const signatureFor = (secret: string) => + `sha256=${NodeCrypto.createHmac("sha256", secret).update(pullRequestBody).digest("hex")}`; + +/** Count recorded by `t3_webhook_deliveries_total` for one outcome and source. */ +const deliveriesCounted = (outcome: string, source: "relay" | "direct") => + Metric.snapshot.pipe( + Effect.map((snapshots) => { + const found = snapshots.find( + (snapshot) => + snapshot.id === "t3_webhook_deliveries_total" && + snapshot.attributes?.outcome === outcome && + snapshot.attributes?.source === source, + ); + return found?.type === "Counter" ? Number(found.state.count) : 0; + }), + ); + +it.effect("counts each handled request by what happened to it", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + signature: { + header: "x-hub-signature-256", + encoding: "hex", + prefix: "sha256=", + secret: "github-secret", + }, + }, + }), + ); + const before = { + accepted: yield* deliveriesCounted("accepted", "direct"), + rejected: yield* deliveriesCounted("rejected_signature", "direct"), + notFound: yield* deliveriesCounted("not_found", "direct"), + relayAccepted: yield* deliveriesCounted("accepted", "relay"), + duplicate: yield* deliveriesCounted("duplicate", "relay"), + }; + const signed = { + "content-type": "application/json", + "x-hub-signature-256": signatureFor("github-secret"), + }; + yield* service.triggerWebhook(requestFor(task, { headers: signed })); + yield* Queue.take(launches); + yield* service.triggerWebhook(requestFor(task)); + yield* service.triggerWebhook(requestFor(task, { token: "wrong" })); + // A request the relay held, then the same request again. + const relayed = requestFor(task, { headers: signed, relayDeliveryId: "relay-1" }); + yield* service.triggerWebhook(relayed); + yield* Queue.take(launches); + yield* service.triggerWebhook(relayed); + + assert.equal((yield* deliveriesCounted("accepted", "direct")) - before.accepted, 1); + assert.equal((yield* deliveriesCounted("rejected_signature", "direct")) - before.rejected, 1); + assert.equal((yield* deliveriesCounted("not_found", "direct")) - before.notFound, 1); + assert.equal((yield* deliveriesCounted("accepted", "relay")) - before.relayAccepted, 1); + assert.equal((yield* deliveriesCounted("duplicate", "relay")) - before.duplicate, 1); + }), + ), +); diff --git a/apps/server/src/scheduledTasks/relayDeliveryProof.ts b/apps/server/src/scheduledTasks/relayDeliveryProof.ts new file mode 100644 index 000000000000..9c277e6998d8 --- /dev/null +++ b/apps/server/src/scheduledTasks/relayDeliveryProof.ts @@ -0,0 +1,74 @@ +/** + * Whether a webhook request really came through this environment's relay. + * + * The relay's delivery id, receive time and trace context arrive as plain + * headers, and the webhook URL can also be called directly, so they are only + * trusted alongside a proof the relay signed with its mint key. A request + * without a valid proof is handled as a direct request. + */ +import { RelayHookDeliveryProofPayload } from "@t3tools/contracts/relay"; +import { + normalizeRelayIssuer, + RELAY_HOOK_DELIVERY_HEADER, + RELAY_HOOK_DELIVERY_TYP, + verifyRelayJwt, +} from "@t3tools/shared/relayJwt"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import { CLOUD_MINT_PUBLIC_KEY, RELAY_ISSUER_SECRET, RELAY_URL_SECRET } from "../cloud/config.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; + +/** Covers the relay's 24-hour hold plus a margin. */ +const MAX_PROOF_AGE_SECONDS = 25 * 60 * 60; + +const decodePayload = Schema.decodeUnknownOption(RelayHookDeliveryProofPayload); +const text = (bytes: Option.Option) => + Option.map(bytes, (value) => new TextDecoder().decode(value)); + +/** The relay's own delivery id and receive time, when the request proves it came from the relay. */ +export const makeRelayDeliveryVerifier = Effect.gen(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + const environment = yield* ServerEnvironment.ServerEnvironment; + return (input: { readonly headers: Readonly>; readonly hookId: string }) => + Effect.gen(function* () { + const proof = input.headers[RELAY_HOOK_DELIVERY_HEADER]; + const deliveryId = input.headers["x-t3-relay-delivery-id"]; + const receivedAt = input.headers["x-t3-relay-received-at"]; + if (proof === undefined || deliveryId === undefined || receivedAt === undefined) { + return Option.none(); + } + const publicKey = text(yield* secrets.get(CLOUD_MINT_PUBLIC_KEY)); + const issuer = Option.orElse(text(yield* secrets.get(RELAY_ISSUER_SECRET)), () => + Option.none(), + ); + const relayUrl = text(yield* secrets.get(RELAY_URL_SECRET)); + const relayIssuer = Option.isSome(issuer) ? issuer : relayUrl; + // Not linked to T3 Connect: no relay can be delivering to us. + if (Option.isNone(publicKey) || Option.isNone(relayIssuer)) return Option.none(); + const environmentId = yield* environment.getEnvironmentId; + const payload = yield* verifyRelayJwt({ + publicKey: publicKey.value, + token: proof, + typ: RELAY_HOOK_DELIVERY_TYP, + issuer: normalizeRelayIssuer(relayIssuer.value), + audience: `t3-env:${environmentId}`, + nowEpochSeconds: Math.floor((yield* Clock.currentTimeMillis) / 1_000), + maxTokenAge: MAX_PROOF_AGE_SECONDS, + }).pipe(Effect.map(decodePayload), Effect.orElseSucceed(Option.none)); + // The proof must name exactly this delivery, so it cannot be lifted onto another one. + if ( + Option.isNone(payload) || + payload.value.environmentId !== environmentId || + payload.value.deliveryId !== deliveryId || + payload.value.receivedAt !== receivedAt || + payload.value.hookId !== input.hookId + ) { + return Option.none(); + } + return Option.some({ deliveryId, receivedAt }); + }).pipe(Effect.orElseSucceed(Option.none), Effect.withSpan("webhook.verifyRelayDelivery")); +}); diff --git a/apps/server/src/scheduledTasks/webhookRoute.test.ts b/apps/server/src/scheduledTasks/webhookRoute.test.ts index e6fb5ff7d1ed..511b4f5ad875 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.test.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "@effect/vitest"; +import * as NodeCrypto from "node:crypto"; +import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Etag from "effect/http/Etag"; import * as HttpPlatform from "effect/http/HttpPlatform"; @@ -8,8 +11,10 @@ import * as HttpRouter from "effect/http/HttpRouter"; import * as HttpApi from "effect/http-api/HttpApi"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; +import { RELAY_HOOK_DELIVERY_TYP, signRelayJwt } from "@t3tools/shared/relayJwt"; import { EnvironmentHttpApi, + EnvironmentId, ScheduledTaskWebhookDeliveryId, ScheduledTaskError, } from "@t3tools/contracts"; @@ -18,19 +23,76 @@ import { type WebhookTriggerRequest, type WebhookTriggerResult, } from "./ScheduledTaskService.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import { CLOUD_MINT_PUBLIC_KEY, RELAY_ISSUER_SECRET } from "../cloud/config.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import { WEBHOOK_MAX_BODY_BYTES, webhookHttpApiLayer } from "./webhookRoute.ts"; class WebhookTestApi extends HttpApi.make("environment").add(EnvironmentHttpApi.groups.webhooks) {} +const environmentId = EnvironmentId.make("env-1"); +const relayIssuer = "https://relay.example.test"; +const mintKeys = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, +}); +/** The secrets a T3 Connect-linked environment holds, keyed by name. */ +const linkedSecrets: ReadonlyMap = new Map([ + [CLOUD_MINT_PUBLIC_KEY, mintKeys.publicKey], + [RELAY_ISSUER_SECRET, relayIssuer], +]); + +/** A proof as the relay signs it for one delivery. */ +const relayProof = Effect.fn("relayProof")(function* (claims: { + readonly deliveryId: string; + readonly receivedAt: string; + readonly hookId: string; + readonly privateKey?: string; +}) { + const now = Math.floor((yield* Clock.currentTimeMillis) / 1_000); + return yield* signRelayJwt({ + privateKey: claims.privateKey ?? mintKeys.privateKey, + typ: RELAY_HOOK_DELIVERY_TYP, + payload: { + iss: relayIssuer, + aud: `t3-env:${environmentId}`, + sub: environmentId, + jti: `proof:${claims.deliveryId}`, + iat: now, + exp: now + 3_600, + environmentId, + deliveryId: claims.deliveryId, + receivedAt: claims.receivedAt, + hookId: claims.hookId, + }, + }); +}); + const handlerFor = ( trigger: ( request: WebhookTriggerRequest, ) => Effect.Effect, + secrets: ReadonlyMap = linkedSecrets, ) => HttpRouter.toWebHandler( HttpApiBuilder.layer(WebhookTestApi).pipe( Layer.provide(webhookHttpApiLayer), Layer.provide(Layer.mock(ScheduledTaskService)({ triggerWebhook: trigger })), + Layer.provide( + Layer.mock(ServerSecretStore.ServerSecretStore)({ + get: (name) => + Effect.succeed( + Option.map(Option.fromUndefinedOr(secrets.get(name)), (value) => + new TextEncoder().encode(value), + ), + ), + }), + ), + Layer.provide( + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getEnvironmentId: Effect.succeed(environmentId), + }), + ), Layer.provide( HttpPlatform.layer.pipe( Layer.provideMerge(NodeServices.layer), @@ -56,6 +118,7 @@ describe("webhook route", () => { return Effect.succeed({ _tag: "accepted", deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"), + outcome: "accepted", }); }); try { @@ -77,25 +140,142 @@ describe("webhook route", () => { } }); - it("maps service outcomes to status codes", async () => { - const cases: ReadonlyArray<[WebhookTriggerResult["_tag"], number]> = [ - ["not_found", 404], - ["rejected_signature", 401], - ["disabled", 409], - ["rate_limited", 429], - ]; - for (const [tag, status] of cases) { - const { handler, dispose } = handlerFor(() => - Effect.succeed({ _tag: tag } as WebhookTriggerResult), + // Live clock: the server checks proofs against real time. + it.live("trusts the relay's delivery id and receive time only with its signed proof", () => + Effect.gen(function* () { + const received: Array = []; + const { handler, dispose } = handlerFor((request) => { + received.push(request); + return Effect.succeed({ + _tag: "accepted", + deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"), + outcome: "accepted", + }); + }); + const send = (path: string, headers: Record) => + Effect.promise(() => handler(post(path, "{}", headers))); + const receivedAt = "2026-10-04T10:00:00.000Z"; + const relayHeaders = { + "x-t3-relay-delivery-id": "relay-1", + "x-t3-relay-received-at": receivedAt, + }; + const forged = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" }); + const forgedProof = yield* relayProof({ + deliveryId: "relay-1", + receivedAt, + hookId: "id", + privateKey: forged.privateKey, + }); + // 0: signed by the relay for exactly this delivery. + yield* send("/api/hooks/id/tok", { ...relayHeaders, "x-t3-relay-delivery": proof }); + // 1: a direct caller claiming to be the relay, without a proof. + yield* send("/api/hooks/id/tok", relayHeaders); + // 2: a proof signed with any other key. + yield* send("/api/hooks/id/tok", { ...relayHeaders, "x-t3-relay-delivery": forgedProof }); + // 3: a real proof lifted onto another delivery id. + yield* send("/api/hooks/id/tok", { + ...relayHeaders, + "x-t3-relay-delivery-id": "relay-2", + "x-t3-relay-delivery": proof, + }); + // 4: a real proof for another hook. + yield* send("/api/hooks/other/tok", { ...relayHeaders, "x-t3-relay-delivery": proof }); + yield* Effect.promise(() => dispose()); + expect(received.map((request) => request.relayDeliveryId)).toEqual([ + "relay-1", + undefined, + undefined, + undefined, + undefined, + ]); + expect(received[0]?.receivedAt).toBe(receivedAt); + expect(received.slice(1).every((request) => request.receivedAt === undefined)).toBe(true); + }), + ); + + // Live clock: the server checks proofs against real time. + it.live("trusts no relay headers on an environment not linked to T3 Connect", () => + Effect.gen(function* () { + const received: Array = []; + const { handler, dispose } = handlerFor((request) => { + received.push(request); + return Effect.succeed({ _tag: "not_found" }); + }, new Map()); + const receivedAt = "2026-10-04T10:00:00.000Z"; + const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" }); + yield* Effect.promise(() => + handler( + post("/api/hooks/id/tok", "{}", { + "x-t3-relay-delivery-id": "relay-1", + "x-t3-relay-received-at": receivedAt, + "x-t3-relay-delivery": proof, + }), + ), ); + yield* Effect.promise(() => dispose()); + expect(received[0]?.relayDeliveryId).toBeUndefined(); + }), + ); + + it("maps service outcomes to status codes and names each outcome for the relay", async () => { + const deliveryId = ScheduledTaskWebhookDeliveryId.make("delivery:1"); + const cases: ReadonlyArray<[WebhookTriggerResult, number, string]> = [ + [{ _tag: "accepted", deliveryId, outcome: "accepted" }, 202, "accepted"], + // Same status as a started run; only the header tells them apart. + [{ _tag: "accepted", deliveryId, outcome: "duplicate" }, 202, "duplicate"], + [{ _tag: "accepted", deliveryId, outcome: "prompt_too_long" }, 202, "prompt_too_long"], + [{ _tag: "not_found" }, 404, "not_found"], + [{ _tag: "rejected_signature" }, 401, "rejected_signature"], + [{ _tag: "disabled" }, 409, "disabled"], + [{ _tag: "rate_limited", outcome: "rate_limited" }, 429, "rate_limited"], + [{ _tag: "rate_limited", outcome: "queue_full" }, 429, "queue_full"], + [{ _tag: "expired" }, 410, "expired"], + ]; + for (const [result, status, outcome] of cases) { + const { handler, dispose } = handlerFor(() => Effect.succeed(result)); try { - expect((await handler(post("/api/hooks/id/tok", "{}"))).status).toBe(status); + const response = await handler(post("/api/hooks/id/tok", "{}")); + expect(response.status).toBe(status); + expect(response.headers.get("x-t3-hook-outcome")).toBe(outcome); } finally { await dispose(); } } }); + // Live clock: the server checks proofs against real time. + it.live("joins the relay's trace only for requests the relay forwarded", () => + Effect.gen(function* () { + const parents: Array = []; + const { handler, dispose } = handlerFor(() => + Effect.gen(function* () { + const span = yield* Effect.currentParentSpan.pipe(Effect.option); + parents.push(span._tag === "Some" ? span.value.traceId : undefined); + return { _tag: "not_found" } as const; + }), + ); + const send = (headers: Record) => + Effect.promise(() => handler(post("/api/hooks/id/tok", "{}", headers))); + const traceparent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; + const receivedAt = "2026-10-04T10:00:00.000Z"; + const relayHeaders = { + "x-t3-relay-delivery-id": "relay-1", + "x-t3-relay-received-at": receivedAt, + }; + const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" }); + yield* send({ ...relayHeaders, traceparent, "x-t3-relay-delivery": proof }); + // Claiming to be the relay without its proof cannot attach to our traces. + yield* send({ ...relayHeaders, traceparent }); + yield* send({ traceparent }); + yield* Effect.promise(() => dispose()); + expect(parents).toEqual(["0af7651916cd43dd8448eb211c80319c", undefined, undefined]); + }), + ); + it("rejects oversized bodies and malformed paths before reaching the service", async () => { let calls = 0; const { handler, dispose } = handlerFor(() => { @@ -132,16 +312,20 @@ describe("webhook route", () => { } }); - it("hides service failures behind a 500", async () => { - const { handler, dispose } = handlerFor(() => + it("hides service failures and defects behind a fixed 500", async () => { + const failures = [ Effect.fail(new ScheduledTaskError({ message: "database locked" })), - ); - try { - const response = await handler(post("/api/hooks/id/tok", "{}")); - expect(response.status).toBe(500); - expect(await response.text()).not.toContain("database"); - } finally { - await dispose(); + Effect.die(new Error("database exploded")), + ]; + for (const failure of failures) { + const { handler, dispose } = handlerFor(() => failure); + try { + const response = await handler(post("/api/hooks/id/tok", "{}")); + expect(response.status).toBe(500); + expect(await response.json()).toEqual({ error: "internal_error" }); + } finally { + await dispose(); + } } }); }); diff --git a/apps/server/src/scheduledTasks/webhookRoute.ts b/apps/server/src/scheduledTasks/webhookRoute.ts index 63d2772c0c82..eee969f2d34c 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.ts @@ -2,27 +2,39 @@ import { EnvironmentHttpApi } from "@t3tools/contracts"; import * as ByteSize from "effect/ByteSize"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; +import * as Tracer from "effect/Tracer"; import * as HttpIncomingMessage from "effect/http/HttpIncomingMessage"; import type * as HttpServerRequest from "effect/http/HttpServerRequest"; import * as HttpServerResponse from "effect/http/HttpServerResponse"; +import * as HttpTraceContext from "effect/http/HttpTraceContext"; +import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; +import * as Metrics from "../observability/Metrics.ts"; +import { makeRelayDeliveryVerifier } from "./relayDeliveryProof.ts"; import * as ScheduledTaskService from "./ScheduledTaskService.ts"; /** Largest request body a webhook accepts. The relay enforces the same cap. */ export const WEBHOOK_MAX_BODY_BYTES = 1024 * 1024; -const json = (status: number, body: Record) => - HttpServerResponse.jsonUnsafe(body, { status }); +/** Response header naming what happened to the request; the relay records it. */ +const WEBHOOK_OUTCOME_HEADER = "x-t3-hook-outcome"; + +const json = (status: number, body: Record, outcome: string) => + HttpServerResponse.jsonUnsafe(body, { status, headers: { [WEBHOOK_OUTCOME_HEADER]: outcome } }); /** * Handles `/api/hooks/:hookId/:token` for every accepted method. The endpoint * is raw so the signature is checked over the exact body bytes; the service * checks the token and signature. It is reachable directly, over the managed - * tunnel, or through the relay's stable `/v1/hooks/...` URL. + * tunnel, or through the relay's stable `/v1/hooks/:endpointKey/:hookId/:token` + * URL, where the endpoint key is this environment's managed tunnel key. */ const handleWebhook = - (scheduledTasks: ScheduledTaskService.ScheduledTaskService["Service"]) => + ( + scheduledTasks: ScheduledTaskService.ScheduledTaskService["Service"], + verifyRelayDelivery: Effect.Success, + ) => ({ params, request, @@ -33,7 +45,7 @@ const handleWebhook = Effect.gen(function* () { const contentLength = Number(request.headers["content-length"] ?? "0"); if (!Number.isFinite(contentLength) || contentLength > WEBHOOK_MAX_BODY_BYTES) { - return json(413, { error: "body_too_large" }); + return json(413, { error: "body_too_large" }, "body_too_large"); } // Chunked requests carry no content-length, so the reader itself is capped. const body = yield* request.arrayBuffer.pipe( @@ -44,9 +56,15 @@ const handleWebhook = ), Effect.option, ); - if (Option.isNone(body)) return json(413, { error: "body_too_large_or_unreadable" }); + // Refused before a task is looked up, so the service never sees them. + const tooLarge = (error: string) => + Metrics.increment(Metrics.webhookDeliveriesTotal, { + outcome: "body_too_large", + source: request.headers["x-t3-relay-delivery-id"] ? "relay" : "direct", + }).pipe(Effect.as(json(413, { error }, "body_too_large"))); + if (Option.isNone(body)) return yield* tooLarge("body_too_large_or_unreadable"); if (body.value.byteLength > WEBHOOK_MAX_BODY_BYTES) { - return json(413, { error: "body_too_large" }); + return yield* tooLarge("body_too_large"); } const headers: Record = {}; @@ -54,7 +72,18 @@ const handleWebhook = if (typeof value === "string") headers[name.toLowerCase()] = value; } const queryIndex = request.url.indexOf("?"); + // The relay's delivery id and receive time count only with its signed + // proof; the URL can also be called directly. The receive time matters + // for requests the relay held while we were offline. + const relay = yield* verifyRelayDelivery({ headers, hookId: params.hookId }); + const relayDeliveryId = Option.isSome(relay) ? relay.value.deliveryId : undefined; + const relayReceivedAt = Option.isSome(relay) ? relay.value.receivedAt : undefined; + // A relay delivery joins the relay's trace, and goes to the T3 Connect + // tracer with it. Anyone else's traceparent is never trusted. + const relayParent = Option.isSome(relay) + ? HttpTraceContext.fromHeaders(request.headers) + : Option.none(); const result = yield* scheduledTasks .triggerWebhook({ hookId: params.hookId, @@ -65,9 +94,16 @@ const handleWebhook = headers, body: body.value, bodyText: new TextDecoder().decode(body.value), + ...(relayDeliveryId ? { relayDeliveryId } : {}), + ...(relayReceivedAt ? { receivedAt: relayReceivedAt } : {}), }) .pipe( - Effect.catch((cause) => + Option.isSome(relayParent) + ? (effect) => + effect.pipe(Effect.withParentSpan(relayParent.value), withRelayClientTracing) + : (effect) => effect, + // Defects too, so the sender only ever sees the fixed error body. + Effect.catchCause((cause) => Effect.logWarning("Webhook delivery failed").pipe( Effect.annotateLogs({ hookId: params.hookId }), Effect.andThen(Effect.logDebug("Webhook delivery failure cause", { cause })), @@ -78,17 +114,19 @@ const handleWebhook = switch (result._tag) { case "accepted": - return json(202, { deliveryId: result.deliveryId }); + return json(202, { deliveryId: result.deliveryId }, result.outcome); case "not_found": - return json(404, { error: "hook_not_found" }); + return json(404, { error: "hook_not_found" }, "not_found"); case "rejected_signature": - return json(401, { error: "invalid_signature" }); + return json(401, { error: "invalid_signature" }, "rejected_signature"); case "disabled": - return json(409, { error: "hook_disabled" }); + return json(409, { error: "hook_disabled" }, "disabled"); case "rate_limited": - return json(429, { error: "rate_limited" }); + return json(429, { error: "rate_limited" }, result.outcome); + case "expired": + return json(410, { error: "delivery_too_old" }, "expired"); case "error": - return json(500, { error: "internal_error" }); + return json(500, { error: "internal_error" }, "error"); } }); @@ -96,7 +134,10 @@ export const webhookHttpApiLayer = HttpApiBuilder.group( EnvironmentHttpApi, "webhooks", Effect.fnUntraced(function* (handlers) { - const handler = handleWebhook(yield* ScheduledTaskService.ScheduledTaskService); + const handler = handleWebhook( + yield* ScheduledTaskService.ScheduledTaskService, + yield* makeRelayDeliveryVerifier, + ); return handlers .handleRaw("webhookPost", handler) .handleRaw("webhookPut", handler) diff --git a/apps/server/src/scheduledTasks/webhookTemplate.test.ts b/apps/server/src/scheduledTasks/webhookTemplate.test.ts index c87fa8b064a2..8b6f11e91f9f 100644 --- a/apps/server/src/scheduledTasks/webhookTemplate.test.ts +++ b/apps/server/src/scheduledTasks/webhookTemplate.test.ts @@ -69,6 +69,31 @@ describe("renderWebhookPrompt", () => { assert.deepEqual(plain.missing, ["body.field"]); }); + it("redacts credentials in whole-request placeholders but not named ones", () => { + const request: WebhookRequest = { + ...githubPullRequest, + query: "source=github&access_token=q-secret", + headers: { + ...githubPullRequest.headers, + authorization: "Bearer h-secret", + "x-hub-signature-256": "sha256=abc", + }, + }; + const whole = renderWebhookPrompt("{{request}}|{{headers}}|{{query}}", request).prompt; + for (const secret of ["q-secret", "h-secret", "sha256=abc"]) { + assert.notInclude(whole, secret); + } + assert.include(whole, "?source=github&access_token=[redacted]\n"); + assert.include(whole, "authorization: [redacted]"); + assert.include(whole, "x-github-event: pull_request"); + + const named = renderWebhookPrompt( + "{{headers.authorization}} {{query.access_token}}", + request, + ).prompt; + assert.equal(named, "Bearer h-secret q-secret"); + }); + it("does not resolve inherited object properties", () => { const rendered = renderWebhookPrompt( "{{body.constructor}}{{body.__proto__}}", diff --git a/apps/server/src/scheduledTasks/webhookTemplate.ts b/apps/server/src/scheduledTasks/webhookTemplate.ts index e365a32858d6..74c0f08ae778 100644 --- a/apps/server/src/scheduledTasks/webhookTemplate.ts +++ b/apps/server/src/scheduledTasks/webhookTemplate.ts @@ -14,6 +14,10 @@ * * Strings and numbers render as text, objects and arrays as JSON. A path with * no value renders empty and is reported in `missing`. + * + * Credential-looking headers and query parameters are redacted wherever the + * whole set renders (`{{request}}`, `{{headers}}`, `{{query}}`), as in the + * delivery log. Naming one (`{{headers.authorization}}`) gives its raw value. */ export interface WebhookRequest { @@ -31,6 +35,40 @@ export interface RenderedWebhookPrompt { readonly missing: ReadonlyArray; } +/** Header and query parameter names that commonly carry credentials. */ +const CREDENTIAL_NAME = + /^(authorization|proxy-authorization|cookie|set-cookie)$|token|secret|signature|key|password|auth/i; +const REDACTED = "[redacted]"; + +export function redactHeaders(headers: Readonly>): Record { + return Object.fromEntries( + Object.entries(headers).map(([name, value]) => [ + name, + CREDENTIAL_NAME.test(name) ? REDACTED : value, + ]), + ); +} + +/** Redacts credential-named values in a raw query string, keeping the rest as sent. */ +export function redactQuery(query: string): string { + if (query === "") return query; + return query + .split("&") + .map((part) => { + const separator = part.indexOf("="); + if (separator === -1) return part; + const name = part.slice(0, separator); + let decoded = name; + try { + decoded = decodeURIComponent(name.replaceAll("+", " ")); + } catch { + // A malformed escape is matched as sent. + } + return CREDENTIAL_NAME.test(decoded) ? `${name}=${REDACTED}` : part; + }) + .join("&"); +} + const PLACEHOLDER = /\{\{\s*([^{}]*?)\s*\}\}/g; function parseBody(request: WebhookRequest): unknown { @@ -64,9 +102,12 @@ function stringify(value: unknown): string { } function formatWebhookRequest(request: WebhookRequest): string { - const headerLines = Object.entries(request.headers).map(([name, value]) => `${name}: ${value}`); + const headerLines = Object.entries(redactHeaders(request.headers)).map( + ([name, value]) => `${name}: ${value}`, + ); + const query = redactQuery(request.query); return [ - `${request.method} ${request.path}${request.query ? `?${request.query}` : ""}`, + `${request.method} ${request.path}${query ? `?${query}` : ""}`, ...headerLines, "", request.bodyText, @@ -90,12 +131,13 @@ export function renderWebhookPrompt( return segments.length === 0 ? request.bodyText : lookup(parsedBody(), segments); case "headers": return segments.length === 0 - ? request.headers + ? redactHeaders(request.headers) : request.headers[segments.join(".").toLowerCase()]; case "query": { - const params = new URLSearchParams(request.query); - if (segments.length === 0) return Object.fromEntries(params); - return params.get(segments.join(".")) ?? undefined; + if (segments.length === 0) { + return Object.fromEntries(new URLSearchParams(redactQuery(request.query))); + } + return new URLSearchParams(request.query).get(segments.join(".")) ?? undefined; } default: return undefined; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index c203fe65c904..3be16c3a2ffb 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -119,8 +119,16 @@ import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http import * as ReplayMarkers from "./auth/replayMarkers.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import { webhookHttpApiLayer } from "./scheduledTasks/webhookRoute.ts"; -import { ScheduledTaskWebhookOrigin } from "./scheduledTasks/ScheduledTaskService.ts"; -import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./cloud/config.ts"; +import * as HeldHooksWaker from "./relay/HeldHooksWaker.ts"; +import { + relayHookBaseUrl, + ScheduledTaskWebhookOrigin, +} from "./scheduledTasks/ScheduledTaskService.ts"; +import { + CLOUD_ENDPOINT_RUNTIME_CONFIG, + decodeRuntimeConfig, + RELAY_URL_SECRET, +} from "./cloud/config.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import { connectHttpApiLayer, @@ -452,24 +460,26 @@ const CloudManagedEndpointRuntimeLive = Layer.mergeAll( // to is configured; otherwise clients show the environment-relative path. const ScheduledTaskWebhookOriginLive = Layer.effect( ScheduledTaskWebhookOrigin, - Effect.map( - Effect.all([ServerEnvironment.ServerEnvironment, ServerSecretStore.ServerSecretStore]), - ([environment, secrets]) => - // The reference holds an effect so each read sees the current link state. - Effect.gen(function* () { - const [relayUrl, tunnelConfig] = yield* Effect.all([ - secrets.get(RELAY_URL_SECRET), - secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), - ]).pipe(Effect.orElseSucceed(() => [Option.none(), Option.none()] as const)); - return { - environmentId: yield* environment.getEnvironmentId, - relayUrl: - Option.isSome(relayUrl) && Option.isSome(tunnelConfig) - ? new TextDecoder().decode(relayUrl.value) || null - : null, - }; - }), - ), + Effect.gen(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + // The reference holds an effect so each read sees the current link state. + return Effect.gen(function* () { + const [relayUrl, tunnelConfig] = yield* Effect.all([ + secrets.get(RELAY_URL_SECRET), + secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + ]).pipe(Effect.orElseSucceed(() => [Option.none(), Option.none()] as const)); + if (Option.isNone(relayUrl) || Option.isNone(tunnelConfig)) { + return { relayHookBaseUrl: null }; + } + const config = decodeRuntimeConfig(new TextDecoder().decode(tunnelConfig.value)); + return { + relayHookBaseUrl: relayHookBaseUrl({ + relayUrl: new TextDecoder().decode(relayUrl.value), + tunnelName: Option.isSome(config) ? config.value.tunnelName : undefined, + }), + }; + }); + }), ); const OrchestrationV2RuntimeLayerLive = OrchestrationV2ProductionLayerLive.pipe( @@ -537,6 +547,8 @@ const ProviderInstallationRefreshLive = Layer.effectDiscard( const RuntimeCoreDependenciesBaseLive = Layer.mergeAll( AgentAwarenessRelay.layer, + // Asks T3 Connect to deliver webhooks it held while this environment was offline. + HeldHooksWaker.layer, ThreadSettlementWorkerLive, Layer.effectDiscard(StorageCleanup.make.pipe(Effect.flatMap((service) => service.start()))).pipe( Layer.provide(ProjectionStoreV2.layer), diff --git a/apps/web/src/cloud/linkEnvironment.ts b/apps/web/src/cloud/linkEnvironment.ts index 40f12cebb9a4..ad9a0c0ea429 100644 --- a/apps/web/src/cloud/linkEnvironment.ts +++ b/apps/web/src/cloud/linkEnvironment.ts @@ -199,13 +199,15 @@ export function readPrimaryCloudLinkState(input: { export function updatePrimaryCloudPreferences(input: { readonly target: CloudLinkTarget; readonly publishAgentActivity: boolean; + readonly holdWebhooksWhileOffline?: boolean; }): Effect.Effect { return Effect.gen(function* () { const client = yield* makeEnvironmentHttpApiClient(input.target.httpBaseUrl); + const { target: _target, ...payload } = input; return yield* client.connect .preferences({ headers: {}, - payload: input, + payload, }) .pipe( Effect.mapError(environmentApiError("Could not update environment cloud preferences.")), diff --git a/apps/web/src/cloud/linkEnvironmentAtoms.ts b/apps/web/src/cloud/linkEnvironmentAtoms.ts index 1094e860e462..4a3541cd3a10 100644 --- a/apps/web/src/cloud/linkEnvironmentAtoms.ts +++ b/apps/web/src/cloud/linkEnvironmentAtoms.ts @@ -41,6 +41,9 @@ export const updatePrimaryEnvironmentPreferences = createRuntimeCommand(connecti label: "web:cloud:update-primary-environment-preferences", scheduler: cloudLinkScheduler, concurrency: cloudLinkConcurrency, - execute: (input: { readonly target: CloudLinkTarget; readonly publishAgentActivity: boolean }) => - updatePrimaryCloudPreferences(input), + execute: (input: { + readonly target: CloudLinkTarget; + readonly publishAgentActivity: boolean; + readonly holdWebhooksWhileOffline?: boolean; + }) => updatePrimaryCloudPreferences(input), }); diff --git a/apps/web/src/cloud/primaryCloudLinkState.ts b/apps/web/src/cloud/primaryCloudLinkState.ts index d46ee641d574..8d9b919db05f 100644 --- a/apps/web/src/cloud/primaryCloudLinkState.ts +++ b/apps/web/src/cloud/primaryCloudLinkState.ts @@ -13,6 +13,7 @@ import { usePrimaryEnvironment } from "../state/environments"; import { runtime } from "../lib/runtime"; import { appAtomRegistry } from "../rpc/atomRegistry"; import { readPrimaryCloudLinkState, type CloudLinkTarget } from "./linkEnvironment"; +import { hasCloudPublicConfig } from "./publicConfig"; const primaryCloudLinkAtomRuntime = Atom.runtime( Layer.effect( @@ -62,9 +63,11 @@ export function usePrimaryCloudLinkState() { : null, [primary], ); - const atom = target - ? primaryCloudLinkStateAtom(targetKey(target)) - : EMPTY_PRIMARY_CLOUD_LINK_STATE_ATOM; + // Builds without T3 Connect have no link to read; skip the request. + const atom = + target && hasCloudPublicConfig() + ? primaryCloudLinkStateAtom(targetKey(target)) + : EMPTY_PRIMARY_CLOUD_LINK_STATE_ATOM; const result = useAtomValue(atom); const refresh = useCallback(() => { refreshPrimaryCloudLinkState(target); diff --git a/apps/web/src/cloud/useCloudLinkController.ts b/apps/web/src/cloud/useCloudLinkController.ts index d91596880850..730159599969 100644 --- a/apps/web/src/cloud/useCloudLinkController.ts +++ b/apps/web/src/cloud/useCloudLinkController.ts @@ -21,6 +21,8 @@ import { resolveRelayClerkTokenOptions } from "./publicConfig"; export interface CloudLinkDesiredState { readonly managedTunnel: boolean; readonly publish: boolean; + /** Omit to leave the webhook-hold setting as it is. */ + readonly holdWebhooksWhileOffline?: boolean; } /** @@ -75,6 +77,7 @@ export function useCloudLinkController() { const managedTunnelActive = primaryCloudLinkState.data?.managedTunnelActive ?? primaryCloudLinkState.data?.linked ?? false; const publishAgentActivity = primaryCloudLinkState.data?.publishAgentActivity ?? false; + const holdWebhooksWhileOffline = primaryCloudLinkState.data?.holdWebhooksWhileOffline ?? false; const linked = primaryCloudLinkState.data?.linked ?? false; const reconcileCloudState = async (desired: CloudLinkDesiredState): Promise => { @@ -132,6 +135,9 @@ export function useCloudLinkController() { const prefResult = await updatePrimaryEnvironmentPreferences({ target, publishAgentActivity: desired.publish, + ...(desired.holdWebhooksWhileOffline === undefined + ? {} + : { holdWebhooksWhileOffline: desired.holdWebhooksWhileOffline }), }); if (prefResult._tag === "Failure") { if (!isAtomCommandInterrupted(prefResult)) { @@ -157,6 +163,7 @@ export function useCloudLinkController() { linked, managedTunnelActive, publishAgentActivity, + holdWebhooksWhileOffline, operationError, reconcileCloudState, }; diff --git a/apps/web/src/components/chat/ThreadAutomationsPanel.tsx b/apps/web/src/components/chat/ThreadAutomationsPanel.tsx index 13fde2d2e38f..d4434fffbf18 100644 --- a/apps/web/src/components/chat/ThreadAutomationsPanel.tsx +++ b/apps/web/src/components/chat/ThreadAutomationsPanel.tsx @@ -186,28 +186,26 @@ export function ThreadAutomationsPanel(props: { /> Edit automation - - void runNow(task)} - > - - - } - /> - Run now - + {/* A webhook task runs from its URL; there is no request to run it with. */} + {task.schedule.type === "webhook" ? null : ( + + void runNow(task)} + > + + + } + /> + Run now + + )} { + setIsUpdatingPreference(true); + const ok = await reconcileCloudState({ + managedTunnel: managedTunnelActive, + publish: publishAgentActivity, + holdWebhooksWhileOffline: enabled, + }); + if (ok) { + toastManager.add({ + type: "success", + title: enabled ? "Webhooks held while offline" : "Webhooks no longer held", + description: enabled + ? "T3 Connect keeps webhook requests for up to 24 hours while this environment is offline." + : "Requests to an offline environment now fail. Anything already held is still delivered.", + }); + } + setIsUpdatingPreference(false); + }; + return ( <> {window.desktopBridge ? ( @@ -1846,6 +1866,21 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b /> } /> + {managedTunnelActive ? ( + void updateHoldWebhooks(enabled)} + /> + } + /> + ) : null} ); } diff --git a/apps/web/src/components/settings/ScheduledTasksSettings.tsx b/apps/web/src/components/settings/ScheduledTasksSettings.tsx index 884ae744c878..d82b6f163694 100644 --- a/apps/web/src/components/settings/ScheduledTasksSettings.tsx +++ b/apps/web/src/components/settings/ScheduledTasksSettings.tsx @@ -23,7 +23,9 @@ import type { ScheduledTaskWebhookDeliverySummary, ThreadId, } from "@t3tools/contracts"; +import { DEFAULT_WEBHOOK_PROMPT } from "@t3tools/client-runtime/scheduled-task-webhook"; import { + MAX_WEBHOOK_DELIVERY_AGE_MINUTES, MIN_SCHEDULED_TASK_INTERVAL_MS, ProviderInstanceId, resolveEnvironmentMachineKind, @@ -41,8 +43,10 @@ import { deriveProviderInstanceEntries, sortProviderInstanceEntries, } from "../../providerInstances"; +import { usePrimaryCloudLinkState } from "../../cloud/primaryCloudLinkState"; import { requestConfirmDialog } from "../../confirmDialog"; import { webhookAddress } from "@t3tools/client-runtime/webhook-address"; +import { Link } from "@tanstack/react-router"; import { useCopyToClipboard } from "../../hooks/useCopyToClipboard"; import { useEnvironment, @@ -57,7 +61,6 @@ import { WorktreeBaseBranchPicker } from "../WorktreeBaseBranchPicker"; import { EnvironmentMachineIcon } from "../EnvironmentMachineIcon"; import { useSettingsScope } from "./SettingsScopeContext"; import { - DEFAULT_WEBHOOK_PROMPT, WEBHOOK_SIGNATURE_DEFAULTS, matchesScheduledTaskScope, scheduleFromDraft, @@ -130,6 +133,7 @@ const EMPTY_DRAFT: DraftState = { signatureEnabled: false, ...WEBHOOK_SIGNATURE_DEFAULTS, signatureSecret: "", + maxDeliveryAgeMinutes: "", }; /** Labelled field: a caption sitting above its control. */ @@ -211,11 +215,14 @@ const DELIVERY_OUTCOME_LABELS: Record {note !== null ?

{note}

: null} + {endpoint.url !== null ? : null} ); } +/** + * Whether T3 Connect forwards requests live or holds them while the + * environment is offline. The setting is per environment and only readable + * for this machine's own environment, so other environments show nothing. + */ +function WebhookDeliveryMode({ environmentId }: { readonly environmentId: EnvironmentId }) { + const cloudLink = usePrimaryCloudLinkState(); + if (cloudLink.target?.environmentId !== environmentId || cloudLink.data === null) return null; + return ( +

+ {cloudLink.data.holdWebhooksWhileOffline + ? "Held for up to 24 hours while this environment is offline. " + : "Forwarded live. Requests fail while this environment is offline. "} + + Change in Connections + +

+ ); +} + function ScheduledTaskEditorDialog({ initialEnvironmentId, task, @@ -830,6 +858,13 @@ function ScheduledTaskEditorDialog({ return; } const schedule = scheduleFromDraft(draft); + if (schedule === null) { + reportFailure( + "Invalid age limit", + `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`, + ); + return; + } if ( schedule.type === "webhook" && schedule.signature && @@ -1126,6 +1161,27 @@ function ScheduledTaskEditorDialog({ "Each request runs the prompt. Use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} or {{request}} in the prompt; only what it names reaches the agent." }

+ + + setDraft((current) => ({ + ...current, + maxDeliveryAgeMinutes: event.target.value, + })) + } + /> +
diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts index 9ccd9881d943..5c4b6c84355e 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts @@ -181,10 +181,15 @@ describe("webhook scheduled tasks", () => { const draft = taskToDraft(webhookTask); expect(draft.scheduleMode).toBe("webhook"); expect(draft.signatureSecret).toBe(""); - expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature }); + expect(scheduleFromDraft(draft)).toEqual({ + type: "webhook", + signature, + maxDeliveryAgeMinutes: null, + }); expect(scheduleFromDraft({ ...draft, signatureSecret: " new " })).toEqual({ type: "webhook", signature: { ...signature, secret: "new" }, + maxDeliveryAgeMinutes: null, }); }); @@ -192,7 +197,28 @@ describe("webhook scheduled tasks", () => { const draft = taskToDraft({ ...webhookTask, schedule: { type: "webhook", signature: null } }); expect(draft.signatureEnabled).toBe(false); expect(draft.signatureHeader).toBe("x-hub-signature-256"); - expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature: null }); + expect(scheduleFromDraft(draft)).toEqual({ + type: "webhook", + signature: null, + maxDeliveryAgeMinutes: null, + }); + }); + + it("round-trips the max age, treats blank as no limit, and rejects an invalid limit", () => { + const draft = taskToDraft({ + ...webhookTask, + schedule: { type: "webhook", signature: null, maxDeliveryAgeMinutes: 90 }, + }); + expect(draft.maxDeliveryAgeMinutes).toBe("90"); + expect(scheduleFromDraft(draft)).toMatchObject({ maxDeliveryAgeMinutes: 90 }); + for (const blank of ["", " "]) { + expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: blank })).toMatchObject({ + maxDeliveryAgeMinutes: null, + }); + } + for (const invalid of ["0", "-5", "1.5", "abc", "1441"]) { + expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: invalid })).toBeNull(); + } }); }); diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts index c6d0cb28b940..6c738ce0eff0 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts @@ -9,6 +9,7 @@ import { type ProviderInteractionMode, type ServerSettings, } from "@t3tools/contracts"; +import { parseMaxDeliveryAge } from "@t3tools/client-runtime/scheduled-task-webhook"; import { resolveProjectSettings, @@ -78,6 +79,8 @@ export interface DraftState { readonly signaturePrefix: string; /** Write-only: empty keeps the secret already stored on the server. */ readonly signatureSecret: string; + /** Minutes as typed; empty runs every held request regardless of age. */ + readonly maxDeliveryAgeMinutes: string; } /** GitHub's signature settings, the most common sender. */ @@ -87,11 +90,11 @@ export const WEBHOOK_SIGNATURE_DEFAULTS = { signaturePrefix: "sha256=", } as const; -/** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */ -export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}"; - -export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule { +/** Null when the draft's webhook age limit is invalid; the caller reports it and does not save. */ +export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule | null { if (draft.scheduleMode === "webhook") { + const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes); + if (maxDeliveryAgeMinutes === undefined) return null; const secret = draft.signatureSecret.trim(); return { type: "webhook", @@ -103,6 +106,7 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul ...(secret ? { secret } : {}), } : null, + maxDeliveryAgeMinutes, }; } if (draft.scheduleMode === "interval") { @@ -159,6 +163,10 @@ export function taskToDraft(task: ScheduledTask): DraftState { } : { signatureEnabled: false, ...WEBHOOK_SIGNATURE_DEFAULTS }), signatureSecret: "", + maxDeliveryAgeMinutes: + schedule.type === "webhook" && schedule.maxDeliveryAgeMinutes != null + ? String(schedule.maxDeliveryAgeMinutes) + : "", }; } diff --git a/apps/web/src/components/settings/settingsSearch.test.ts b/apps/web/src/components/settings/settingsSearch.test.ts index 76f26b2eeab6..17bc6407d14b 100644 --- a/apps/web/src/components/settings/settingsSearch.test.ts +++ b/apps/web/src/components/settings/settingsSearch.test.ts @@ -170,6 +170,7 @@ describe("searchSettings", () => { "source-control-writer-model", "source-control-writing-style", "t3-connect", + "hold-webhooks-while-offline", "tailscale-https", "wsl-backend", "auto-settle-inactive-threads", @@ -221,6 +222,24 @@ describe("searchSettings", () => { expect(browser).toContain("publish-agent-activity"); }); + it("offers webhook holding only while the managed tunnel is on, like its row", () => { + const availability = { + hasCloudPublicConfig: true, + hasEnvironment: true, + hasProviderSettingsEnvironment: true, + hasMacProviderSettingsEnvironment: false, + canManageLocalBackend: true, + isWslSettingsRowVisible: false, + hasThreadAutoSettlement: false, + }; + const itemIds = (managedTunnelActive: boolean) => + filterAvailableSettingsSearchItems({ ...availability, managedTunnelActive }).map( + (item) => item.id, + ); + expect(itemIds(false)).not.toContain("hold-webhooks-while-offline"); + expect(itemIds(true)).toContain("hold-webhooks-while-offline"); + }); + it("shows automatic settlement settings when the server supports them", () => { const available = filterAvailableSettingsSearchItems({ hasCloudPublicConfig: false, diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 6a198a066f72..be1eb15a6c1d 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -60,6 +60,8 @@ export interface SettingsSearchItem { readonly localBackendManagementOnly?: boolean; readonly localEnvironmentOnly?: boolean; readonly wslAvailableOnly?: boolean; + // Its row only renders while this environment's T3 Connect managed tunnel is on. + readonly managedTunnelOnly?: boolean; /** * Sorts after every other match. Keybinding commands mirror rows on other * surfaces, so "model" must still lead with Default model, not Model Picker. @@ -77,6 +79,7 @@ export interface SettingsSearchAvailability { readonly canManageLocalBackend: boolean; readonly isWslSettingsRowVisible: boolean; readonly hasThreadAutoSettlement: boolean; + readonly managedTunnelActive?: boolean; } /** @@ -836,6 +839,16 @@ export const SETTINGS_SEARCH_ITEMS = [ desktopOnly: true, cloudOnly: true, }, + { + id: "hold-webhooks-while-offline", + localEnvironmentOnly: true, + title: "Hold webhooks while offline", + to: "/settings/connections", + targetId: "connections-environment", + searchTerms: ["webhook automations offline queue mailbox t3 connect"], + cloudOnly: true, + managedTunnelOnly: true, + }, { id: "publish-agent-activity", localEnvironmentOnly: true, @@ -1018,7 +1031,8 @@ export function filterAvailableSettingsSearchItems( (!item.localBackendManagementOnly || availability.canManageLocalBackend) && (!item.localEnvironmentOnly || !availability.localEnvironmentDisabled) && (!item.wslAvailableOnly || availability.isWslSettingsRowVisible) && - (!item.requiresThreadAutoSettlement || availability.hasThreadAutoSettlement), + (!item.requiresThreadAutoSettlement || availability.hasThreadAutoSettlement) && + (!item.managedTunnelOnly || availability.managedTunnelActive === true), ); } diff --git a/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts b/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts index a5be950b81ab..3820357e59fa 100644 --- a/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts +++ b/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts @@ -1,6 +1,7 @@ import { useMemo } from "react"; import { AuthAccessWriteScope } from "@t3tools/contracts"; +import { usePrimaryCloudLinkState } from "~/cloud/primaryCloudLinkState"; import { hasCloudPublicConfig } from "~/cloud/publicConfig"; import { isElectron } from "~/env"; import { isLocalEnvironmentDisabled } from "~/localEnvironment"; @@ -23,6 +24,10 @@ export function useAvailableSettingsSearchItems(scopeSearch: SettingsScopeSearch const desktopWsl = useEnvironmentQuery( isElectron && !localEnvironmentDisabled ? desktopWslStateAtom : null, ); + const cloudLinkState = usePrimaryCloudLinkState().data; + // Same fallback as the Connections row: older servers imply a tunnel from `linked`. + const managedTunnelActive = + cloudLinkState?.managedTunnelActive ?? cloudLinkState?.linked ?? false; const canManageLocalBackend = !localEnvironmentDisabled && (isElectron || @@ -59,8 +64,10 @@ export function useAvailableSettingsSearchItems(scopeSearch: SettingsScopeSearch }), hasThreadAutoSettlement: getThreadAutoSettlementSearchAvailability(environments).eligibleEnvironmentIds.length > 0, + managedTunnelActive, }), [ + managedTunnelActive, canManageLocalBackend, desktopWsl.data, desktopWsl.error, diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index a857bd95819d..54fdf6dc310c 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -5,12 +5,30 @@ credentials for reaching environments, and managed tunnel allocations. After bootstrap, clients send application traffic through the environment's tunnel hostname; the relay Worker does not proxy their HTTP or WebSocket sessions. The one exception is automation webhooks: the relay forwards -`/v1/hooks/:environmentId/:hookId/:token` statelessly to the environment's -tunnel so senders get a stable URL. It stores nothing, keeps bodies and tokens -out of its traces, and leaves token and signature verification to the -environment +`/v1/hooks/:environmentId/:hookId/:token` to the environment's tunnel so +senders get a stable URL. It keeps bodies and tokens out of its traces and +leaves token and signature verification to the environment ([forwarder](../../infra/relay/src/hooks/HookForwarder.ts)). +By default the forwarder stores nothing. An environment can opt in to having +the relay hold requests while it is offline +(`hold_webhooks_while_offline` on its link). Only then does the relay store the +raw request, including the hook token in the path, in a Durable Object for +that environment, with SQLite storage. The object pushes held requests back +through the tunnel from its alarm, oldest first, and backs off while the +environment stays away. When the tunnel reconnects, the environment asks the +relay to deliver right away. Requests are deleted once the environment +answers, after 24 hours, or when no user has the environment linked. The relay +still never checks the token; delivery goes through the same environment route. +Every forward carries `x-t3-relay-delivery-id`, so a request that reached the +environment before a timeout and is delivered again later runs once +([inbox object](../../infra/relay/src/hooks/HookInboxObject.ts)). + +A Durable Object, not Postgres or Queues, because held requests are write-once, +read-once bodies of up to 1 MiB that need per-environment order, caps, and +retry timing. Queues cap messages at 128 KB and cannot hold one environment's +requests back while it is away. + Clerk, deployment, and native authentication setup live in the [Connect setup runbook](../operations/connect-setup.md). diff --git a/docs/operations/observability.md b/docs/operations/observability.md index b83a9015abb2..7a39ac3dd601 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -386,6 +386,21 @@ Counters tell you volume and failure rate: - `t3_provider_turns_total` - `t3_git_commands_total` +Webhooks have their own families: + +- `t3_webhook_deliveries_total` by `outcome` and `source` (`relay` or `direct`). Beyond what the + sender sees, `queue_full` means a task already had its limit of deliveries waiting, + `prompt_too_long` means the filled-in prompt passed the provider limit, and `duplicate` means + the relay delivered a request this environment had already run. +- `t3_webhook_runs_total` by `outcome` (`started`, `skipped`, `failed`) for the runs those + deliveries start, which happen after the sender has its answer. +- `t3_webhook_held_delay` for how long requests the relay held waited before arriving. + +`ScheduledTaskService.triggerWebhook` spans carry the same outcome per request, and each run +started from a delivery is its own `ScheduledTaskService.runWebhookDelivery` trace. For a request +the relay forwarded, the span also goes to the T3 Connect trace export as a child of the relay's +span; requests that reach the environment directly never join a sender's trace. + Use metrics when the question is: - "is this always slow?" diff --git a/docs/operations/relay-observability.md b/docs/operations/relay-observability.md index c7f84d4ef825..dd8537abb132 100644 --- a/docs/operations/relay-observability.md +++ b/docs/operations/relay-observability.md @@ -57,3 +57,54 @@ failure means that a signed proof was too old or too far in the future for the r window. It can point to a date or time problem on either device, but it can also result from a delayed request. The client uses this category, and the absence of a category from an older relay, to decide whether clock skew is confirmed or only one possible cause. + +## Webhooks + +A public webhook request is one `relay.hooks.forward` span. Its `relay.hook.outcome` says what +happened: `forwarded`, `held`, `rate_limited`, `inbox_full`, `not_found`, `payload_too_large`, +`environment_unavailable`, or `environment_timeout`. `relay.hook.endpoint_key` identifies the +managed endpoint, and with it the environment. On a forward, `relay.hook.upstream_status` or +`relay.hook.upstream_error` records the environment's answer, and +`relay.hook.upstream_outcome` what it did with the request (`accepted`, `duplicate`, +`prompt_too_long`, `queue_full`, `rejected_signature`, `expired`, `disabled`, ...), from its +`x-t3-hook-outcome` response header. A held request's delivery records the same on its +`relay.inbox.deliver` span. The relay sends its own `traceparent` with each forward and drops any a +sender supplied, and signs each forward (`x-t3-relay-delivery`) so the environment trusts the +relay's delivery id, receive time and trace context only from the relay, so on environments that export to T3 Connect, the environment's +`ScheduledTaskService.triggerWebhook` span lands in the same trace. `relay.hook.rate_limit` says which +budget ran out: `endpoint` or `hook`. `relay.hook.rate_limiter_failed_open` is set when the +Cloudflare rate limiter was unavailable and the request went through unlimited. + +Held requests are handled in the endpoint's `HookInboxObject`, and each call into it is its own +trace, not a child of the forward span: `relay.inbox.hold`, `relay.inbox.wake`, and +`relay.inbox.deliver` for each alarm run. Each carries `relay.hook.endpoint_key`, so they join to +forwards. The details are on the child spans: `HookInboxStore.hold` carries +`relay.inbox.refused` when the inbox refused a request (`max_per_hook`, `max_requests`, +`max_bytes`, or `already_held`). `HookInboxStore.deliverDue` carries `relay.inbox.run_result` +(`drained`, `more_pending`, `busy`, or `unreachable`), how many requests it sent and delivered, +the consecutive failures behind the retry delay, the longest time a delivered request waited, and +the backlog left. A run that errored outright has `relay.inbox.run_result = failed` on the root +`relay.inbox.deliver` span instead. + +Questions these answer: + +```apl +// Webhook outcomes per hour +['t3-code-relay-traces-prod'] +| where name == 'relay.hooks.forward' +| summarize count() by bin(_time, 1h), outcome = tostring(['attributes.custom']['relay.hook.outcome']) + +// Inboxes stuck behind an unreachable environment, by endpoint +['t3-code-relay-traces-prod'] +| where name == 'HookInboxStore.deliverDue' +| extend c = ['attributes.custom'] +| where tostring(c['relay.inbox.run_result']) == 'unreachable' +| project trace_id, failures = toint(c['relay.inbox.consecutive_failures']), + backlog = toint(c['relay.inbox.held_count']) +| join kind=inner ( + ['t3-code-relay-traces-prod'] + | where name == 'relay.inbox.deliver' + | project trace_id, endpoint = tostring(['attributes.custom']['relay.hook.endpoint_key']) + ) on trace_id +| summarize runs = count(), failures = max(failures), backlog = max(backlog) by endpoint +``` diff --git a/docs/user/project-settings.md b/docs/user/project-settings.md index bf30f0ba637a..d1affc3f0793 100644 --- a/docs/user/project-settings.md +++ b/docs/user/project-settings.md @@ -61,16 +61,18 @@ using its project, model, and workspace settings. Fixed-time schedules use that environment's time zone, which may differ from your phone's. You can edit, pause, resume, run immediately, or delete a task from the list. +Webhook tasks only run when their URL is called, so they can't be run +immediately. Leaving an edited form asks before discarding unsaved changes. ## Webhook automations -In **Settings → Scheduled tasks**, choose **On webhook** as a task's schedule to run it whenever another service -calls its URL, such as GitHub on a new pull request or a CI job that failed. -After you save the task, copy its URL from the editor. If the environment uses a -[T3 Connect](remote-access.md) managed tunnel, the URL is public; otherwise it -works anywhere the environment itself is reachable. **Rotate** replaces the URL and -the old one stops working. +In **Settings → Scheduled tasks**, choose **On webhook** +as a task's schedule to run it whenever another service calls its URL, such as +GitHub on a new pull request or a CI job that failed. A public URL needs a +[T3 Connect](remote-access.md) managed tunnel; after you save the task, copy +its URL from the editor. Without one, the editor shows only the URL's path. +**Rotate** replaces the URL and the old one stops working. The prompt decides what the agent sees. Placeholders pull values out of the request: `{{body.path}}` for a JSON or form field, `{{headers.name}}`, @@ -81,12 +83,21 @@ pull request link. A placeholder with no value is left empty. For GitHub, turn on **Require signature**, keep the header `x-hub-signature-256`, hex encoding and the `sha256=` prefix, and enter the same secret in the repository's webhook settings with content type -`application/json`. Requests without a valid signature are rejected. +`application/json`. Requests without a valid signature are rejected. Set this +up on desktop or web; mobile keeps an existing signature check but can't turn +one on. On desktop and web, pick **Deliveries** from a task's menu to see recent -requests and the prompt each one produced. If the environment is offline, the sender gets an error and -nothing runs; redeliver from the sender, such as GitHub's **Recent Deliveries**, -once it is back. +requests and the prompt each one produced. + +If the environment is offline, the sender gets an error and nothing runs; +redeliver from the sender, such as GitHub's **Recent Deliveries**, once it is +back. To have T3 Connect keep requests instead, turn on **Hold webhooks while +offline** in **Settings → Connections**. T3 Connect then stores requests to a +T3 Connect URL for up to 24 hours and delivers them when the environment +returns. Leave it off if you don't want request bodies stored outside your +machine. To skip requests that waited too long, set **Skip requests older +than** on the task. ## Defaults and inheritance diff --git a/infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/migration.sql b/infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/migration.sql new file mode 100644 index 000000000000..b5561429664b --- /dev/null +++ b/infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "relay_environment_links" ADD COLUMN "hold_webhooks_while_offline" boolean DEFAULT false NOT NULL; \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/snapshot.json b/infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/snapshot.json new file mode 100644 index 000000000000..21c7a3747177 --- /dev/null +++ b/infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/snapshot.json @@ -0,0 +1,1581 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "13972659-45db-4ce9-9818-523b450bceca", + "prevIds": ["3809c51e-3821-4a08-818d-e5d28dd3e9b4"], + "ddl": [ + { + "isRlsEnabled": false, + "name": "relay_agent_activity_rows", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_delivery_attempts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_dpop_proofs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_credentials", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_links", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_live_activities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_endpoint_allocations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_tunnel_limits", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_mobile_devices", + "entityType": "tables", + "schema": "public" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state_json", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(36)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_job_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_suffix", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_status", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_reason", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "transport_error", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thumbprint", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "jti", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "iat", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_hash", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'T3 Environment'", + "generated": null, + "identity": null, + "name": "environment_label", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_http_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_ws_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(32)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_provider_kind", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "notifications_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "live_activities_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "managed_tunnels_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "hold_webhooks_while_offline", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activity_push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_start_queued_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_started_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ended_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_aggregate_json", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_live_activity_delivery_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "hostname", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_name", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dns_record_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ready_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_enabled_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "generation", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "max_tunnels", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'iOS device'", + "generated": null, + "identity": null, + "name": "label", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "platform", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ios_major_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "android_api_level", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "bundle_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "aps_environment", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_to_start_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "preferences_json", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "updated_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_agent_activity_rows_updated", + "entityType": "indexes", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "thread_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_source_job", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_dpop_proofs_expires_at", + "entityType": "indexes", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "credential_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_hash", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "environment_public_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment_key", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_links_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_links" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "activity_push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_live_activities_activity_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_live_activities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "hostname", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_hostname", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "tunnel_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_tunnel_name", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_to_start_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_to_start_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["environment_id", "environment_public_key", "thread_id"], + "nameExplicit": false, + "name": "relay_agent_activity_rows_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "columns": ["thumbprint", "jti"], + "nameExplicit": false, + "name": "relay_dpop_proofs_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_environment_links_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_environment_links" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_live_activities_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_live_activities" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_managed_endpoint_allocations_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_mobile_devices_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "relay_delivery_attempts_pkey", + "schema": "public", + "table": "relay_delivery_attempts", + "entityType": "pks" + }, + { + "columns": ["credential_id"], + "nameExplicit": false, + "name": "relay_environment_credentials_pkey", + "schema": "public", + "table": "relay_environment_credentials", + "entityType": "pks" + }, + { + "columns": ["user_id"], + "nameExplicit": false, + "name": "relay_managed_tunnel_limits_pkey", + "schema": "public", + "table": "relay_managed_tunnel_limits", + "entityType": "pks" + } + ], + "renames": [] +} diff --git a/infra/relay/package.json b/infra/relay/package.json index 03976ea81df8..ae3c806c4d24 100644 --- a/infra/relay/package.json +++ b/infra/relay/package.json @@ -13,6 +13,7 @@ "dependencies": { "@clerk/backend": "catalog:", "@effect/sql-pg": "catalog:", + "@effect/sql-sqlite-do": "catalog:", "@noble/curves": "catalog:", "@noble/hashes": "catalog:", "@t3tools/client-runtime": "workspace:*", diff --git a/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts b/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts index 5f2c9463d4da..2c44a2809d9e 100644 --- a/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts +++ b/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts @@ -95,6 +95,7 @@ function makeEnvironmentLinks( listForUser: () => Effect.succeed([]), getForUser: () => Effect.succeed(null), findActiveManagedForEnvironment: () => Effect.succeed([]), + setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: () => Effect.succeed(false), ...overrides, }; diff --git a/infra/relay/src/agentActivity/FcmDeliveries.test.ts b/infra/relay/src/agentActivity/FcmDeliveries.test.ts index ee903db8f382..65216cbe5308 100644 --- a/infra/relay/src/agentActivity/FcmDeliveries.test.ts +++ b/infra/relay/src/agentActivity/FcmDeliveries.test.ts @@ -172,6 +172,7 @@ function harness() { ), listForUser: () => Effect.succeed([]), findActiveManagedForEnvironment: () => Effect.succeed([]), + setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: () => Effect.succeed(false), getForUser: (input) => Effect.sync(() => diff --git a/infra/relay/src/agentActivity/MobileRegistrations.test.ts b/infra/relay/src/agentActivity/MobileRegistrations.test.ts index 57c5d6e55928..b649c09c7b56 100644 --- a/infra/relay/src/agentActivity/MobileRegistrations.test.ts +++ b/infra/relay/src/agentActivity/MobileRegistrations.test.ts @@ -117,6 +117,7 @@ function makeEnvironmentLinks( listForUser: () => Effect.succeed([]), getForUser: () => Effect.succeed(null), findActiveManagedForEnvironment: () => Effect.succeed([]), + setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: () => Effect.succeed(false), ...overrides, }; diff --git a/infra/relay/src/deploymentConfig.ts b/infra/relay/src/deploymentConfig.ts index 961c6f1b2f4a..e6ad6c4a8ee4 100644 --- a/infra/relay/src/deploymentConfig.ts +++ b/infra/relay/src/deploymentConfig.ts @@ -124,3 +124,16 @@ export function managedEndpointTunnelNamePrefix(stage: string): string { export function managedEndpointTunnelName(stage: string, hash: string): string { return `${managedEndpointTunnelNamePrefix(stage)}${stableSuffix(hash)}`; } + +/** + * A managed endpoint's public key in webhook URLs: the hash suffix its tunnel + * name ends with. The hash covers user and environment, so one key names + * exactly one link, unlike the environment id, which any account can claim. + */ +export const MANAGED_ENDPOINT_KEY_PATTERN = new RegExp( + `^[0-9a-f]{${MANAGED_ENDPOINT_HASH_LENGTH}}$`, +); + +export function managedEndpointTunnelNameForKey(stage: string, endpointKey: string): string { + return `${managedEndpointTunnelNamePrefix(stage)}${endpointKey}`; +} diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index 1a088b829d98..a0da2c17d1d9 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -194,6 +194,7 @@ function makeAllocations( }, ): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] { return { + getByTunnelName: () => Effect.die("unused getByTunnelName"), get: () => Effect.succeed(allocation), reserve: () => Effect.die("unused"), recordTunnel: () => Effect.die("unused"), @@ -230,6 +231,7 @@ function makeLinks( ...overrides, }), findActiveManagedForEnvironment: () => Effect.succeed([]), + setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: () => Effect.succeed(false), }; } diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index 912ffc30f70b..f068cd1fc499 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -127,6 +127,7 @@ function testLayer(input?: { listForUser: () => Effect.succeed([]), getForUser: () => Effect.succeed(null), findActiveManagedForEnvironment: () => Effect.succeed([]), + setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: () => Effect.succeed(false), }), Layer.succeed(EnvironmentCredentials.EnvironmentCredentials, { diff --git a/infra/relay/src/environments/EnvironmentLinks.test.ts b/infra/relay/src/environments/EnvironmentLinks.test.ts index 0a856bcaf7f5..2757b2fc00f0 100644 --- a/infra/relay/src/environments/EnvironmentLinks.test.ts +++ b/infra/relay/src/environments/EnvironmentLinks.test.ts @@ -122,6 +122,45 @@ describe("EnvironmentLinks", () => { ); }); + it.effect("carries the webhook-hold opt-in over only from links with the same key", () => { + const inserted: Array> = []; + const fakeDb = { + insert: () => ({ + values: (values: Record) => { + inserted.push(values); + return { onConflictDoUpdate: () => Effect.void }; + }, + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const links = yield* EnvironmentLinks.EnvironmentLinks; + yield* links.upsert({ + userId: "user-1", + request: { + notificationsEnabled: false, + liveActivitiesEnabled: false, + managedTunnelsEnabled: true, + } as never, + proof: { + environmentId: "env-1", + environmentPublicKey: "public-key-1", + descriptor: { label: "Laptop" }, + } as never, + endpoint: { httpBaseUrl: "https://a.example", wsBaseUrl: "wss://a.example" } as never, + }); + const query = new PgDialect().sqlToQuery(inserted[0]?.holdWebhooksWhileOffline as never); + // An environment id is public: another account linking it with its own + // key must not switch the opt-in on for this one. + expect(query.sql).toContain("environment_public_key"); + expect(query.params).toEqual(["env-1", "public-key-1"]); + }).pipe( + Effect.provide( + EnvironmentLinks.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, fakeDb))), + ), + ); + }); + it.effect("revokes only the active link owned by the requesting user", () => { const updateValues: Array> = []; const whereConditions: Array = []; diff --git a/infra/relay/src/environments/EnvironmentLinks.ts b/infra/relay/src/environments/EnvironmentLinks.ts index c6832a8025f3..afa15b917ff8 100644 --- a/infra/relay/src/environments/EnvironmentLinks.ts +++ b/infra/relay/src/environments/EnvironmentLinks.ts @@ -9,7 +9,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; -import { and, eq, isNull, or } from "drizzle-orm"; +import { and, eq, isNull, or, sql } from "drizzle-orm"; import * as RelayDb from "../db.ts"; import { relayEnvironmentLinks } from "../persistence/schema.ts"; @@ -126,13 +126,30 @@ export class EnvironmentLinks extends Context.Service< readonly userId: string; readonly environmentId: string; }) => Effect.Effect; - /** Active relay-managed links for an environment, across all users (webhook forwarding). */ + /** + * Active relay-managed links for an environment, narrowed to one user or to + * links proven by one environment key. The environment id alone is public + * and any account can link it, so callers acting on it must narrow. + */ readonly findActiveManagedForEnvironment: (input: { readonly environmentId: string; + readonly userId?: string; + readonly environmentPublicKey?: string; }) => Effect.Effect< - ReadonlyArray, + ReadonlyArray< + RelayLinkedEnvironmentRecord & { + readonly userId: string; + readonly holdWebhooksWhileOffline: boolean; + } + >, EnvironmentLinkEnvironmentLookupPersistenceError >; + /** Sets the webhook-hold opt-in on the active links proven by one environment key. */ + readonly setHoldWebhooksWhileOffline: (input: { + readonly environmentId: string; + readonly environmentPublicKey: string; + readonly holdWebhooksWhileOffline: boolean; + }) => Effect.Effect; readonly revokeForUser: (input: { readonly userId: string; readonly environmentId: string; @@ -166,12 +183,24 @@ const make = Effect.gen(function* () { const now = DateTime.formatIso(yield* DateTime.now); const { request, proof } = input; const environmentId = proof.environmentId; + // The webhook-hold opt-in belongs to the environment: a new or re-made + // link carries it over from the environment's other active links. Only + // links proven by the same key count; an environment id is public, so + // anyone can link one and switch the opt-in on for their own link. + const inheritedHoldWebhooks = sql`EXISTS ( + SELECT 1 FROM ${relayEnvironmentLinks} AS other + WHERE other.environment_id = ${environmentId} + AND other.environment_public_key = ${proof.environmentPublicKey} + AND other.revoked_at IS NULL + AND other.hold_webhooks_while_offline + )`; const { endpoint } = input; yield* db .insert(relayEnvironmentLinks) .values({ userId: input.userId, environmentId, + holdWebhooksWhileOffline: inheritedHoldWebhooks, environmentLabel: proof.descriptor.label, environmentPublicKey: proof.environmentPublicKey, endpointHttpBaseUrl: endpoint.httpBaseUrl, @@ -199,6 +228,7 @@ const make = Effect.gen(function* () { createdByDeviceId: request.deviceId ?? null, revokedAt: null, updatedAt: now, + holdWebhooksWhileOffline: inheritedHoldWebhooks, }, }) .pipe( @@ -355,6 +385,7 @@ const make = Effect.gen(function* () { endpointWsBaseUrl: relayEnvironmentLinks.endpointWsBaseUrl, endpointProviderKind: relayEnvironmentLinks.endpointProviderKind, createdAt: relayEnvironmentLinks.createdAt, + holdWebhooksWhileOffline: relayEnvironmentLinks.holdWebhooksWhileOffline, }) .from(relayEnvironmentLinks) .where( @@ -363,10 +394,13 @@ const make = Effect.gen(function* () { isNull(relayEnvironmentLinks.revokedAt), eq(relayEnvironmentLinks.endpointProviderKind, "cloudflare_tunnel"), eq(relayEnvironmentLinks.managedTunnelsEnabled, true), + input.userId === undefined ? undefined : eq(relayEnvironmentLinks.userId, input.userId), + input.environmentPublicKey === undefined + ? undefined + : eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey), ), ) - // One row per user who linked this environment; every row is checked - // until one has a ready endpoint, so none may be cut off. + // At most one row per user who linked this environment. .pipe( Effect.map((rows) => rows.map((row) => ({ @@ -382,6 +416,7 @@ const make = Effect.gen(function* () { }, environmentPublicKey: row.environmentPublicKey, linkedAt: row.createdAt, + holdWebhooksWhileOffline: row.holdWebhooksWhileOffline, })), ), Effect.mapError( @@ -394,6 +429,31 @@ const make = Effect.gen(function* () { ); }), + setHoldWebhooksWhileOffline: Effect.fn( + "relay.environment_links.set_hold_webhooks_while_offline", + )(function* (input) { + yield* Effect.annotateCurrentSpan({ "relay.environment_id": input.environmentId }); + yield* db + .update(relayEnvironmentLinks) + .set({ holdWebhooksWhileOffline: input.holdWebhooksWhileOffline }) + .where( + and( + eq(relayEnvironmentLinks.environmentId, input.environmentId), + eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey), + isNull(relayEnvironmentLinks.revokedAt), + ), + ) + .pipe( + Effect.mapError( + (cause) => + new EnvironmentLinkEnvironmentLookupPersistenceError({ + environmentId: input.environmentId, + cause, + }), + ), + ); + }), + revokeForUser: Effect.fn("relay.environment_links.revoke_for_user")(function* (input) { yield* Effect.annotateCurrentSpan({ "relay.environment_id": input.environmentId, diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 1b4d3744963b..e2d8475a1b2a 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -64,6 +64,7 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro "claim-deprovision", "remove", "remove-claimed", + "get-by-tunnel-name", ]), stage: Schema.Literals(["database-request", "resolve-reservation"]), userId: Schema.String, @@ -132,6 +133,10 @@ export class ManagedEndpointAllocations extends Context.Service< readonly get: ( input: ManagedEndpointAllocationKey, ) => Effect.Effect; + /** The allocation that owns a tunnel name; tunnel names are unique. */ + readonly getByTunnelName: ( + tunnelName: string, + ) => Effect.Effect; readonly reserve: ( input: ReserveManagedEndpointAllocationInput, ) => Effect.Effect; @@ -230,6 +235,29 @@ export const make = Effect.gen(function* () { ), ); }), + getByTunnelName: Effect.fn("relay.managed_endpoint_allocations.get_by_tunnel_name")(function* ( + tunnelName: string, + ) { + return yield* db + .select(allocationSelection) + .from(relayManagedEndpointAllocations) + .where(eq(relayManagedEndpointAllocations.tunnelName, tunnelName)) + .limit(1) + .pipe( + Effect.map((rows) => rows[0] ?? null), + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "get-by-tunnel-name", + stage: "database-request", + userId: "", + environmentId: "", + tunnelName, + cause, + }), + ), + ); + }), reserve: Effect.fn("relay.managed_endpoint_allocations.reserve")(function* ( input: ReserveManagedEndpointAllocationInput, ) { diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index becaf11488c8..94675125ded6 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -204,6 +204,7 @@ function makeAllocations(calls: AllocationCall[] = []) { } }; return ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + getByTunnelName: () => Effect.die("unused getByTunnelName"), get: (input) => Effect.sync(() => { calls.push({ operation: "get", input }); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index 9a2d62e36067..6f4d38d6a06d 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -166,6 +166,7 @@ function harness(input?: { }), }); const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + getByTunnelName: () => Effect.die("unused"), get: () => Effect.die("unused"), reserve: () => Effect.die("unused"), recordTunnel: () => Effect.die("unused"), diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index 6c4383d10df3..aab7c991af1e 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -1,9 +1,13 @@ import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform"; import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeCrypto from "node:crypto"; +import * as EffectNodeCrypto from "@effect/platform-node/NodeCrypto"; import { describe, expect, it } from "@effect/vitest"; import { RelayApi } from "@t3tools/contracts/relay"; +import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; +import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; @@ -18,6 +22,7 @@ import * as Etag from "effect/http/Etag"; import * as HttpRouter from "effect/http/HttpRouter"; import * as HttpApi from "effect/http-api/HttpApi"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; +import * as HttpMiddleware from "effect/http/HttpMiddleware"; import * as HttpServerRequest from "effect/http/HttpServerRequest"; import * as HttpServerResponse from "effect/http/HttpServerResponse"; @@ -31,6 +36,15 @@ import { traceRelayHttpRequestWith, } from "../http/Api.ts"; import * as HookForwarder from "./HookForwarder.ts"; +import { RELAY_HOOK_DELIVERY_TYP, verifyRelayJwt } from "@t3tools/shared/relayJwt"; +import * as HookInbox from "./HookInbox.ts"; +import type { HeldHook } from "./HookInboxStore.ts"; +import { RELAY_HOOK_UPSTREAM_TIMEOUT_MS } from "./upstream.ts"; + +const mintKeys = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, +}); const settings: RelayConfiguration.RelayConfiguration["Service"] = { relayIssuer: "https://relay.example.test", @@ -39,19 +53,20 @@ const settings: RelayConfiguration.RelayConfiguration["Service"] = { clerkPublishableKey: "pk_test_test", clerkJwtAudience: "t3-code-relay", apnsDeliveryJobSigningSecret: Redacted.make("apns-delivery-secret"), - cloudMintPrivateKey: Redacted.make("cloud-mint-private-key"), - cloudMintPublicKey: "cloud-mint-public-key", + cloudMintPrivateKey: Redacted.make(mintKeys.privateKey), + cloudMintPublicKey: mintKeys.publicKey, managedEndpointBaseDomain: "example.test", - managedEndpointNamespace: undefined, + managedEndpointNamespace: "dev", }; const environmentId = "env-hook"; +const endpointKey = "0123456789abcdef"; const readyAllocation: ManagedEndpointAllocations.ManagedEndpointAllocation = { userId: "user_1", environmentId, hostname: "env.example.test", tunnelId: "tunnel-id", - tunnelName: "tunnel-name", + tunnelName: `t3coderelay-managedendpoint-dev-${endpointKey}`, dnsRecordId: "dns-record-id", readyAt: "2026-05-25T00:00:00.000Z", origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, @@ -70,6 +85,7 @@ const managedLink = { }, environmentPublicKey: "public-key", linkedAt: "2026-05-25T00:00:00.000Z", + holdWebhooksWhileOffline: false, }; interface Harness { @@ -79,11 +95,15 @@ interface Harness { readonly links?: ReadonlyArray; readonly allocation?: ManagedEndpointAllocations.ManagedEndpointAllocation | null; readonly allow?: (key: string) => boolean; + readonly allowEndpoint?: (endpointKey: string) => boolean; + /** Inbox capacity; hold reports full once this many requests are held. */ + readonly inboxCapacity?: number; } function makeHarness(options: Harness = {}) { const sent: Array = []; const rateLimitKeys: Array = []; + const held: Array = []; const execute = options.execute ?? ((request: HttpClientRequest.HttpClientRequest) => @@ -100,12 +120,19 @@ function makeHarness(options: Harness = {}) { Layer.mock(EnvironmentLinks.EnvironmentLinks, { findActiveManagedForEnvironment: (input) => Effect.succeed( - input.environmentId === environmentId ? (options.links ?? [managedLink]) : [], + (options.links ?? [managedLink]).filter( + (link) => + link.environmentId === input.environmentId && + (input.userId === undefined || link.userId === input.userId), + ), ), }), Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, { - get: () => - Effect.succeed(options.allocation === undefined ? readyAllocation : options.allocation), + getByTunnelName: (tunnelName) => { + const allocation = + options.allocation === undefined ? readyAllocation : options.allocation; + return Effect.succeed(allocation?.tunnelName === tunnelName ? allocation : null); + }, }), Layer.succeed( HttpClient.HttpClient, @@ -114,12 +141,23 @@ function makeHarness(options: Harness = {}) { return execute(request); }), ), + Layer.mock(HookInbox.HookInbox, { + hold: ({ hook, baseUrl }) => + Effect.sync(() => { + if (held.length >= (options.inboxCapacity ?? Infinity)) return false; + held.push({ ...hook, baseUrl }); + return true; + }), + }), + EffectNodeCrypto.layer, Layer.succeed(HookForwarder.HookRateLimiter, { - allow: (key) => + allowHook: (key) => Effect.sync(() => { rateLimitKeys.push(key); return options.allow ? options.allow(key) : true; }), + allowEndpoint: (key) => + Effect.sync(() => (options.allowEndpoint ? options.allowEndpoint(key) : true)), }), ), ), @@ -144,11 +182,11 @@ function makeHarness(options: Harness = {}) { ), ); }); - return { sent, rateLimitKeys, send, httpEffect }; + return { sent, rateLimitKeys, held, send, httpEffect }; } const hookUrl = (path = "hook-1/secret-token", query = "") => - `https://relay.test/v1/hooks/${environmentId}/${path}${query}`; + `https://relay.test/v1/hooks/${endpointKey}/${path}${query}`; const readBody = (response: HttpServerResponse.HttpServerResponse) => Effect.promise(() => HttpServerResponse.toWeb(response).arrayBuffer()).pipe( @@ -210,6 +248,36 @@ describe("HookForwarder", () => { }), ); + it.effect("signs each forward for the environment, replacing any copy a sender sent", () => + Effect.gen(function* () { + const harness = makeHarness(); + yield* harness.send( + new Request(hookUrl("hook-1/tok"), { + method: "POST", + headers: { "x-t3-relay-delivery": "forged", "x-t3-relay-delivery-id": "forged" }, + body: "{}", + }), + ); + const sent = harness.sent[0]!; + const payload = yield* verifyRelayJwt({ + publicKey: mintKeys.publicKey, + token: sent.headers["x-t3-relay-delivery"]!, + typ: RELAY_HOOK_DELIVERY_TYP, + issuer: settings.relayIssuer, + audience: `t3-env:${environmentId}`, + nowEpochSeconds: Math.floor((yield* Clock.currentTimeMillis) / 1_000), + }); + // The proof names exactly the delivery the environment receives. + expect(payload).toMatchObject({ + environmentId, + hookId: "hook-1", + deliveryId: sent.headers["x-t3-relay-delivery-id"], + receivedAt: sent.headers["x-t3-relay-received-at"], + }); + expect(sent.headers["x-t3-relay-delivery-id"]).not.toBe("forged"); + }), + ); + it.effect("passes upstream status, body and content-type through", () => Effect.gen(function* () { const harness = makeHarness({ @@ -229,11 +297,24 @@ describe("HookForwarder", () => { expect(response.headers["content-type"]).toBe("application/json"); expect(response.headers["set-cookie"]).toBeUndefined(); expect(response.headers["access-control-allow-origin"]).toBeUndefined(); + // Served from the relay's origin, so it may never render or run there. + expect(response.headers["x-content-type-options"]).toBe("nosniff"); + expect(response.headers["content-security-policy"]).toBe("sandbox; default-src 'none'"); expect(new TextDecoder().decode(yield* readBody(response))).toBe('{"error":"bad_signature"}'); expect(harness.sent[0]?.method).toBe("GET"); }), ); + it.effect("never forwards or holds HEAD, which can carry no body", () => + Effect.gen(function* () { + const harness = makeHarness({ execute: () => Effect.die("must not be sent") }); + const response = yield* harness.send(new Request(hookUrl(), { method: "HEAD" })); + expect(response.status).toBeGreaterThanOrEqual(400); + expect(response.status).toBeLessThan(500); + expect(harness.held).toHaveLength(0); + }), + ); + it.effect("does not follow or relay upstream redirects", () => Effect.gen(function* () { const harness = makeHarness({ @@ -284,14 +365,17 @@ describe("HookForwarder", () => { }), ); - it.effect("returns 404 for unknown environments and unready endpoints", () => + it.effect("returns 404 for unknown endpoints and unready endpoints", () => Effect.gen(function* () { const unknown = makeHarness(); - const response = yield* unknown.send( - new Request("https://relay.test/v1/hooks/other-env/hook-1/token", { method: "POST" }), - ); - expect(response.status).toBe(404); - expect(yield* readJson(response)).toEqual({ error: "hook_not_found" }); + for (const key of ["fedcba9876543210", environmentId]) { + const response = yield* unknown.send( + new Request(`https://relay.test/v1/hooks/${key}/hook-1/token`, { method: "POST" }), + ); + expect(response.status).toBe(404); + expect(yield* readJson(response)).toEqual({ error: "hook_not_found" }); + } + expect(unknown.sent).toHaveLength(0); const unready = makeHarness({ allocation: { ...readyAllocation, readyAt: null } }); const unreadyResponse = yield* unready.send(new Request(hookUrl(), { method: "POST" })); @@ -300,6 +384,42 @@ describe("HookForwarder", () => { }), ); + it.effect("forwards only to the link that owns the endpoint key", () => + Effect.gen(function* () { + // Another account linked the same environment id under its own key; its + // link must not receive this endpoint's hooks, whatever order rows come in. + const intruder = { + ...managedLink, + userId: "user_attacker", + environmentPublicKey: "attacker-key", + endpoint: { ...managedLink.endpoint, httpBaseUrl: "https://attacker.example.test/" }, + }; + const harness = makeHarness({ links: [intruder, managedLink] }); + const response = yield* harness.send(new Request(hookUrl(), { method: "POST", body: "{}" })); + expect(response.status).toBe(200); + expect(harness.sent.map((request) => new URL(request.url).host)).toEqual([ + "env.example.test", + ]); + + // Without the owner's link, the key resolves to nothing at all. + const orphaned = makeHarness({ links: [intruder] }); + const orphanedResponse = yield* orphaned.send( + new Request(hookUrl(), { method: "POST", body: "{}" }), + ); + expect(orphanedResponse.status).toBe(404); + expect(orphaned.sent).toHaveLength(0); + }), + ); + + it.effect("returns 429 when the endpoint's overall budget is spent", () => + Effect.gen(function* () { + const harness = makeHarness({ allowEndpoint: () => false }); + const response = yield* harness.send(new Request(hookUrl(), { method: "POST" })); + expect(response.status).toBe(429); + expect(harness.sent).toHaveLength(0); + }), + ); + it.effect("maps tunnel-offline and network failures to 503", () => Effect.gen(function* () { const offline = makeHarness({ @@ -353,13 +473,32 @@ describe("HookForwarder", () => { .send(new Request(hookUrl(), { method: "POST", body: "{}" })) .pipe(Effect.forkChild); yield* Deferred.await(reachedUpstream); - yield* TestClock.adjust(Duration.millis(HookForwarder.RELAY_HOOK_UPSTREAM_TIMEOUT_MS)); + yield* TestClock.adjust(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)); const response = yield* Fiber.join(fiber); expect(response.status).toBe(504); expect(yield* readJson(response)).toEqual({ error: "environment_timeout" }); }), ); + it.effect("holds a timed-out request with the time it arrived", () => + Effect.gen(function* () { + const reachedUpstream = yield* Deferred.make(); + const harness = makeHarness({ + links: [{ ...managedLink, holdWebhooksWhileOffline: true }], + execute: () => + Deferred.succeed(reachedUpstream, undefined).pipe(Effect.andThen(Effect.never)), + }); + const arrivedAt = DateTime.formatIso(yield* DateTime.now); + const fiber = yield* harness + .send(new Request(hookUrl(), { method: "POST", body: "{}" })) + .pipe(Effect.forkChild); + yield* Deferred.await(reachedUpstream); + yield* TestClock.adjust(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)); + expect((yield* Fiber.join(fiber)).status).toBe(202); + expect(harness.held[0]?.receivedAt).toBe(arrivedAt); + }), + ); + it.effect("answers OPTIONS without a CORS preflight or forwarding", () => Effect.gen(function* () { const harness = makeHarness(); @@ -454,9 +593,215 @@ describe("HookForwarder", () => { expect(serialized).not.toContain("also-secret"); expect(serialized).not.toContain("header-secret"); const server = spans.find((span) => span.kind === "server"); - expect(server?.attributes.get("url.path")).toBe( - `/v1/hooks/${environmentId}/hook-1/`, + expect(server?.attributes.get("url.path")).toBe(`/v1/hooks/${endpointKey}/hook-1/`); + }), + ); + + it.effect("records one redacted server span even inside the worker's own HTTP tracer", () => + Effect.gen(function* () { + const spans: Array = []; + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, + }); + const harness = makeHarness(); + const handler = yield* harness.httpEffect; + // As the worker runtime runs it: its own tracer around ours, off for hook + // paths. This checks the predicate; whether alchemy applies it per event + // is only visible on a deployed worker (see worker.ts). + yield* HttpMiddleware.tracer( + traceRelayHttpRequestWith(handler, Layer.succeed(Tracer.Tracer, tracer)), + ).pipe( + Effect.provideService(HttpMiddleware.TracerDisabledWhen, (request) => + HookForwarder.isRelayHookPath(request.url), + ), + Effect.withTracer(tracer), + Effect.provideService( + HttpServerRequest.HttpServerRequest, + HttpServerRequest.fromWeb( + new Request(hookUrl("hook-1/super-secret-token"), { + method: "POST", + headers: { traceparent: "00-11111111111111111111111111111111-2222222222222222-01" }, + body: "{}", + }), + ), + ), + ); + yield* Effect.yieldNow; + const servers = spans.filter((span) => span.kind === "server"); + expect(servers).toHaveLength(1); + expect(servers[0]?.attributes.get("url.path")).toBe( + `/v1/hooks/${endpointKey}/hook-1/`, ); + expect(spans.every((span) => span.traceId !== "11111111111111111111111111111111")).toBe(true); + }), + ); + + it.effect("joins the environment to its trace and records what the environment did", () => + Effect.gen(function* () { + const spans: Array = []; + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, + }); + const respondWith = (outcome: string) => + makeHarness({ + execute: (request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response('{"deliveryId":"d"}', { + status: 202, + headers: { "x-t3-hook-outcome": outcome }, + }), + ), + ), + }); + const forward = (harness: ReturnType) => + Effect.gen(function* () { + const handler = yield* harness.httpEffect; + return yield* traceRelayHttpRequestWith( + handler, + Layer.succeed(Tracer.Tracer, tracer), + ).pipe( + Effect.provideService( + HttpServerRequest.HttpServerRequest, + HttpServerRequest.fromWeb( + new Request(hookUrl(), { + method: "POST", + // A sender's own trace context never reaches the environment. + headers: { + traceparent: "00-11111111111111111111111111111111-2222222222222222-01", + "x-b3-traceid": "33333333333333333333333333333333", + }, + body: "{}", + }), + ), + ), + ); + }); + + const duplicate = respondWith("duplicate"); + expect((yield* forward(duplicate)).status).toBe(202); + yield* Effect.yieldNow; + const forwardSpan = spans.find((span) => span.name === "relay.hooks.forward"); + expect(forwardSpan?.attributes.get("relay.hook.upstream_outcome")).toBe("duplicate"); + const sent = duplicate.sent[0]!; + expect(sent.headers.traceparent).toContain(forwardSpan!.traceId); + expect(sent.headers.traceparent).not.toContain("1111111111"); + expect(sent.headers["x-b3-traceid"]).toBeUndefined(); + + // Only a plain outcome name is recorded. + spans.length = 0; + yield* forward(respondWith("")); + yield* Effect.yieldNow; + expect( + spans + .find((span) => span.name === "relay.hooks.forward") + ?.attributes.has("relay.hook.upstream_outcome"), + ).toBe(false); }), ); + + describe("holding requests while the environment is offline", () => { + const offline = (request: HttpClientRequest.HttpClientRequest) => + Effect.fail( + new HttpClientError.HttpClientError({ + reason: new HttpClientError.TransportError({ request, cause: new Error("offline") }), + }), + ); + + it.effect("holds when cloudflared answers that the local server is down", () => + Effect.gen(function* () { + for (const status of [502, 503, 504, 530]) { + const harness = makeHarness({ + links: [{ ...managedLink, holdWebhooksWhileOffline: true }], + execute: (request) => + Effect.succeed(HttpClientResponse.fromWeb(request, new Response("", { status }))), + }); + const response = yield* harness.send( + new Request(hookUrl(), { method: "POST", body: "{}" }), + ); + expect(response.status).toBe(202); + expect(harness.held).toHaveLength(1); + } + }), + ); + + it.effect("stays a plain proxy when the environment has not opted in", () => + Effect.gen(function* () { + const harness = makeHarness({ execute: offline }); + const response = yield* harness.send( + new Request(hookUrl(), { method: "POST", body: "{}" }), + ); + expect(response.status).toBe(503); + expect(harness.held).toHaveLength(0); + }), + ); + + it.effect("holds the exact request and answers 202 once opted in", () => + Effect.gen(function* () { + const harness = makeHarness({ + execute: offline, + links: [{ ...managedLink, holdWebhooksWhileOffline: true }], + }); + const body = new Uint8Array([0, 255, 10]); + const response = yield* harness.send( + new Request(hookUrl("%68ook-1/tok%2Fen", "?a=1"), { + method: "POST", + body, + headers: { "x-t3-relay-delivery-id": "forged", "x-sig": "s" }, + }), + ); + expect(response.status).toBe(202); + const [hook] = harness.held; + expect(hook?.baseUrl).toBe("https://env.example.test/"); + expect(hook?.rawToken).toBe("tok%2Fen"); + expect(hook?.query).toBe("a=1"); + expect([...(hook?.body ?? [])]).toEqual([0, 255, 10]); + expect(hook?.headers["x-sig"]).toBe("s"); + // Every spelling of the hook id shares one per-hook cap. + expect(hook?.hookKey).toBe("hook-1"); + // The sender cannot choose the delivery id. + expect(hook?.headers["x-t3-relay-delivery-id"]).toBeUndefined(); + expect(hook?.id).not.toBe("forged"); + }), + ); + + it.effect("answers 503 inbox_full when the environment's inbox is full", () => + Effect.gen(function* () { + const harness = makeHarness({ + execute: offline, + links: [{ ...managedLink, holdWebhooksWhileOffline: true }], + inboxCapacity: 0, + }); + const response = yield* harness.send( + new Request(hookUrl(), { method: "POST", body: "{}" }), + ); + expect(response.status).toBe(503); + expect(yield* readJson(response)).toEqual({ error: "inbox_full" }); + }), + ); + + it.effect("tags every forward with a relay delivery id", () => + Effect.gen(function* () { + const harness = makeHarness(); + yield* harness.send( + new Request(hookUrl(), { + method: "POST", + body: "{}", + headers: { "x-t3-relay-delivery-id": "forged" }, + }), + ); + const id = harness.sent[0]?.headers["x-t3-relay-delivery-id"]; + expect(id).toMatch(/^[0-9a-f-]{36}$/); + }), + ); + }); }); diff --git a/infra/relay/src/hooks/HookForwarder.ts b/infra/relay/src/hooks/HookForwarder.ts index 7f33a7d2513f..f5031389dfc7 100644 --- a/infra/relay/src/hooks/HookForwarder.ts +++ b/infra/relay/src/hooks/HookForwarder.ts @@ -1,28 +1,57 @@ +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; -import * as Duration from "effect/Duration"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import * as HttpClient from "effect/http/HttpClient"; -import * as HttpClientRequest from "effect/http/HttpClientRequest"; -import type * as HttpClientResponse from "effect/http/HttpClientResponse"; import type * as HttpServerRequest from "effect/http/HttpServerRequest"; import * as HttpServerResponse from "effect/http/HttpServerResponse"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; -import { RelayApi } from "@t3tools/contracts/relay"; +import { EnvironmentId } from "@t3tools/contracts"; +import { RelayApi, type RelayHookDeliveryProofPayload } from "@t3tools/contracts/relay"; +import { + normalizeRelayIssuer, + RELAY_HOOK_DELIVERY_HEADER, + RELAY_HOOK_DELIVERY_TYP, + signRelayJwt, +} from "@t3tools/shared/relayJwt"; import * as RelayConfiguration from "../Config.ts"; -import { validateManagedEndpoint, withoutRedirects } from "../environments/EnvironmentConnector.ts"; +import { + MANAGED_ENDPOINT_KEY_PATTERN, + managedEndpointTunnelNameForKey, +} from "../deploymentConfig.ts"; +import { validateManagedEndpoint } from "../environments/EnvironmentConnector.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; +import * as HookInbox from "./HookInbox.ts"; +import { sendUpstream, TUNNEL_OFFLINE_STATUS } from "./upstream.ts"; export const RELAY_HOOK_PATH_PREFIX = "/v1/hooks/"; export const RELAY_HOOK_MAX_BODY_BYTES = 1_048_576; -export const RELAY_HOOK_UPSTREAM_TIMEOUT_MS = 8_000; export const RELAY_HOOK_RATE_LIMIT = { limit: 60, periodSeconds: 60 } as const; +/** + * Hook budgets are per URL, and a sender who knows an endpoint key can mint + * new URLs for free, so every endpoint also has one overall budget. + */ +export const RELAY_HOOK_ENDPOINT_RATE_LIMIT = { limit: 600, periodSeconds: 60 } as const; +/** + * Upstream statuses that mean the environment did not take the request: the + * tunnel has no origin (530) or cloudflared cannot reach the local server + * (502, 503, 504) while it restarts. + */ +export const ENVIRONMENT_UNREACHABLE_STATUSES: ReadonlySet = new Set([ + 502, + 503, + 504, + TUNNEL_OFFLINE_STATUS, +]); const DROPPED_REQUEST_HEADERS = new Set([ "host", @@ -34,12 +63,16 @@ const DROPPED_REQUEST_HEADERS = new Set([ "content-length", "cookie", "x-real-ip", + // Only the relay may set this; a sender could otherwise collide delivery ids. + "x-t3-relay-delivery-id", + "x-t3-relay-received-at", + RELAY_HOOK_DELIVERY_HEADER, + // The environment trusts trace context only from the relay, which sets its own. + "traceparent", + "tracestate", + "b3", ]); -const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-"]; -// Cloudflare answers 530 when the tunnel for a hostname has no connected origin. -const TUNNEL_OFFLINE_STATUS = 530; -/** The environment answers with a small JSON status; anything past this is cut off. */ -const MAX_RESPONSE_BYTES = 64 * 1024; +const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-", "x-b3-"]; export const isRelayHookPath = (url: string): boolean => url.startsWith(RELAY_HOOK_PATH_PREFIX); @@ -55,13 +88,13 @@ export const redactRelayHookUrl = (url: string): string => { /** * Request budget for public hook forwarding, keyed by a hash of the hook URL - * (environment, hook and token). Requests with a wrong token get their own + * (endpoint, hook and token). Requests with a wrong token get their own * budget, so they cannot use up a real sender's; the environment rejects them. * Built from decoded segments, because the environment decodes them too: two * spellings of one token (`token`, `%74oken`) must share one budget. */ const hookBudgetKey = (hook: { - readonly environmentId: string; + readonly endpointKey: string; readonly hookId: string; readonly token: string; }) => @@ -70,7 +103,7 @@ const hookBudgetKey = (hook: { "SHA-256", // Length-prefixed, so no segment contents can make two keys collide. new TextEncoder().encode( - [hook.environmentId, hook.hookId, hook.token] + [hook.endpointKey, hook.hookId, hook.token] .map((part) => `${part.length}:${part}`) .join(""), ), @@ -83,7 +116,12 @@ const hookBudgetKey = (hook: { export class HookRateLimiter extends Context.Service< HookRateLimiter, - { readonly allow: (key: string) => Effect.Effect } + { + /** One hook URL's budget, keyed by `hookBudgetKey`. */ + readonly allowHook: (key: string) => Effect.Effect; + /** One endpoint's overall budget, keyed by its endpoint key. */ + readonly allowEndpoint: (endpointKey: string) => Effect.Effect; + } >()("t3code-relay/hooks/HookForwarder/HookRateLimiter") {} export class HookForwarder extends Context.Service< @@ -96,25 +134,69 @@ export class HookForwarder extends Context.Service< >()("t3code-relay/hooks/HookForwarder") {} class HookBodyTooLarge extends Schema.TaggedError()("HookBodyTooLarge", {}) {} -class ResponseTooLarge extends Schema.TaggedError()("ResponseTooLarge", {}) {} const errorResponse = (status: number, error: string, headers?: Record) => HttpServerResponse.jsonUnsafe({ error }, { status, ...(headers ? { headers } : {}) }); const hookNotFound = () => errorResponse(404, "hook_not_found"); +/** Longer than the inbox holds a request, so a held delivery's proof still verifies. */ +const DELIVERY_PROOF_LIFETIME_SECONDS = 25 * 60 * 60; + +const signDeliveryProof = (input: { + readonly settings: RelayConfiguration.RelayConfiguration["Service"]; + readonly environmentId: string; + readonly deliveryId: string; + readonly receivedAt: string; + readonly hookId: string; + readonly jti: string; +}) => + Effect.gen(function* () { + const now = Math.floor((yield* Clock.currentTimeMillis) / 1_000); + return yield* signRelayJwt({ + privateKey: Redacted.value(input.settings.cloudMintPrivateKey), + typ: RELAY_HOOK_DELIVERY_TYP, + payload: { + iss: normalizeRelayIssuer(input.settings.relayIssuer), + aud: `t3-env:${input.environmentId}`, + sub: input.environmentId, + jti: input.jti, + iat: now, + exp: now + DELIVERY_PROOF_LIFETIME_SECONDS, + environmentId: EnvironmentId.make(input.environmentId), + deliveryId: input.deliveryId, + receivedAt: input.receivedAt, + hookId: input.hookId, + } satisfies RelayHookDeliveryProofPayload, + }); + }).pipe(Effect.orDie); + +/** Methods a webhook can arrive with; HEAD reaches the GET route and is refused. */ +const FORWARDED_METHODS = new Set(["GET", "POST", "PUT", "PATCH"]); + +/** + * Whatever the environment answers is served from the relay's own origin, so + * a body must never render or run there. + */ +const SANDBOXED_RESPONSE_HEADERS = { + "x-content-type-options": "nosniff", + "content-security-policy": "sandbox; default-src 'none'", +} as const; + function parseHookPath(url: string) { const queryIndex = url.indexOf("?"); const path = queryIndex === -1 ? url : url.slice(0, queryIndex); const search = queryIndex === -1 ? "" : url.slice(queryIndex); const segments = path.split("/"); - // ["", "v1", "hooks", environmentId, hookId, token] + // ["", "v1", "hooks", endpointKey, hookId, token] if (segments.length !== 6) return null; - const [, , , rawEnvironmentId, rawHookId, rawToken] = segments; - if (!rawEnvironmentId || !rawHookId || !rawToken) return null; + const [, , , endpointKey, rawHookId, rawToken] = segments; + if (!endpointKey || !MANAGED_ENDPOINT_KEY_PATTERN.test(endpointKey) || !rawHookId || !rawToken) { + return null; + } try { return { - environmentId: decodeURIComponent(rawEnvironmentId), + endpointKey, hookId: decodeURIComponent(rawHookId), token: decodeURIComponent(rawToken), // Forward the encoded segments byte-for-byte; the environment decodes them. @@ -156,20 +238,18 @@ function forwardedHeaders(headers: Readonly>): Record request.source instanceof Request && request.source.body === null; -/** Collects a byte stream, failing with `tooLarge` once it passes `maxBytes` rather than buffering it all. */ -const collectCapped = ( - stream: Stream.Stream, - maxBytes: number, - tooLarge: () => E2, -) => +const readCappedBody = (request: HttpServerRequest.HttpServerRequest) => Effect.suspend(() => { + if (hasNoBody(request)) { + return Effect.succeed(new Uint8Array(0)); + } const chunks: Array = []; let total = 0; - return stream.pipe( + return request.stream.pipe( Stream.runForEach((chunk) => { total += chunk.length; - if (total > maxBytes) { - return Effect.fail(tooLarge()); + if (total > RELAY_HOOK_MAX_BODY_BYTES) { + return Effect.fail(new HookBodyTooLarge()); } chunks.push(chunk); return Effect.void; @@ -186,19 +266,47 @@ const collectCapped = ( ); }); -const readCappedBody = (request: HttpServerRequest.HttpServerRequest) => - hasNoBody(request) - ? Effect.succeed(new Uint8Array(0)) - : collectCapped(request.stream, RELAY_HOOK_MAX_BODY_BYTES, () => new HookBodyTooLarge()); - -const readCappedResponse = (response: HttpClientResponse.HttpClientResponse) => - collectCapped(response.stream, MAX_RESPONSE_BYTES, () => new ResponseTooLarge()).pipe( - // A response without a body, such as a redirect, has no stream at all. - Effect.catchIf( - (error) => error._tag === "HttpClientError" && error.reason._tag === "EmptyBodyError", - () => Effect.succeed(new Uint8Array(0)), - ), +/** + * The ready managed endpoint a webhook URL's endpoint key names, with whether + * its link opted in to holding webhooks while offline. The key is the tunnel + * name's hash of user and environment, so it names exactly one allocation and + * at most one active link; nobody else can link their way onto it. + */ +export const resolveHookEndpoint = Effect.fn("relay.hooks.resolve_endpoint")(function* ( + endpointKey: string, +) { + const links = yield* EnvironmentLinks.EnvironmentLinks; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const settings = yield* RelayConfiguration.RelayConfiguration; + if (!settings.managedEndpointNamespace) return null; + const allocation = yield* allocations.getByTunnelName( + managedEndpointTunnelNameForKey(settings.managedEndpointNamespace, endpointKey), ); + if (allocation === null) return null; + const [link] = yield* links.findActiveManagedForEnvironment({ + environmentId: allocation.environmentId, + userId: allocation.userId, + }); + if (!link) return null; + const result = validateManagedEndpoint({ + link, + allocation, + baseDomain: settings.managedEndpointBaseDomain, + }); + if (Result.isFailure(result)) return null; + return { + ...result.success, + environmentId: allocation.environmentId, + holdWhileOffline: link.holdWebhooksWhileOffline, + }; +}); + +/** The endpoint key of an environment's own managed endpoint, for authenticated callers. */ +export const endpointKeyForTunnelName = (namespace: string, tunnelName: string): string | null => { + const prefix = managedEndpointTunnelNameForKey(namespace, ""); + const key = tunnelName.startsWith(prefix) ? tunnelName.slice(prefix.length) : ""; + return MANAGED_ENDPOINT_KEY_PATTERN.test(key) ? key : null; +}; const make = Effect.gen(function* () { const links = yield* EnvironmentLinks.EnvironmentLinks; @@ -206,39 +314,38 @@ const make = Effect.gen(function* () { const settings = yield* RelayConfiguration.RelayConfiguration; const httpClient = yield* HttpClient.HttpClient; const rateLimiter = yield* HookRateLimiter; - - const resolveEndpoint = Effect.fn("relay.hooks.resolve_endpoint")(function* ( - environmentId: string, - ) { - const candidates = yield* links.findActiveManagedForEnvironment({ environmentId }); - for (const link of candidates) { - const allocation = yield* allocations.get({ userId: link.userId, environmentId }); - const result = validateManagedEndpoint({ - link, - allocation, - baseDomain: settings.managedEndpointBaseDomain, - }); - if (Result.isSuccess(result)) { - return result.success; - } - } - return null; - }); + const inbox = yield* HookInbox.HookInbox; + const crypto = yield* Crypto.Crypto; const handle = Effect.fn("relay.hooks.forward")(function* ( request: HttpServerRequest.HttpServerRequest, ) { const outcome = (value: string) => Effect.annotateCurrentSpan({ "relay.hook.outcome": value }); + if (!FORWARDED_METHODS.has(request.method)) { + yield* outcome("method_not_allowed"); + return errorResponse(405, "method_not_allowed", { allow: "GET, POST, PUT, PATCH" }); + } + // When the sender called, not when the environment failed to answer. + const receivedAt = DateTime.formatIso(yield* DateTime.now); const parsed = parseHookPath(request.url); if (!parsed) { yield* outcome("invalid_path"); return hookNotFound(); } yield* Effect.annotateCurrentSpan({ - "relay.environment_id": parsed.environmentId, + "relay.hook.endpoint_key": parsed.endpointKey, "relay.hook_id": parsed.hookId, }); - if (!(yield* rateLimiter.allow(yield* hookBudgetKey(parsed)))) { + // A coarse budget per endpoint first, so minting new hook ids or tokens + // cannot buy unlimited lookups and forwards, or fill the inbox. + const endpointAllowed = yield* rateLimiter.allowEndpoint(parsed.endpointKey); + const hookAllowed = + endpointAllowed && (yield* rateLimiter.allowHook(yield* hookBudgetKey(parsed))); + if (!endpointAllowed || !hookAllowed) { + // Which budget ran out: the whole endpoint's, or this one hook URL's. + yield* Effect.annotateCurrentSpan({ + "relay.hook.rate_limit": endpointAllowed ? "hook" : "endpoint", + }); yield* outcome("rate_limited"); return errorResponse(429, "rate_limited", { "retry-after": String(RELAY_HOOK_RATE_LIMIT.periodSeconds), @@ -250,10 +357,13 @@ const make = Effect.gen(function* () { return errorResponse(413, "payload_too_large"); } - const endpoint = yield* resolveEndpoint(parsed.environmentId).pipe( + const endpoint = yield* resolveHookEndpoint(parsed.endpointKey).pipe( + Effect.provideService(EnvironmentLinks.EnvironmentLinks, links), + Effect.provideService(ManagedEndpointAllocations.ManagedEndpointAllocations, allocations), + Effect.provideService(RelayConfiguration.RelayConfiguration, settings), Effect.catch((error) => Effect.logWarning("Failed to resolve hook endpoint", { - environmentId: parsed.environmentId, + endpointKey: parsed.endpointKey, errorTag: error._tag, }).pipe(Effect.as(null)), ), @@ -262,6 +372,10 @@ const make = Effect.gen(function* () { yield* outcome("not_found"); return hookNotFound(); } + yield* Effect.annotateCurrentSpan({ + "relay.environment_id": endpoint.environmentId, + "relay.hook.hold_while_offline": endpoint.holdWhileOffline, + }); const body = request.method === "GET" @@ -276,67 +390,101 @@ const make = Effect.gen(function* () { return errorResponse(400, "invalid_body"); } - const baseUrl = endpoint.httpBaseUrl.endsWith("/") - ? endpoint.httpBaseUrl - : `${endpoint.httpBaseUrl}/`; - const headers = forwardedHeaders(request.headers); - let upstreamRequest = HttpClientRequest.make( - request.method as "GET" | "POST" | "PUT" | "PATCH", - )(`${baseUrl}api/hooks/${parsed.rawHookId}/${parsed.rawToken}${parsed.search}`, { headers }); - if (request.method !== "GET") { - upstreamRequest = HttpClientRequest.bodyUint8Array( - upstreamRequest, - body.success, - headers["content-type"], - ); - } + // One id per request, so a request that reached the environment before a + // timeout and is later delivered from the inbox runs only once. + yield* Effect.annotateCurrentSpan({ "relay.hook.body_bytes": body.success.byteLength }); + const deliveryId = yield* crypto.randomUUIDv4.pipe(Effect.orDie); + // Proves to the environment that this delivery id, receive time and + // trace context came from the relay. Signed once here and stored with a + // held request, so the inbox never needs the signing key. + const proof = yield* signDeliveryProof({ + settings, + environmentId: endpoint.environmentId, + deliveryId, + receivedAt, + hookId: parsed.hookId, + jti: yield* crypto.randomUUIDv4.pipe(Effect.orDie), + }); + const hook = { + id: deliveryId, + receivedAt, + method: request.method, + rawHookId: parsed.rawHookId, + rawToken: parsed.rawToken, + hookKey: parsed.hookId, + query: parsed.search.replace(/^\?/, ""), + headers: { ...forwardedHeaders(request.headers), [RELAY_HOOK_DELIVERY_HEADER]: proof }, + body: body.success, + }; + // Held only for environments that opted in; otherwise the relay is a plain proxy. + const holdOrFail = (status: 503 | 504, error: string) => + Effect.gen(function* () { + if (!endpoint.holdWhileOffline) { + yield* outcome(error); + return errorResponse(status, error); + } + const stored = yield* inbox + .hold({ + endpointKey: parsed.endpointKey, + baseUrl: endpoint.httpBaseUrl, + hook, + }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("Could not hold webhook request", { + environmentId: endpoint.environmentId, + errorTag: cause._tag, + }).pipe(Effect.as(null)), + ), + ); + if (stored === null) { + yield* outcome(error); + return errorResponse(status, error); + } + if (!stored) { + // The inbox span carries which cap refused it (relay.inbox.refused). + yield* outcome("inbox_full"); + return errorResponse(503, "inbox_full"); + } + yield* outcome("held"); + return HttpServerResponse.jsonUnsafe({ queued: true }, { status: 202 }); + }); - const upstream = yield* httpClient.execute(upstreamRequest).pipe( - Effect.flatMap((response) => - readCappedResponse(response).pipe( - Effect.map((body) => ({ - status: response.status, - contentType: response.headers["content-type"], - body, - })), - ), - ), - withoutRedirects, - // The client span would record url.full, which carries the token. - Effect.provideService(HttpClient.TracerDisabledWhen, () => true), - Effect.timeoutOption(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)), + const upstream = yield* sendUpstream(endpoint.httpBaseUrl, hook).pipe( + Effect.provideService(HttpClient.HttpClient, httpClient), Effect.result, ); if (Result.isFailure(upstream)) { - yield* outcome("environment_unavailable"); - return errorResponse(503, "environment_unavailable"); + yield* Effect.annotateCurrentSpan({ "relay.hook.upstream_error": upstream.failure._tag }); + return yield* holdOrFail(503, "environment_unavailable"); } if (Option.isNone(upstream.success)) { - yield* outcome("environment_timeout"); - return errorResponse(504, "environment_timeout"); + return yield* holdOrFail(504, "environment_timeout"); } const response = upstream.success.value; - if (response.status === TUNNEL_OFFLINE_STATUS) { - yield* outcome("environment_unavailable"); - return errorResponse(503, "environment_unavailable"); + if (ENVIRONMENT_UNREACHABLE_STATUSES.has(response.status)) { + yield* Effect.annotateCurrentSpan({ "relay.hook.upstream_status": response.status }); + return yield* holdOrFail(503, "environment_unavailable"); } yield* Effect.annotateCurrentSpan({ "relay.hook.outcome": "forwarded", "relay.hook.upstream_status": response.status, + ...(response.outcome === undefined + ? {} + : { "relay.hook.upstream_outcome": response.outcome }), }); // Only content-type is passed through: no location (redirects are never // followed or relayed), no cookies, no upstream infrastructure headers. - const contentTypeHeaders = response.contentType - ? { "content-type": response.contentType } - : undefined; + const headers = { + ...SANDBOXED_RESPONSE_HEADERS, + ...(response.contentType ? { "content-type": response.contentType } : {}), + }; if (response.body.length === 0) { - return HttpServerResponse.empty({ - status: response.status, - ...(contentTypeHeaders ? { headers: contentTypeHeaders } : {}), - }); + return HttpServerResponse.empty({ status: response.status, headers }); } return HttpServerResponse.uint8Array(response.body, { status: response.status, + headers, ...(response.contentType ? { contentType: response.contentType } : {}), }); }); diff --git a/infra/relay/src/hooks/HookInbox.ts b/infra/relay/src/hooks/HookInbox.ts new file mode 100644 index 000000000000..cc3d1517df4c --- /dev/null +++ b/infra/relay/src/hooks/HookInbox.ts @@ -0,0 +1,40 @@ +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import type { HeldHook } from "./HookInboxStore.ts"; + +export class HookInboxError extends Schema.TaggedError()("HookInboxError", { + operation: Schema.Literals(["hold", "wake", "clear"]), + endpointKey: Schema.String, + cause: Schema.Defect(), +}) { + override get message(): string { + return `Hook inbox '${this.operation}' failed for endpoint '${this.endpointKey}'`; + } +} + +/** + * Webhook requests held for environments that opted in, while they are + * offline. Each managed endpoint's requests live in its own `HookInboxObject`, + * named by endpoint key, which delivers them itself once the environment is back. + */ +export class HookInbox extends Context.Service< + HookInbox, + { + /** False when the environment's inbox is full and nothing was stored. */ + readonly hold: (input: { + readonly endpointKey: string; + readonly baseUrl: string; + readonly hook: HeldHook; + }) => Effect.Effect; + /** Delivers what is waiting now; true when anything was waiting. */ + readonly wake: (input: { + readonly endpointKey: string; + readonly baseUrl: string; + }) => Effect.Effect; + readonly clear: (input: { + readonly endpointKey: string; + }) => Effect.Effect; + } +>()("t3code-relay/hooks/HookInbox") {} diff --git a/infra/relay/src/hooks/HookInboxObject.ts b/infra/relay/src/hooks/HookInboxObject.ts new file mode 100644 index 000000000000..82897e24d5c8 --- /dev/null +++ b/infra/relay/src/hooks/HookInboxObject.ts @@ -0,0 +1,154 @@ +import * as SqliteClient from "@effect/sql-sqlite-do/SqliteClient"; +import type * as Alchemy from "alchemy"; +import * as Cloudflare from "alchemy/Cloudflare"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Result from "effect/Result"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; + +import * as HookInboxStore from "./HookInboxStore.ts"; +import { sendUpstream, TUNNEL_OFFLINE_STATUS } from "./upstream.ts"; + +/** Statuses that mean the environment is not there; the whole inbox waits and backs off. */ +const UNREACHABLE_STATUSES = new Set([502, 503, 504, TUNNEL_OFFLINE_STATUS]); +/** + * Statuses that mean the environment is there but did not take this request + * yet: its task's queue is full (429) or it failed while handling it (500). + * The environment drops a delivery id it has already run, so retrying is safe. + */ +const BUSY_STATUSES = new Set([429, 500]); +/** When a run itself fails, the next one is tried after this long. */ +const RUN_FAILURE_RETRY_MS = 60_000; + +type Call = Effect.Effect; + +export interface HookInboxObjectShape { + /** Holds a request for `baseUrl`; false when the inbox is full and nothing was stored. */ + readonly hold: (hook: HookInboxStore.HeldHook, baseUrl: string) => Call; + /** The environment is back at `baseUrl`: deliver what is waiting now. */ + readonly wake: (baseUrl: string) => Call; + readonly clear: () => Call; +} + +/** + * One per managed endpoint, addressed by endpoint key. Holds webhook requests + * the environment could not take, in SQLite, and pushes them back through + * its tunnel from the alarm, oldest first, backing off while it stays away. + */ +export class HookInboxObject extends Cloudflare.DurableObject< + HookInboxObject, + HookInboxObjectShape +>()("HookInboxObject") {} + +/** + * Each call into an inbox is its own trace: the alarm has no parent, and a + * hold arrives over Durable Object RPC without the forwarding span. The + * object's name is the endpoint key, which the forward spans carry too. + */ +const withInboxSpan = + (name: string, inboxId: string) => + (effect: Effect.Effect) => + effect.pipe( + Effect.withSpan(name, { root: true, attributes: { "relay.hook.endpoint_key": inboxId } }), + ); + +const deliver = (baseUrl: string, hook: HookInboxStore.HeldHook) => + sendUpstream(baseUrl, hook).pipe( + Effect.result, + Effect.map((result) => { + // Unreachable or timed out: a timeout may still have run it, and the + // environment drops a delivery id it has already seen. + if (Result.isFailure(result)) { + return { outcome: "unreachable" as const, reason: result.failure._tag }; + } + if (Option.isNone(result.success)) + return { outcome: "unreachable" as const, reason: "timeout" }; + const status = result.success.value.status; + const upstreamOutcome = result.success.value.outcome; + const outcome: HookInboxStore.DeliveryOutcome = UNREACHABLE_STATUSES.has(status) + ? "unreachable" + : BUSY_STATUSES.has(status) + ? "busy" + : "delivered"; + return { outcome, reason: `status ${status}`, upstreamOutcome }; + }), + Effect.tap(({ upstreamOutcome }) => + upstreamOutcome === undefined + ? Effect.void + : Effect.annotateCurrentSpan({ "relay.hook.upstream_outcome": upstreamOutcome }), + ), + Effect.tap(({ outcome, reason }) => + outcome === "delivered" + ? Effect.void + : Effect.logInfo("Held webhook not delivered yet", { + outcome, + reason, + deliveryId: hook.id, + }), + ), + Effect.map(({ outcome }) => outcome), + Effect.provide(FetchHttpClient.layer), + ); + +export const HookInboxObjectLive = HookInboxObject.make( + Effect.gen(function* () { + const state = yield* Cloudflare.DurableObjectState; + // The init phase returns the per-instance Effect, which alchemy runs once + // per object; only that inner Effect may touch storage. + // @effect-diagnostics-next-line returnEffectInGen:off + return Effect.gen(function* () { + const sql = SqliteClient.layer({ storage: state.raw.storage }); + // Inboxes are opened by endpoint key, so spans line up with the + // forward spans that held their requests. + const inboxId = state.raw.id.name ?? state.raw.id.toString(); + const run = (effect: Effect.Effect) => + effect.pipe(Effect.provide(sql), Effect.orDie); + yield* run(HookInboxStore.migrate); + + /** Moves the alarm to `at`, unless one is already due sooner. */ + const scheduleBy = (at: number) => + Effect.gen(function* () { + const current = yield* state.storage.getAlarm(); + if (current === null || current > at) yield* state.storage.setAlarm(at); + }); + + return { + hold: (hook: HookInboxStore.HeldHook, baseUrl: string) => + Effect.gen(function* () { + const dueAt = yield* run(HookInboxStore.hold(hook, baseUrl)); + yield* Effect.annotateCurrentSpan({ "relay.inbox.stored": dueAt !== null }); + if (dueAt === null) return false; + yield* scheduleBy(dueAt); + return true; + }).pipe(withInboxSpan("relay.inbox.hold", inboxId)), + wake: (baseUrl: string) => + Effect.gen(function* () { + const pending = yield* run(HookInboxStore.wake(baseUrl)); + yield* Effect.annotateCurrentSpan({ "relay.inbox.pending": pending }); + if (pending) yield* state.storage.setAlarm(yield* Clock.currentTimeMillis); + return pending; + }).pipe(withInboxSpan("relay.inbox.wake", inboxId)), + clear: () => + Effect.gen(function* () { + yield* run(HookInboxStore.clear); + yield* state.storage.deleteAlarm(); + }).pipe(withInboxSpan("relay.inbox.clear", inboxId)), + alarm: () => + run(HookInboxStore.deliverDue(deliver)).pipe( + // A wake during this run may already have asked for an earlier + // run; a backoff must not push it out. + Effect.flatMap((nextAt) => (nextAt === null ? Effect.void : scheduleBy(nextAt))), + Effect.catchCause((cause) => + Effect.logWarning("Held webhook delivery run failed", { cause }).pipe( + Effect.andThen(Effect.annotateCurrentSpan({ "relay.inbox.run_result": "failed" })), + Effect.andThen(Clock.currentTimeMillis), + Effect.flatMap((now) => scheduleBy(now + RUN_FAILURE_RETRY_MS)), + ), + ), + withInboxSpan("relay.inbox.deliver", inboxId), + ), + }; + }); + }), +); diff --git a/infra/relay/src/hooks/HookInboxStore.test.ts b/infra/relay/src/hooks/HookInboxStore.test.ts new file mode 100644 index 000000000000..121ce84a686c --- /dev/null +++ b/infra/relay/src/hooks/HookInboxStore.test.ts @@ -0,0 +1,260 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Clock from "effect/Clock"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; + +import * as HookInboxStore from "./HookInboxStore.ts"; + +const BASE_URL = "https://env.example.test/"; + +const hook = (id: string, overrides: Partial = {}) => + Effect.map(Clock.currentTimeMillis, (now): HookInboxStore.HeldHook => ({ + id, + receivedAt: DateTime.formatIso(DateTime.makeUnsafe(now)), + method: "POST", + rawHookId: "hook-1", + hookKey: "hook-1", + rawToken: "tok%2Fen", + query: "a=1", + headers: { "content-type": "application/json", "x-sig": "s" }, + body: new Uint8Array([0, 255, 10]), + ...overrides, + })); + +/** Delivers with `outcome` per request and records what the environment was sent. */ +const deliverer = (outcome: (hook: HookInboxStore.HeldHook) => HookInboxStore.DeliveryOutcome) => { + const sent: Array<{ readonly baseUrl: string; readonly hook: HookInboxStore.HeldHook }> = []; + const send = (baseUrl: string, held: HookInboxStore.HeldHook) => + Effect.sync(() => { + sent.push({ baseUrl, hook: held }); + return outcome(held); + }); + return { sent, send }; +}; + +const withInbox = (effect: Effect.Effect) => + HookInboxStore.migrate.pipe( + Effect.andThen(effect), + Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" })), + ); + +describe("HookInboxStore", () => { + it.effect("delivers held requests oldest first, byte for byte", () => + withInbox( + Effect.gen(function* () { + const first = yield* hook("first"); + expect(yield* HookInboxStore.hold(first, BASE_URL)).not.toBeNull(); + yield* HookInboxStore.hold(yield* hook("second"), BASE_URL); + const { sent, send } = deliverer(() => "delivered"); + expect(yield* HookInboxStore.deliverDue(send)).toBeNull(); + expect(sent.map((entry) => entry.hook.id)).toEqual(["first", "second"]); + expect(sent[0]).toEqual({ baseUrl: BASE_URL, hook: first }); + // Delivered requests are gone. + expect(yield* HookInboxStore.deliverDue(send)).toBeNull(); + expect(sent).toHaveLength(2); + }), + ), + ); + + it.effect("keeps a request the environment did not take and backs off", () => + withInbox( + Effect.gen(function* () { + yield* HookInboxStore.hold(yield* hook("first"), BASE_URL); + yield* HookInboxStore.hold(yield* hook("second"), BASE_URL); + const offline = deliverer(() => "unreachable"); + const now = yield* Clock.currentTimeMillis; + // Stops at the first failure, so order is kept. + expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000); + expect(offline.sent.map((entry) => entry.hook.id)).toEqual(["first"]); + expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000); + + const online = deliverer(() => "delivered"); + expect(yield* HookInboxStore.deliverDue(online.send)).toBeNull(); + expect(online.sent.map((entry) => entry.hook.id)).toEqual(["first", "second"]); + }), + ), + ); + + it.effect("lets other hooks through while one hook's environment is busy", () => + withInbox( + Effect.gen(function* () { + const stuck = { rawHookId: "stuck", hookKey: "stuck" }; + yield* HookInboxStore.hold(yield* hook("stuck-1", stuck), BASE_URL); + yield* HookInboxStore.hold(yield* hook("other-1"), BASE_URL); + yield* HookInboxStore.hold(yield* hook("stuck-2", stuck), BASE_URL); + const busy = deliverer((held) => (held.hookKey === "stuck" ? "busy" : "delivered")); + const now = yield* Clock.currentTimeMillis; + // The other hook is delivered; the busy hook keeps its order and runs again soon. + expect(yield* HookInboxStore.deliverDue(busy.send)).toBe(now); + expect(busy.sent.map((entry) => entry.hook.id)).toEqual(["stuck-1", "other-1"]); + // Only the busy hook is left, so the next run waits rather than spinning. + expect(yield* HookInboxStore.deliverDue(busy.send)).toBe(now + 30_000); + + const drained = deliverer(() => "delivered"); + expect(yield* HookInboxStore.deliverDue(drained.send)).toBeNull(); + expect(drained.sent.map((entry) => entry.hook.id)).toEqual(["stuck-1", "stuck-2"]); + }), + ), + ); + + it.effect("reaches a hook queued behind busy hooks' full backlogs in the same run", () => + withInbox( + Effect.gen(function* () { + for (const name of ["stuck-a", "stuck-b"]) { + const stuck = { rawHookId: name, hookKey: name }; + for (let index = 0; index < HookInboxStore.HOOK_INBOX_MAX_PER_HOOK; index++) { + yield* HookInboxStore.hold(yield* hook(`${name}-${index}`, stuck), BASE_URL); + } + } + yield* HookInboxStore.hold(yield* hook("other-1"), BASE_URL); + const busy = deliverer((held) => (held.hookKey.startsWith("stuck") ? "busy" : "delivered")); + yield* HookInboxStore.deliverDue(busy.send); + expect(busy.sent.map((entry) => entry.hook.id)).toEqual([ + "stuck-a-0", + "stuck-b-0", + "other-1", + ]); + }), + ), + ); + + it.effect("drops a held request it cannot read instead of stalling on it", () => + withInbox( + Effect.gen(function* () { + yield* HookInboxStore.hold(yield* hook("broken"), BASE_URL); + yield* HookInboxStore.hold(yield* hook("fine"), BASE_URL); + const sql = yield* SqlClient.SqlClient; + yield* sql`UPDATE held_hooks SET headers = 'not json' WHERE id = 'broken'`; + const { sent, send } = deliverer(() => "delivered"); + expect(yield* HookInboxStore.deliverDue(send)).toBeNull(); + expect(sent.map((entry) => entry.hook.id)).toEqual(["fine"]); + }), + ), + ); + + it.effect("retries every 10 s for 3 minutes, then backs off to 10 minutes", () => + Effect.gen(function* () { + const delays = yield* Effect.forEach( + [1, 18, 19, 20, 21, 23, 24, 40], + HookInboxStore.retryDelayMs, + ); + expect(delays).toEqual([10_000, 10_000, 30_000, 60_000, 120_000, 480_000, 600_000, 600_000]); + }), + ); + + it.effect("waking resets the backoff and reports whether anything waits", () => + withInbox( + Effect.gen(function* () { + expect(yield* HookInboxStore.wake(BASE_URL)).toBe(false); + yield* HookInboxStore.hold(yield* hook("first"), "https://old.example.test/"); + const offline = deliverer(() => "unreachable"); + yield* HookInboxStore.deliverDue(offline.send); + yield* HookInboxStore.deliverDue(offline.send); + + // Past the fast phase: the environment was away a while. + for (let failure = 0; failure < 20; failure++) { + yield* HookInboxStore.deliverDue(offline.send); + } + const now = yield* Clock.currentTimeMillis; + expect(yield* HookInboxStore.deliverDue(offline.send)).toBeGreaterThan(now + 60_000); + + // A wake means its tunnel just connected, which Cloudflare may not + // route to for a few minutes: back to retrying every 10 s, sent to + // where the environment is now. + expect(yield* HookInboxStore.wake(BASE_URL)).toBe(true); + expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000); + expect(offline.sent.at(-1)?.baseUrl).toBe(BASE_URL); + }), + ), + ); + + it.effect("starts the schedule over once the inbox drains", () => + withInbox( + Effect.gen(function* () { + yield* HookInboxStore.hold(yield* hook("old"), BASE_URL); + const offline = deliverer(() => "unreachable"); + for (let failure = 0; failure < 30; failure++) { + yield* HookInboxStore.deliverDue(offline.send); + } + // Nothing got through before the request expired. + yield* TestClock.adjust(Duration.hours(25)); + expect(yield* HookInboxStore.deliverDue(offline.send)).toBeNull(); + + const now = yield* Clock.currentTimeMillis; + expect(yield* HookInboxStore.hold(yield* hook("new"), BASE_URL)).toBe(now + 10_000); + }), + ), + ); + + it.effect("drops requests older than 24 hours", () => + withInbox( + Effect.gen(function* () { + yield* HookInboxStore.hold(yield* hook("old"), BASE_URL); + yield* TestClock.adjust(Duration.hours(1)); + yield* HookInboxStore.hold(yield* hook("new"), BASE_URL); + yield* TestClock.adjust(Duration.minutes(23 * 60 + 1)); + const { sent, send } = deliverer(() => "delivered"); + yield* HookInboxStore.deliverDue(send); + expect(sent.map((entry) => entry.hook.id)).toEqual(["new"]); + }), + ), + ); + + it.effect("refuses requests past the per-hook cap", () => + withInbox( + Effect.gen(function* () { + for (let index = 0; index < HookInboxStore.HOOK_INBOX_MAX_PER_HOOK; index++) { + expect(yield* HookInboxStore.hold(yield* hook(`a-${index}`), BASE_URL)).not.toBeNull(); + } + expect(yield* HookInboxStore.hold(yield* hook("one-too-many"), BASE_URL)).toBeNull(); + // Another hook still has room. + const other = yield* hook("other", { rawHookId: "hook-2", hookKey: "hook-2" }); + expect(yield* HookInboxStore.hold(other, BASE_URL)).not.toBeNull(); + }), + ), + ); + + it.effect("refuses a request that would pass the byte cap", () => + withInbox( + Effect.gen(function* () { + const big = new Uint8Array(HookInboxStore.HOOK_INBOX_MAX_BYTES - 10); + expect( + yield* HookInboxStore.hold(yield* hook("big", { body: big }), BASE_URL), + ).not.toBeNull(); + const small = new Uint8Array(11); + expect( + yield* HookInboxStore.hold( + yield* hook("small", { rawHookId: "x", hookKey: "x", body: small }), + BASE_URL, + ), + ).toBeNull(); + }), + ), + ); + + it.effect("stores a request id once", () => + withInbox( + Effect.gen(function* () { + const first = yield* hook("same"); + expect(yield* HookInboxStore.hold(first, BASE_URL)).not.toBeNull(); + expect(yield* HookInboxStore.hold(first, BASE_URL)).toBeNull(); + }), + ), + ); + + it.effect("clear drops everything held", () => + withInbox( + Effect.gen(function* () { + yield* HookInboxStore.hold(yield* hook("first"), BASE_URL); + yield* HookInboxStore.clear; + const { sent, send } = deliverer(() => "delivered"); + expect(yield* HookInboxStore.deliverDue(send)).toBeNull(); + expect(sent).toHaveLength(0); + }), + ), + ); +}); diff --git a/infra/relay/src/hooks/HookInboxStore.ts b/infra/relay/src/hooks/HookInboxStore.ts new file mode 100644 index 000000000000..64049f6b86ab --- /dev/null +++ b/infra/relay/src/hooks/HookInboxStore.ts @@ -0,0 +1,380 @@ +import * as Clock from "effect/Clock"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Pull from "effect/Pull"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; + +/** + * The storage and delivery rules of one environment's held webhook requests. + * Runs inside the environment's `HookInboxObject` Durable Object against its SQLite + * storage; written against `SqlClient` so tests can run it on any SQLite. + */ + +/** How long a held request waits for its environment. */ +const HOOK_INBOX_TTL_MS = 24 * 60 * 60 * 1000; +/** Requests one environment may have waiting at once. */ +const HOOK_INBOX_MAX_REQUESTS = 1_000; +/** Body bytes one environment may have waiting at once. */ +export const HOOK_INBOX_MAX_BYTES = 50 * 1_048_576; +/** + * Requests one hook may have waiting at once. The relay cannot check tokens, + * so this keeps junk sent to one hook id from crowding out the others. + */ +export const HOOK_INBOX_MAX_PER_HOOK = 100; +/** Requests pushed per alarm run; the next run starts right away while more wait. */ +const DELIVERIES_PER_RUN = 20; +/** Requests read at a time; a run keeps reading past busy hooks' backlogs. */ +const ROWS_READ_PER_PAGE = 50; +/** Wait before trying a hook whose environment answered busy again. */ +const BUSY_RETRY_MS = 30_000; +const MAX_RETRY_DELAY = Duration.minutes(10); +/** Past this many failures the schedule is at its cap, so stepping further changes nothing. */ +const MAX_COUNTED_FAILURES = 25; + +/** + * Delays between delivery attempts while the environment is unreachable, + * indexed by consecutive failures since it was last reached or woke us. + * A wake means its tunnel just connected, but Cloudflare can take a few + * minutes to route the hostname to it, so the first 3 minutes retry every + * 10 s. After that the environment is likely gone again: 30 s, 1 min, 2 min, + * ... up to 10 min. + */ +export const retrySchedule = Schedule.spaced("10 seconds").pipe( + Schedule.upTo({ times: 18 }), + Schedule.concat( + Schedule.exponential("30 seconds").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(Duration.fromInputUnsafe(duration), MAX_RETRY_DELAY)), + ), + ), + ), +); + +export interface HeldHook { + readonly id: string; + readonly receivedAt: string; + readonly method: string; + /** Path segments exactly as the sender sent them; the environment decodes them. */ + readonly rawHookId: string; + readonly rawToken: string; + /** The decoded hook id, so every spelling of one hook shares its cap. */ + readonly hookKey: string; + /** Without the leading `?`. */ + readonly query: string; + readonly headers: Readonly>; + readonly body: Uint8Array; +} + +/** + * `delivered` deletes the request. `unreachable` keeps it and backs off the + * whole inbox, since nothing else will get through either. `busy` keeps it + * and its hook's later requests for the next run, but lets other hooks' + * requests go ahead, so one stuck task cannot hold up the rest. + */ +export type DeliveryOutcome = "delivered" | "busy" | "unreachable"; + +/** The `retrySchedule` delay after `failures` consecutive unreachable attempts. */ +export const retryDelayMs = Effect.fn("HookInboxStore.retryDelayMs")(function* (failures: number) { + const step = yield* Schedule.toStep(retrySchedule); + let delay = MAX_RETRY_DELAY; + for ( + let attempt = 0; + attempt < Math.min(Math.max(1, failures), MAX_COUNTED_FAILURES); + attempt++ + ) { + const next = yield* step(0, undefined).pipe( + Effect.map(([, duration]) => Option.some(duration)), + // The schedule never ends; stay at the cap if it ever does. + Pull.catchDone(() => Effect.succeedNone), + ); + if (Option.isNone(next)) break; + delay = next.value; + } + return Duration.toMillis(delay); +}); + +const HeadersJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)); +const encodeHeaders = Schema.encodeSync(HeadersJson); +const decodeHeaders = Schema.decodeUnknownOption(HeadersJson); + +interface HeldHookRow { + readonly seq: number; + readonly id: string; + readonly received_at: string; + readonly method: string; + readonly raw_hook_id: string; + readonly raw_token: string; + readonly hook_key: string; + readonly query: string; + readonly headers: string; + readonly body: Uint8Array; +} + +interface TargetRow { + readonly base_url: string; + readonly failures: number; +} + +const iso = (epochMillis: number) => DateTime.formatIso(DateTime.makeUnsafe(epochMillis)); + +export const migrate = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql` + CREATE TABLE IF NOT EXISTS held_hooks ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT NOT NULL UNIQUE, + received_at TEXT NOT NULL, + method TEXT NOT NULL, + raw_hook_id TEXT NOT NULL, + raw_token TEXT NOT NULL, + hook_key TEXT NOT NULL, + query TEXT NOT NULL, + headers TEXT NOT NULL, + body BLOB NOT NULL + ) + `; + yield* sql`CREATE INDEX IF NOT EXISTS held_hooks_hook ON held_hooks (hook_key)`; + // Where to push, and how many attempts in a row have failed. One row. + yield* sql` + CREATE TABLE IF NOT EXISTS held_hooks_target ( + id INTEGER PRIMARY KEY CHECK (id = 1), + base_url TEXT NOT NULL, + failures INTEGER NOT NULL DEFAULT 0 + ) + `; +}); + +const setTarget = (baseUrl: string, options: { readonly resetFailures: boolean }) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* options.resetFailures + ? sql` + INSERT INTO held_hooks_target (id, base_url) VALUES (1, ${baseUrl}) + ON CONFLICT (id) DO UPDATE SET base_url = excluded.base_url, failures = 0 + ` + : sql` + INSERT INTO held_hooks_target (id, base_url) VALUES (1, ${baseUrl}) + ON CONFLICT (id) DO UPDATE SET base_url = excluded.base_url + `; + }); + +const readTarget = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql`SELECT base_url, failures FROM held_hooks_target WHERE id = 1`; + return rows[0] ?? null; +}); + +const resetFailures = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`UPDATE held_hooks_target SET failures = 0 WHERE id = 1`; +}); + +/** + * Counts one more unreachable attempt. Incremented in place, so a wake that + * reset the count while this run's request was in flight is not overwritten. + */ +const countFailure = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ readonly failures: number }>` + UPDATE held_hooks_target SET failures = min(failures + 1, ${MAX_COUNTED_FAILURES}) + WHERE id = 1 RETURNING failures + `; + return rows[0]?.failures ?? 1; +}); + +const hasPending = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ readonly id: string }>`SELECT id FROM held_hooks LIMIT 1`; + return rows.length > 0; +}); + +/** + * Stores a request for later delivery to `baseUrl`. Returns the time the + * first attempt is due, or null when the inbox is full and nothing was stored. + */ +export const hold = Effect.fn("HookInboxStore.hold")(function* (hook: HeldHook, baseUrl: string) { + const sql = yield* SqlClient.SqlClient; + // One statement, so the caps hold however many requests arrive at once. + const inserted = yield* sql<{ readonly id: string }>` + INSERT INTO held_hooks + (id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body) + SELECT ${hook.id}, ${hook.receivedAt}, ${hook.method}, ${hook.rawHookId}, ${hook.rawToken}, + ${hook.hookKey}, ${hook.query}, ${encodeHeaders(hook.headers)}, ${hook.body} + WHERE (SELECT count(*) FROM held_hooks) < ${HOOK_INBOX_MAX_REQUESTS} + AND (SELECT count(*) FROM held_hooks WHERE hook_key = ${hook.hookKey}) + < ${HOOK_INBOX_MAX_PER_HOOK} + AND (SELECT coalesce(sum(length(body)), 0) FROM held_hooks) + ${hook.body.byteLength} + <= ${HOOK_INBOX_MAX_BYTES} + ON CONFLICT (id) DO NOTHING + RETURNING id + `; + if (inserted.length === 0) { + yield* Effect.annotateCurrentSpan({ "relay.inbox.refused": yield* refusalReason(hook) }); + return null; + } + yield* setTarget(baseUrl, { resetFailures: false }); + const target = yield* readTarget; + const backlog = yield* backlogSize; + yield* Effect.annotateCurrentSpan({ + "relay.inbox.held_count": backlog.count, + "relay.inbox.held_bytes": backlog.bytes, + }); + return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(target?.failures ?? 0)); +}); + +const backlogSize = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ readonly count: number; readonly bytes: number }>` + SELECT count(*) AS count, coalesce(sum(length(body)), 0) AS bytes FROM held_hooks + `; + return rows[0] ?? { count: 0, bytes: 0 }; +}); + +/** Which cap refused a request, or that it was already held, for traces. */ +const refusalReason = (hook: HeldHook) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const existing = yield* sql<{ readonly id: string }>` + SELECT id FROM held_hooks WHERE id = ${hook.id} + `; + if (existing.length > 0) return "already_held"; + const backlog = yield* backlogSize; + if (backlog.count >= HOOK_INBOX_MAX_REQUESTS) return "max_requests"; + if (backlog.bytes + hook.body.byteLength > HOOK_INBOX_MAX_BYTES) return "max_bytes"; + return "max_per_hook"; + }); + +/** + * The environment is reachable again at `baseUrl`. Returns whether anything + * is waiting, so the caller can deliver right away. + */ +export const wake = Effect.fn("HookInboxStore.wake")(function* (baseUrl: string) { + if (!(yield* hasPending)) return false; + yield* setTarget(baseUrl, { resetFailures: true }); + return true; +}); + +export const clear = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`DELETE FROM held_hooks`; + yield* sql`DELETE FROM held_hooks_target`; +}); + +/** + * Pushes the oldest held requests to the environment, one at a time, in + * order per hook. Stops at the first sign the environment is unreachable; + * skips past a hook whose environment answered busy. Drops requests older + * than the TTL. Returns when the next run is due, or null when nothing is left. + */ +export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( + send: (baseUrl: string, hook: HeldHook) => Effect.Effect, +) { + const sql = yield* SqlClient.SqlClient; + const startedAt = yield* Clock.currentTimeMillis; + const expired = yield* sql<{ readonly id: string }>` + DELETE FROM held_hooks WHERE received_at < ${iso(startedAt - HOOK_INBOX_TTL_MS)} + RETURNING id + `; + const target = yield* readTarget; + /** Oldest held requests after `afterSeq`, skipping hooks already found busy. */ + const readPage = (afterSeq: number, skip: ReadonlySet) => + sql` + SELECT seq, id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body + FROM held_hooks + WHERE seq > ${afterSeq} + ${skip.size === 0 ? sql`` : sql`AND hook_key NOT IN ${sql.in([...skip])}`} + ORDER BY seq LIMIT ${ROWS_READ_PER_PAGE} + `; + let batch = yield* readPage(0, new Set()); + if (batch.length === 0 || target === null) { + if (target === null) yield* sql`DELETE FROM held_hooks`; + // Empty, so the next request held starts the schedule from the top. + else yield* resetFailures; + yield* Effect.annotateCurrentSpan({ "relay.inbox.expired": expired.length }); + return null; + } + + let failures = target.failures; + let sent = 0; + let delivered = 0; + let unreadable = 0; + let longestWaitMs = 0; + const busyHooks = new Set(); + /** What this run did, for the alarm's span; never request contents. */ + const annotateRun = (result: string) => + Effect.gen(function* () { + const backlog = yield* backlogSize; + yield* Effect.annotateCurrentSpan({ + "relay.inbox.run_result": result, + "relay.inbox.sent": sent, + "relay.inbox.delivered": delivered, + "relay.inbox.busy_hooks": busyHooks.size, + "relay.inbox.expired": expired.length, + "relay.inbox.unreadable": unreadable, + "relay.inbox.consecutive_failures": failures, + "relay.inbox.longest_wait_ms": longestWaitMs, + "relay.inbox.held_count": backlog.count, + "relay.inbox.held_bytes": backlog.bytes, + }); + }); + // Pages run out only once every non-busy request has been tried, so a + // backlog behind busy hooks never hides another hook's requests. + pages: while (batch.length > 0) { + for (const row of batch) { + if (sent === DELIVERIES_PER_RUN) break pages; + // Later requests to a busy hook wait their turn, so its order is kept. + if (busyHooks.has(row.hook_key)) continue; + const headers = decodeHeaders(row.headers); + if (Option.isNone(headers)) { + // Unreadable: it can never be delivered, and must not block the rest. + yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`; + unreadable += 1; + continue; + } + sent += 1; + const outcome = yield* send(target.base_url, { + id: row.id, + receivedAt: row.received_at, + method: row.method, + rawHookId: row.raw_hook_id, + rawToken: row.raw_token, + hookKey: row.hook_key, + query: row.query, + headers: headers.value, + body: row.body, + }); + if (outcome === "unreachable") { + failures = yield* countFailure; + yield* annotateRun("unreachable"); + return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures)); + } + if (failures !== 0) { + failures = 0; + yield* resetFailures; + } + if (outcome === "busy") { + busyHooks.add(row.hook_key); + continue; + } + yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`; + delivered += 1; + const receivedAtMs = DateTime.toEpochMillis(DateTime.makeUnsafe(row.received_at)); + longestWaitMs = Math.max(longestWaitMs, startedAt - receivedAtMs); + } + batch = yield* readPage(batch.at(-1)!.seq, busyHooks); + } + if (!(yield* hasPending)) { + yield* resetFailures; + yield* annotateRun("drained"); + return null; + } + // Everything left this run was busy: give those tasks a moment to drain. + const now = yield* Clock.currentTimeMillis; + const idle = delivered === 0 && busyHooks.size > 0; + yield* annotateRun(idle ? "busy" : "more_pending"); + return idle ? now + BUSY_RETRY_MS : now; +}); diff --git a/infra/relay/src/hooks/upstream.ts b/infra/relay/src/hooks/upstream.ts new file mode 100644 index 000000000000..9871317f99bf --- /dev/null +++ b/infra/relay/src/hooks/upstream.ts @@ -0,0 +1,125 @@ +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientRequest from "effect/http/HttpClientRequest"; +import * as HttpMethod from "effect/http/HttpMethod"; +import * as HttpTraceContext from "effect/http/HttpTraceContext"; +import type * as HttpClientResponse from "effect/http/HttpClientResponse"; + +import { withoutRedirects } from "../environments/EnvironmentConnector.ts"; + +/** Set by the relay on every forward; the environment uses it as the delivery id. */ +const RELAY_DELIVERY_ID_HEADER = "x-t3-relay-delivery-id"; +/** When the relay received the request; the environment trusts it only with the delivery id. */ +const RELAY_RECEIVED_AT_HEADER = "x-t3-relay-received-at"; +/** The environment answers with a small JSON status; anything past this is cut off. */ +const MAX_RESPONSE_BYTES = 64 * 1024; +export const RELAY_HOOK_UPSTREAM_TIMEOUT_MS = 8_000; +// Cloudflare answers 530 when the tunnel for a hostname has no connected origin. +export const TUNNEL_OFFLINE_STATUS = 530; + +/** A webhook request as the relay sends it on to the environment. */ +export interface UpstreamHook { + readonly id: string; + readonly receivedAt: string; + readonly method: string; + /** Path segments exactly as the sender sent them; the environment decodes them. */ + readonly rawHookId: string; + readonly rawToken: string; + /** Without the leading `?`. */ + readonly query: string; + readonly headers: Readonly>; + readonly body: Uint8Array; +} + +export interface UpstreamResponse { + readonly status: number; + readonly contentType: string | undefined; + /** What the environment did with the request (`x-t3-hook-outcome`). */ + readonly outcome: string | undefined; + readonly body: Uint8Array; +} + +/** Outcome names are short identifiers; anything else is not recorded. */ +const OUTCOME_PATTERN = /^[a-z_]{1,32}$/; + +class ResponseTooLarge extends Schema.TaggedError()("ResponseTooLarge", {}) {} + +/** Reads a response body, failing once it passes the cap rather than buffering it all. */ +const readCapped = (response: HttpClientResponse.HttpClientResponse) => + Effect.suspend(() => { + const chunks: Array = []; + let total = 0; + return response.stream.pipe( + Stream.runForEach((chunk) => { + total += chunk.length; + if (total > MAX_RESPONSE_BYTES) return Effect.fail(new ResponseTooLarge()); + chunks.push(chunk); + return Effect.void; + }), + Effect.map(() => { + const body = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.length; + } + return body; + }), + // A response without a body, such as a redirect, has no stream at all. + Effect.catchIf( + (error) => error._tag === "HttpClientError" && error.reason._tag === "EmptyBodyError", + () => Effect.succeed(new Uint8Array(0)), + ), + ); + }); + +/** + * Sends a webhook request through the environment's tunnel. Fails when the + * environment cannot be reached, and succeeds with None on timeout. + */ +export const sendUpstream = (baseUrl: string, hook: UpstreamHook) => + Effect.gen(function* () { + const httpClient = yield* HttpClient.HttpClient; + const base = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`; + // The environment's span joins this trace. Set by hand: the client span + // that would propagate it is off, because it records the token in url.full. + const parent = yield* Effect.currentSpan.pipe(Effect.option); + // `hook.headers` carries the signed delivery proof, set once when the + // relay received the request, so a held request sends the same proof. + const headers: Record = { + ...hook.headers, + ...(Option.isSome(parent) ? HttpTraceContext.toHeaders(parent.value) : {}), + [RELAY_DELIVERY_ID_HEADER]: hook.id, + [RELAY_RECEIVED_AT_HEADER]: hook.receivedAt, + }; + let request = HttpClientRequest.make(hook.method as "GET" | "POST" | "PUT" | "PATCH")( + `${base}api/hooks/${hook.rawHookId}/${hook.rawToken}${hook.query ? `?${hook.query}` : ""}`, + { headers }, + ); + if (HttpMethod.hasBody(request.method)) { + request = HttpClientRequest.bodyUint8Array(request, hook.body, headers["content-type"]); + } + return yield* httpClient.execute(request).pipe( + Effect.flatMap((response) => + readCapped(response).pipe( + Effect.map((body): UpstreamResponse => { + const outcome = response.headers["x-t3-hook-outcome"]; + return { + status: response.status, + contentType: response.headers["content-type"], + outcome: outcome !== undefined && OUTCOME_PATTERN.test(outcome) ? outcome : undefined, + body, + }; + }), + ), + ), + withoutRedirects, + // The client span would record url.full, which carries the token. + Effect.provideService(HttpClient.TracerDisabledWhen, () => true), + Effect.timeoutOption(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)), + ); + }); diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 6220c27c4b30..4f1cd17e094c 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -61,6 +61,7 @@ import { import * as RelayConfiguration from "../Config.ts"; import * as RelayDb from "../db.ts"; import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts"; +import * as HookInbox from "../hooks/HookInbox.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts"; @@ -124,6 +125,7 @@ describe("device listing compatibility", () => { Layer.mock(EnvironmentLinks.EnvironmentLinks, {}), Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}), Layer.mock(RelayDb.RelayTransactions, {}), + Layer.mock(HookInbox.HookInbox, {}), ), ), Layer.provide( @@ -317,8 +319,12 @@ function relayUnlinkTestLayer(input?: { readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"]; readonly reconcileOrigin?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["reconcileOrigin"]; readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]; + readonly clearInbox?: HookInbox.HookInbox["Service"]["clear"]; }) { return Layer.mergeAll( + Layer.mock(HookInbox.HookInbox, { + clear: input?.clearInbox ?? (() => Effect.void), + }), Layer.succeed( RelayDb.RelayTransactions, RelayDb.RelayTransactions.of({ @@ -332,7 +338,8 @@ function relayUnlinkTestLayer(input?: { listDeliveryUsersForEnvironment: () => Effect.die("unused listDeliveryUsersForEnvironment"), listForUser: () => Effect.die("unused listForUser"), getForUser: input?.getForUser ?? (() => Effect.succeed(null)), - findActiveManagedForEnvironment: () => Effect.succeed([]), + findActiveManagedForEnvironment: () => Effect.die("unused findActiveManagedForEnvironment"), + setHoldWebhooksWhileOffline: () => Effect.die("unused setHoldWebhooksWhileOffline"), revokeForUser: input?.revokeForUser ?? (() => Effect.succeed(false)), }), ), @@ -899,6 +906,53 @@ describe("relay environment unlink", () => { ); }); + it.effect("drops the unlinked endpoint's held webhooks, even if clearing fails", () => { + const cleared: Array = []; + const endpointKey = "0123456789abcdef"; + const unlink = (clearFails: boolean) => + unlinkEnvironmentRecord({ + userId: "user-1", + environmentId: "environment-1", + managedEndpointNamespace: "dev", + }).pipe( + Effect.provide( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + revokeForUser: () => Effect.succeed(true), + prepareDeprovision: () => + Effect.succeed({ + userId: "user-1", + environmentId: "environment-1", + hostname: "dev-0123456789abcdef.example.test", + tunnelId: "tunnel-1", + tunnelName: `t3coderelay-managedendpoint-dev-${endpointKey}`, + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + origin: null, + updatedAt: "2026-07-28T00:00:00.000Z", + generation: 1, + }), + clearInbox: (input) => + clearFails + ? Effect.fail( + new HookInbox.HookInboxError({ + operation: "clear", + endpointKey: input.endpointKey, + cause: new Error("unavailable"), + }), + ) + : Effect.sync(() => void cleared.push(input.endpointKey)), + }), + ), + ); + return Effect.gen(function* () { + expect(yield* unlink(false)).toBe(true); + expect(cleared).toEqual([endpointKey]); + // The link is already revoked; a failed clear must not fail the unlink. + expect(yield* unlink(true)).toBe(true); + }); + }); + it.effect("commits database revocation before deprovisioning the managed endpoint", () => { const calls: Array = []; const deprovisionTarget = { @@ -1215,7 +1269,14 @@ describe("relay routing fallback", () => { Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}), ), ), - Layer.provide([publisher, signatures]), + Layer.provide([ + publisher, + signatures, + Layer.mock(EnvironmentLinks.EnvironmentLinks, {}), + Layer.mock(HookInbox.HookInbox, {}), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, {}), + Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings), + ]), ), ), Layer.provide(auth), diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 5c3b1fe5cfde..7b308653d510 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -66,6 +66,8 @@ import * as DpopProofs from "../auth/DpopProofs.ts"; import * as RelayTokens from "../auth/RelayTokens.ts"; import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; +import * as HookForwarder from "../hooks/HookForwarder.ts"; +import * as HookInbox from "../hooks/HookInbox.ts"; import * as LiveActivities from "../agentActivity/LiveActivities.ts"; import * as RelayConfiguration from "../Config.ts"; import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher.ts"; @@ -188,6 +190,12 @@ export const relayDocsRedirectRoute = HttpRouter.add( // contains the exact child span that stalled, and the response still carries // the traceparent back to the client. export const RELAY_REQUEST_DEADLINE_MS = 9_000; +/** + * Webhook forwarding reads a body of up to 1 MiB, waits up to the upstream + * timeout, and may then hold the request, so it needs more room than an API + * call; cutting it off before the hold would drop a request it should keep. + */ +export const RELAY_HOOK_REQUEST_DEADLINE_MS = 25_000; const relayRequestDeadline = ( httpEffect: Effect.Effect< @@ -195,9 +203,10 @@ const relayRequestDeadline = ( E, HttpServerRequest.HttpServerRequest | R >, + deadlineMs = RELAY_REQUEST_DEADLINE_MS, ) => httpEffect.pipe( - Effect.timeoutOption(Duration.millis(RELAY_REQUEST_DEADLINE_MS)), + Effect.timeoutOption(Duration.millis(deadlineMs)), Effect.flatMap( Option.match({ onNone: () => @@ -206,7 +215,7 @@ const relayRequestDeadline = ( yield* Effect.logError("relay request exceeded deadline", { "http.method": request.method, "http.url": request.url, - "relay.request.deadline_ms": RELAY_REQUEST_DEADLINE_MS, + "relay.request.deadline_ms": deadlineMs, }); yield* Effect.annotateCurrentSpan({ "relay.request.deadline_exceeded": true, @@ -221,6 +230,9 @@ const relayRequestDeadline = ( ), ); +/** Trace context headers in every format the tracer reads (W3C and B3). */ +const SENDER_TRACE_HEADER = /^(traceparent|tracestate|b3|x-b3-.*)$/i; + export const traceRelayHttpRequest = ( httpEffect: Effect.Effect< HttpServerResponse.HttpServerResponse, @@ -238,18 +250,29 @@ export const traceRelayHttpRequest = ( return yield* HttpMiddleware.tracer(traced).pipe(Effect.ensuring(Effect.yieldNow)); } // Hook URLs carry a secret token: the tracer and deadline log see a redacted - // request, while the route itself still receives the original. - const redacted = request.modify({ url: redactRelayHookUrl(request.url) }); + // request, while the route itself still receives the original. A webhook + // sender's trace context is dropped, so it cannot pick the trace our relay + // and environment spans land in. + const redacted = request.modify({ + url: redactRelayHookUrl(request.url), + headers: Headers.removeMany( + request.headers, + Object.keys(request.headers).filter((name) => SENDER_TRACE_HEADER.test(name)), + ), + }); return yield* HttpMiddleware.tracer( appendRelayTraceContextResponseHeader.pipe( Effect.andThen( relayRequestDeadline( httpEffect.pipe(Effect.provideService(HttpServerRequest.HttpServerRequest, request)), + RELAY_HOOK_REQUEST_DEADLINE_MS, ), ), ), ).pipe( Effect.provideService(HttpServerRequest.HttpServerRequest, redacted), + // The worker disables its own span for hook paths; this one is ours. + Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false), Effect.ensuring(Effect.yieldNow), ); }); @@ -494,7 +517,12 @@ export const revokeEnvironmentLinkRecord = Effect.fn( }); export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnvironmentRecord")( - function* (input: { readonly userId: string; readonly environmentId: string }) { + function* (input: { + readonly userId: string; + readonly environmentId: string; + /** The stage's tunnel-name namespace, to find this link's held webhook requests. */ + readonly managedEndpointNamespace?: string | undefined; + }) { const links = yield* EnvironmentLinks.EnvironmentLinks; const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider; const deprovisionTarget = yield* managedEndpointProvider.prepareDeprovision({ @@ -513,7 +541,6 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron environmentId: link.environmentId, environmentPublicKey: link.environmentPublicKey, }); - // External teardown cannot share the SQL transaction. Run it only after // revocation commits so a database failure leaves a fully usable active // link. Still run teardown when the link is already revoked, allowing a @@ -523,10 +550,31 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron environmentId: input.environmentId, target: deprovisionTarget, }); + // Requests held for this link's endpoint go with it. Best effort: the link + // is already gone, and its inbox drops anything left after its TTL. + const endpointKey = + deprovisionTarget && input.managedEndpointNamespace + ? HookForwarder.endpointKeyForTunnelName( + input.managedEndpointNamespace, + deprovisionTarget.tunnelName, + ) + : null; + if (endpointKey !== null) { + const inbox = yield* HookInbox.HookInbox; + yield* inbox.clear({ endpointKey }).pipe( + Effect.catch((error) => + Effect.logWarning("Could not clear held webhook requests", { + environmentId: input.environmentId, + errorTag: error._tag, + }), + ), + ); + } if (!deprovisioned) { - const retryTarget = yield* managedEndpointProvider.prepareDeprovision(input); - if (retryTarget !== null && (yield* links.getForUser(input)) === null) { - yield* managedEndpointProvider.deprovision({ ...input, target: retryTarget }); + const key = { userId: input.userId, environmentId: input.environmentId }; + const retryTarget = yield* managedEndpointProvider.prepareDeprovision(key); + if (retryTarget !== null && (yield* links.getForUser(key)) === null) { + yield* managedEndpointProvider.deprovision({ ...key, target: retryTarget }); } } return unlinked; @@ -926,6 +974,7 @@ export const clientApi = HttpApiBuilder.group( const unlinked = yield* unlinkEnvironmentRecord({ userId, environmentId: params.environmentId, + managedEndpointNamespace: config.managedEndpointNamespace, }).pipe( Effect.catchTags({ SqlError: () => relayInternalErrorResponse("internal_error"), @@ -1127,6 +1176,40 @@ export const serverApi = HttpApiBuilder.group( Effect.fnUntraced(function* (handlers) { const publisher = yield* AgentActivityPublisher.AgentActivityPublisher; const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures; + const links = yield* EnvironmentLinks.EnvironmentLinks; + const inbox = yield* HookInbox.HookInbox; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const settings = yield* RelayConfiguration.RelayConfiguration; + const requireOwnEnvironment = (environmentId: string) => + Effect.gen(function* () { + const principal = yield* RelayEnvironmentPrincipal; + if (principal.environmentId !== environmentId) { + return yield* new HttpApiError.Unauthorized({}); + } + }); + /** + * Endpoint keys of the managed links the calling environment key proved. + * The environment id alone would also match other accounts' links of it. + */ + const ownEndpointKeys = (environmentId: string) => + Effect.gen(function* () { + const principal = yield* RelayEnvironmentPrincipal; + const namespace = settings.managedEndpointNamespace; + if (!namespace) return []; + const ownLinks = yield* links.findActiveManagedForEnvironment({ + environmentId, + environmentPublicKey: principal.environmentPublicKey, + }); + const keys: Array = []; + for (const link of ownLinks) { + const allocation = yield* allocations.get({ userId: link.userId, environmentId }); + const key = allocation + ? HookForwarder.endpointKeyForTunnelName(namespace, allocation.tunnelName) + : null; + if (key !== null) keys.push(key); + } + return keys; + }); const activityHandlers = handlers.handle( "publishAgentActivity", Effect.fn("relay.api.server.publishAgentActivity")( @@ -1336,6 +1419,48 @@ export const serverApi = HttpApiBuilder.group( }), mapRelayCommonApiErrors("not_authorized"), ), + ) + .handle( + "updateLinkPreferences", + Effect.fn("relay.api.server.updateLinkPreferences")(function* ({ params, payload }) { + yield* requireOwnEnvironment(params.environmentId); + const principal = yield* RelayEnvironmentPrincipal; + yield* links.setHoldWebhooksWhileOffline({ + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + holdWebhooksWhileOffline: payload.holdWebhooksWhileOffline, + }); + // Opting out also drops what is already held, rather than delivering + // it later to an environment that said it does not want it. + if (!payload.holdWebhooksWhileOffline) { + for (const endpointKey of yield* ownEndpointKeys(params.environmentId)) { + yield* inbox.clear({ endpointKey }); + } + } + return payload; + }, mapRelayCommonApiErrors("not_authorized")), + ) + .handle( + "wakeHeldHooks", + Effect.fn("relay.api.server.wakeHeldHooks")(function* ({ params }) { + yield* requireOwnEnvironment(params.environmentId); + let pending = false; + for (const endpointKey of yield* ownEndpointKeys(params.environmentId)) { + const endpoint = yield* HookForwarder.resolveHookEndpoint(endpointKey).pipe( + Effect.provideService(EnvironmentLinks.EnvironmentLinks, links), + Effect.provideService( + ManagedEndpointAllocations.ManagedEndpointAllocations, + allocations, + ), + Effect.provideService(RelayConfiguration.RelayConfiguration, settings), + ); + if (endpoint === null) continue; + if (yield* inbox.wake({ endpointKey, baseUrl: endpoint.httpBaseUrl })) { + pending = true; + } + } + return { pending }; + }, mapRelayCommonApiErrors("not_authorized")), ); }), ); @@ -1378,6 +1503,8 @@ const RelayCommonPersistenceError = Schema.Union([ AgentActivityRows.AgentActivityRowListPersistenceError, LiveActivities.LiveActivityDeliveryMarkPersistenceError, DeliveryAttempts.DeliveryAttemptRecordPersistenceError, + EnvironmentLinks.EnvironmentLinkEnvironmentLookupPersistenceError, + HookInbox.HookInboxError, ]); type RelayCommonPersistenceError = typeof RelayCommonPersistenceError.Type; const isRelayCommonPersistenceError = Schema.is(RelayCommonPersistenceError); diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index 1f5d8c4d94e2..e24bb1a5c4c0 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -74,6 +74,8 @@ export const relayEnvironmentLinks = pgTable( notificationsEnabled: boolean("notifications_enabled").notNull().default(true), liveActivitiesEnabled: boolean("live_activities_enabled").notNull().default(true), managedTunnelsEnabled: boolean("managed_tunnels_enabled").notNull().default(false), + // Opt-in: hold webhook requests while the environment is offline. + holdWebhooksWhileOffline: boolean("hold_webhooks_while_offline").notNull().default(false), createdByDeviceId: varchar("created_by_device_id", { length: 191 }), revokedAt: varchar("revoked_at", { length: 64 }), createdAt: varchar("created_at", { length: 64 }).notNull(), diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index cc03abd5bee8..25cdd257f166 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -1,4 +1,5 @@ import * as Alchemy from "alchemy"; +import * as Axiom from "alchemy/Axiom"; import * as Cloudflare from "alchemy/Cloudflare"; import * as Drizzle from "alchemy/Drizzle/Postgres"; import * as Config from "effect/Config"; @@ -11,6 +12,7 @@ import * as Option from "effect/Option"; import * as Redacted from "effect/Redacted"; import * as Stream from "effect/Stream"; import * as Etag from "effect/http/Etag"; +import * as HttpMiddleware from "effect/http/HttpMiddleware"; import * as HttpPlatform from "effect/http/HttpPlatform"; import * as HttpRouter from "effect/http/HttpRouter"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; @@ -74,6 +76,8 @@ import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts" import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts"; import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts"; import * as HookForwarder from "./hooks/HookForwarder.ts"; +import * as HookInbox from "./hooks/HookInbox.ts"; +import { HookInboxObject, HookInboxObjectLive } from "./hooks/HookInboxObject.ts"; const webcryptoLayer = Layer.succeed( Crypto.Crypto, @@ -190,6 +194,8 @@ export const ApiLive = Api.make( const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone); const managedEndpointZoneName = yield* managedEndpointZone.name; const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig; + // Keys are endpoint keys or hashes over them, which already differ per + // stage, so stages sharing an account cannot collide in these namespaces. const hookRateLimit = yield* Cloudflare.RateLimit("HOOK_RATE_LIMIT", { namespaceId: 1001, simple: { @@ -197,6 +203,14 @@ export const ApiLive = Api.make( period: HookForwarder.RELAY_HOOK_RATE_LIMIT.periodSeconds, }, }); + const hookEndpointRateLimit = yield* Cloudflare.RateLimit("HOOK_ENDPOINT_RATE_LIMIT", { + namespaceId: 1002, + simple: { + limit: HookForwarder.RELAY_HOOK_ENDPOINT_RATE_LIMIT.limit, + period: HookForwarder.RELAY_HOOK_ENDPOINT_RATE_LIMIT.periodSeconds, + }, + }); + const hookInboxes = yield* HookInboxObject; // // 3. Runtime layers and app construction @@ -228,6 +242,31 @@ export const ApiLive = Api.make( }).pipe(Effect.map(makeRelayTraceLayer)), ); + // Each managed endpoint's held webhook requests live in its own Durable Object. + const inboxCall = + (operation: HookInbox.HookInboxError["operation"], endpointKey: string) => + (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), + Effect.catchCause((cause) => + Effect.fail( + new HookInbox.HookInboxError({ + operation, + endpointKey, + cause: Cause.squash(cause), + }), + ), + ), + ); + const hookInboxLayer = Layer.succeed(HookInbox.HookInbox, { + hold: ({ endpointKey, baseUrl, hook }) => + hookInboxes.getByName(endpointKey).hold(hook, baseUrl).pipe(inboxCall("hold", endpointKey)), + wake: ({ endpointKey, baseUrl }) => + hookInboxes.getByName(endpointKey).wake(baseUrl).pipe(inboxCall("wake", endpointKey)), + clear: ({ endpointKey }) => + hookInboxes.getByName(endpointKey).clear().pipe(inboxCall("clear", endpointKey)), + }); + const runtimeLayer = Layer.empty.pipe( Layer.provideMerge(MobileRegistrations.layer), Layer.provideMerge(AgentActivityPublisher.layer), @@ -269,7 +308,7 @@ export const ApiLive = Api.make( Layer.provideMerge( ApnsDeliveryQueue.layerCloudflareQueues(apnsDeliveryQueueSender, alchemyRuntimeContext), ), - Layer.provideMerge(Layer.mergeAll(AgentActivityRows.layer, Devices.layer)), + Layer.provideMerge(Layer.mergeAll(AgentActivityRows.layer, Devices.layer, hookInboxLayer)), Layer.provideMerge(EnvironmentCredentials.layer), Layer.provideMerge( Layer.mergeAll( @@ -291,17 +330,25 @@ export const ApiLive = Api.make( ); // Fails open: a limiter outage must not drop webhooks the environment would accept. - const hookRateLimiterLayer = Layer.succeed(HookForwarder.HookRateLimiter, { - allow: (key) => - hookRateLimit.limit({ key }).pipe( + const allowWith = + (limiter: typeof hookRateLimit) => + (key: string): Effect.Effect => + limiter.limit({ key }).pipe( Effect.map((result) => result.success), Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), Effect.catch((error) => Effect.logWarning("Hook rate limiter unavailable", { error: error.message }).pipe( + // Visible on the forward span, so an outage that disables limits shows up. + Effect.andThen( + Effect.annotateCurrentSpan({ "relay.hook.rate_limiter_failed_open": true }), + ), Effect.as(true), ), ), - ), + ); + const hookRateLimiterLayer = Layer.succeed(HookForwarder.HookRateLimiter, { + allowHook: allowWith(hookRateLimit), + allowEndpoint: allowWith(hookEndpointRateLimit), }); const appLayer = Layer.merge( @@ -426,6 +473,32 @@ export const ApiLive = Api.make( Layer.provideMerge(Cloudflare.Tunnel.ReadWriteTunnelBinding), Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp), Layer.provideMerge(Cloudflare.Workers.RateLimitBinding), + Layer.provideMerge(HookInboxObjectLive), + // The worker runtime opens its own HTTP span around ours. For webhook + // paths it would record the raw URL, token included, and adopt the + // sender's traceparent, so only our redacted span covers those. + // Registered as telemetry: request-time context is assembled per + // event, and only these layers are built into it. + Layer.provideMerge( + Alchemy.Telemetry.layer( + Layer.succeed(HttpMiddleware.TracerDisabledWhen)((request) => + HookForwarder.isRelayHookPath(request.url), + ), + ), + ), + // Exports spans from events the HTTP tracer does not wrap, notably + // HookInboxObject calls and alarms, to the same Axiom dataset. + Layer.provideMerge( + Layer.unwrap( + Effect.map(RelayObservability, (observability) => + Axiom.Telemetry({ + serviceName: "t3code-relay", + token: observability.workerIngestToken, + traces: observability.traces, + }), + ), + ), + ), ), ), ), diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 7063fed96ca8..6f95926cd1d1 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -35,6 +35,10 @@ "types": "./src/userMessage.ts", "default": "./src/userMessage.ts" }, + "./scheduled-task-webhook": { + "types": "./src/scheduledTaskWebhook.ts", + "default": "./src/scheduledTaskWebhook.ts" + }, "./connection": { "types": "./src/connection/index.ts", "default": "./src/connection/index.ts" diff --git a/packages/client-runtime/src/scheduledTaskWebhook.test.ts b/packages/client-runtime/src/scheduledTaskWebhook.test.ts new file mode 100644 index 000000000000..c2de7d240bc7 --- /dev/null +++ b/packages/client-runtime/src/scheduledTaskWebhook.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { parseMaxDeliveryAge } from "./scheduledTaskWebhook.ts"; + +describe("parseMaxDeliveryAge", () => { + it("treats blank as no limit and rejects values the server would not accept", () => { + expect(parseMaxDeliveryAge("")).toBeNull(); + expect(parseMaxDeliveryAge(" 45 ")).toBe(45); + for (const invalid of ["0", "1.5", "-3", "abc", "1441"]) { + expect(parseMaxDeliveryAge(invalid)).toBeUndefined(); + } + }); +}); diff --git a/packages/client-runtime/src/scheduledTaskWebhook.ts b/packages/client-runtime/src/scheduledTaskWebhook.ts new file mode 100644 index 000000000000..785a3bcfe8d1 --- /dev/null +++ b/packages/client-runtime/src/scheduledTaskWebhook.ts @@ -0,0 +1,13 @@ +import { MAX_WEBHOOK_DELIVERY_AGE_MINUTES } from "@t3tools/contracts"; + +/** Prompt a new webhook task starts with: just the body, so request headers stay out unless the user adds them. */ +export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{body}}"; + +/** Blank means "no limit"; undefined means the input is not a valid limit, which blocks saving. */ +export function parseMaxDeliveryAge(value: string): number | null | undefined { + if (value.trim() === "") return null; + const minutes = Number(value.trim()); + return Number.isInteger(minutes) && minutes > 0 && minutes <= MAX_WEBHOOK_DELIVERY_AGE_MINUTES + ? minutes + : undefined; +} diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 411b10574b02..b530c5c4c898 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -404,11 +404,16 @@ export const EnvironmentCloudLinkStateResult = Schema.Struct({ // Optional so newer clients tolerate older environment servers. managedTunnelActive: Schema.optional(Schema.Boolean), publishAgentActivity: Schema.Boolean, + // Opt-in: T3 Connect holds webhook requests while this environment is + // offline. Optional so newer clients tolerate older environment servers. + holdWebhooksWhileOffline: Schema.optional(Schema.Boolean), }); export type EnvironmentCloudLinkStateResult = typeof EnvironmentCloudLinkStateResult.Type; export const EnvironmentCloudPreferencesRequest = Schema.Struct({ publishAgentActivity: Schema.Boolean, + // Omit to leave the current value unchanged. + holdWebhooksWhileOffline: Schema.optional(Schema.Boolean), }); export type EnvironmentCloudPreferencesRequest = typeof EnvironmentCloudPreferencesRequest.Type; diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 95d680bac246..f5cb83b4f8dc 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -864,6 +864,21 @@ export const RelayCloudEnvironmentHealthProofPayload = Schema.Struct({ export type RelayCloudEnvironmentHealthProofPayload = typeof RelayCloudEnvironmentHealthProofPayload.Type; +/** + * Sent with every webhook the relay forwards, signed with the relay's mint key + * (`x-t3-relay-delivery`). The environment trusts the relay's delivery id, + * receive time and trace context only when this verifies, since its webhook + * URL can also be called directly. + */ +export const RelayHookDeliveryProofPayload = Schema.Struct({ + ...RelaySignedJwtRegisteredClaims, + environmentId: EnvironmentId, + deliveryId: TrimmedNonEmptyString, + receivedAt: TrimmedNonEmptyString, + hookId: TrimmedNonEmptyString, +}); +export type RelayHookDeliveryProofPayload = typeof RelayHookDeliveryProofPayload.Type; + export const RelayCloudEnvironmentHealthProof = TrimmedNonEmptyString; export type RelayCloudEnvironmentHealthProof = typeof RelayCloudEnvironmentHealthProof.Type; @@ -1136,6 +1151,18 @@ const RelayDpopClientGroup = HttpApiGroup.make("dpopClient") .annotate(OpenApi.Description, "DPoP-authenticated client access to linked environments.") .middleware(RelayDpopClientAuth); +export const RelayEnvironmentLinkPreferencesRequest = Schema.Struct({ + holdWebhooksWhileOffline: Schema.Boolean, +}); +export type RelayEnvironmentLinkPreferencesRequest = + typeof RelayEnvironmentLinkPreferencesRequest.Type; + +export const RelayWakeHeldHooksResponse = Schema.Struct({ + /** True when the relay was holding requests and has started delivering them. */ + pending: Schema.Boolean, +}); +export type RelayWakeHeldHooksResponse = typeof RelayWakeHeldHooksResponse.Type; + const RelayServerGroup = HttpApiGroup.make("server") .add( HttpApiEndpoint.post( @@ -1171,6 +1198,24 @@ const RelayServerGroup = HttpApiGroup.make("server") error: RelayAgentActivityPublishErrors, }, ).annotate(OpenApi.Summary, "Publish agent activity"), + HttpApiEndpoint.post( + "updateLinkPreferences", + "/v1/environments/:environmentId/link-preferences", + { + params: Schema.Struct({ environmentId: EnvironmentId }), + payload: RelayEnvironmentLinkPreferencesRequest, + success: RelayEnvironmentLinkPreferencesRequest, + error: RelayAuthAndInternalErrors, + }, + ).annotate(OpenApi.Summary, "Update an environment's link preferences"), + HttpApiEndpoint.post("wakeHeldHooks", "/v1/environments/:environmentId/hooks/wake", { + params: Schema.Struct({ environmentId: EnvironmentId }), + success: RelayWakeHeldHooksResponse, + error: RelayAuthAndInternalErrors, + }).annotate( + OpenApi.Summary, + "Deliver webhook requests held while the environment was offline now", + ), ) .annotate(OpenApi.Description, "Environment-authenticated activity publication.") .middleware(RelayEnvironmentAuth); @@ -1179,14 +1224,16 @@ const RelayServerGroup = HttpApiGroup.make("server") * Public, stateless webhook forwarding to an environment's managed tunnel. * Unauthenticated: the token in the path is the environment's credential, and * the relay only routes and forwards. Raw, so the body reaches the - * environment byte for byte and signatures still verify there. + * environment byte for byte and signatures still verify there. `endpointKey` + * names one managed endpoint (its tunnel's hash of user and environment), not + * the environment id, which any account can link. */ const RelayHookParams = Schema.Struct({ - environmentId: Schema.String, + endpointKey: Schema.String, hookId: Schema.String, token: Schema.String, }); -const RELAY_HOOK_PATH = "/v1/hooks/:environmentId/:hookId/:token"; +const RELAY_HOOK_PATH = "/v1/hooks/:endpointKey/:hookId/:token"; const relayHookEndpoint = { params: RelayHookParams } as const; const RelayHooksGroup = HttpApiGroup.make("hooks") .add( diff --git a/packages/contracts/src/scheduledTask.ts b/packages/contracts/src/scheduledTask.ts index f00ce5da93b8..3140ff8a79cf 100644 --- a/packages/contracts/src/scheduledTask.ts +++ b/packages/contracts/src/scheduledTask.ts @@ -73,11 +73,23 @@ export const ScheduledTaskWebhookSignature = Schema.Struct( ).annotate({ description: "Optional HMAC-SHA256 signature check over the raw request body." }); export type ScheduledTaskWebhookSignature = typeof ScheduledTaskWebhookSignature.Type; +/** Matches how long the relay holds a request for an offline environment. */ +export const MAX_WEBHOOK_DELIVERY_AGE_MINUTES = 24 * 60; + +const WebhookMaxDeliveryAgeMinutes = Schema.Int.check( + Schema.isBetween({ minimum: 1, maximum: MAX_WEBHOOK_DELIVERY_AGE_MINUTES }), +).annotate({ + description: + "Skip requests the relay held longer than this many minutes while the environment was offline. Null runs every request.", +}); + const ScheduledTaskWebhookSchedule = Schema.Struct({ type: Schema.Literal("webhook").annotate({ description: "Run when the task's webhook URL receives a request.", }), signature: Schema.NullOr(ScheduledTaskWebhookSignature), + // Optional so rows saved before this setting existed still decode. + maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)), }).annotate({ description: "Run on each request to the task's webhook URL. The prompt may use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} and {{request}} placeholders.", @@ -99,6 +111,7 @@ const ScheduledTaskUpsertWebhookSchedule = Schema.Struct({ ).annotate({ description: "Signature check; omit or null to accept requests by URL token only.", }), + maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)), }).annotate({ description: "Run on each request to the task's webhook URL. The prompt may use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} and {{request}} placeholders.", @@ -241,6 +254,7 @@ export const ScheduledTaskWebhookDeliveryOutcome = Schema.Literals([ "rejected_signature", "disabled", "rate_limited", + "expired", ]); export type ScheduledTaskWebhookDeliveryOutcome = typeof ScheduledTaskWebhookDeliveryOutcome.Type; diff --git a/packages/shared/src/relayJwt.ts b/packages/shared/src/relayJwt.ts index cabe340d7009..0b06c00d4b5d 100644 --- a/packages/shared/src/relayJwt.ts +++ b/packages/shared/src/relayJwt.ts @@ -11,6 +11,9 @@ export const RELAY_MINT_RESPONSE_TYP = "t3-env-mint+jwt"; export const RELAY_HEALTH_RESPONSE_TYP = "t3-env-health+jwt"; export const RELAY_ACTIVITY_PUBLISH_TYP = "t3-env-activity+jwt"; export const RELAY_MANAGED_TUNNEL_RECOVERY_TYP = "t3-env-managed-tunnel-recovery+jwt"; +export const RELAY_HOOK_DELIVERY_TYP = "t3-relay-hook-delivery+jwt"; +/** Header carrying the signed proof that a webhook request came from the relay. */ +export const RELAY_HOOK_DELIVERY_HEADER = "x-t3-relay-delivery"; export class RelayJwtError extends Schema.TaggedError()("RelayJwtError", { operation: Schema.Literals(["sign", "verify"]), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0b81889e809e..403cf940ff9c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,6 +9,9 @@ catalogs: '@effect/openapi-generator': specifier: 4.0.1 version: 4.0.1 + '@effect/sql-sqlite-do': + specifier: 4.0.1 + version: 4.0.1 '@effect/tsgo': specifier: 0.41.0 version: 0.41.0 @@ -837,6 +840,9 @@ importers: '@effect/sql-pg': specifier: 4.0.1 version: 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) + '@effect/sql-sqlite-do': + specifier: 'catalog:' + version: 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@noble/curves': specifier: 'catalog:' version: 1.9.1 @@ -854,10 +860,10 @@ importers: version: link:../../packages/shared alchemy: specifier: 2.0.0-beta.80 - version: 2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(824b7c95c4257cf54626a40f00f69c9b) + version: 2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(d02b4e60a6a3c6b542dfa700209cf5ce) drizzle-orm: specifier: 1.0.0-rc.5-ab785fc - version: 1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d) + version: 1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c) effect: specifier: 4.0.1 version: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f) @@ -2362,8 +2368,8 @@ packages: peerDependencies: effect: 4.0.1 - '@effect/sql-sqlite-do@4.0.0': - resolution: {integrity: sha512-XkRZmQOPblzwjmAJ0RWSOiLlTsKkf1EigZsxUNuIuk4uZ/pnufSqvDpXubCwv8iyRTbOLFMmM+EyOEH5V5XqAw==} + '@effect/sql-sqlite-do@4.0.1': + resolution: {integrity: sha512-tS+qQok3xKsycnnhvIDX+6nvlM4Y2ouvru3voOzxtDQqX96NSOMX2xReLf4Feb6GN3jnzgEEtVGDDWk3YLrpsg==} peerDependencies: effect: 4.0.1 @@ -13051,7 +13057,7 @@ snapshots: effect: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f) optional: true - '@effect/sql-sqlite-do@4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))': + '@effect/sql-sqlite-do@4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))': dependencies: effect: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f) @@ -16625,7 +16631,7 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 - alchemy@2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(824b7c95c4257cf54626a40f00f69c9b): + alchemy@2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(d02b4e60a6a3c6b542dfa700209cf5ce): dependencies: '@alchemy.run/cloudflare-runtime': 2.0.0-beta.80(@distilled.cloud/cloudflare@1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)))(@effect/platform-node@4.0.1(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(redis@6.2.1)(utf-8-validate@6.0.6))(@types/node@24.12.4)(@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0))(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(rolldown@1.2.5)(typescript@7.0.2) '@alchemy.run/floci': 2.0.0-beta.80(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) @@ -16648,7 +16654,7 @@ snapshots: '@distilled.cloud/stripe': 1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@distilled.cloud/zerossl': 1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@effect/sql-d1': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) - '@effect/sql-sqlite-do': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) + '@effect/sql-sqlite-do': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@libsql/client': 0.17.3(bufferutil@4.1.0)(utf-8-validate@6.0.6) '@neon/functions': 0.11.0(hono@4.13.7) '@octokit/rest': 22.0.1 @@ -16668,7 +16674,7 @@ snapshots: '@effect/sql-pg': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@effect/vitest': 4.0.1(patch_hash=359f6fb2f7b3ec145bb72208edb9034f02489791aa2491a55cdbd69bd56ee0d2)(@types/node@24.12.4)(@vitest/ui@5.0.1)(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) drizzle-kit: 1.0.0-rc.5-ab785fc - drizzle-orm: 1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d) + drizzle-orm: 1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c) mongodb: 6.21.0(@aws-sdk/credential-providers@3.1062.0)(socks@2.8.9) pg: 8.23.0 vite: '@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0)' @@ -17825,13 +17831,13 @@ snapshots: get-tsconfig: 4.14.3 jiti: 2.7.0 - drizzle-orm@1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d): + drizzle-orm@1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c): optionalDependencies: '@cloudflare/workers-types': 4.20260604.1 '@effect/sql-d1': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@effect/sql-pg': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@effect/sql-sqlite-bun': 4.0.0-rc.112(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) - '@effect/sql-sqlite-do': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) + '@effect/sql-sqlite-do': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@electric-sql/pglite': 0.3.15 '@libsql/client': 0.17.3(bufferutil@4.1.0)(utf-8-validate@6.0.6) bun-types: 1.3.14 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 460a39ca541e..7f74dc5ba16b 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -35,6 +35,7 @@ catalog: "@effect/platform-node": 4.0.1 "@effect/platform-node-shared": 4.0.1 "@effect/sql-pg": 4.0.1 + "@effect/sql-sqlite-do": 4.0.1 "@effect/tsgo": 0.41.0 "@effect/vitest": 4.0.1 "@legendapp/list": 3.3.5 @@ -75,6 +76,7 @@ minimumReleaseAgeExclude: - "@effect/platform-node-shared@4.0.1" - "@effect/platform-node@4.0.1" - "@effect/sql-pg@4.0.1" + - "@effect/sql-sqlite-do@4.0.1" - "@effect/vitest@4.0.1" - alchemy@2.0.0-beta.80 - effect@4.0.1