From f65694ad44d5cf22cb51b268a6a1a93cb6a8cb90 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:28:48 -0700 Subject: [PATCH 01/35] wip(relay): hold webhook requests for environments that opt in Co-Authored-By: Claude Opus 5.5 (1M context) --- .../20261004030842_hook_mailbox/migration.sql | 16 + .../20261004030842_hook_mailbox/snapshot.json | 1803 +++++++++++++++++ .../AgentActivityPublisher.test.ts | 1 + .../src/agentActivity/FcmDeliveries.test.ts | 1 + .../agentActivity/MobileRegistrations.test.ts | 1 + .../environments/EnvironmentConnector.test.ts | 1 + .../environments/EnvironmentLinker.test.ts | 1 + .../src/environments/EnvironmentLinks.ts | 38 +- infra/relay/src/hooks/HookForwarder.test.ts | 93 +- infra/relay/src/hooks/HookForwarder.ts | 66 +- infra/relay/src/hooks/HookMailbox.test.ts | 128 ++ infra/relay/src/hooks/HookMailbox.ts | 163 ++ infra/relay/src/http/Api.test.ts | 14 +- infra/relay/src/http/Api.ts | 65 + infra/relay/src/persistence/schema.ts | 28 + infra/relay/src/worker.ts | 11 +- packages/contracts/src/environmentHttp.ts | 5 + packages/contracts/src/relay.ts | 57 + packages/contracts/src/scheduledTask.ts | 14 + 19 files changed, 2494 insertions(+), 12 deletions(-) create mode 100644 infra/relay/migrations/postgres/20261004030842_hook_mailbox/migration.sql create mode 100644 infra/relay/migrations/postgres/20261004030842_hook_mailbox/snapshot.json create mode 100644 infra/relay/src/hooks/HookMailbox.test.ts create mode 100644 infra/relay/src/hooks/HookMailbox.ts diff --git a/infra/relay/migrations/postgres/20261004030842_hook_mailbox/migration.sql b/infra/relay/migrations/postgres/20261004030842_hook_mailbox/migration.sql new file mode 100644 index 000000000000..d66698deb8a8 --- /dev/null +++ b/infra/relay/migrations/postgres/20261004030842_hook_mailbox/migration.sql @@ -0,0 +1,16 @@ +CREATE TABLE "relay_hook_mailbox" ( + "id" varchar(36) PRIMARY KEY, + "environment_id" varchar(191) NOT NULL, + "received_at" varchar(64) NOT NULL, + "expires_at" varchar(64) NOT NULL, + "method" varchar(16) NOT NULL, + "raw_hook_id" varchar(512) NOT NULL, + "raw_token" varchar(512) NOT NULL, + "query" text NOT NULL, + "headers" jsonb NOT NULL, + "body" bytea NOT NULL +); +--> statement-breakpoint +ALTER TABLE "relay_environment_links" ADD COLUMN "hold_webhooks_while_offline" boolean DEFAULT false NOT NULL;--> statement-breakpoint +CREATE INDEX "idx_relay_hook_mailbox_environment" ON "relay_hook_mailbox" ("environment_id","received_at");--> statement-breakpoint +CREATE INDEX "idx_relay_hook_mailbox_expires" ON "relay_hook_mailbox" ("expires_at"); \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20261004030842_hook_mailbox/snapshot.json b/infra/relay/migrations/postgres/20261004030842_hook_mailbox/snapshot.json new file mode 100644 index 000000000000..c8fdaeb0a80a --- /dev/null +++ b/infra/relay/migrations/postgres/20261004030842_hook_mailbox/snapshot.json @@ -0,0 +1,1803 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "fcfe51b7-4f3e-45a5-9e83-ac96f1ced4e6", + "prevIds": [ + "3809c51e-3821-4a08-818d-e5d28dd3e9b4" + ], + "ddl": [ + { + "isRlsEnabled": false, + "name": "relay_agent_activity_rows", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_delivery_attempts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_dpop_proofs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_credentials", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_links", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_hook_mailbox", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_live_activities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_endpoint_allocations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_tunnel_limits", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_mobile_devices", + "entityType": "tables", + "schema": "public" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state_json", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(36)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_job_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_suffix", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_status", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_reason", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "transport_error", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thumbprint", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "jti", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "iat", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_hash", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'T3 Environment'", + "generated": null, + "identity": null, + "name": "environment_label", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_http_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_ws_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(32)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_provider_kind", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "notifications_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "live_activities_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "managed_tunnels_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "hold_webhooks_while_offline", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(36)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "received_at", + "entityType": "columns", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "method", + "entityType": "columns", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "raw_hook_id", + "entityType": "columns", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "raw_token", + "entityType": "columns", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "query", + "entityType": "columns", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "headers", + "entityType": "columns", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "type": "bytea", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "body", + "entityType": "columns", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activity_push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_start_queued_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_started_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ended_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_aggregate_json", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_live_activity_delivery_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "hostname", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_name", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dns_record_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ready_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_enabled_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "generation", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "max_tunnels", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'iOS device'", + "generated": null, + "identity": null, + "name": "label", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "platform", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ios_major_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "android_api_level", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "bundle_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "aps_environment", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_to_start_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "preferences_json", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "updated_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_agent_activity_rows_updated", + "entityType": "indexes", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "thread_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_source_job", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_dpop_proofs_expires_at", + "entityType": "indexes", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "credential_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_hash", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "environment_public_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment_key", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_links_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_links" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "received_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_hook_mailbox_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_hook_mailbox_expires", + "entityType": "indexes", + "schema": "public", + "table": "relay_hook_mailbox" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "activity_push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_live_activities_activity_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_live_activities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "hostname", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_hostname", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "tunnel_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_tunnel_name", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_to_start_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_to_start_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": [ + "environment_id", + "environment_public_key", + "thread_id" + ], + "nameExplicit": false, + "name": "relay_agent_activity_rows_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "columns": [ + "thumbprint", + "jti" + ], + "nameExplicit": false, + "name": "relay_dpop_proofs_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "columns": [ + "user_id", + "environment_id" + ], + "nameExplicit": false, + "name": "relay_environment_links_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_environment_links" + }, + { + "columns": [ + "user_id", + "device_id" + ], + "nameExplicit": false, + "name": "relay_live_activities_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_live_activities" + }, + { + "columns": [ + "user_id", + "environment_id" + ], + "nameExplicit": false, + "name": "relay_managed_endpoint_allocations_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "columns": [ + "user_id", + "device_id" + ], + "nameExplicit": false, + "name": "relay_mobile_devices_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "relay_delivery_attempts_pkey", + "schema": "public", + "table": "relay_delivery_attempts", + "entityType": "pks" + }, + { + "columns": [ + "credential_id" + ], + "nameExplicit": false, + "name": "relay_environment_credentials_pkey", + "schema": "public", + "table": "relay_environment_credentials", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "relay_hook_mailbox_pkey", + "schema": "public", + "table": "relay_hook_mailbox", + "entityType": "pks" + }, + { + "columns": [ + "user_id" + ], + "nameExplicit": false, + "name": "relay_managed_tunnel_limits_pkey", + "schema": "public", + "table": "relay_managed_tunnel_limits", + "entityType": "pks" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts b/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts index 5f2c9463d4da..2c44a2809d9e 100644 --- a/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts +++ b/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts @@ -95,6 +95,7 @@ function makeEnvironmentLinks( listForUser: () => Effect.succeed([]), getForUser: () => Effect.succeed(null), findActiveManagedForEnvironment: () => Effect.succeed([]), + setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: () => Effect.succeed(false), ...overrides, }; diff --git a/infra/relay/src/agentActivity/FcmDeliveries.test.ts b/infra/relay/src/agentActivity/FcmDeliveries.test.ts index ee903db8f382..65216cbe5308 100644 --- a/infra/relay/src/agentActivity/FcmDeliveries.test.ts +++ b/infra/relay/src/agentActivity/FcmDeliveries.test.ts @@ -172,6 +172,7 @@ function harness() { ), listForUser: () => Effect.succeed([]), findActiveManagedForEnvironment: () => Effect.succeed([]), + setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: () => Effect.succeed(false), getForUser: (input) => Effect.sync(() => diff --git a/infra/relay/src/agentActivity/MobileRegistrations.test.ts b/infra/relay/src/agentActivity/MobileRegistrations.test.ts index 57c5d6e55928..b649c09c7b56 100644 --- a/infra/relay/src/agentActivity/MobileRegistrations.test.ts +++ b/infra/relay/src/agentActivity/MobileRegistrations.test.ts @@ -117,6 +117,7 @@ function makeEnvironmentLinks( listForUser: () => Effect.succeed([]), getForUser: () => Effect.succeed(null), findActiveManagedForEnvironment: () => Effect.succeed([]), + setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: () => Effect.succeed(false), ...overrides, }; diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index 1a088b829d98..7746cbeaae22 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -230,6 +230,7 @@ function makeLinks( ...overrides, }), findActiveManagedForEnvironment: () => Effect.succeed([]), + setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: () => Effect.succeed(false), }; } diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index 912ffc30f70b..f068cd1fc499 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -127,6 +127,7 @@ function testLayer(input?: { listForUser: () => Effect.succeed([]), getForUser: () => Effect.succeed(null), findActiveManagedForEnvironment: () => Effect.succeed([]), + setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: () => Effect.succeed(false), }), Layer.succeed(EnvironmentCredentials.EnvironmentCredentials, { diff --git a/infra/relay/src/environments/EnvironmentLinks.ts b/infra/relay/src/environments/EnvironmentLinks.ts index c6832a8025f3..f033157c3c7f 100644 --- a/infra/relay/src/environments/EnvironmentLinks.ts +++ b/infra/relay/src/environments/EnvironmentLinks.ts @@ -130,9 +130,19 @@ export class EnvironmentLinks extends Context.Service< readonly findActiveManagedForEnvironment: (input: { readonly environmentId: string; }) => Effect.Effect< - ReadonlyArray, + ReadonlyArray< + RelayLinkedEnvironmentRecord & { + readonly userId: string; + readonly holdWebhooksWhileOffline: boolean; + } + >, EnvironmentLinkEnvironmentLookupPersistenceError >; + /** Sets the webhook-hold opt-in on every active link of an environment. */ + readonly setHoldWebhooksWhileOffline: (input: { + readonly environmentId: string; + readonly holdWebhooksWhileOffline: boolean; + }) => Effect.Effect; readonly revokeForUser: (input: { readonly userId: string; readonly environmentId: string; @@ -355,6 +365,7 @@ const make = Effect.gen(function* () { endpointWsBaseUrl: relayEnvironmentLinks.endpointWsBaseUrl, endpointProviderKind: relayEnvironmentLinks.endpointProviderKind, createdAt: relayEnvironmentLinks.createdAt, + holdWebhooksWhileOffline: relayEnvironmentLinks.holdWebhooksWhileOffline, }) .from(relayEnvironmentLinks) .where( @@ -382,6 +393,7 @@ const make = Effect.gen(function* () { }, environmentPublicKey: row.environmentPublicKey, linkedAt: row.createdAt, + holdWebhooksWhileOffline: row.holdWebhooksWhileOffline, })), ), Effect.mapError( @@ -394,6 +406,30 @@ const make = Effect.gen(function* () { ); }), + setHoldWebhooksWhileOffline: Effect.fn( + "relay.environment_links.set_hold_webhooks_while_offline", + )(function* (input) { + yield* Effect.annotateCurrentSpan({ "relay.environment_id": input.environmentId }); + yield* db + .update(relayEnvironmentLinks) + .set({ holdWebhooksWhileOffline: input.holdWebhooksWhileOffline }) + .where( + and( + eq(relayEnvironmentLinks.environmentId, input.environmentId), + isNull(relayEnvironmentLinks.revokedAt), + ), + ) + .pipe( + Effect.mapError( + (cause) => + new EnvironmentLinkEnvironmentLookupPersistenceError({ + environmentId: input.environmentId, + cause, + }), + ), + ); + }), + revokeForUser: Effect.fn("relay.environment_links.revoke_for_user")(function* (input) { yield* Effect.annotateCurrentSpan({ "relay.environment_id": input.environmentId, diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index 6c4383d10df3..ba8617fb8baf 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -1,5 +1,6 @@ import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform"; import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { describe, expect, it } from "@effect/vitest"; import { RelayApi } from "@t3tools/contracts/relay"; import * as Deferred from "effect/Deferred"; @@ -31,6 +32,7 @@ import { traceRelayHttpRequestWith, } from "../http/Api.ts"; import * as HookForwarder from "./HookForwarder.ts"; +import * as HookMailbox from "./HookMailbox.ts"; const settings: RelayConfiguration.RelayConfiguration["Service"] = { relayIssuer: "https://relay.example.test", @@ -70,6 +72,7 @@ const managedLink = { }, environmentPublicKey: "public-key", linkedAt: "2026-05-25T00:00:00.000Z", + holdWebhooksWhileOffline: false, }; interface Harness { @@ -79,11 +82,14 @@ interface Harness { readonly links?: ReadonlyArray; readonly allocation?: ManagedEndpointAllocations.ManagedEndpointAllocation | null; readonly allow?: (key: string) => boolean; + /** Mailbox capacity; enqueue reports full once this many requests are held. */ + readonly mailboxCapacity?: number; } function makeHarness(options: Harness = {}) { const sent: Array = []; const rateLimitKeys: Array = []; + const held: Array = []; const execute = options.execute ?? ((request: HttpClientRequest.HttpClientRequest) => @@ -114,6 +120,15 @@ function makeHarness(options: Harness = {}) { return execute(request); }), ), + Layer.mock(HookMailbox.HookMailbox, { + enqueue: (hook) => + Effect.sync(() => { + if (held.length >= (options.mailboxCapacity ?? Infinity)) return false; + held.push(hook); + return true; + }), + }), + NodeCrypto.layer, Layer.succeed(HookForwarder.HookRateLimiter, { allow: (key) => Effect.sync(() => { @@ -144,7 +159,7 @@ function makeHarness(options: Harness = {}) { ), ); }); - return { sent, rateLimitKeys, send, httpEffect }; + return { sent, rateLimitKeys, held, send, httpEffect }; } const hookUrl = (path = "hook-1/secret-token", query = "") => @@ -459,4 +474,80 @@ describe("HookForwarder", () => { ); }), ); + + describe("holding requests while the environment is offline", () => { + const offline = (request: HttpClientRequest.HttpClientRequest) => + Effect.fail( + new HttpClientError.HttpClientError({ + reason: new HttpClientError.TransportError({ request, cause: new Error("offline") }), + }), + ); + + it.effect("stays a plain proxy when the environment has not opted in", () => + Effect.gen(function* () { + const harness = makeHarness({ execute: offline }); + const response = yield* harness.send( + new Request(hookUrl(), { method: "POST", body: "{}" }), + ); + expect(response.status).toBe(503); + expect(harness.held).toHaveLength(0); + }), + ); + + it.effect("holds the exact request and answers 202 once opted in", () => + Effect.gen(function* () { + const harness = makeHarness({ + execute: offline, + links: [{ ...managedLink, holdWebhooksWhileOffline: true }], + }); + const body = new Uint8Array([0, 255, 10]); + const response = yield* harness.send( + new Request(hookUrl("hook-1/tok%2Fen", "?a=1"), { + method: "POST", + body, + headers: { "x-t3-relay-delivery-id": "forged", "x-sig": "s" }, + }), + ); + expect(response.status).toBe(202); + const [hook] = harness.held; + expect(hook?.rawToken).toBe("tok%2Fen"); + expect(hook?.query).toBe("a=1"); + expect([...(hook?.body ?? [])]).toEqual([0, 255, 10]); + expect(hook?.headers["x-sig"]).toBe("s"); + // The sender cannot choose the delivery id. + expect(hook?.headers["x-t3-relay-delivery-id"]).toBeUndefined(); + expect(hook?.id).not.toBe("forged"); + }), + ); + + it.effect("answers 503 mailbox_full when the environment's mailbox is full", () => + Effect.gen(function* () { + const harness = makeHarness({ + execute: offline, + links: [{ ...managedLink, holdWebhooksWhileOffline: true }], + mailboxCapacity: 0, + }); + const response = yield* harness.send( + new Request(hookUrl(), { method: "POST", body: "{}" }), + ); + expect(response.status).toBe(503); + expect(yield* readJson(response)).toEqual({ error: "mailbox_full" }); + }), + ); + + it.effect("tags every forward with a relay delivery id", () => + Effect.gen(function* () { + const harness = makeHarness(); + yield* harness.send( + new Request(hookUrl(), { + method: "POST", + body: "{}", + headers: { "x-t3-relay-delivery-id": "forged" }, + }), + ); + const id = harness.sent[0]?.headers["x-t3-relay-delivery-id"]; + expect(id).toMatch(/^[0-9a-f-]{36}$/); + }), + ); + }); }); diff --git a/infra/relay/src/hooks/HookForwarder.ts b/infra/relay/src/hooks/HookForwarder.ts index 7f33a7d2513f..a2d9ee18ba43 100644 --- a/infra/relay/src/hooks/HookForwarder.ts +++ b/infra/relay/src/hooks/HookForwarder.ts @@ -1,4 +1,6 @@ import * as Context from "effect/Context"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -18,8 +20,11 @@ import * as RelayConfiguration from "../Config.ts"; import { validateManagedEndpoint, withoutRedirects } from "../environments/EnvironmentConnector.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; +import * as HookMailbox from "./HookMailbox.ts"; export const RELAY_HOOK_PATH_PREFIX = "/v1/hooks/"; +/** Set by the relay on every forward; the environment uses it as the delivery id. */ +export const RELAY_DELIVERY_ID_HEADER = "x-t3-relay-delivery-id"; export const RELAY_HOOK_MAX_BODY_BYTES = 1_048_576; export const RELAY_HOOK_UPSTREAM_TIMEOUT_MS = 8_000; export const RELAY_HOOK_RATE_LIMIT = { limit: 60, periodSeconds: 60 } as const; @@ -34,6 +39,8 @@ const DROPPED_REQUEST_HEADERS = new Set([ "content-length", "cookie", "x-real-ip", + // Only the relay may set this; a sender could otherwise collide delivery ids. + "x-t3-relay-delivery-id", ]); const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-"]; // Cloudflare answers 530 when the tunnel for a hostname has no connected origin. @@ -206,6 +213,8 @@ const make = Effect.gen(function* () { const settings = yield* RelayConfiguration.RelayConfiguration; const httpClient = yield* HttpClient.HttpClient; const rateLimiter = yield* HookRateLimiter; + const mailbox = yield* HookMailbox.HookMailbox; + const crypto = yield* Crypto.Crypto; const resolveEndpoint = Effect.fn("relay.hooks.resolve_endpoint")(function* ( environmentId: string, @@ -219,7 +228,7 @@ const make = Effect.gen(function* () { baseDomain: settings.managedEndpointBaseDomain, }); if (Result.isSuccess(result)) { - return result.success; + return { ...result.success, holdWhileOffline: link.holdWebhooksWhileOffline }; } } return null; @@ -279,7 +288,51 @@ const make = Effect.gen(function* () { const baseUrl = endpoint.httpBaseUrl.endsWith("/") ? endpoint.httpBaseUrl : `${endpoint.httpBaseUrl}/`; - const headers = forwardedHeaders(request.headers); + // One id per attempt, so a request that reached the environment before a + // timeout and is later replayed from the mailbox dispatches only once. + const deliveryId = yield* crypto.randomUUIDv4.pipe(Effect.orDie); + const headers: Record = { + ...forwardedHeaders(request.headers), + [RELAY_DELIVERY_ID_HEADER]: deliveryId, + }; + // Held only for environments that opted in; otherwise the relay is a plain proxy. + const holdOrFail = (status: 503 | 504, error: string) => + Effect.gen(function* () { + if (!endpoint.holdWhileOffline) { + yield* outcome(error); + return errorResponse(status, error); + } + const stored = yield* mailbox + .enqueue({ + id: deliveryId, + environmentId: parsed.environmentId, + receivedAt: DateTime.formatIso(yield* DateTime.now), + method: request.method, + rawHookId: parsed.rawHookId, + rawToken: parsed.rawToken, + query: parsed.search.replace(/^\?/, ""), + headers: forwardedHeaders(request.headers), + body: body.success, + }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("Could not hold webhook request", { + environmentId: parsed.environmentId, + errorTag: cause._tag, + }).pipe(Effect.as(null)), + ), + ); + if (stored === null) { + yield* outcome(error); + return errorResponse(status, error); + } + if (!stored) { + yield* outcome("mailbox_full"); + return errorResponse(503, "mailbox_full"); + } + yield* outcome("held"); + return HttpServerResponse.jsonUnsafe({ queued: true }, { status: 202 }); + }); let upstreamRequest = HttpClientRequest.make( request.method as "GET" | "POST" | "PUT" | "PATCH", )(`${baseUrl}api/hooks/${parsed.rawHookId}/${parsed.rawToken}${parsed.search}`, { headers }); @@ -308,17 +361,14 @@ const make = Effect.gen(function* () { Effect.result, ); if (Result.isFailure(upstream)) { - yield* outcome("environment_unavailable"); - return errorResponse(503, "environment_unavailable"); + return yield* holdOrFail(503, "environment_unavailable"); } if (Option.isNone(upstream.success)) { - yield* outcome("environment_timeout"); - return errorResponse(504, "environment_timeout"); + return yield* holdOrFail(504, "environment_timeout"); } const response = upstream.success.value; if (response.status === TUNNEL_OFFLINE_STATUS) { - yield* outcome("environment_unavailable"); - return errorResponse(503, "environment_unavailable"); + return yield* holdOrFail(503, "environment_unavailable"); } yield* Effect.annotateCurrentSpan({ "relay.hook.outcome": "forwarded", diff --git a/infra/relay/src/hooks/HookMailbox.test.ts b/infra/relay/src/hooks/HookMailbox.test.ts new file mode 100644 index 000000000000..9cec3624ecba --- /dev/null +++ b/infra/relay/src/hooks/HookMailbox.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from "@effect/vitest"; +import { PgDialect } from "drizzle-orm/pg-core"; +import type { SQL } from "drizzle-orm"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as RelayDb from "../db.ts"; +import { relayHookMailbox } from "../persistence/schema.ts"; +import * as HookMailbox from "./HookMailbox.ts"; + +const dialect = new PgDialect(); +const render = (condition: unknown) => dialect.sqlToQuery(condition as SQL); + +const layerWithDb = (db: RelayDb.RelayDb["Service"]) => + HookMailbox.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db))); + +const hook: HookMailbox.HeldHook = { + id: "delivery-1", + environmentId: "environment-1", + receivedAt: "2026-10-04T10:00:00.000Z", + method: "POST", + rawHookId: "task", + rawToken: "token", + query: "", + headers: { "content-type": "application/json" }, + body: new Uint8Array([1, 2, 3]), +}; + +describe("HookMailbox", () => { + it.effect("stores a request with a 24 hour expiry while there is room", () => { + const inserted: Array> = []; + const db = { + select: () => ({ + from: () => ({ where: () => Effect.succeed([{ value: 3 }]) }), + }), + insert: (table: unknown) => { + expect(table).toBe(relayHookMailbox); + return { + values: (values: Record) => { + inserted.push(values); + return { onConflictDoNothing: () => Effect.void }; + }, + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const mailbox = yield* HookMailbox.HookMailbox; + expect(yield* mailbox.enqueue(hook)).toBe(true); + expect(inserted[0]?.expiresAt).toBe("2026-10-05T10:00:00.000Z"); + expect([...(inserted[0]?.body as Uint8Array)]).toEqual([1, 2, 3]); + }).pipe(Effect.provide(layerWithDb(db))); + }); + + it.effect("refuses a request once the environment's mailbox is full", () => { + let insertedAny = false; + const db = { + select: () => ({ + from: () => ({ + where: () => Effect.succeed([{ value: HookMailbox.HOOK_MAILBOX_MAX_PER_ENVIRONMENT }]), + }), + }), + insert: () => { + insertedAny = true; + return { values: () => ({ onConflictDoNothing: () => Effect.void }) }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const mailbox = yield* HookMailbox.HookMailbox; + expect(yield* mailbox.enqueue(hook)).toBe(false); + expect(insertedAny).toBe(false); + }).pipe(Effect.provide(layerWithDb(db))); + }); + + it.effect("lists and acks only the calling environment's requests", () => { + const conditions: Array = []; + let limit = 0; + const db = { + select: () => ({ + from: () => ({ + where: (condition: unknown) => { + conditions.push(condition); + return { + orderBy: () => ({ + limit: (value: number) => { + limit = value; + return Effect.succeed([]); + }, + }), + }; + }, + }), + }), + delete: () => ({ + where: (condition: unknown) => { + conditions.push(condition); + return { returning: () => Effect.succeed([{ id: "delivery-1" }]) }; + }, + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const mailbox = yield* HookMailbox.HookMailbox; + yield* mailbox.listPending({ environmentId: "environment-1", limit: 1_000 }); + expect(limit).toBe(HookMailbox.HOOK_MAILBOX_MAX_PULL); + expect(yield* mailbox.ack({ environmentId: "environment-1", ids: ["delivery-1"] })).toBe(1); + for (const condition of conditions) { + const query = render(condition); + expect(query.sql).toContain('"relay_hook_mailbox"."environment_id" = $1'); + expect(query.params[0]).toBe("environment-1"); + } + }).pipe(Effect.provide(layerWithDb(db))); + }); + + it.effect("acks nothing for an empty id list", () => { + const db = { + delete: () => { + throw new Error("no delete expected"); + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const mailbox = yield* HookMailbox.HookMailbox; + expect(yield* mailbox.ack({ environmentId: "environment-1", ids: [] })).toBe(0); + }).pipe(Effect.provide(layerWithDb(db))); + }); +}); diff --git a/infra/relay/src/hooks/HookMailbox.ts b/infra/relay/src/hooks/HookMailbox.ts new file mode 100644 index 000000000000..22d117ee0360 --- /dev/null +++ b/infra/relay/src/hooks/HookMailbox.ts @@ -0,0 +1,163 @@ +import { and, asc, count, eq, inArray, lt } from "drizzle-orm"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import * as RelayDb from "../db.ts"; +import { relayHookMailbox } from "../persistence/schema.ts"; + +/** How long a held request waits for its environment. */ +export const HOOK_MAILBOX_TTL_MS = 24 * 60 * 60 * 1000; +/** Requests one environment may have waiting at once. */ +export const HOOK_MAILBOX_MAX_PER_ENVIRONMENT = 500; +/** Requests returned per pull; bodies are up to 1 MiB each. */ +export const HOOK_MAILBOX_MAX_PULL = 10; + +export class HookMailboxPersistenceError extends Schema.TaggedError()( + "HookMailboxPersistenceError", + { + operation: Schema.Literals(["enqueue", "list", "ack", "prune", "clear"]), + environmentId: Schema.optional(Schema.String), + cause: Schema.Defect(), + }, +) { + override get message(): string { + return `Hook mailbox '${this.operation}' failed${this.environmentId ? ` for environment '${this.environmentId}'` : ""}`; + } +} + +export interface HeldHook { + readonly id: string; + readonly environmentId: string; + readonly receivedAt: string; + readonly method: string; + readonly rawHookId: string; + readonly rawToken: string; + readonly query: string; + readonly headers: Readonly>; + readonly body: Uint8Array; +} + +export class HookMailbox extends Context.Service< + HookMailbox, + { + /** Stores a request; returns false when the environment's mailbox is full. */ + readonly enqueue: (hook: HeldHook) => Effect.Effect; + /** Oldest first, never another environment's requests. */ + readonly listPending: (input: { + readonly environmentId: string; + readonly limit: number; + }) => Effect.Effect, HookMailboxPersistenceError>; + readonly ack: (input: { + readonly environmentId: string; + readonly ids: ReadonlyArray; + }) => Effect.Effect; + readonly pruneExpired: (input: { + readonly now: string; + }) => Effect.Effect; + /** Deletes everything held for an environment, used when it is unlinked. */ + readonly clearEnvironment: (input: { + readonly environmentId: string; + }) => Effect.Effect; + } +>()("t3code-relay/hooks/HookMailbox") {} + +export const make = Effect.gen(function* () { + const db = yield* RelayDb.RelayDb; + const fail = + (operation: HookMailboxPersistenceError["operation"], environmentId?: string) => + (cause: unknown) => + new HookMailboxPersistenceError({ + operation, + ...(environmentId === undefined ? {} : { environmentId }), + cause, + }); + + return HookMailbox.of({ + enqueue: Effect.fn("relay.hook_mailbox.enqueue")(function* (hook) { + yield* Effect.annotateCurrentSpan({ "relay.environment_id": hook.environmentId }); + const [held] = yield* db + .select({ value: count() }) + .from(relayHookMailbox) + .where(eq(relayHookMailbox.environmentId, hook.environmentId)) + .pipe(Effect.mapError(fail("enqueue", hook.environmentId))); + if ((held?.value ?? 0) >= HOOK_MAILBOX_MAX_PER_ENVIRONMENT) return false; + const expiresAt = DateTime.formatIso( + DateTime.add(DateTime.makeUnsafe(hook.receivedAt), { milliseconds: HOOK_MAILBOX_TTL_MS }), + ); + yield* db + .insert(relayHookMailbox) + .values({ + id: hook.id, + environmentId: hook.environmentId, + receivedAt: hook.receivedAt, + expiresAt, + method: hook.method, + rawHookId: hook.rawHookId, + rawToken: hook.rawToken, + query: hook.query, + headers: { ...hook.headers }, + body: Buffer.from(hook.body), + }) + .onConflictDoNothing() + .pipe(Effect.mapError(fail("enqueue", hook.environmentId))); + return true; + }), + + listPending: Effect.fn("relay.hook_mailbox.list_pending")(function* (input) { + yield* Effect.annotateCurrentSpan({ "relay.environment_id": input.environmentId }); + const rows = yield* db + .select() + .from(relayHookMailbox) + .where(eq(relayHookMailbox.environmentId, input.environmentId)) + .orderBy(asc(relayHookMailbox.receivedAt), asc(relayHookMailbox.id)) + .limit(Math.max(1, Math.min(input.limit, HOOK_MAILBOX_MAX_PULL))) + .pipe(Effect.mapError(fail("list", input.environmentId))); + return rows.map((row) => ({ + id: row.id, + environmentId: row.environmentId, + receivedAt: row.receivedAt, + method: row.method, + rawHookId: row.rawHookId, + rawToken: row.rawToken, + query: row.query, + headers: row.headers, + body: new Uint8Array(row.body), + })); + }), + + ack: Effect.fn("relay.hook_mailbox.ack")(function* (input) { + yield* Effect.annotateCurrentSpan({ "relay.environment_id": input.environmentId }); + if (input.ids.length === 0) return 0; + const deleted = yield* db + .delete(relayHookMailbox) + .where( + and( + eq(relayHookMailbox.environmentId, input.environmentId), + inArray(relayHookMailbox.id, [...input.ids]), + ), + ) + .returning({ id: relayHookMailbox.id }) + .pipe(Effect.mapError(fail("ack", input.environmentId))); + return deleted.length; + }), + + pruneExpired: Effect.fn("relay.hook_mailbox.prune_expired")(function* (input) { + yield* db + .delete(relayHookMailbox) + .where(lt(relayHookMailbox.expiresAt, input.now)) + .pipe(Effect.mapError(fail("prune"))); + }), + + clearEnvironment: Effect.fn("relay.hook_mailbox.clear_environment")(function* (input) { + yield* db + .delete(relayHookMailbox) + .where(eq(relayHookMailbox.environmentId, input.environmentId)) + .pipe(Effect.mapError(fail("clear", input.environmentId))); + }), + }); +}); + +export const layer = Layer.effect(HookMailbox, make); diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 6220c27c4b30..adf02955ca5a 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -61,6 +61,7 @@ import { import * as RelayConfiguration from "../Config.ts"; import * as RelayDb from "../db.ts"; import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts"; +import * as HookMailbox from "../hooks/HookMailbox.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts"; @@ -124,6 +125,7 @@ describe("device listing compatibility", () => { Layer.mock(EnvironmentLinks.EnvironmentLinks, {}), Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}), Layer.mock(RelayDb.RelayTransactions, {}), + Layer.mock(HookMailbox.HookMailbox, {}), ), ), Layer.provide( @@ -317,8 +319,12 @@ function relayUnlinkTestLayer(input?: { readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"]; readonly reconcileOrigin?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["reconcileOrigin"]; readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]; + readonly clearMailbox?: HookMailbox.HookMailbox["Service"]["clearEnvironment"]; }) { return Layer.mergeAll( + Layer.mock(HookMailbox.HookMailbox, { + clearEnvironment: input?.clearMailbox ?? (() => Effect.void), + }), Layer.succeed( RelayDb.RelayTransactions, RelayDb.RelayTransactions.of({ @@ -333,6 +339,7 @@ function relayUnlinkTestLayer(input?: { listForUser: () => Effect.die("unused listForUser"), getForUser: input?.getForUser ?? (() => Effect.succeed(null)), findActiveManagedForEnvironment: () => Effect.succeed([]), + setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: input?.revokeForUser ?? (() => Effect.succeed(false)), }), ), @@ -1215,7 +1222,12 @@ describe("relay routing fallback", () => { Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}), ), ), - Layer.provide([publisher, signatures]), + Layer.provide([ + publisher, + signatures, + Layer.mock(EnvironmentLinks.EnvironmentLinks, {}), + Layer.mock(HookMailbox.HookMailbox, {}), + ]), ), ), Layer.provide(auth), diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 5c3b1fe5cfde..aa514f7b2f9f 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -66,6 +66,7 @@ import * as DpopProofs from "../auth/DpopProofs.ts"; import * as RelayTokens from "../auth/RelayTokens.ts"; import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; +import * as HookMailbox from "../hooks/HookMailbox.ts"; import * as LiveActivities from "../agentActivity/LiveActivities.ts"; import * as RelayConfiguration from "../Config.ts"; import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher.ts"; @@ -513,6 +514,15 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron environmentId: link.environmentId, environmentPublicKey: link.environmentPublicKey, }); + // Held webhook requests belong to the environment, not one user's link; + // drop them once no user has it linked any more. + const remaining = yield* links.findActiveManagedForEnvironment({ + environmentId: input.environmentId, + }); + if (remaining.length === 0) { + const mailbox = yield* HookMailbox.HookMailbox; + yield* mailbox.clearEnvironment({ environmentId: input.environmentId }); + } // External teardown cannot share the SQL transaction. Run it only after // revocation commits so a database failure leaves a fully usable active @@ -1127,6 +1137,15 @@ export const serverApi = HttpApiBuilder.group( Effect.fnUntraced(function* (handlers) { const publisher = yield* AgentActivityPublisher.AgentActivityPublisher; const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures; + const links = yield* EnvironmentLinks.EnvironmentLinks; + const mailbox = yield* HookMailbox.HookMailbox; + const requireOwnEnvironment = (environmentId: string) => + Effect.gen(function* () { + const principal = yield* RelayEnvironmentPrincipal; + if (principal.environmentId !== environmentId) { + return yield* new HttpApiError.Unauthorized({}); + } + }); const activityHandlers = handlers.handle( "publishAgentActivity", Effect.fn("relay.api.server.publishAgentActivity")( @@ -1336,6 +1355,50 @@ export const serverApi = HttpApiBuilder.group( }), mapRelayCommonApiErrors("not_authorized"), ), + ) + .handle( + "updateLinkPreferences", + Effect.fn("relay.api.server.updateLinkPreferences")(function* ({ params, payload }) { + yield* requireOwnEnvironment(params.environmentId); + yield* links.setHoldWebhooksWhileOffline({ + environmentId: params.environmentId, + holdWebhooksWhileOffline: payload.holdWebhooksWhileOffline, + }); + return payload; + }, mapRelayCommonApiErrors("not_authorized")), + ) + .handle( + "listPendingHooks", + Effect.fn("relay.api.server.listPendingHooks")(function* ({ params, query }) { + yield* requireOwnEnvironment(params.environmentId); + const held = yield* mailbox.listPending({ + environmentId: params.environmentId, + limit: query.limit ?? HookMailbox.HOOK_MAILBOX_MAX_PULL, + }); + return { + deliveries: held.map((hook) => ({ + id: hook.id, + receivedAt: hook.receivedAt, + method: hook.method, + rawHookId: hook.rawHookId, + rawToken: hook.rawToken, + query: hook.query, + headers: hook.headers, + bodyBase64: Buffer.from(hook.body).toString("base64"), + })), + }; + }, mapRelayCommonApiErrors("not_authorized")), + ) + .handle( + "ackPendingHooks", + Effect.fn("relay.api.server.ackPendingHooks")(function* ({ params, payload }) { + yield* requireOwnEnvironment(params.environmentId); + const deleted = yield* mailbox.ack({ + environmentId: params.environmentId, + ids: payload.ids, + }); + return { deleted }; + }, mapRelayCommonApiErrors("not_authorized")), ); }), ); @@ -1378,6 +1441,8 @@ const RelayCommonPersistenceError = Schema.Union([ AgentActivityRows.AgentActivityRowListPersistenceError, LiveActivities.LiveActivityDeliveryMarkPersistenceError, DeliveryAttempts.DeliveryAttemptRecordPersistenceError, + EnvironmentLinks.EnvironmentLinkEnvironmentLookupPersistenceError, + HookMailbox.HookMailboxPersistenceError, ]); type RelayCommonPersistenceError = typeof RelayCommonPersistenceError.Type; const isRelayCommonPersistenceError = Schema.is(RelayCommonPersistenceError); diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index 1f5d8c4d94e2..c86d14923fcd 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -6,6 +6,7 @@ import type { } from "@t3tools/contracts/relay"; import { boolean, + bytea, index, integer, jsonb, @@ -74,6 +75,8 @@ export const relayEnvironmentLinks = pgTable( notificationsEnabled: boolean("notifications_enabled").notNull().default(true), liveActivitiesEnabled: boolean("live_activities_enabled").notNull().default(true), managedTunnelsEnabled: boolean("managed_tunnels_enabled").notNull().default(false), + // Opt-in: hold webhook requests while the environment is offline. + holdWebhooksWhileOffline: boolean("hold_webhooks_while_offline").notNull().default(false), createdByDeviceId: varchar("created_by_device_id", { length: 191 }), revokedAt: varchar("revoked_at", { length: 64 }), createdAt: varchar("created_at", { length: 64 }).notNull(), @@ -195,3 +198,28 @@ export const relayDpopProofs = pgTable( index("idx_relay_dpop_proofs_expires_at").on(table.expiresAt), ], ); + +/** + * Webhook requests held for an environment that opted in, while it was + * offline. Rows are deleted once the environment acks them, after 24 hours, or + * when the environment is unlinked. + */ +export const relayHookMailbox = pgTable( + "relay_hook_mailbox", + { + id: varchar("id", { length: 36 }).primaryKey(), + environmentId: varchar("environment_id", { length: 191 }).notNull(), + receivedAt: varchar("received_at", { length: 64 }).notNull(), + expiresAt: varchar("expires_at", { length: 64 }).notNull(), + method: varchar("method", { length: 16 }).notNull(), + rawHookId: varchar("raw_hook_id", { length: 512 }).notNull(), + rawToken: varchar("raw_token", { length: 512 }).notNull(), + query: text("query").notNull(), + headers: jsonb("headers").notNull().$type>(), + body: bytea("body").notNull(), + }, + (table) => [ + index("idx_relay_hook_mailbox_environment").on(table.environmentId, table.receivedAt), + index("idx_relay_hook_mailbox_expires").on(table.expiresAt), + ], +); diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index cc03abd5bee8..9b13cb278c8a 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -74,6 +74,7 @@ import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts" import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts"; import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts"; import * as HookForwarder from "./hooks/HookForwarder.ts"; +import * as HookMailbox from "./hooks/HookMailbox.ts"; const webcryptoLayer = Layer.succeed( Crypto.Crypto, @@ -269,7 +270,7 @@ export const ApiLive = Api.make( Layer.provideMerge( ApnsDeliveryQueue.layerCloudflareQueues(apnsDeliveryQueueSender, alchemyRuntimeContext), ), - Layer.provideMerge(Layer.mergeAll(AgentActivityRows.layer, Devices.layer)), + Layer.provideMerge(Layer.mergeAll(AgentActivityRows.layer, Devices.layer, HookMailbox.layer)), Layer.provideMerge(EnvironmentCredentials.layer), Layer.provideMerge( Layer.mergeAll( @@ -371,6 +372,14 @@ export const ApiLive = Api.make( ), ), ), + // Held webhook requests expire 24 hours after the relay received them. + Effect.andThen( + Effect.all([HookMailbox.HookMailbox, DateTime.now]).pipe( + Effect.flatMap(([mailbox, now]) => + mailbox.pruneExpired({ now: DateTime.formatIso(now) }), + ), + ), + ), Effect.catchCause((cause) => Cause.hasInterrupts(cause) ? Effect.interrupt diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 411b10574b02..b530c5c4c898 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -404,11 +404,16 @@ export const EnvironmentCloudLinkStateResult = Schema.Struct({ // Optional so newer clients tolerate older environment servers. managedTunnelActive: Schema.optional(Schema.Boolean), publishAgentActivity: Schema.Boolean, + // Opt-in: T3 Connect holds webhook requests while this environment is + // offline. Optional so newer clients tolerate older environment servers. + holdWebhooksWhileOffline: Schema.optional(Schema.Boolean), }); export type EnvironmentCloudLinkStateResult = typeof EnvironmentCloudLinkStateResult.Type; export const EnvironmentCloudPreferencesRequest = Schema.Struct({ publishAgentActivity: Schema.Boolean, + // Omit to leave the current value unchanged. + holdWebhooksWhileOffline: Schema.optional(Schema.Boolean), }); export type EnvironmentCloudPreferencesRequest = typeof EnvironmentCloudPreferencesRequest.Type; diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 95d680bac246..693c78c34c42 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -1136,6 +1136,41 @@ const RelayDpopClientGroup = HttpApiGroup.make("dpopClient") .annotate(OpenApi.Description, "DPoP-authenticated client access to linked environments.") .middleware(RelayDpopClientAuth); +export const RelayEnvironmentLinkPreferencesRequest = Schema.Struct({ + holdWebhooksWhileOffline: Schema.Boolean, +}); +export type RelayEnvironmentLinkPreferencesRequest = + typeof RelayEnvironmentLinkPreferencesRequest.Type; + +/** A webhook request the relay held because the environment was offline. */ +export const RelayPendingHook = Schema.Struct({ + id: Schema.String, + receivedAt: Schema.String, + method: Schema.String, + /** Path segments exactly as the sender sent them; the environment decodes them. */ + rawHookId: Schema.String, + rawToken: Schema.String, + query: Schema.String, + headers: Schema.Record(Schema.String, Schema.String), + bodyBase64: Schema.String, +}); +export type RelayPendingHook = typeof RelayPendingHook.Type; + +export const RelayPendingHooksResponse = Schema.Struct({ + deliveries: Schema.Array(RelayPendingHook), +}); +export type RelayPendingHooksResponse = typeof RelayPendingHooksResponse.Type; + +export const RelayAckHooksRequest = Schema.Struct({ + ids: Schema.Array(Schema.String), +}); +export type RelayAckHooksRequest = typeof RelayAckHooksRequest.Type; + +export const RelayAckHooksResponse = Schema.Struct({ + deleted: Schema.Number, +}); +export type RelayAckHooksResponse = typeof RelayAckHooksResponse.Type; + const RelayServerGroup = HttpApiGroup.make("server") .add( HttpApiEndpoint.post( @@ -1171,6 +1206,28 @@ const RelayServerGroup = HttpApiGroup.make("server") error: RelayAgentActivityPublishErrors, }, ).annotate(OpenApi.Summary, "Publish agent activity"), + HttpApiEndpoint.post( + "updateLinkPreferences", + "/v1/environments/:environmentId/link-preferences", + { + params: Schema.Struct({ environmentId: EnvironmentId }), + payload: RelayEnvironmentLinkPreferencesRequest, + success: RelayEnvironmentLinkPreferencesRequest, + error: RelayAuthAndInternalErrors, + }, + ).annotate(OpenApi.Summary, "Update an environment's link preferences"), + HttpApiEndpoint.get("listPendingHooks", "/v1/environments/:environmentId/hooks/pending", { + params: Schema.Struct({ environmentId: EnvironmentId }), + query: Schema.Struct({ limit: Schema.optional(Schema.NumberFromString) }), + success: RelayPendingHooksResponse, + error: RelayAuthAndInternalErrors, + }).annotate(OpenApi.Summary, "List webhook requests held while the environment was offline"), + HttpApiEndpoint.post("ackPendingHooks", "/v1/environments/:environmentId/hooks/ack", { + params: Schema.Struct({ environmentId: EnvironmentId }), + payload: RelayAckHooksRequest, + success: RelayAckHooksResponse, + error: RelayAuthAndInternalErrors, + }).annotate(OpenApi.Summary, "Delete delivered webhook requests"), ) .annotate(OpenApi.Description, "Environment-authenticated activity publication.") .middleware(RelayEnvironmentAuth); diff --git a/packages/contracts/src/scheduledTask.ts b/packages/contracts/src/scheduledTask.ts index f00ce5da93b8..3140ff8a79cf 100644 --- a/packages/contracts/src/scheduledTask.ts +++ b/packages/contracts/src/scheduledTask.ts @@ -73,11 +73,23 @@ export const ScheduledTaskWebhookSignature = Schema.Struct( ).annotate({ description: "Optional HMAC-SHA256 signature check over the raw request body." }); export type ScheduledTaskWebhookSignature = typeof ScheduledTaskWebhookSignature.Type; +/** Matches how long the relay holds a request for an offline environment. */ +export const MAX_WEBHOOK_DELIVERY_AGE_MINUTES = 24 * 60; + +const WebhookMaxDeliveryAgeMinutes = Schema.Int.check( + Schema.isBetween({ minimum: 1, maximum: MAX_WEBHOOK_DELIVERY_AGE_MINUTES }), +).annotate({ + description: + "Skip requests the relay held longer than this many minutes while the environment was offline. Null runs every request.", +}); + const ScheduledTaskWebhookSchedule = Schema.Struct({ type: Schema.Literal("webhook").annotate({ description: "Run when the task's webhook URL receives a request.", }), signature: Schema.NullOr(ScheduledTaskWebhookSignature), + // Optional so rows saved before this setting existed still decode. + maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)), }).annotate({ description: "Run on each request to the task's webhook URL. The prompt may use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} and {{request}} placeholders.", @@ -99,6 +111,7 @@ const ScheduledTaskUpsertWebhookSchedule = Schema.Struct({ ).annotate({ description: "Signature check; omit or null to accept requests by URL token only.", }), + maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)), }).annotate({ description: "Run on each request to the task's webhook URL. The prompt may use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} and {{request}} placeholders.", @@ -241,6 +254,7 @@ export const ScheduledTaskWebhookDeliveryOutcome = Schema.Literals([ "rejected_signature", "disabled", "rate_limited", + "expired", ]); export type ScheduledTaskWebhookDeliveryOutcome = typeof ScheduledTaskWebhookDeliveryOutcome.Type; From 0fafa4615b08f8b9cf28eec13e7022ac57a74294 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:46:18 -0700 Subject: [PATCH 02/35] feat(relay,server,web,mobile): opt-in to hold webhooks while offline With the new Connect preference on, the relay stores webhook requests for an offline environment for up to 24 hours instead of answering 503; the environment pulls them with its credential when it is back and runs each through the usual token and signature checks. Off by default, so the relay stays a plain proxy unless an environment opts in. Every forward carries x-t3-relay-delivery-id, so a request that reached the environment before a timeout and is later replayed runs once. Webhook tasks gain an optional max age to skip requests that waited too long. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../SettingsScheduledTasksRouteScreen.tsx | 27 ++- .../settings/scheduledTaskDraft.test.ts | 19 +- .../features/settings/scheduledTaskDraft.ts | 21 +- apps/server/src/cloud/config.ts | 31 +++ apps/server/src/cloud/http.ts | 76 ++++++-- .../server/src/relay/HookMailboxDrain.test.ts | 98 ++++++++++ apps/server/src/relay/HookMailboxDrain.ts | 180 ++++++++++++++++++ .../src/relay/relayEnvironmentClient.ts | 21 ++ .../scheduledTasks/ScheduledTaskService.ts | 46 ++++- .../ScheduledTaskService.webhook.test.ts | 76 ++++++++ .../server/src/scheduledTasks/webhookRoute.ts | 5 + apps/server/src/server.ts | 9 + apps/web/src/cloud/linkEnvironment.ts | 4 +- apps/web/src/cloud/linkEnvironmentAtoms.ts | 7 +- apps/web/src/cloud/useCloudLinkController.ts | 7 + .../settings/ConnectionsSettings.tsx | 35 ++++ .../settings/ScheduledTasksSettings.tsx | 28 ++- .../scheduledTasksSettings.logic.test.ts | 27 ++- .../settings/scheduledTasksSettings.logic.ts | 13 ++ .../src/components/settings/settingsSearch.ts | 9 + docs/internals/t3-connect.md | 19 +- docs/user/project-settings.md | 13 +- 22 files changed, 726 insertions(+), 45 deletions(-) create mode 100644 apps/server/src/relay/HookMailboxDrain.test.ts create mode 100644 apps/server/src/relay/HookMailboxDrain.ts create mode 100644 apps/server/src/relay/relayEnvironmentClient.ts diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index b3a949e65e59..62aaaf252ccd 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -910,13 +910,26 @@ function TaskForm({ /> ) : draft.schedule.mode === "webhook" ? ( - task.id === draft.task?.id) ?? draft.task ?? null - } - signatureConfigured={draft.schedule.signature !== null} - /> + <> + task.id === draft.task?.id) ?? draft.task ?? null + } + signatureConfigured={draft.schedule.signature !== null} + /> + + setDraft({ ...draft, schedule: { ...draft.schedule, maxDeliveryAgeMinutes } }) + } + /> + ) : ( <> { it("round-trips a webhook schedule without a signature", () => { const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null } }); expect(draft.mode).toBe("webhook"); - expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature: null }); + expect(scheduleFromDraft(draft)).toEqual({ + type: "webhook", + signature: null, + maxDeliveryAgeMinutes: null, + }); + }); + + it("round-trips a webhook max age and treats blank input as no limit", () => { + const draft = scheduleDraftForTask({ + schedule: { type: "webhook", signature: null, maxDeliveryAgeMinutes: 45 }, + }); + expect(draft.maxDeliveryAgeMinutes).toBe("45"); + expect(scheduleFromDraft(draft)).toMatchObject({ maxDeliveryAgeMinutes: 45 }); + expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "" })).toMatchObject({ + maxDeliveryAgeMinutes: null, + }); }); it("keeps a webhook signature on save without sending a secret", () => { @@ -44,7 +59,7 @@ describe("scheduleDraftForTask", () => { const saved = scheduleFromDraft( scheduleDraftForTask({ schedule: { type: "webhook", signature } }), ); - expect(saved).toEqual({ type: "webhook", signature }); + expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null }); expect(saved?.type === "webhook" && saved.signature && "secret" in saved.signature).toBe(false); }); }); diff --git a/apps/mobile/src/features/settings/scheduledTaskDraft.ts b/apps/mobile/src/features/settings/scheduledTaskDraft.ts index 2843700620e8..b2e9509436b5 100644 --- a/apps/mobile/src/features/settings/scheduledTaskDraft.ts +++ b/apps/mobile/src/features/settings/scheduledTaskDraft.ts @@ -44,6 +44,8 @@ export type ScheduleDraft = { readonly intervalMinutes: string; /** A webhook signature check configured elsewhere; mobile keeps it but does not edit it. */ readonly signature: ScheduledTaskWebhookSignature | null; + /** Minutes as typed; empty runs every held request regardless of age. */ + readonly maxDeliveryAgeMinutes: string; }; export const DEFAULT_SCHEDULE: ScheduleDraft = { @@ -52,6 +54,7 @@ export const DEFAULT_SCHEDULE: ScheduleDraft = { weekdays: [1, 2, 3, 4, 5], intervalMinutes: "15", signature: null, + maxDeliveryAgeMinutes: "", }; /** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */ @@ -74,10 +77,24 @@ export function scheduleDraftForTask(task: Pick): Sch intervalMinutes: String(Math.max(1, task.schedule.everyMs / 60_000)), }; case "webhook": - return { ...DEFAULT_SCHEDULE, mode: "webhook", signature: task.schedule.signature }; + return { + ...DEFAULT_SCHEDULE, + mode: "webhook", + signature: task.schedule.signature, + maxDeliveryAgeMinutes: + task.schedule.maxDeliveryAgeMinutes == null + ? "" + : String(task.schedule.maxDeliveryAgeMinutes), + }; } } +/** Blank or invalid input means "no limit"; the server rejects values past the relay's TTL. */ +function parseMaxDeliveryAge(value: string): number | null { + const minutes = Number(value.trim()); + return value.trim() !== "" && Number.isInteger(minutes) && minutes > 0 ? minutes : null; +} + export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSchedule | null { if (draft.mode === "webhook") { // No secret is sent, so the server keeps the stored one. @@ -91,6 +108,7 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche encoding: draft.signature.encoding, prefix: draft.signature.prefix, }, + maxDeliveryAgeMinutes: parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes), }; } if (draft.mode === "interval") { @@ -145,6 +163,7 @@ function draftSignature(draft: ScheduledTaskDraft): string { draft.schedule.timeOfDay, [...draft.schedule.weekdays].sort((a, b) => a - b), draft.schedule.intervalMinutes, + draft.schedule.maxDeliveryAgeMinutes, draft.workspace, draft.baseRef, draft.checkoutPath, diff --git a/apps/server/src/cloud/config.ts b/apps/server/src/cloud/config.ts index 9b1b281ba2da..92bd9c856ca6 100644 --- a/apps/server/src/cloud/config.ts +++ b/apps/server/src/cloud/config.ts @@ -16,6 +16,7 @@ export const RELAY_URL_SECRET = "cloud-relay-url"; export const RELAY_ISSUER_SECRET = "cloud-relay-issuer"; export const RELAY_ENVIRONMENT_CREDENTIAL_SECRET = "cloud-relay-environment-credential"; export const PUBLISH_AGENT_ACTIVITY_SECRET = "cloud-publish-agent-activity"; +export const HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET = "cloud-hold-webhooks-while-offline"; export const encodeEndpointRuntimeConfigJson = Schema.encodeEffect( Schema.fromJsonString(RelayManagedEndpointRuntimeConfig), @@ -73,3 +74,33 @@ export const readAgentActivityPublishingActive = ( environmentCredential !== "" ); }).pipe(Effect.orElseSucceed(() => false)); + +const readSecretString = ( + secrets: ServerSecretStore.ServerSecretStore["Service"], + name: string, +): Effect.Effect => + secrets.get(name).pipe( + Effect.map((bytes) => + Option.isSome(bytes) && bytes.value.length > 0 ? new TextDecoder().decode(bytes.value) : null, + ), + Effect.orElseSucceed(() => null), + ); + +/** The relay URL and environment credential, or null when not linked to T3 Connect. */ +export const readRelayConnection = (secrets: ServerSecretStore.ServerSecretStore["Service"]) => + Effect.all([ + readSecretString(secrets, RELAY_URL_SECRET), + readSecretString(secrets, RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + ]).pipe( + Effect.map(([url, environmentCredential]) => + url && environmentCredential ? { url, environmentCredential } : null, + ), + ); + +/** Whether this environment opted in to T3 Connect holding webhooks while it is offline. */ +export const readHoldWebhooksWhileOffline = ( + secrets: ServerSecretStore.ServerSecretStore["Service"], +) => + readSecretString(secrets, HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET).pipe( + Effect.map((value) => value === "true"), + ); diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 82337df5b6a7..dc33f1369854 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -8,6 +8,7 @@ import { EnvironmentCloudRelayConfigResult, EnvironmentHttpApi, EnvironmentHttpBadRequestError, + type EnvironmentCloudPreferencesRequest, EnvironmentHttpConflictError, EnvironmentHttpInternalServerError, EnvironmentHttpUnauthorizedError, @@ -69,6 +70,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { requireEnvironmentScope } from "../auth/http.ts"; import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import { makeRelayEnvironmentClient } from "../relay/relayEnvironmentClient.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { @@ -86,6 +88,8 @@ import { encodeEndpointRuntimeConfigJson, encodeConfirmedOriginJson, PUBLISH_AGENT_ACTIVITY_SECRET, + HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, + readRelayConnection, RELAY_ENVIRONMENT_CREDENTIAL_SECRET, RELAY_ISSUER_SECRET, RELAY_URL_SECRET, @@ -1277,17 +1281,24 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* ( dependencies: CloudHttpDependencies, ) { - const [cloudUserId, relayUrl, relayIssuer, endpointRuntimeConfig, publishAgentActivity] = - yield* Effect.all( - [ - dependencies.secrets.get(CLOUD_LINKED_USER_ID), - dependencies.secrets.get(RELAY_URL_SECRET), - dependencies.secrets.get(RELAY_ISSUER_SECRET), - dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), - dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET), - ], - { concurrency: 5 }, - ); + const [ + cloudUserId, + relayUrl, + relayIssuer, + endpointRuntimeConfig, + publishAgentActivity, + holdWebhooks, + ] = yield* Effect.all( + [ + dependencies.secrets.get(CLOUD_LINKED_USER_ID), + dependencies.secrets.get(RELAY_URL_SECRET), + dependencies.secrets.get(RELAY_ISSUER_SECRET), + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET), + dependencies.secrets.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET), + ], + { concurrency: 6 }, + ); return { linked: Option.isSome(cloudUserId), cloudUserId: Option.isSome(cloudUserId) ? bytesToString(cloudUserId.value) : null, @@ -1299,6 +1310,8 @@ const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function publishAgentActivity: Option.isSome(publishAgentActivity) ? bytesToString(publishAgentActivity.value) === "true" : false, + holdWebhooksWhileOffline: + Option.isSome(holdWebhooks) && bytesToString(holdWebhooks.value) === "true", } satisfies EnvironmentCloudLinkStateResult; }); @@ -1329,8 +1342,9 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")( dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN), dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), + dependencies.secrets.remove(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET), ], - { concurrency: 8 }, + { concurrency: 9 }, ); yield* setCliDesiredCloudLink(false); return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; @@ -1343,12 +1357,42 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")( ), ); +const pushHoldWebhooksWhileOffline = Effect.fn("environment.cloud.pushHoldWebhooksWhileOffline")( + function* (dependencies: CloudHttpDependencies, holdWebhooksWhileOffline: boolean) { + const connection = yield* readRelayConnection(dependencies.secrets); + if (connection === null) { + return yield* new EnvironmentHttpBadRequestError({ + message: "Link this environment to T3 Connect first.", + }); + } + const environmentId = yield* dependencies.environment.getEnvironmentId; + const client = yield* makeRelayEnvironmentClient(connection); + yield* client.server + .updateLinkPreferences({ + params: { environmentId }, + payload: { holdWebhooksWhileOffline }, + }) + .pipe( + Effect.timeout("10 seconds"), + Effect.catch( + failEnvironmentCloudInternalError("Could not update T3 Connect webhook settings."), + ), + ); + }, +); + const cloudPreferencesHandler = Effect.fn("environment.cloud.preferences")( - function* ( - dependencies: CloudHttpDependencies, - payload: { readonly publishAgentActivity: boolean }, - ) { + function* (dependencies: CloudHttpDependencies, payload: EnvironmentCloudPreferencesRequest) { yield* requireEnvironmentScope(AuthRelayWriteScope); + if (payload.holdWebhooksWhileOffline !== undefined) { + // The relay decides whether to hold a request, so it is told first; the + // local copy is only saved once the relay has the same value. + yield* pushHoldWebhooksWhileOffline(dependencies, payload.holdWebhooksWhileOffline); + yield* dependencies.secrets.set( + HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, + stringToBytes(String(payload.holdWebhooksWhileOffline)), + ); + } yield* dependencies.secrets.set( PUBLISH_AGENT_ACTIVITY_SECRET, stringToBytes(String(payload.publishAgentActivity)), diff --git a/apps/server/src/relay/HookMailboxDrain.test.ts b/apps/server/src/relay/HookMailboxDrain.test.ts new file mode 100644 index 000000000000..8744334e92de --- /dev/null +++ b/apps/server/src/relay/HookMailboxDrain.test.ts @@ -0,0 +1,98 @@ +import { assert, it } from "@effect/vitest"; +import type { RelayPendingHook } from "@t3tools/contracts/relay"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { + ScheduledTaskService, + type WebhookTriggerRequest, + type WebhookTriggerResult, +} from "../scheduledTasks/ScheduledTaskService.ts"; +import { drainOnce, HookMailboxRelay, toTriggerRequest } from "./HookMailboxDrain.ts"; + +const held = (id: string, overrides: Partial = {}): RelayPendingHook => ({ + id, + receivedAt: "2026-10-04T10:00:00.000Z", + method: "POST", + rawHookId: "scheduled-task%3Ahook", + rawToken: "tok%2Fen", + query: "a=1", + headers: { "Content-Type": "application/json" }, + bodyBase64: Buffer.from('{"x":1}').toString("base64"), + ...overrides, +}); + +/** Runs one drain pass against a relay holding `pages` and a service answering `outcome`. */ +const drain = ( + pages: ReadonlyArray> | null, + outcome: (request: WebhookTriggerRequest) => WebhookTriggerResult["_tag"], +) => + Effect.gen(function* () { + const acked: Array = []; + const triggered: Array = []; + let page = 0; + const relay = Layer.succeed(HookMailboxRelay, { + pending: Effect.sync(() => (pages === null ? null : (pages[page++] ?? []))), + ack: (ids) => Effect.sync(() => void acked.push(...ids)), + }); + const service = Layer.mock(ScheduledTaskService)({ + triggerWebhook: (request) => + Effect.sync(() => { + triggered.push(request); + return { _tag: outcome(request) } as WebhookTriggerResult; + }), + }); + const delivered = yield* drainOnce().pipe(Effect.provide(Layer.merge(relay, service))); + return { acked, triggered, delivered }; + }); + +it("rebuilds the request the webhook route would have built", () => { + const request = toTriggerRequest(held("d1")); + assert.equal(request?.hookId, "scheduled-task:hook"); + assert.equal(request?.token, "tok/en"); + assert.equal(request?.headers["content-type"], "application/json"); + assert.equal(request?.bodyText, '{"x":1}'); + assert.equal(request?.relayDeliveryId, "d1"); + assert.equal(request?.receivedAt, "2026-10-04T10:00:00.000Z"); +}); + +it.effect("delivers held requests and acks every final outcome", () => + Effect.gen(function* () { + const result = yield* drain([[held("ok"), held("gone"), held("bad-sig")]], (request) => + request.relayDeliveryId === "ok" + ? "accepted" + : request.relayDeliveryId === "gone" + ? "not_found" + : "rejected_signature", + ); + assert.deepEqual(result.acked, ["ok", "gone", "bad-sig"]); + }), +); + +it.effect("leaves rate-limited requests held for the next pass", () => + Effect.gen(function* () { + const result = yield* drain( + [[held("first"), held("limited")], [held("never-pulled")]], + (request) => (request.relayDeliveryId === "limited" ? "rate_limited" : "accepted"), + ); + assert.deepEqual(result.acked, ["first"]); + // The pass stops instead of pulling the next page. + assert.isFalse(result.triggered.some((request) => request.relayDeliveryId === "never-pulled")); + }), +); + +it.effect("acks and drops a request whose path cannot be decoded", () => + Effect.gen(function* () { + const result = yield* drain([[held("broken", { rawToken: "%E0" })]], () => "accepted"); + assert.deepEqual(result.acked, ["broken"]); + assert.equal(result.triggered.length, 0); + }), +); + +it.effect("does nothing when the environment is not linked", () => + Effect.gen(function* () { + const result = yield* drain(null, () => "accepted"); + assert.equal(result.delivered, 0); + assert.equal(result.triggered.length, 0); + }), +); diff --git a/apps/server/src/relay/HookMailboxDrain.ts b/apps/server/src/relay/HookMailboxDrain.ts new file mode 100644 index 000000000000..0767aa17da27 --- /dev/null +++ b/apps/server/src/relay/HookMailboxDrain.ts @@ -0,0 +1,180 @@ +import type { RelayPendingHook } from "@t3tools/contracts/relay"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; + +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import { readHoldWebhooksWhileOffline, readRelayConnection } from "../cloud/config.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; +import { WEBHOOK_MAX_BODY_BYTES } from "../scheduledTasks/webhookRoute.ts"; +import * as Scheduler from "../scheduling/Scheduler.ts"; +import { makeRelayEnvironmentClient } from "./relayEnvironmentClient.ts"; + +const DRAIN_INTERVAL_MS = 30_000; +const MAX_BACKOFF_MS = 5 * 60_000; +const PULL_LIMIT = 10; + +export class HookMailboxRelayError extends Schema.TaggedError()( + "HookMailboxRelayError", + { operation: Schema.Literals(["pending", "ack"]), cause: Schema.Defect() }, +) {} + +/** + * Talks to the relay's held-webhook endpoints. A service so tests can stand in + * for the relay without a network. + */ +export class HookMailboxRelay extends Context.Service< + HookMailboxRelay, + { + /** Null when this environment is not linked to T3 Connect. */ + readonly pending: Effect.Effect | null, HookMailboxRelayError>; + readonly ack: (ids: ReadonlyArray) => Effect.Effect; + } +>()("t3/relay/HookMailboxDrain/HookMailboxRelay") {} + +export const relayLayer = Layer.effect( + HookMailboxRelay, + Effect.gen(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + const environment = yield* ServerEnvironment.ServerEnvironment; + const client = Effect.gen(function* () { + const connection = yield* readRelayConnection(secrets); + if (connection === null) return null; + return { + api: yield* makeRelayEnvironmentClient(connection), + environmentId: yield* environment.getEnvironmentId, + }; + }); + return HookMailboxRelay.of({ + pending: Effect.gen(function* () { + const relay = yield* client; + if (relay === null) return null; + const { deliveries } = yield* relay.api.server.listPendingHooks({ + params: { environmentId: relay.environmentId }, + query: { limit: PULL_LIMIT }, + }); + return deliveries; + }).pipe( + Effect.mapError((cause) => new HookMailboxRelayError({ operation: "pending", cause })), + ), + ack: (ids) => + Effect.gen(function* () { + const relay = yield* client; + if (relay === null || ids.length === 0) return; + yield* relay.api.server.ackPendingHooks({ + params: { environmentId: relay.environmentId }, + payload: { ids: [...ids] }, + }); + }).pipe(Effect.mapError((cause) => new HookMailboxRelayError({ operation: "ack", cause }))), + }); + }), +); + +/** Turns a held request back into what the webhook route would have built. */ +export function toTriggerRequest( + hook: RelayPendingHook, +): ScheduledTaskService.WebhookTriggerRequest | null { + let hookId: string; + let token: string; + try { + hookId = decodeURIComponent(hook.rawHookId); + token = decodeURIComponent(hook.rawToken); + } catch { + return null; + } + const body = new Uint8Array(Buffer.from(hook.bodyBase64, "base64")); + if (body.byteLength > WEBHOOK_MAX_BODY_BYTES) return null; + const headers: Record = {}; + for (const [name, value] of Object.entries(hook.headers)) headers[name.toLowerCase()] = value; + return { + hookId, + token, + method: hook.method, + path: `${ScheduledTaskService.WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(hookId)}`, + query: hook.query, + headers, + body, + bodyText: new TextDecoder().decode(body), + relayDeliveryId: hook.id, + receivedAt: hook.receivedAt, + }; +} + +/** + * Delivers webhook requests T3 Connect held while this environment was + * offline. Runs once at startup and then every 30 seconds while the opt-in is + * on. Rate-limited requests stay held for the next pass; every other outcome, + * including a request that cannot be decoded, is acked. + */ +export const drainOnce = Effect.fn("HookMailboxDrain.drainOnce")(function* () { + const relay = yield* HookMailboxRelay; + const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService; + let delivered = 0; + // Bounded so one pass cannot run forever against a relay that keeps returning work. + for (let page = 0; page < 50; page++) { + const pending = yield* relay.pending; + if (pending === null || pending.length === 0) return delivered; + const ack: Array = []; + let rateLimited = false; + for (const hook of pending) { + const request = toTriggerRequest(hook); + if (request === null) { + ack.push(hook.id); + continue; + } + const result = yield* scheduledTasks.triggerWebhook(request); + if (result._tag === "rate_limited") { + rateLimited = true; + continue; + } + ack.push(hook.id); + delivered++; + } + yield* relay.ack(ack); + // Rate-limited requests are still held; stop until the next pass. + if (rateLimited || ack.length === 0) return delivered; + } + return delivered; +}); + +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const scheduler = yield* Scheduler.Scheduler; + const secrets = yield* ServerSecretStore.ServerSecretStore; + const relay = yield* HookMailboxRelay; + const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService; + const state = yield* Ref.make({ nextAt: 0, failures: 0, wasEnabled: false }); + + const tick = Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const current = yield* Ref.get(state); + if (now < current.nextAt) return; + const enabled = yield* readHoldWebhooksWhileOffline(secrets); + // One last pass after the opt-in is turned off, so nothing already held + // is stranded; after that the loop stays idle. + if (!enabled && !current.wasEnabled) return; + const outcome = yield* drainOnce().pipe( + Effect.provideService(HookMailboxRelay, relay), + Effect.provideService(ScheduledTaskService.ScheduledTaskService, scheduledTasks), + Effect.result, + ); + const failures = outcome._tag === "Failure" ? current.failures + 1 : 0; + if (outcome._tag === "Failure") { + yield* Effect.logWarning("Could not deliver held webhook requests", { + cause: outcome.failure, + }); + } + const delay = + failures === 0 + ? DRAIN_INTERVAL_MS + : Math.min(DRAIN_INTERVAL_MS * 2 ** failures, MAX_BACKOFF_MS); + yield* Ref.set(state, { nextAt: now + delay, failures, wasEnabled: enabled }); + }); + + yield* scheduler.register("relay-hook-mailbox", tick); + }), +); diff --git a/apps/server/src/relay/relayEnvironmentClient.ts b/apps/server/src/relay/relayEnvironmentClient.ts new file mode 100644 index 000000000000..a63741c240dc --- /dev/null +++ b/apps/server/src/relay/relayEnvironmentClient.ts @@ -0,0 +1,21 @@ +import { RelayApi } from "@t3tools/contracts/relay"; +import * as Effect from "effect/Effect"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientRequest from "effect/http/HttpClientRequest"; +import * as HttpApiClient from "effect/http-api/HttpApiClient"; + +/** + * A typed RelayApi client that authenticates as this environment, for the + * environment-credential endpoints (link preferences, held webhooks). + */ +export const makeRelayEnvironmentClient = (connection: { + readonly url: string; + readonly environmentCredential: string; +}) => + HttpApiClient.make(RelayApi, { + baseUrl: connection.url, + transformClient: HttpClient.mapRequest( + HttpClientRequest.setHeader("authorization", `Bearer ${connection.environmentCredential}`), + ), + }).pipe(Effect.provide(FetchHttpClient.layer)); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index dca44f1e4112..d432f10ac412 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -83,6 +83,10 @@ export interface WebhookTriggerRequest extends WebhookRequest { readonly hookId: string; readonly token: string; readonly body: Uint8Array; + /** Set by T3 Connect; the same id is never dispatched twice. */ + readonly relayDeliveryId?: string; + /** When the relay received a held request; defaults to now. */ + readonly receivedAt?: string; } /** What the HTTP route should answer. `not_found` covers unknown hooks and wrong tokens alike. */ @@ -91,7 +95,8 @@ export type WebhookTriggerResult = | { readonly _tag: "not_found" } | { readonly _tag: "rejected_signature" } | { readonly _tag: "disabled" } - | { readonly _tag: "rate_limited" }; + | { readonly _tag: "rate_limited" } + | { readonly _tag: "expired" }; const decodeTask = Schema.decodeUnknownEffect(ScheduledTask); const decodeTaskId = Schema.decodeUnknownOption(ScheduledTaskId); @@ -985,6 +990,7 @@ export const layer = Layer.effect( encoding: input.schedule.signature.encoding, prefix: input.schedule.signature.prefix, }, + maxDeliveryAgeMinutes: input.schedule.maxDeliveryAgeMinutes ?? null, } : input.schedule; const webhook = @@ -1317,11 +1323,31 @@ export const layer = Layer.effect( const task = yield* decodeRow(row); if (task.schedule.type !== "webhook") return { _tag: "not_found" as const }; - const receivedAt = yield* localNow; - const deliveryUuid = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => taskError("Could not generate delivery id.", { cause })), + const now = yield* localNow; + const receivedAt = + request.receivedAt === undefined + ? now + : Option.getOrElse(DateTime.make(request.receivedAt), () => now); + const deliveryId = ScheduledTaskWebhookDeliveryId.make( + request.relayDeliveryId === undefined + ? `delivery:${yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => taskError("Could not generate delivery id.", { cause })), + )}` + : `delivery:relay:${request.relayDeliveryId}`, ); - const deliveryId = ScheduledTaskWebhookDeliveryId.make(`delivery:${deliveryUuid}`); + // A held request may already have reached this environment directly + // before a timeout; it runs once. + if (request.relayDeliveryId !== undefined) { + const seen = yield* sql<{ delivery_id: string }>` + SELECT delivery_id FROM scheduled_task_webhook_deliveries + WHERE delivery_id = ${deliveryId} + `.pipe( + Effect.mapError((cause) => + taskError("Could not load webhook delivery.", { taskId: task.id, cause }), + ), + ); + if (seen.length > 0) return { _tag: "accepted" as const, deliveryId }; + } const log = ( outcome: ScheduledTaskWebhookDeliveryOutcome, details: { @@ -1343,7 +1369,7 @@ export const layer = Layer.effect( // Only the first rejected request in a window is logged, so a flood // cannot write rows or push the real deliveries out of the log. - const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(receivedAt)); + const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(now)); if (slot !== "allowed") { if (slot === "first_rejected") yield* log("rate_limited"); return { _tag: "rate_limited" as const }; @@ -1367,6 +1393,14 @@ export const layer = Layer.effect( return { _tag: "rejected_signature" as const }; } } + const maxAgeMinutes = task.schedule.maxDeliveryAgeMinutes ?? null; + if ( + maxAgeMinutes !== null && + DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt) > maxAgeMinutes * 60_000 + ) { + yield* log("expired", { signatureVerified: signature !== null }); + return { _tag: "expired" as const }; + } const rendered = renderWebhookPrompt(task.prompt, request); // Bound the deliveries one task holds, so steady traffic to a stuck diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index 7108d7b804f5..ba91db274aca 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -3,6 +3,7 @@ import * as NodeCrypto from "node:crypto"; import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto"; import { assert, it } from "@effect/vitest"; import { ScheduledTaskUpsertInput } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -168,6 +169,7 @@ it.effect("checks the configured signature and keeps the secret write-only", () assert.deepEqual(task.schedule, { type: "webhook", signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" }, + maxDeliveryAgeMinutes: null, }); assert.isTrue(task.webhook?.hasSecret); @@ -454,6 +456,80 @@ it.effect("logs a body's first 64 KiB by bytes, not characters", () => ), ); +it.effect("a held request already delivered directly runs only once", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + const direct = yield* service.triggerWebhook( + requestFor(task, { relayDeliveryId: "relay-1" }), + ); + const replayed = yield* service.triggerWebhook( + requestFor(task, { relayDeliveryId: "relay-1", receivedAt: "2026-10-04T10:00:00.000Z" }), + ); + assert.equal(direct._tag, "accepted"); + assert.deepEqual(replayed, direct); + yield* Queue.take(launches); + const logged = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + assert.equal(logged.length, 1); + }), + ), +); + +it.effect("logs a held request at the time the relay received it", () => + withService(({ service }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); + yield* service.triggerWebhook( + requestFor(task, { relayDeliveryId: "relay-2", receivedAt: "2026-10-04T10:00:00.000Z" }), + ); + const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + assert.equal(delivery?.receivedAt, "2026-10-04T10:00:00.000Z"); + }), + ), +); + +it.effect("skips a held request older than the task's max age", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ schedule: { type: "webhook", maxDeliveryAgeMinutes: 30 } }), + ); + const now = yield* DateTime.now; + const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 })); + const fresh = DateTime.formatIso(DateTime.subtract(now, { minutes: 5 })); + const tooOld = yield* service.triggerWebhook( + requestFor(task, { relayDeliveryId: "old", receivedAt: old }), + ); + assert.equal(tooOld._tag, "expired"); + assert.equal(yield* Queue.size(launches), 0); + const ok = yield* service.triggerWebhook( + requestFor(task, { relayDeliveryId: "fresh", receivedAt: fresh }), + ); + assert.equal(ok._tag, "accepted"); + const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map( + (delivery) => delivery.outcome, + ); + assert.includeMembers(outcomes, ["expired", "accepted"]); + }), + ), +); + +it.effect("runs a held request of any age when no max age is set", () => + withService(({ service }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + const now = yield* DateTime.now; + const result = yield* service.triggerWebhook( + requestFor(task, { + relayDeliveryId: "ancient", + receivedAt: DateTime.formatIso(DateTime.subtract(now, { hours: 23 })), + }), + ); + assert.equal(result._tag, "accepted"); + }), + ), +); + it.effect("deleting a task removes its delivery log", () => withService(({ service }) => Effect.gen(function* () { diff --git a/apps/server/src/scheduledTasks/webhookRoute.ts b/apps/server/src/scheduledTasks/webhookRoute.ts index 63d2772c0c82..c85c878ba334 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.ts @@ -54,6 +54,8 @@ const handleWebhook = if (typeof value === "string") headers[name.toLowerCase()] = value; } const queryIndex = request.url.indexOf("?"); + // Only the relay sets this; it strips any copy a sender supplied. + const relayDeliveryId = headers["x-t3-relay-delivery-id"]; const result = yield* scheduledTasks .triggerWebhook({ @@ -65,6 +67,7 @@ const handleWebhook = headers, body: body.value, bodyText: new TextDecoder().decode(body.value), + ...(relayDeliveryId ? { relayDeliveryId } : {}), }) .pipe( Effect.catch((cause) => @@ -87,6 +90,8 @@ const handleWebhook = return json(409, { error: "hook_disabled" }); case "rate_limited": return json(429, { error: "rate_limited" }); + case "expired": + return json(410, { error: "delivery_too_old" }); case "error": return json(500, { error: "internal_error" }); } diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index c203fe65c904..5d96bd53797d 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -119,6 +119,8 @@ import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http import * as ReplayMarkers from "./auth/replayMarkers.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import { webhookHttpApiLayer } from "./scheduledTasks/webhookRoute.ts"; +import * as HookMailboxDrain from "./relay/HookMailboxDrain.ts"; +import * as Scheduler from "./scheduling/Scheduler.ts"; import { ScheduledTaskWebhookOrigin } from "./scheduledTasks/ScheduledTaskService.ts"; import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./cloud/config.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; @@ -535,8 +537,15 @@ const ProviderInstallationRefreshLive = Layer.effectDiscard( }), ); +// Delivers webhook requests T3 Connect held while this environment was offline. +const HookMailboxDrainLive = HookMailboxDrain.layer.pipe( + Layer.provide(HookMailboxDrain.relayLayer), + Layer.provide(Scheduler.layer), +); + const RuntimeCoreDependenciesBaseLive = Layer.mergeAll( AgentAwarenessRelay.layer, + HookMailboxDrainLive, ThreadSettlementWorkerLive, Layer.effectDiscard(StorageCleanup.make.pipe(Effect.flatMap((service) => service.start()))).pipe( Layer.provide(ProjectionStoreV2.layer), diff --git a/apps/web/src/cloud/linkEnvironment.ts b/apps/web/src/cloud/linkEnvironment.ts index 40f12cebb9a4..ad9a0c0ea429 100644 --- a/apps/web/src/cloud/linkEnvironment.ts +++ b/apps/web/src/cloud/linkEnvironment.ts @@ -199,13 +199,15 @@ export function readPrimaryCloudLinkState(input: { export function updatePrimaryCloudPreferences(input: { readonly target: CloudLinkTarget; readonly publishAgentActivity: boolean; + readonly holdWebhooksWhileOffline?: boolean; }): Effect.Effect { return Effect.gen(function* () { const client = yield* makeEnvironmentHttpApiClient(input.target.httpBaseUrl); + const { target: _target, ...payload } = input; return yield* client.connect .preferences({ headers: {}, - payload: input, + payload, }) .pipe( Effect.mapError(environmentApiError("Could not update environment cloud preferences.")), diff --git a/apps/web/src/cloud/linkEnvironmentAtoms.ts b/apps/web/src/cloud/linkEnvironmentAtoms.ts index 1094e860e462..4a3541cd3a10 100644 --- a/apps/web/src/cloud/linkEnvironmentAtoms.ts +++ b/apps/web/src/cloud/linkEnvironmentAtoms.ts @@ -41,6 +41,9 @@ export const updatePrimaryEnvironmentPreferences = createRuntimeCommand(connecti label: "web:cloud:update-primary-environment-preferences", scheduler: cloudLinkScheduler, concurrency: cloudLinkConcurrency, - execute: (input: { readonly target: CloudLinkTarget; readonly publishAgentActivity: boolean }) => - updatePrimaryCloudPreferences(input), + execute: (input: { + readonly target: CloudLinkTarget; + readonly publishAgentActivity: boolean; + readonly holdWebhooksWhileOffline?: boolean; + }) => updatePrimaryCloudPreferences(input), }); diff --git a/apps/web/src/cloud/useCloudLinkController.ts b/apps/web/src/cloud/useCloudLinkController.ts index d91596880850..730159599969 100644 --- a/apps/web/src/cloud/useCloudLinkController.ts +++ b/apps/web/src/cloud/useCloudLinkController.ts @@ -21,6 +21,8 @@ import { resolveRelayClerkTokenOptions } from "./publicConfig"; export interface CloudLinkDesiredState { readonly managedTunnel: boolean; readonly publish: boolean; + /** Omit to leave the webhook-hold setting as it is. */ + readonly holdWebhooksWhileOffline?: boolean; } /** @@ -75,6 +77,7 @@ export function useCloudLinkController() { const managedTunnelActive = primaryCloudLinkState.data?.managedTunnelActive ?? primaryCloudLinkState.data?.linked ?? false; const publishAgentActivity = primaryCloudLinkState.data?.publishAgentActivity ?? false; + const holdWebhooksWhileOffline = primaryCloudLinkState.data?.holdWebhooksWhileOffline ?? false; const linked = primaryCloudLinkState.data?.linked ?? false; const reconcileCloudState = async (desired: CloudLinkDesiredState): Promise => { @@ -132,6 +135,9 @@ export function useCloudLinkController() { const prefResult = await updatePrimaryEnvironmentPreferences({ target, publishAgentActivity: desired.publish, + ...(desired.holdWebhooksWhileOffline === undefined + ? {} + : { holdWebhooksWhileOffline: desired.holdWebhooksWhileOffline }), }); if (prefResult._tag === "Failure") { if (!isAtomCommandInterrupted(prefResult)) { @@ -157,6 +163,7 @@ export function useCloudLinkController() { linked, managedTunnelActive, publishAgentActivity, + holdWebhooksWhileOffline, operationError, reconcileCloudState, }; diff --git a/apps/web/src/components/settings/ConnectionsSettings.tsx b/apps/web/src/components/settings/ConnectionsSettings.tsx index 1959857d12f6..42084ab8d5a4 100644 --- a/apps/web/src/components/settings/ConnectionsSettings.tsx +++ b/apps/web/src/components/settings/ConnectionsSettings.tsx @@ -1761,6 +1761,7 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b linkState: primaryCloudLinkState, managedTunnelActive, publishAgentActivity, + holdWebhooksWhileOffline, operationError, reconcileCloudState, } = useCloudLinkController(); @@ -1812,6 +1813,25 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b setIsUpdatingPreference(false); }; + const updateHoldWebhooks = async (enabled: boolean) => { + setIsUpdatingPreference(true); + const ok = await reconcileCloudState({ + managedTunnel: managedTunnelActive, + publish: publishAgentActivity, + holdWebhooksWhileOffline: enabled, + }); + if (ok) { + toastManager.add({ + type: "success", + title: enabled ? "Webhooks held while offline" : "Webhooks no longer held", + description: enabled + ? "T3 Connect keeps webhook requests for up to 24 hours while this environment is offline." + : "Requests to an offline environment now fail. Anything already held is still delivered.", + }); + } + setIsUpdatingPreference(false); + }; + return ( <> {window.desktopBridge ? ( @@ -1846,6 +1866,21 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b /> } /> + {managedTunnelActive ? ( + void updateHoldWebhooks(enabled)} + /> + } + /> + ) : null} ); } diff --git a/apps/web/src/components/settings/ScheduledTasksSettings.tsx b/apps/web/src/components/settings/ScheduledTasksSettings.tsx index 884ae744c878..6c29c5945b73 100644 --- a/apps/web/src/components/settings/ScheduledTasksSettings.tsx +++ b/apps/web/src/components/settings/ScheduledTasksSettings.tsx @@ -24,6 +24,7 @@ import type { ThreadId, } from "@t3tools/contracts"; import { + MAX_WEBHOOK_DELIVERY_AGE_MINUTES, MIN_SCHEDULED_TASK_INTERVAL_MS, ProviderInstanceId, resolveEnvironmentMachineKind, @@ -130,6 +131,7 @@ const EMPTY_DRAFT: DraftState = { signatureEnabled: false, ...WEBHOOK_SIGNATURE_DEFAULTS, signatureSecret: "", + maxDeliveryAgeMinutes: "", }; /** Labelled field: a caption sitting above its control. */ @@ -211,11 +213,14 @@ const DELIVERY_OUTCOME_LABELS: Record + + + setDraft((current) => ({ + ...current, + maxDeliveryAgeMinutes: event.target.value, + })) + } + /> +
diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts index 9ccd9881d943..bda8ea0e0563 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts @@ -181,10 +181,15 @@ describe("webhook scheduled tasks", () => { const draft = taskToDraft(webhookTask); expect(draft.scheduleMode).toBe("webhook"); expect(draft.signatureSecret).toBe(""); - expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature }); + expect(scheduleFromDraft(draft)).toEqual({ + type: "webhook", + signature, + maxDeliveryAgeMinutes: null, + }); expect(scheduleFromDraft({ ...draft, signatureSecret: " new " })).toEqual({ type: "webhook", signature: { ...signature, secret: "new" }, + maxDeliveryAgeMinutes: null, }); }); @@ -192,7 +197,25 @@ describe("webhook scheduled tasks", () => { const draft = taskToDraft({ ...webhookTask, schedule: { type: "webhook", signature: null } }); expect(draft.signatureEnabled).toBe(false); expect(draft.signatureHeader).toBe("x-hub-signature-256"); - expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature: null }); + expect(scheduleFromDraft(draft)).toEqual({ + type: "webhook", + signature: null, + maxDeliveryAgeMinutes: null, + }); + }); + + it("round-trips the max age and treats a blank or invalid entry as no limit", () => { + const draft = taskToDraft({ + ...webhookTask, + schedule: { type: "webhook", signature: null, maxDeliveryAgeMinutes: 90 }, + }); + expect(draft.maxDeliveryAgeMinutes).toBe("90"); + expect(scheduleFromDraft(draft)).toMatchObject({ maxDeliveryAgeMinutes: 90 }); + for (const blank of ["", " ", "0", "-5", "1.5", "abc"]) { + expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: blank })).toMatchObject({ + maxDeliveryAgeMinutes: null, + }); + } }); }); diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts index c6d0cb28b940..719f814e384b 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts @@ -78,6 +78,8 @@ export interface DraftState { readonly signaturePrefix: string; /** Write-only: empty keeps the secret already stored on the server. */ readonly signatureSecret: string; + /** Minutes as typed; empty runs every held request regardless of age. */ + readonly maxDeliveryAgeMinutes: string; } /** GitHub's signature settings, the most common sender. */ @@ -90,6 +92,12 @@ export const WEBHOOK_SIGNATURE_DEFAULTS = { /** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */ export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}"; +/** Blank or invalid input means "no limit"; the server rejects values past the relay's TTL. */ +function parseMaxDeliveryAge(value: string): number | null { + const minutes = Number(value.trim()); + return value.trim() !== "" && Number.isInteger(minutes) && minutes > 0 ? minutes : null; +} + export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule { if (draft.scheduleMode === "webhook") { const secret = draft.signatureSecret.trim(); @@ -103,6 +111,7 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul ...(secret ? { secret } : {}), } : null, + maxDeliveryAgeMinutes: parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes), }; } if (draft.scheduleMode === "interval") { @@ -159,6 +168,10 @@ export function taskToDraft(task: ScheduledTask): DraftState { } : { signatureEnabled: false, ...WEBHOOK_SIGNATURE_DEFAULTS }), signatureSecret: "", + maxDeliveryAgeMinutes: + schedule.type === "webhook" && schedule.maxDeliveryAgeMinutes != null + ? String(schedule.maxDeliveryAgeMinutes) + : "", }; } diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 6a198a066f72..3a36bf9cb705 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -836,6 +836,15 @@ export const SETTINGS_SEARCH_ITEMS = [ desktopOnly: true, cloudOnly: true, }, + { + id: "hold-webhooks-while-offline", + localEnvironmentOnly: true, + title: "Hold webhooks while offline", + to: "/settings/connections", + targetId: "connections-environment", + searchTerms: ["webhook automations offline queue mailbox t3 connect"], + cloudOnly: true, + }, { id: "publish-agent-activity", localEnvironmentOnly: true, diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index a857bd95819d..c12553e67b5d 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -5,12 +5,23 @@ credentials for reaching environments, and managed tunnel allocations. After bootstrap, clients send application traffic through the environment's tunnel hostname; the relay Worker does not proxy their HTTP or WebSocket sessions. The one exception is automation webhooks: the relay forwards -`/v1/hooks/:environmentId/:hookId/:token` statelessly to the environment's -tunnel so senders get a stable URL. It stores nothing, keeps bodies and tokens -out of its traces, and leaves token and signature verification to the -environment +`/v1/hooks/:environmentId/:hookId/:token` to the environment's tunnel so +senders get a stable URL. It keeps bodies and tokens out of its traces and +leaves token and signature verification to the environment ([forwarder](../../infra/relay/src/hooks/HookForwarder.ts)). +By default the forwarder stores nothing. An environment can opt in to having +the relay hold requests while it is offline +(`hold_webhooks_while_offline` on its link). Only then does the relay store the +raw request, including the hook token in the path, in `relay_hook_mailbox`. +Rows are deleted when the environment acks them, after 24 hours, or when no +user has the environment linked. The relay still never checks the token: the +environment pulls held requests with its credential and runs them through the +same verification. Every forward carries `x-t3-relay-delivery-id` so a request +that reached the environment before a timeout and is later replayed runs once +([mailbox](../../infra/relay/src/hooks/HookMailbox.ts), +[drain](../../apps/server/src/relay/HookMailboxDrain.ts)). + Clerk, deployment, and native authentication setup live in the [Connect setup runbook](../operations/connect-setup.md). diff --git a/docs/user/project-settings.md b/docs/user/project-settings.md index bf30f0ba637a..43401e8800a8 100644 --- a/docs/user/project-settings.md +++ b/docs/user/project-settings.md @@ -84,9 +84,16 @@ same secret in the repository's webhook settings with content type `application/json`. Requests without a valid signature are rejected. On desktop and web, pick **Deliveries** from a task's menu to see recent -requests and the prompt each one produced. If the environment is offline, the sender gets an error and -nothing runs; redeliver from the sender, such as GitHub's **Recent Deliveries**, -once it is back. +requests and the prompt each one produced. + +If the environment is offline, the sender gets an error and nothing runs; +redeliver from the sender, such as GitHub's **Recent Deliveries**, once it is +back. To have T3 Connect keep requests instead, turn on **Hold webhooks while +offline** in **Settings → Connections**. T3 Connect then stores requests to a +T3 Connect URL for up to 24 hours and delivers them when the environment +returns. Leave it off if you don't want request bodies stored outside your +machine. To skip requests that waited too long, set **Skip requests older +than** on the task. ## Defaults and inheritance From 0e9d168e691ad9e107459ec1313e8fcd56e15c78 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:30:43 -0700 Subject: [PATCH 03/35] fix(server): relay deliveries run once and rate-limited ones can retry Relay delivery ids are claimed in their own table, kept 48 hours, because the delivery log keeps only 50 rows per task and could forget a delivery before the relay retries it. A rate-limited delivery releases its claim so a later attempt is not mistaken for a duplicate. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/persistence/Migrations.ts | 2 + .../Migrations/055_OrchestrationV2.test.ts | 4 +- .../Migrations/058_WebhookRelayDeliveries.ts | 22 ++++++++++ .../reconcileV2PreviewMigration.test.ts | 2 + .../scheduledTasks/ScheduledTaskService.ts | 42 +++++++++++++++---- .../ScheduledTaskService.webhook.test.ts | 39 +++++++++++++++++ 6 files changed, 102 insertions(+), 9 deletions(-) create mode 100644 apps/server/src/persistence/Migrations/058_WebhookRelayDeliveries.ts diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 1067c2416085..ccfe7f50055a 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -71,6 +71,7 @@ import Migration0054 from "./Migrations/054_ProjectionThreadsAutoSettleDisabledA import Migration0055 from "./Migrations/055_OrchestrationV2.ts"; import Migration0056 from "./Migrations/056_RemoveRedundantProjectionIndexes.ts"; import Migration0057 from "./Migrations/057_ScheduledTaskWebhooks.ts"; +import Migration0058 from "./Migrations/058_WebhookRelayDeliveries.ts"; /** * Migration loader with all migrations defined inline. @@ -142,6 +143,7 @@ export const migrationEntries = [ [55, "OrchestrationV2", Migration0055], [56, "RemoveRedundantProjectionIndexes", Migration0056], [57, "ScheduledTaskWebhooks", Migration0057], + [58, "WebhookRelayDeliveries", Migration0058], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts index e3839e3f0d17..6ee7c94cf1ba 100644 --- a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts +++ b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts @@ -13,7 +13,7 @@ layer("055_OrchestrationV2", (it) => { Effect.sync(() => { assert.deepStrictEqual( migrationEntries.map(([id]) => id), - Array.from({ length: 57 }, (_, index) => index + 1), + Array.from({ length: 58 }, (_, index) => index + 1), ); }), ); @@ -29,6 +29,7 @@ layer("055_OrchestrationV2", (it) => { [55, "OrchestrationV2"], [56, "RemoveRedundantProjectionIndexes"], [57, "ScheduledTaskWebhooks"], + [58, "WebhookRelayDeliveries"], ]); assert.deepStrictEqual(yield* runMigrations(), []); @@ -52,6 +53,7 @@ layer("055_OrchestrationV2", (it) => { { migration_id: 55, name: "OrchestrationV2" }, { migration_id: 56, name: "RemoveRedundantProjectionIndexes" }, { migration_id: 57, name: "ScheduledTaskWebhooks" }, + { migration_id: 58, name: "WebhookRelayDeliveries" }, ]); const tables = yield* sql<{ readonly name: string }>` diff --git a/apps/server/src/persistence/Migrations/058_WebhookRelayDeliveries.ts b/apps/server/src/persistence/Migrations/058_WebhookRelayDeliveries.ts new file mode 100644 index 000000000000..6e1c264c1647 --- /dev/null +++ b/apps/server/src/persistence/Migrations/058_WebhookRelayDeliveries.ts @@ -0,0 +1,22 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + // Relay delivery ids already handled, kept longer than the relay holds a + // request so a replay can never run twice. The delivery log keeps only the + // newest 50 rows per task, which is too short for that. + yield* sql` + CREATE TABLE IF NOT EXISTS scheduled_task_webhook_relay_deliveries ( + relay_delivery_id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + seen_at TEXT NOT NULL + ) + `; + + yield* sql` + CREATE INDEX IF NOT EXISTS idx_scheduled_task_webhook_relay_deliveries_seen + ON scheduled_task_webhook_relay_deliveries(seen_at) + `; +}); diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index e872cb553bfa..fef7bd808aa1 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -38,6 +38,7 @@ describe("V2 preview upgrade", () => { [54, "ProjectionThreadsAutoSettleDisabledAt"], [56, "RemoveRedundantProjectionIndexes"], [57, "ScheduledTaskWebhooks"], + [58, "WebhookRelayDeliveries"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -118,6 +119,7 @@ describe("V2 preview upgrade", () => { [54, "ProjectionThreadsAutoSettleDisabledAt"], [56, "RemoveRedundantProjectionIndexes"], [57, "ScheduledTaskWebhooks"], + [58, "WebhookRelayDeliveries"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index d432f10ac412..f7ccf79eb751 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -1227,6 +1227,14 @@ export const layer = Layer.effect( ${input.signatureVerified ? 1 : 0}, ${encodeMissingFieldsJson(input.missing)}, ${input.renderedPrompt}, NULL WHERE EXISTS (SELECT 1 FROM scheduled_tasks WHERE task_id = ${input.taskId}) + -- A held request retried after a rate limit reuses its relay + -- delivery id; the newer attempt replaces the logged one. + ON CONFLICT (delivery_id) DO UPDATE SET + outcome = excluded.outcome, + signature_verified = excluded.signature_verified, + missing_fields_json = excluded.missing_fields_json, + rendered_prompt = excluded.rendered_prompt, + error = NULL `; yield* sql` DELETE FROM scheduled_task_webhook_deliveries @@ -1336,17 +1344,35 @@ export const layer = Layer.effect( : `delivery:relay:${request.relayDeliveryId}`, ); // A held request may already have reached this environment directly - // before a timeout; it runs once. + // before a timeout; it runs once. Claimed in its own table, kept longer + // than the relay holds a request, because the delivery log is trimmed. + // A rate-limited delivery stays held on the relay, so it must give up + // its claim or the next pass would treat it as already delivered. + const releaseClaim = (result: A) => + request.relayDeliveryId === undefined + ? Effect.succeed(result) + : sql` + DELETE FROM scheduled_task_webhook_relay_deliveries + WHERE relay_delivery_id = ${request.relayDeliveryId} + `.pipe(Effect.ignore, Effect.as(result)); if (request.relayDeliveryId !== undefined) { - const seen = yield* sql<{ delivery_id: string }>` - SELECT delivery_id FROM scheduled_task_webhook_deliveries - WHERE delivery_id = ${deliveryId} + const claimed = yield* sql<{ relay_delivery_id: string }>` + INSERT INTO scheduled_task_webhook_relay_deliveries + (relay_delivery_id, task_id, seen_at) + VALUES (${request.relayDeliveryId}, ${task.id}, ${iso(now)}) + ON CONFLICT (relay_delivery_id) DO NOTHING + RETURNING relay_delivery_id `.pipe( Effect.mapError((cause) => - taskError("Could not load webhook delivery.", { taskId: task.id, cause }), + taskError("Could not record webhook delivery.", { taskId: task.id, cause }), ), ); - if (seen.length > 0) return { _tag: "accepted" as const, deliveryId }; + if (claimed.length === 0) return { _tag: "accepted" as const, deliveryId }; + // Older claims can no longer be replayed by the relay. + yield* sql` + DELETE FROM scheduled_task_webhook_relay_deliveries + WHERE seen_at < ${iso(DateTime.subtract(now, { hours: 48 }))} + `.pipe(Effect.ignore); } const log = ( outcome: ScheduledTaskWebhookDeliveryOutcome, @@ -1372,7 +1398,7 @@ export const layer = Layer.effect( const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(now)); if (slot !== "allowed") { if (slot === "first_rejected") yield* log("rate_limited"); - return { _tag: "rate_limited" as const }; + return yield* releaseClaim({ _tag: "rate_limited" as const }); } if (!task.enabled) { yield* log("disabled"); @@ -1413,7 +1439,7 @@ export const layer = Layer.effect( ? ([false, counts] as const) : ([true, new Map(counts).set(queueKey, count + 1)] as const); }); - if (!queued) return { _tag: "rate_limited" as const }; + if (!queued) return yield* releaseClaim({ _tag: "rate_limited" as const }); // Entries leave the map when their count reaches zero, so a deleted // task's key does not linger once its last delivery finishes. const release = Ref.update(webhookQueued, (counts) => { diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index ba91db274aca..fc15b222a1b1 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -475,6 +475,45 @@ it.effect("a held request already delivered directly runs only once", () => ), ); +it.effect("a held request runs once even after the log has trimmed it", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + const first = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" })); + assert.equal(first._tag, "accepted"); + yield* Queue.take(launches); + // Push the original row out of the 50-row delivery log. + const paused = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); + yield* Effect.forEach(Array.from({ length: 55 }), () => + service.triggerWebhook(requestFor(paused.task)), + ); + yield* service.upsert(yield* webhookTaskInput()); + const replay = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" })); + assert.equal(replay._tag, "accepted"); + assert.equal(yield* Queue.size(launches), 0); + }), + ), +); + +it.effect("a rate-limited held request can run on a later pass", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); + // Spend the task's 60-a-minute budget. + yield* Effect.forEach(Array.from({ length: 60 }), () => + service.triggerWebhook(requestFor(task)), + ); + const limited = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "later" })); + assert.equal(limited._tag, "rate_limited"); + yield* service.upsert(yield* webhookTaskInput()); + yield* TestClock.adjust("61 seconds"); + const retried = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "later" })); + assert.equal(retried._tag, "accepted"); + yield* Queue.take(launches); + }), + ), +); + it.effect("logs a held request at the time the relay received it", () => withService(({ service }) => Effect.gen(function* () { From 7e1db6e0dc4374bffc3345da489f63e71c385b26 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:26:53 -0700 Subject: [PATCH 04/35] refactor(relay,server): hold webhooks in a Durable Object that pushes them back Held webhook requests move out of Postgres into one Durable Object per environment with SQLite storage. Its alarm pushes them back through the tunnel, oldest first, with backoff (30 s up to 10 min) while the environment stays away, and drops them after 24 hours. When the tunnel reconnects, the environment calls the new wake endpoint so delivery starts right away. This replaces the pull drain and the pending/ack endpoints. The Postgres migration now only adds the opt-in column. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/cloud/ManagedEndpointRuntime.test.ts | 30 +++ .../src/cloud/ManagedEndpointRuntime.ts | 9 +- apps/server/src/cloud/http.test.ts | 2 + apps/server/src/relay/HeldHooksWaker.ts | 48 ++++ .../server/src/relay/HookMailboxDrain.test.ts | 98 ------- apps/server/src/relay/HookMailboxDrain.ts | 180 ------------- apps/server/src/server.ts | 12 +- docs/internals/t3-connect.md | 23 +- .../20261004030842_hook_mailbox/migration.sql | 16 -- .../migration.sql | 1 + .../snapshot.json | 246 +----------------- infra/relay/package.json | 1 + .../src/environments/EnvironmentLinks.ts | 12 +- infra/relay/src/hooks/HookForwarder.test.ts | 27 +- infra/relay/src/hooks/HookForwarder.ts | 157 ++++------- infra/relay/src/hooks/HookInbox.ts | 40 +++ infra/relay/src/hooks/HookInboxObject.ts | 103 ++++++++ infra/relay/src/hooks/HookInboxStore.test.ts | 171 ++++++++++++ infra/relay/src/hooks/HookInboxStore.ts | 215 +++++++++++++++ infra/relay/src/hooks/HookMailbox.test.ts | 128 --------- infra/relay/src/hooks/HookMailbox.ts | 163 ------------ infra/relay/src/hooks/upstream.ts | 67 +++++ infra/relay/src/http/Api.test.ts | 45 +++- infra/relay/src/http/Api.ts | 54 ++-- infra/relay/src/persistence/schema.ts | 26 -- infra/relay/src/worker.ts | 43 ++- packages/contracts/src/relay.ts | 47 +--- pnpm-lock.yaml | 26 +- pnpm-workspace.yaml | 2 + 29 files changed, 920 insertions(+), 1072 deletions(-) create mode 100644 apps/server/src/relay/HeldHooksWaker.ts delete mode 100644 apps/server/src/relay/HookMailboxDrain.test.ts delete mode 100644 apps/server/src/relay/HookMailboxDrain.ts delete mode 100644 infra/relay/migrations/postgres/20261004030842_hook_mailbox/migration.sql create mode 100644 infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/migration.sql rename infra/relay/migrations/postgres/{20261004030842_hook_mailbox => 20261004061459_hold_webhooks_while_offline}/snapshot.json (87%) create mode 100644 infra/relay/src/hooks/HookInbox.ts create mode 100644 infra/relay/src/hooks/HookInboxObject.ts create mode 100644 infra/relay/src/hooks/HookInboxStore.test.ts create mode 100644 infra/relay/src/hooks/HookInboxStore.ts delete mode 100644 infra/relay/src/hooks/HookMailbox.test.ts delete mode 100644 infra/relay/src/hooks/HookMailbox.ts create mode 100644 infra/relay/src/hooks/upstream.ts diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index aebefed787c1..a0490e77446f 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -209,6 +209,36 @@ describe("CloudManagedEndpointRuntime", () => { }), ); + it.effect("signals each registered tunnel connection", () => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const handle = makeHandle({ + pid: 700, + onKill: () => {}, + output: Stream.fromQueue(output), + }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const runtime = yield* buildCloudManagedEndpointRuntime(spawner); + yield* runtime.applyConfig({ + providerKind: "cloudflare_tunnel", + connectorToken: "token", + tunnelId: "tunnel-1", + }); + yield* Queue.offer( + output, + new TextEncoder().encode( + "2026-10-04T06:30:43Z INF Registered tunnel connection connIndex=0 event=0\n", + ), + ); + expect(Option.isSome(yield* Stream.runHead(runtime.tunnelConnected))).toBe(true); + }), + ); + it.effect("recovers a rejected tunnel without waiting for the connector to exit", () => Effect.gen(function* () { const output = yield* Queue.unbounded(); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 25426fcba4ce..6b54b7ee3bdf 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -47,6 +47,8 @@ export class CloudManagedEndpointRuntime extends Context.Service< ) => Effect.Effect; readonly recoveryRequests: Stream.Stream; readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect; + /** Emits when the connector registers a tunnel connection, i.e. the relay can reach us again. */ + readonly tunnelConnected: Stream.Stream; readonly withLinkStateLock: (effect: Effect.Effect) => Effect.Effect; } >()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {} @@ -134,6 +136,7 @@ export const make = Effect.gen(function* () { const activeRef = yield* Ref.make(null); const desiredConfigRef = yield* Ref.make(null); const recoveryRequests = yield* Queue.sliding(1); + const tunnelConnections = yield* Queue.sliding(1); const reconcileSemaphore = yield* Semaphore.make(1); const restartDelayRef = yield* Ref.make(0); const linkStateSemaphore = yield* Semaphore.make(1); @@ -236,7 +239,10 @@ export const make = Effect.gen(function* () { switch (classifyRelayClientOutput(line)) { case "connected": rejectedRegistrations = 0; - return Effect.logInfo("Relay client tunnel connection registered", attributes); + return Effect.logInfo("Relay client tunnel connection registered", attributes).pipe( + Effect.andThen(Queue.offer(tunnelConnections, undefined)), + Effect.asVoid, + ); case "warning": if (isRejectedRelayClientTunnelOutput(line)) { rejectedRegistrations += 1; @@ -412,6 +418,7 @@ export const make = Effect.gen(function* () { applyConfig, recoveryRequests: Stream.fromQueue(recoveryRequests), requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid), + tunnelConnected: Stream.fromQueue(tunnelConnections), withLinkStateLock: linkStateSemaphore.withPermits(1), }); diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 77d432308477..085cac39184d 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -249,6 +249,7 @@ describe("reconcileDesiredCloudLink", () => { applyConfig: unusedSecretStoreOperation, recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, + tunnelConnected: Stream.empty, withLinkStateLock: (effect) => effect, } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]), ), @@ -410,6 +411,7 @@ describe("releaseManagedTunnelOnShutdown", () => { }), recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, + tunnelConnected: Stream.empty, withLinkStateLock: (effect) => effect, }), ), diff --git a/apps/server/src/relay/HeldHooksWaker.ts b/apps/server/src/relay/HeldHooksWaker.ts new file mode 100644 index 000000000000..c240f9cd52e3 --- /dev/null +++ b/apps/server/src/relay/HeldHooksWaker.ts @@ -0,0 +1,48 @@ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schedule from "effect/Schedule"; +import * as Stream from "effect/Stream"; + +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as CloudManagedEndpointRuntime from "../cloud/ManagedEndpointRuntime.ts"; +import { readHoldWebhooksWhileOffline, readRelayConnection } from "../cloud/config.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import { makeRelayEnvironmentClient } from "./relayEnvironmentClient.ts"; + +/** + * Tells T3 Connect this environment is reachable again, so the relay delivers + * the webhook requests it held while we were offline now rather than at its + * next backoff step. Nothing happens unless the environment opted in. + */ +const wakeHeldHooks = Effect.fn("HeldHooksWaker.wake")(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + if (!(yield* readHoldWebhooksWhileOffline(secrets))) return false; + const connection = yield* readRelayConnection(secrets); + if (connection === null) return false; + const environmentId = yield* (yield* ServerEnvironment.ServerEnvironment).getEnvironmentId; + const client = yield* makeRelayEnvironmentClient(connection); + const { pending } = yield* client.server.wakeHeldHooks({ params: { environmentId } }); + return pending; +}); + +/** Wakes held webhooks each time the managed tunnel connects. */ +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const runtime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; + const wake = wakeHeldHooks().pipe( + Effect.timeout("10 seconds"), + // The relay retries on its own schedule too, so a few tries are enough. + Effect.retry({ schedule: Schedule.exponential("2 seconds"), times: 3 }), + Effect.tap((pending) => + pending ? Effect.logInfo("T3 Connect is delivering held webhook requests") : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning("Could not ask T3 Connect to deliver held webhook requests", { cause }), + ), + ); + yield* runtime.tunnelConnected.pipe( + Stream.runForEach(() => wake), + Effect.forkScoped, + ); + }), +); diff --git a/apps/server/src/relay/HookMailboxDrain.test.ts b/apps/server/src/relay/HookMailboxDrain.test.ts deleted file mode 100644 index 8744334e92de..000000000000 --- a/apps/server/src/relay/HookMailboxDrain.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { assert, it } from "@effect/vitest"; -import type { RelayPendingHook } from "@t3tools/contracts/relay"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; - -import { - ScheduledTaskService, - type WebhookTriggerRequest, - type WebhookTriggerResult, -} from "../scheduledTasks/ScheduledTaskService.ts"; -import { drainOnce, HookMailboxRelay, toTriggerRequest } from "./HookMailboxDrain.ts"; - -const held = (id: string, overrides: Partial = {}): RelayPendingHook => ({ - id, - receivedAt: "2026-10-04T10:00:00.000Z", - method: "POST", - rawHookId: "scheduled-task%3Ahook", - rawToken: "tok%2Fen", - query: "a=1", - headers: { "Content-Type": "application/json" }, - bodyBase64: Buffer.from('{"x":1}').toString("base64"), - ...overrides, -}); - -/** Runs one drain pass against a relay holding `pages` and a service answering `outcome`. */ -const drain = ( - pages: ReadonlyArray> | null, - outcome: (request: WebhookTriggerRequest) => WebhookTriggerResult["_tag"], -) => - Effect.gen(function* () { - const acked: Array = []; - const triggered: Array = []; - let page = 0; - const relay = Layer.succeed(HookMailboxRelay, { - pending: Effect.sync(() => (pages === null ? null : (pages[page++] ?? []))), - ack: (ids) => Effect.sync(() => void acked.push(...ids)), - }); - const service = Layer.mock(ScheduledTaskService)({ - triggerWebhook: (request) => - Effect.sync(() => { - triggered.push(request); - return { _tag: outcome(request) } as WebhookTriggerResult; - }), - }); - const delivered = yield* drainOnce().pipe(Effect.provide(Layer.merge(relay, service))); - return { acked, triggered, delivered }; - }); - -it("rebuilds the request the webhook route would have built", () => { - const request = toTriggerRequest(held("d1")); - assert.equal(request?.hookId, "scheduled-task:hook"); - assert.equal(request?.token, "tok/en"); - assert.equal(request?.headers["content-type"], "application/json"); - assert.equal(request?.bodyText, '{"x":1}'); - assert.equal(request?.relayDeliveryId, "d1"); - assert.equal(request?.receivedAt, "2026-10-04T10:00:00.000Z"); -}); - -it.effect("delivers held requests and acks every final outcome", () => - Effect.gen(function* () { - const result = yield* drain([[held("ok"), held("gone"), held("bad-sig")]], (request) => - request.relayDeliveryId === "ok" - ? "accepted" - : request.relayDeliveryId === "gone" - ? "not_found" - : "rejected_signature", - ); - assert.deepEqual(result.acked, ["ok", "gone", "bad-sig"]); - }), -); - -it.effect("leaves rate-limited requests held for the next pass", () => - Effect.gen(function* () { - const result = yield* drain( - [[held("first"), held("limited")], [held("never-pulled")]], - (request) => (request.relayDeliveryId === "limited" ? "rate_limited" : "accepted"), - ); - assert.deepEqual(result.acked, ["first"]); - // The pass stops instead of pulling the next page. - assert.isFalse(result.triggered.some((request) => request.relayDeliveryId === "never-pulled")); - }), -); - -it.effect("acks and drops a request whose path cannot be decoded", () => - Effect.gen(function* () { - const result = yield* drain([[held("broken", { rawToken: "%E0" })]], () => "accepted"); - assert.deepEqual(result.acked, ["broken"]); - assert.equal(result.triggered.length, 0); - }), -); - -it.effect("does nothing when the environment is not linked", () => - Effect.gen(function* () { - const result = yield* drain(null, () => "accepted"); - assert.equal(result.delivered, 0); - assert.equal(result.triggered.length, 0); - }), -); diff --git a/apps/server/src/relay/HookMailboxDrain.ts b/apps/server/src/relay/HookMailboxDrain.ts deleted file mode 100644 index 0767aa17da27..000000000000 --- a/apps/server/src/relay/HookMailboxDrain.ts +++ /dev/null @@ -1,180 +0,0 @@ -import type { RelayPendingHook } from "@t3tools/contracts/relay"; -import * as Clock from "effect/Clock"; -import * as Context from "effect/Context"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; -import * as Ref from "effect/Ref"; -import * as Schema from "effect/Schema"; - -import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; -import { readHoldWebhooksWhileOffline, readRelayConnection } from "../cloud/config.ts"; -import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; -import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; -import { WEBHOOK_MAX_BODY_BYTES } from "../scheduledTasks/webhookRoute.ts"; -import * as Scheduler from "../scheduling/Scheduler.ts"; -import { makeRelayEnvironmentClient } from "./relayEnvironmentClient.ts"; - -const DRAIN_INTERVAL_MS = 30_000; -const MAX_BACKOFF_MS = 5 * 60_000; -const PULL_LIMIT = 10; - -export class HookMailboxRelayError extends Schema.TaggedError()( - "HookMailboxRelayError", - { operation: Schema.Literals(["pending", "ack"]), cause: Schema.Defect() }, -) {} - -/** - * Talks to the relay's held-webhook endpoints. A service so tests can stand in - * for the relay without a network. - */ -export class HookMailboxRelay extends Context.Service< - HookMailboxRelay, - { - /** Null when this environment is not linked to T3 Connect. */ - readonly pending: Effect.Effect | null, HookMailboxRelayError>; - readonly ack: (ids: ReadonlyArray) => Effect.Effect; - } ->()("t3/relay/HookMailboxDrain/HookMailboxRelay") {} - -export const relayLayer = Layer.effect( - HookMailboxRelay, - Effect.gen(function* () { - const secrets = yield* ServerSecretStore.ServerSecretStore; - const environment = yield* ServerEnvironment.ServerEnvironment; - const client = Effect.gen(function* () { - const connection = yield* readRelayConnection(secrets); - if (connection === null) return null; - return { - api: yield* makeRelayEnvironmentClient(connection), - environmentId: yield* environment.getEnvironmentId, - }; - }); - return HookMailboxRelay.of({ - pending: Effect.gen(function* () { - const relay = yield* client; - if (relay === null) return null; - const { deliveries } = yield* relay.api.server.listPendingHooks({ - params: { environmentId: relay.environmentId }, - query: { limit: PULL_LIMIT }, - }); - return deliveries; - }).pipe( - Effect.mapError((cause) => new HookMailboxRelayError({ operation: "pending", cause })), - ), - ack: (ids) => - Effect.gen(function* () { - const relay = yield* client; - if (relay === null || ids.length === 0) return; - yield* relay.api.server.ackPendingHooks({ - params: { environmentId: relay.environmentId }, - payload: { ids: [...ids] }, - }); - }).pipe(Effect.mapError((cause) => new HookMailboxRelayError({ operation: "ack", cause }))), - }); - }), -); - -/** Turns a held request back into what the webhook route would have built. */ -export function toTriggerRequest( - hook: RelayPendingHook, -): ScheduledTaskService.WebhookTriggerRequest | null { - let hookId: string; - let token: string; - try { - hookId = decodeURIComponent(hook.rawHookId); - token = decodeURIComponent(hook.rawToken); - } catch { - return null; - } - const body = new Uint8Array(Buffer.from(hook.bodyBase64, "base64")); - if (body.byteLength > WEBHOOK_MAX_BODY_BYTES) return null; - const headers: Record = {}; - for (const [name, value] of Object.entries(hook.headers)) headers[name.toLowerCase()] = value; - return { - hookId, - token, - method: hook.method, - path: `${ScheduledTaskService.WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(hookId)}`, - query: hook.query, - headers, - body, - bodyText: new TextDecoder().decode(body), - relayDeliveryId: hook.id, - receivedAt: hook.receivedAt, - }; -} - -/** - * Delivers webhook requests T3 Connect held while this environment was - * offline. Runs once at startup and then every 30 seconds while the opt-in is - * on. Rate-limited requests stay held for the next pass; every other outcome, - * including a request that cannot be decoded, is acked. - */ -export const drainOnce = Effect.fn("HookMailboxDrain.drainOnce")(function* () { - const relay = yield* HookMailboxRelay; - const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService; - let delivered = 0; - // Bounded so one pass cannot run forever against a relay that keeps returning work. - for (let page = 0; page < 50; page++) { - const pending = yield* relay.pending; - if (pending === null || pending.length === 0) return delivered; - const ack: Array = []; - let rateLimited = false; - for (const hook of pending) { - const request = toTriggerRequest(hook); - if (request === null) { - ack.push(hook.id); - continue; - } - const result = yield* scheduledTasks.triggerWebhook(request); - if (result._tag === "rate_limited") { - rateLimited = true; - continue; - } - ack.push(hook.id); - delivered++; - } - yield* relay.ack(ack); - // Rate-limited requests are still held; stop until the next pass. - if (rateLimited || ack.length === 0) return delivered; - } - return delivered; -}); - -export const layer = Layer.effectDiscard( - Effect.gen(function* () { - const scheduler = yield* Scheduler.Scheduler; - const secrets = yield* ServerSecretStore.ServerSecretStore; - const relay = yield* HookMailboxRelay; - const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService; - const state = yield* Ref.make({ nextAt: 0, failures: 0, wasEnabled: false }); - - const tick = Effect.gen(function* () { - const now = yield* Clock.currentTimeMillis; - const current = yield* Ref.get(state); - if (now < current.nextAt) return; - const enabled = yield* readHoldWebhooksWhileOffline(secrets); - // One last pass after the opt-in is turned off, so nothing already held - // is stranded; after that the loop stays idle. - if (!enabled && !current.wasEnabled) return; - const outcome = yield* drainOnce().pipe( - Effect.provideService(HookMailboxRelay, relay), - Effect.provideService(ScheduledTaskService.ScheduledTaskService, scheduledTasks), - Effect.result, - ); - const failures = outcome._tag === "Failure" ? current.failures + 1 : 0; - if (outcome._tag === "Failure") { - yield* Effect.logWarning("Could not deliver held webhook requests", { - cause: outcome.failure, - }); - } - const delay = - failures === 0 - ? DRAIN_INTERVAL_MS - : Math.min(DRAIN_INTERVAL_MS * 2 ** failures, MAX_BACKOFF_MS); - yield* Ref.set(state, { nextAt: now + delay, failures, wasEnabled: enabled }); - }); - - yield* scheduler.register("relay-hook-mailbox", tick); - }), -); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 5d96bd53797d..8328f35fafbe 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -119,8 +119,7 @@ import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http import * as ReplayMarkers from "./auth/replayMarkers.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import { webhookHttpApiLayer } from "./scheduledTasks/webhookRoute.ts"; -import * as HookMailboxDrain from "./relay/HookMailboxDrain.ts"; -import * as Scheduler from "./scheduling/Scheduler.ts"; +import * as HeldHooksWaker from "./relay/HeldHooksWaker.ts"; import { ScheduledTaskWebhookOrigin } from "./scheduledTasks/ScheduledTaskService.ts"; import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./cloud/config.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; @@ -537,15 +536,10 @@ const ProviderInstallationRefreshLive = Layer.effectDiscard( }), ); -// Delivers webhook requests T3 Connect held while this environment was offline. -const HookMailboxDrainLive = HookMailboxDrain.layer.pipe( - Layer.provide(HookMailboxDrain.relayLayer), - Layer.provide(Scheduler.layer), -); - const RuntimeCoreDependenciesBaseLive = Layer.mergeAll( AgentAwarenessRelay.layer, - HookMailboxDrainLive, + // Asks T3 Connect to deliver webhooks it held while this environment was offline. + HeldHooksWaker.layer, ThreadSettlementWorkerLive, Layer.effectDiscard(StorageCleanup.make.pipe(Effect.flatMap((service) => service.start()))).pipe( Layer.provide(ProjectionStoreV2.layer), diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index c12553e67b5d..54fdf6dc310c 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -13,14 +13,21 @@ leaves token and signature verification to the environment By default the forwarder stores nothing. An environment can opt in to having the relay hold requests while it is offline (`hold_webhooks_while_offline` on its link). Only then does the relay store the -raw request, including the hook token in the path, in `relay_hook_mailbox`. -Rows are deleted when the environment acks them, after 24 hours, or when no -user has the environment linked. The relay still never checks the token: the -environment pulls held requests with its credential and runs them through the -same verification. Every forward carries `x-t3-relay-delivery-id` so a request -that reached the environment before a timeout and is later replayed runs once -([mailbox](../../infra/relay/src/hooks/HookMailbox.ts), -[drain](../../apps/server/src/relay/HookMailboxDrain.ts)). +raw request, including the hook token in the path, in a Durable Object for +that environment, with SQLite storage. The object pushes held requests back +through the tunnel from its alarm, oldest first, and backs off while the +environment stays away. When the tunnel reconnects, the environment asks the +relay to deliver right away. Requests are deleted once the environment +answers, after 24 hours, or when no user has the environment linked. The relay +still never checks the token; delivery goes through the same environment route. +Every forward carries `x-t3-relay-delivery-id`, so a request that reached the +environment before a timeout and is delivered again later runs once +([inbox object](../../infra/relay/src/hooks/HookInboxObject.ts)). + +A Durable Object, not Postgres or Queues, because held requests are write-once, +read-once bodies of up to 1 MiB that need per-environment order, caps, and +retry timing. Queues cap messages at 128 KB and cannot hold one environment's +requests back while it is away. Clerk, deployment, and native authentication setup live in the [Connect setup runbook](../operations/connect-setup.md). diff --git a/infra/relay/migrations/postgres/20261004030842_hook_mailbox/migration.sql b/infra/relay/migrations/postgres/20261004030842_hook_mailbox/migration.sql deleted file mode 100644 index d66698deb8a8..000000000000 --- a/infra/relay/migrations/postgres/20261004030842_hook_mailbox/migration.sql +++ /dev/null @@ -1,16 +0,0 @@ -CREATE TABLE "relay_hook_mailbox" ( - "id" varchar(36) PRIMARY KEY, - "environment_id" varchar(191) NOT NULL, - "received_at" varchar(64) NOT NULL, - "expires_at" varchar(64) NOT NULL, - "method" varchar(16) NOT NULL, - "raw_hook_id" varchar(512) NOT NULL, - "raw_token" varchar(512) NOT NULL, - "query" text NOT NULL, - "headers" jsonb NOT NULL, - "body" bytea NOT NULL -); ---> statement-breakpoint -ALTER TABLE "relay_environment_links" ADD COLUMN "hold_webhooks_while_offline" boolean DEFAULT false NOT NULL;--> statement-breakpoint -CREATE INDEX "idx_relay_hook_mailbox_environment" ON "relay_hook_mailbox" ("environment_id","received_at");--> statement-breakpoint -CREATE INDEX "idx_relay_hook_mailbox_expires" ON "relay_hook_mailbox" ("expires_at"); \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/migration.sql b/infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/migration.sql new file mode 100644 index 000000000000..b5561429664b --- /dev/null +++ b/infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "relay_environment_links" ADD COLUMN "hold_webhooks_while_offline" boolean DEFAULT false NOT NULL; \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20261004030842_hook_mailbox/snapshot.json b/infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/snapshot.json similarity index 87% rename from infra/relay/migrations/postgres/20261004030842_hook_mailbox/snapshot.json rename to infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/snapshot.json index c8fdaeb0a80a..21c7a3747177 100644 --- a/infra/relay/migrations/postgres/20261004030842_hook_mailbox/snapshot.json +++ b/infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/snapshot.json @@ -1,10 +1,8 @@ { "version": "8", "dialect": "postgres", - "id": "fcfe51b7-4f3e-45a5-9e83-ac96f1ced4e6", - "prevIds": [ - "3809c51e-3821-4a08-818d-e5d28dd3e9b4" - ], + "id": "13972659-45db-4ce9-9818-523b450bceca", + "prevIds": ["3809c51e-3821-4a08-818d-e5d28dd3e9b4"], "ddl": [ { "isRlsEnabled": false, @@ -36,12 +34,6 @@ "entityType": "tables", "schema": "public" }, - { - "isRlsEnabled": false, - "name": "relay_hook_mailbox", - "entityType": "tables", - "schema": "public" - }, { "isRlsEnabled": false, "name": "relay_live_activities", @@ -664,136 +656,6 @@ "schema": "public", "table": "relay_environment_links" }, - { - "type": "varchar(36)", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "id", - "entityType": "columns", - "schema": "public", - "table": "relay_hook_mailbox" - }, - { - "type": "varchar(191)", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "environment_id", - "entityType": "columns", - "schema": "public", - "table": "relay_hook_mailbox" - }, - { - "type": "varchar(64)", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "received_at", - "entityType": "columns", - "schema": "public", - "table": "relay_hook_mailbox" - }, - { - "type": "varchar(64)", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "expires_at", - "entityType": "columns", - "schema": "public", - "table": "relay_hook_mailbox" - }, - { - "type": "varchar(16)", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "method", - "entityType": "columns", - "schema": "public", - "table": "relay_hook_mailbox" - }, - { - "type": "varchar(512)", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "raw_hook_id", - "entityType": "columns", - "schema": "public", - "table": "relay_hook_mailbox" - }, - { - "type": "varchar(512)", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "raw_token", - "entityType": "columns", - "schema": "public", - "table": "relay_hook_mailbox" - }, - { - "type": "text", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "query", - "entityType": "columns", - "schema": "public", - "table": "relay_hook_mailbox" - }, - { - "type": "jsonb", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "headers", - "entityType": "columns", - "schema": "public", - "table": "relay_hook_mailbox" - }, - { - "type": "bytea", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "body", - "entityType": "columns", - "schema": "public", - "table": "relay_hook_mailbox" - }, { "type": "varchar(255)", "typeSchema": null, @@ -1537,55 +1399,6 @@ "schema": "public", "table": "relay_environment_links" }, - { - "nameExplicit": true, - "columns": [ - { - "value": "environment_id", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - }, - { - "value": "received_at", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_relay_hook_mailbox_environment", - "entityType": "indexes", - "schema": "public", - "table": "relay_hook_mailbox" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "expires_at", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_relay_hook_mailbox_expires", - "entityType": "indexes", - "schema": "public", - "table": "relay_hook_mailbox" - }, { "nameExplicit": true, "columns": [ @@ -1692,11 +1505,7 @@ "table": "relay_mobile_devices" }, { - "columns": [ - "environment_id", - "environment_public_key", - "thread_id" - ], + "columns": ["environment_id", "environment_public_key", "thread_id"], "nameExplicit": false, "name": "relay_agent_activity_rows_pkey", "entityType": "pks", @@ -1704,10 +1513,7 @@ "table": "relay_agent_activity_rows" }, { - "columns": [ - "thumbprint", - "jti" - ], + "columns": ["thumbprint", "jti"], "nameExplicit": false, "name": "relay_dpop_proofs_pkey", "entityType": "pks", @@ -1715,10 +1521,7 @@ "table": "relay_dpop_proofs" }, { - "columns": [ - "user_id", - "environment_id" - ], + "columns": ["user_id", "environment_id"], "nameExplicit": false, "name": "relay_environment_links_pkey", "entityType": "pks", @@ -1726,10 +1529,7 @@ "table": "relay_environment_links" }, { - "columns": [ - "user_id", - "device_id" - ], + "columns": ["user_id", "device_id"], "nameExplicit": false, "name": "relay_live_activities_pkey", "entityType": "pks", @@ -1737,10 +1537,7 @@ "table": "relay_live_activities" }, { - "columns": [ - "user_id", - "environment_id" - ], + "columns": ["user_id", "environment_id"], "nameExplicit": false, "name": "relay_managed_endpoint_allocations_pkey", "entityType": "pks", @@ -1748,10 +1545,7 @@ "table": "relay_managed_endpoint_allocations" }, { - "columns": [ - "user_id", - "device_id" - ], + "columns": ["user_id", "device_id"], "nameExplicit": false, "name": "relay_mobile_devices_pkey", "entityType": "pks", @@ -1759,9 +1553,7 @@ "table": "relay_mobile_devices" }, { - "columns": [ - "id" - ], + "columns": ["id"], "nameExplicit": false, "name": "relay_delivery_attempts_pkey", "schema": "public", @@ -1769,9 +1561,7 @@ "entityType": "pks" }, { - "columns": [ - "credential_id" - ], + "columns": ["credential_id"], "nameExplicit": false, "name": "relay_environment_credentials_pkey", "schema": "public", @@ -1779,19 +1569,7 @@ "entityType": "pks" }, { - "columns": [ - "id" - ], - "nameExplicit": false, - "name": "relay_hook_mailbox_pkey", - "schema": "public", - "table": "relay_hook_mailbox", - "entityType": "pks" - }, - { - "columns": [ - "user_id" - ], + "columns": ["user_id"], "nameExplicit": false, "name": "relay_managed_tunnel_limits_pkey", "schema": "public", @@ -1800,4 +1578,4 @@ } ], "renames": [] -} \ No newline at end of file +} diff --git a/infra/relay/package.json b/infra/relay/package.json index 03976ea81df8..ae3c806c4d24 100644 --- a/infra/relay/package.json +++ b/infra/relay/package.json @@ -13,6 +13,7 @@ "dependencies": { "@clerk/backend": "catalog:", "@effect/sql-pg": "catalog:", + "@effect/sql-sqlite-do": "catalog:", "@noble/curves": "catalog:", "@noble/hashes": "catalog:", "@t3tools/client-runtime": "workspace:*", diff --git a/infra/relay/src/environments/EnvironmentLinks.ts b/infra/relay/src/environments/EnvironmentLinks.ts index f033157c3c7f..c2e68fa13b1b 100644 --- a/infra/relay/src/environments/EnvironmentLinks.ts +++ b/infra/relay/src/environments/EnvironmentLinks.ts @@ -9,7 +9,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; -import { and, eq, isNull, or } from "drizzle-orm"; +import { and, eq, isNull, or, sql } from "drizzle-orm"; import * as RelayDb from "../db.ts"; import { relayEnvironmentLinks } from "../persistence/schema.ts"; @@ -176,12 +176,21 @@ const make = Effect.gen(function* () { const now = DateTime.formatIso(yield* DateTime.now); const { request, proof } = input; const environmentId = proof.environmentId; + // The webhook-hold opt-in belongs to the environment: a new or re-made + // link carries it over from the environment's other active links. + const inheritedHoldWebhooks = sql`EXISTS ( + SELECT 1 FROM ${relayEnvironmentLinks} AS other + WHERE other.environment_id = ${environmentId} + AND other.revoked_at IS NULL + AND other.hold_webhooks_while_offline + )`; const { endpoint } = input; yield* db .insert(relayEnvironmentLinks) .values({ userId: input.userId, environmentId, + holdWebhooksWhileOffline: inheritedHoldWebhooks, environmentLabel: proof.descriptor.label, environmentPublicKey: proof.environmentPublicKey, endpointHttpBaseUrl: endpoint.httpBaseUrl, @@ -209,6 +218,7 @@ const make = Effect.gen(function* () { createdByDeviceId: request.deviceId ?? null, revokedAt: null, updatedAt: now, + holdWebhooksWhileOffline: inheritedHoldWebhooks, }, }) .pipe( diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index ba8617fb8baf..0eedf0ed6cbb 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -32,7 +32,9 @@ import { traceRelayHttpRequestWith, } from "../http/Api.ts"; import * as HookForwarder from "./HookForwarder.ts"; -import * as HookMailbox from "./HookMailbox.ts"; +import * as HookInbox from "./HookInbox.ts"; +import type { HeldHook } from "./HookInboxStore.ts"; +import { RELAY_HOOK_UPSTREAM_TIMEOUT_MS } from "./upstream.ts"; const settings: RelayConfiguration.RelayConfiguration["Service"] = { relayIssuer: "https://relay.example.test", @@ -82,14 +84,14 @@ interface Harness { readonly links?: ReadonlyArray; readonly allocation?: ManagedEndpointAllocations.ManagedEndpointAllocation | null; readonly allow?: (key: string) => boolean; - /** Mailbox capacity; enqueue reports full once this many requests are held. */ - readonly mailboxCapacity?: number; + /** Inbox capacity; hold reports full once this many requests are held. */ + readonly inboxCapacity?: number; } function makeHarness(options: Harness = {}) { const sent: Array = []; const rateLimitKeys: Array = []; - const held: Array = []; + const held: Array = []; const execute = options.execute ?? ((request: HttpClientRequest.HttpClientRequest) => @@ -120,11 +122,11 @@ function makeHarness(options: Harness = {}) { return execute(request); }), ), - Layer.mock(HookMailbox.HookMailbox, { - enqueue: (hook) => + Layer.mock(HookInbox.HookInbox, { + hold: ({ hook, baseUrl }) => Effect.sync(() => { - if (held.length >= (options.mailboxCapacity ?? Infinity)) return false; - held.push(hook); + if (held.length >= (options.inboxCapacity ?? Infinity)) return false; + held.push({ ...hook, baseUrl }); return true; }), }), @@ -368,7 +370,7 @@ describe("HookForwarder", () => { .send(new Request(hookUrl(), { method: "POST", body: "{}" })) .pipe(Effect.forkChild); yield* Deferred.await(reachedUpstream); - yield* TestClock.adjust(Duration.millis(HookForwarder.RELAY_HOOK_UPSTREAM_TIMEOUT_MS)); + yield* TestClock.adjust(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)); const response = yield* Fiber.join(fiber); expect(response.status).toBe(504); expect(yield* readJson(response)).toEqual({ error: "environment_timeout" }); @@ -510,6 +512,7 @@ describe("HookForwarder", () => { ); expect(response.status).toBe(202); const [hook] = harness.held; + expect(hook?.baseUrl).toBe("https://env.example.test/"); expect(hook?.rawToken).toBe("tok%2Fen"); expect(hook?.query).toBe("a=1"); expect([...(hook?.body ?? [])]).toEqual([0, 255, 10]); @@ -520,18 +523,18 @@ describe("HookForwarder", () => { }), ); - it.effect("answers 503 mailbox_full when the environment's mailbox is full", () => + it.effect("answers 503 inbox_full when the environment's inbox is full", () => Effect.gen(function* () { const harness = makeHarness({ execute: offline, links: [{ ...managedLink, holdWebhooksWhileOffline: true }], - mailboxCapacity: 0, + inboxCapacity: 0, }); const response = yield* harness.send( new Request(hookUrl(), { method: "POST", body: "{}" }), ); expect(response.status).toBe(503); - expect(yield* readJson(response)).toEqual({ error: "mailbox_full" }); + expect(yield* readJson(response)).toEqual({ error: "inbox_full" }); }), ); diff --git a/infra/relay/src/hooks/HookForwarder.ts b/infra/relay/src/hooks/HookForwarder.ts index a2d9ee18ba43..15b46f18c82d 100644 --- a/infra/relay/src/hooks/HookForwarder.ts +++ b/infra/relay/src/hooks/HookForwarder.ts @@ -1,7 +1,6 @@ import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; -import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -9,24 +8,20 @@ import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import * as HttpClient from "effect/http/HttpClient"; -import * as HttpClientRequest from "effect/http/HttpClientRequest"; -import type * as HttpClientResponse from "effect/http/HttpClientResponse"; import type * as HttpServerRequest from "effect/http/HttpServerRequest"; import * as HttpServerResponse from "effect/http/HttpServerResponse"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import { RelayApi } from "@t3tools/contracts/relay"; import * as RelayConfiguration from "../Config.ts"; -import { validateManagedEndpoint, withoutRedirects } from "../environments/EnvironmentConnector.ts"; +import { validateManagedEndpoint } from "../environments/EnvironmentConnector.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; -import * as HookMailbox from "./HookMailbox.ts"; +import * as HookInbox from "./HookInbox.ts"; +import { sendUpstream, TUNNEL_OFFLINE_STATUS } from "./upstream.ts"; export const RELAY_HOOK_PATH_PREFIX = "/v1/hooks/"; -/** Set by the relay on every forward; the environment uses it as the delivery id. */ -export const RELAY_DELIVERY_ID_HEADER = "x-t3-relay-delivery-id"; export const RELAY_HOOK_MAX_BODY_BYTES = 1_048_576; -export const RELAY_HOOK_UPSTREAM_TIMEOUT_MS = 8_000; export const RELAY_HOOK_RATE_LIMIT = { limit: 60, periodSeconds: 60 } as const; const DROPPED_REQUEST_HEADERS = new Set([ @@ -43,10 +38,6 @@ const DROPPED_REQUEST_HEADERS = new Set([ "x-t3-relay-delivery-id", ]); const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-"]; -// Cloudflare answers 530 when the tunnel for a hostname has no connected origin. -const TUNNEL_OFFLINE_STATUS = 530; -/** The environment answers with a small JSON status; anything past this is cut off. */ -const MAX_RESPONSE_BYTES = 64 * 1024; export const isRelayHookPath = (url: string): boolean => url.startsWith(RELAY_HOOK_PATH_PREFIX); @@ -103,7 +94,6 @@ export class HookForwarder extends Context.Service< >()("t3code-relay/hooks/HookForwarder") {} class HookBodyTooLarge extends Schema.TaggedError()("HookBodyTooLarge", {}) {} -class ResponseTooLarge extends Schema.TaggedError()("ResponseTooLarge", {}) {} const errorResponse = (status: number, error: string, headers?: Record) => HttpServerResponse.jsonUnsafe({ error }, { status, ...(headers ? { headers } : {}) }); @@ -163,20 +153,18 @@ function forwardedHeaders(headers: Readonly>): Record request.source instanceof Request && request.source.body === null; -/** Collects a byte stream, failing with `tooLarge` once it passes `maxBytes` rather than buffering it all. */ -const collectCapped = ( - stream: Stream.Stream, - maxBytes: number, - tooLarge: () => E2, -) => +const readCappedBody = (request: HttpServerRequest.HttpServerRequest) => Effect.suspend(() => { + if (hasNoBody(request)) { + return Effect.succeed(new Uint8Array(0)); + } const chunks: Array = []; let total = 0; - return stream.pipe( + return request.stream.pipe( Stream.runForEach((chunk) => { total += chunk.length; - if (total > maxBytes) { - return Effect.fail(tooLarge()); + if (total > RELAY_HOOK_MAX_BODY_BYTES) { + return Effect.fail(new HookBodyTooLarge()); } chunks.push(chunk); return Effect.void; @@ -193,19 +181,30 @@ const collectCapped = ( ); }); -const readCappedBody = (request: HttpServerRequest.HttpServerRequest) => - hasNoBody(request) - ? Effect.succeed(new Uint8Array(0)) - : collectCapped(request.stream, RELAY_HOOK_MAX_BODY_BYTES, () => new HookBodyTooLarge()); - -const readCappedResponse = (response: HttpClientResponse.HttpClientResponse) => - collectCapped(response.stream, MAX_RESPONSE_BYTES, () => new ResponseTooLarge()).pipe( - // A response without a body, such as a redirect, has no stream at all. - Effect.catchIf( - (error) => error._tag === "HttpClientError" && error.reason._tag === "EmptyBodyError", - () => Effect.succeed(new Uint8Array(0)), - ), - ); +/** + * The environment's ready managed endpoint, across every user that linked it, + * with whether it opted in to holding webhooks while offline. + */ +export const resolveHookEndpoint = Effect.fn("relay.hooks.resolve_endpoint")(function* ( + environmentId: string, +) { + const links = yield* EnvironmentLinks.EnvironmentLinks; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const settings = yield* RelayConfiguration.RelayConfiguration; + const candidates = yield* links.findActiveManagedForEnvironment({ environmentId }); + for (const link of candidates) { + const allocation = yield* allocations.get({ userId: link.userId, environmentId }); + const result = validateManagedEndpoint({ + link, + allocation, + baseDomain: settings.managedEndpointBaseDomain, + }); + if (Result.isSuccess(result)) { + return { ...result.success, holdWhileOffline: link.holdWebhooksWhileOffline }; + } + } + return null; +}); const make = Effect.gen(function* () { const links = yield* EnvironmentLinks.EnvironmentLinks; @@ -213,27 +212,9 @@ const make = Effect.gen(function* () { const settings = yield* RelayConfiguration.RelayConfiguration; const httpClient = yield* HttpClient.HttpClient; const rateLimiter = yield* HookRateLimiter; - const mailbox = yield* HookMailbox.HookMailbox; + const inbox = yield* HookInbox.HookInbox; const crypto = yield* Crypto.Crypto; - const resolveEndpoint = Effect.fn("relay.hooks.resolve_endpoint")(function* ( - environmentId: string, - ) { - const candidates = yield* links.findActiveManagedForEnvironment({ environmentId }); - for (const link of candidates) { - const allocation = yield* allocations.get({ userId: link.userId, environmentId }); - const result = validateManagedEndpoint({ - link, - allocation, - baseDomain: settings.managedEndpointBaseDomain, - }); - if (Result.isSuccess(result)) { - return { ...result.success, holdWhileOffline: link.holdWebhooksWhileOffline }; - } - } - return null; - }); - const handle = Effect.fn("relay.hooks.forward")(function* ( request: HttpServerRequest.HttpServerRequest, ) { @@ -259,7 +240,10 @@ const make = Effect.gen(function* () { return errorResponse(413, "payload_too_large"); } - const endpoint = yield* resolveEndpoint(parsed.environmentId).pipe( + const endpoint = yield* resolveHookEndpoint(parsed.environmentId).pipe( + Effect.provideService(EnvironmentLinks.EnvironmentLinks, links), + Effect.provideService(ManagedEndpointAllocations.ManagedEndpointAllocations, allocations), + Effect.provideService(RelayConfiguration.RelayConfiguration, settings), Effect.catch((error) => Effect.logWarning("Failed to resolve hook endpoint", { environmentId: parsed.environmentId, @@ -285,15 +269,16 @@ const make = Effect.gen(function* () { return errorResponse(400, "invalid_body"); } - const baseUrl = endpoint.httpBaseUrl.endsWith("/") - ? endpoint.httpBaseUrl - : `${endpoint.httpBaseUrl}/`; - // One id per attempt, so a request that reached the environment before a - // timeout and is later replayed from the mailbox dispatches only once. - const deliveryId = yield* crypto.randomUUIDv4.pipe(Effect.orDie); - const headers: Record = { - ...forwardedHeaders(request.headers), - [RELAY_DELIVERY_ID_HEADER]: deliveryId, + // One id per request, so a request that reached the environment before a + // timeout and is later delivered from the inbox runs only once. + const hook = { + id: yield* crypto.randomUUIDv4.pipe(Effect.orDie), + method: request.method, + rawHookId: parsed.rawHookId, + rawToken: parsed.rawToken, + query: parsed.search.replace(/^\?/, ""), + headers: forwardedHeaders(request.headers), + body: body.success, }; // Held only for environments that opted in; otherwise the relay is a plain proxy. const holdOrFail = (status: 503 | 504, error: string) => @@ -302,17 +287,11 @@ const make = Effect.gen(function* () { yield* outcome(error); return errorResponse(status, error); } - const stored = yield* mailbox - .enqueue({ - id: deliveryId, + const stored = yield* inbox + .hold({ environmentId: parsed.environmentId, - receivedAt: DateTime.formatIso(yield* DateTime.now), - method: request.method, - rawHookId: parsed.rawHookId, - rawToken: parsed.rawToken, - query: parsed.search.replace(/^\?/, ""), - headers: forwardedHeaders(request.headers), - body: body.success, + baseUrl: endpoint.httpBaseUrl, + hook: { ...hook, receivedAt: DateTime.formatIso(yield* DateTime.now) }, }) .pipe( Effect.catch((cause) => @@ -327,37 +306,15 @@ const make = Effect.gen(function* () { return errorResponse(status, error); } if (!stored) { - yield* outcome("mailbox_full"); - return errorResponse(503, "mailbox_full"); + yield* outcome("inbox_full"); + return errorResponse(503, "inbox_full"); } yield* outcome("held"); return HttpServerResponse.jsonUnsafe({ queued: true }, { status: 202 }); }); - let upstreamRequest = HttpClientRequest.make( - request.method as "GET" | "POST" | "PUT" | "PATCH", - )(`${baseUrl}api/hooks/${parsed.rawHookId}/${parsed.rawToken}${parsed.search}`, { headers }); - if (request.method !== "GET") { - upstreamRequest = HttpClientRequest.bodyUint8Array( - upstreamRequest, - body.success, - headers["content-type"], - ); - } - const upstream = yield* httpClient.execute(upstreamRequest).pipe( - Effect.flatMap((response) => - readCappedResponse(response).pipe( - Effect.map((body) => ({ - status: response.status, - contentType: response.headers["content-type"], - body, - })), - ), - ), - withoutRedirects, - // The client span would record url.full, which carries the token. - Effect.provideService(HttpClient.TracerDisabledWhen, () => true), - Effect.timeoutOption(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)), + const upstream = yield* sendUpstream(endpoint.httpBaseUrl, hook).pipe( + Effect.provideService(HttpClient.HttpClient, httpClient), Effect.result, ); if (Result.isFailure(upstream)) { diff --git a/infra/relay/src/hooks/HookInbox.ts b/infra/relay/src/hooks/HookInbox.ts new file mode 100644 index 000000000000..61f332612dfe --- /dev/null +++ b/infra/relay/src/hooks/HookInbox.ts @@ -0,0 +1,40 @@ +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import type { HeldHook } from "./HookInboxStore.ts"; + +export class HookInboxError extends Schema.TaggedError()("HookInboxError", { + operation: Schema.Literals(["hold", "wake", "clear"]), + environmentId: Schema.String, + cause: Schema.Defect(), +}) { + override get message(): string { + return `Hook inbox '${this.operation}' failed for environment '${this.environmentId}'`; + } +} + +/** + * Webhook requests held for environments that opted in, while they are + * offline. Each environment's requests live in its own `HookInboxObject`, + * which delivers them itself once the environment is back. + */ +export class HookInbox extends Context.Service< + HookInbox, + { + /** False when the environment's inbox is full and nothing was stored. */ + readonly hold: (input: { + readonly environmentId: string; + readonly baseUrl: string; + readonly hook: HeldHook; + }) => Effect.Effect; + /** Delivers what is waiting now; true when anything was waiting. */ + readonly wake: (input: { + readonly environmentId: string; + readonly baseUrl: string; + }) => Effect.Effect; + readonly clear: (input: { + readonly environmentId: string; + }) => Effect.Effect; + } +>()("t3code-relay/hooks/HookInbox") {} diff --git a/infra/relay/src/hooks/HookInboxObject.ts b/infra/relay/src/hooks/HookInboxObject.ts new file mode 100644 index 000000000000..afe1f5c461b1 --- /dev/null +++ b/infra/relay/src/hooks/HookInboxObject.ts @@ -0,0 +1,103 @@ +import * as SqliteClient from "@effect/sql-sqlite-do/SqliteClient"; +import type * as Alchemy from "alchemy"; +import * as Cloudflare from "alchemy/Cloudflare"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Result from "effect/Result"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; + +import * as HookInboxStore from "./HookInboxStore.ts"; +import { sendUpstream, TUNNEL_OFFLINE_STATUS } from "./upstream.ts"; + +/** Statuses that mean the environment did not get to the request; it is tried again later. */ +const RETRY_STATUSES = new Set([429, 502, 503, 504, TUNNEL_OFFLINE_STATUS]); +/** When a run itself fails, the next one is tried after this long. */ +const RUN_FAILURE_RETRY_MS = 60_000; + +type Call = Effect.Effect; + +export interface HookInboxObjectShape { + /** Holds a request for `baseUrl`; false when the inbox is full and nothing was stored. */ + readonly hold: (hook: HookInboxStore.HeldHook, baseUrl: string) => Call; + /** The environment is back at `baseUrl`: deliver what is waiting now. */ + readonly wake: (baseUrl: string) => Call; + readonly clear: () => Call; +} + +/** + * One per environment, addressed by environment id. Holds webhook requests + * the environment could not take, in SQLite, and pushes them back through + * its tunnel from the alarm, oldest first, backing off while it stays away. + */ +export class HookInboxObject extends Cloudflare.DurableObject< + HookInboxObject, + HookInboxObjectShape +>()("HookInboxObject") {} + +const deliver = (baseUrl: string, hook: HookInboxStore.HeldHook) => + sendUpstream(baseUrl, hook).pipe( + Effect.result, + Effect.map((result): HookInboxStore.DeliveryOutcome => { + // Unreachable or timed out: a timeout may still have run it, and the + // environment drops a delivery id it has already seen. + if (Result.isFailure(result) || Option.isNone(result.success)) return "retry"; + return RETRY_STATUSES.has(result.success.value.status) ? "retry" : "delivered"; + }), + Effect.provide(FetchHttpClient.layer), + ); + +export const HookInboxObjectLive = HookInboxObject.make( + Effect.gen(function* () { + const state = yield* Cloudflare.DurableObjectState; + // The init phase returns the per-instance Effect, which alchemy runs once + // per object; only that inner Effect may touch storage. + // @effect-diagnostics-next-line returnEffectInGen:off + return Effect.gen(function* () { + const sql = SqliteClient.layer({ storage: state.raw.storage }); + const run = (effect: Effect.Effect) => + effect.pipe(Effect.provide(sql), Effect.orDie); + yield* run(HookInboxStore.migrate); + + /** Moves the alarm to `at`, unless one is already due sooner. */ + const scheduleBy = (at: number) => + Effect.gen(function* () { + const current = yield* state.storage.getAlarm(); + if (current === null || current > at) yield* state.storage.setAlarm(at); + }); + + return { + hold: (hook: HookInboxStore.HeldHook, baseUrl: string) => + Effect.gen(function* () { + const dueAt = yield* run(HookInboxStore.hold(hook, baseUrl)); + if (dueAt === null) return false; + yield* scheduleBy(dueAt); + return true; + }), + wake: (baseUrl: string) => + Effect.gen(function* () { + const pending = yield* run(HookInboxStore.wake(baseUrl)); + if (pending) yield* state.storage.setAlarm(yield* Clock.currentTimeMillis); + return pending; + }), + clear: () => + Effect.gen(function* () { + yield* run(HookInboxStore.clear); + yield* state.storage.deleteAlarm(); + }), + alarm: () => + run(HookInboxStore.deliverDue(deliver)).pipe( + Effect.flatMap((nextAt) => + nextAt === null ? Effect.void : state.storage.setAlarm(nextAt), + ), + Effect.catchCause((cause) => + Effect.logWarning("Held webhook delivery run failed", { cause }).pipe( + Effect.andThen(Clock.currentTimeMillis), + Effect.flatMap((now) => state.storage.setAlarm(now + RUN_FAILURE_RETRY_MS)), + ), + ), + ), + }; + }); + }), +); diff --git a/infra/relay/src/hooks/HookInboxStore.test.ts b/infra/relay/src/hooks/HookInboxStore.test.ts new file mode 100644 index 000000000000..37c85c983db2 --- /dev/null +++ b/infra/relay/src/hooks/HookInboxStore.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Clock from "effect/Clock"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as TestClock from "effect/testing/TestClock"; + +import * as HookInboxStore from "./HookInboxStore.ts"; + +const BASE_URL = "https://env.example.test/"; + +const hook = (id: string, overrides: Partial = {}) => + Effect.map(Clock.currentTimeMillis, (now): HookInboxStore.HeldHook => ({ + id, + receivedAt: DateTime.formatIso(DateTime.makeUnsafe(now)), + method: "POST", + rawHookId: "hook-1", + rawToken: "tok%2Fen", + query: "a=1", + headers: { "content-type": "application/json", "x-sig": "s" }, + body: new Uint8Array([0, 255, 10]), + ...overrides, + })); + +/** Delivers with `outcome` per request and records what the environment was sent. */ +const deliverer = (outcome: (hook: HookInboxStore.HeldHook) => HookInboxStore.DeliveryOutcome) => { + const sent: Array<{ readonly baseUrl: string; readonly hook: HookInboxStore.HeldHook }> = []; + const send = (baseUrl: string, held: HookInboxStore.HeldHook) => + Effect.sync(() => { + sent.push({ baseUrl, hook: held }); + return outcome(held); + }); + return { sent, send }; +}; + +const withInbox = (effect: Effect.Effect) => + HookInboxStore.migrate.pipe( + Effect.andThen(effect), + Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" })), + ); + +describe("HookInboxStore", () => { + it.effect("delivers held requests oldest first, byte for byte", () => + withInbox( + Effect.gen(function* () { + const first = yield* hook("first"); + expect(yield* HookInboxStore.hold(first, BASE_URL)).not.toBeNull(); + yield* HookInboxStore.hold(yield* hook("second"), BASE_URL); + const { sent, send } = deliverer(() => "delivered"); + expect(yield* HookInboxStore.deliverDue(send)).toBeNull(); + expect(sent.map((entry) => entry.hook.id)).toEqual(["first", "second"]); + expect(sent[0]).toEqual({ baseUrl: BASE_URL, hook: first }); + // Delivered requests are gone. + expect(yield* HookInboxStore.deliverDue(send)).toBeNull(); + expect(sent).toHaveLength(2); + }), + ), + ); + + it.effect("keeps a request the environment did not take and backs off", () => + withInbox( + Effect.gen(function* () { + yield* HookInboxStore.hold(yield* hook("first"), BASE_URL); + yield* HookInboxStore.hold(yield* hook("second"), BASE_URL); + const offline = deliverer(() => "retry"); + const now = yield* Clock.currentTimeMillis; + // Stops at the first failure, so order is kept. + expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 30_000); + expect(offline.sent.map((entry) => entry.hook.id)).toEqual(["first"]); + expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 60_000); + expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 120_000); + + const online = deliverer(() => "delivered"); + expect(yield* HookInboxStore.deliverDue(online.send)).toBeNull(); + expect(online.sent.map((entry) => entry.hook.id)).toEqual(["first", "second"]); + }), + ), + ); + + it("caps the wait between attempts at 10 minutes", () => { + expect(HookInboxStore.retryDelayMs(1)).toBe(30_000); + expect(HookInboxStore.retryDelayMs(20)).toBe(10 * 60_000); + }); + + it.effect("waking resets the backoff and reports whether anything waits", () => + withInbox( + Effect.gen(function* () { + expect(yield* HookInboxStore.wake(BASE_URL)).toBe(false); + yield* HookInboxStore.hold(yield* hook("first"), "https://old.example.test/"); + const offline = deliverer(() => "retry"); + yield* HookInboxStore.deliverDue(offline.send); + yield* HookInboxStore.deliverDue(offline.send); + + expect(yield* HookInboxStore.wake(BASE_URL)).toBe(true); + const now = yield* Clock.currentTimeMillis; + // Back to the first step, and sent to where the environment is now. + expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 30_000); + expect(offline.sent.at(-1)?.baseUrl).toBe(BASE_URL); + }), + ), + ); + + it.effect("drops requests older than 24 hours", () => + withInbox( + Effect.gen(function* () { + yield* HookInboxStore.hold(yield* hook("old"), BASE_URL); + yield* TestClock.adjust(Duration.hours(1)); + yield* HookInboxStore.hold(yield* hook("new"), BASE_URL); + yield* TestClock.adjust(Duration.minutes(23 * 60 + 1)); + const { sent, send } = deliverer(() => "delivered"); + yield* HookInboxStore.deliverDue(send); + expect(sent.map((entry) => entry.hook.id)).toEqual(["new"]); + }), + ), + ); + + it.effect("refuses requests past the per-hook cap", () => + withInbox( + Effect.gen(function* () { + for (let index = 0; index < HookInboxStore.HOOK_INBOX_MAX_PER_HOOK; index++) { + expect(yield* HookInboxStore.hold(yield* hook(`a-${index}`), BASE_URL)).not.toBeNull(); + } + expect(yield* HookInboxStore.hold(yield* hook("one-too-many"), BASE_URL)).toBeNull(); + // Another hook still has room. + const other = yield* hook("other", { rawHookId: "hook-2" }); + expect(yield* HookInboxStore.hold(other, BASE_URL)).not.toBeNull(); + }), + ), + ); + + it.effect("refuses a request that would pass the byte cap", () => + withInbox( + Effect.gen(function* () { + const big = new Uint8Array(HookInboxStore.HOOK_INBOX_MAX_BYTES - 10); + expect( + yield* HookInboxStore.hold(yield* hook("big", { body: big }), BASE_URL), + ).not.toBeNull(); + const small = new Uint8Array(11); + expect( + yield* HookInboxStore.hold( + yield* hook("small", { rawHookId: "x", body: small }), + BASE_URL, + ), + ).toBeNull(); + }), + ), + ); + + it.effect("stores a request id once", () => + withInbox( + Effect.gen(function* () { + const first = yield* hook("same"); + expect(yield* HookInboxStore.hold(first, BASE_URL)).not.toBeNull(); + expect(yield* HookInboxStore.hold(first, BASE_URL)).toBeNull(); + }), + ), + ); + + it.effect("clear drops everything held", () => + withInbox( + Effect.gen(function* () { + yield* HookInboxStore.hold(yield* hook("first"), BASE_URL); + yield* HookInboxStore.clear; + const { sent, send } = deliverer(() => "delivered"); + expect(yield* HookInboxStore.deliverDue(send)).toBeNull(); + expect(sent).toHaveLength(0); + }), + ), + ); +}); diff --git a/infra/relay/src/hooks/HookInboxStore.ts b/infra/relay/src/hooks/HookInboxStore.ts new file mode 100644 index 000000000000..b5c086facac6 --- /dev/null +++ b/infra/relay/src/hooks/HookInboxStore.ts @@ -0,0 +1,215 @@ +import * as Clock from "effect/Clock"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; + +/** + * The storage and delivery rules of one environment's held webhook requests. + * Runs inside the environment's `HookInboxObject` Durable Object against its SQLite + * storage; written against `SqlClient` so tests can run it on any SQLite. + */ + +/** How long a held request waits for its environment. */ +const HOOK_INBOX_TTL_MS = 24 * 60 * 60 * 1000; +/** Requests one environment may have waiting at once. */ +const HOOK_INBOX_MAX_REQUESTS = 1_000; +/** Body bytes one environment may have waiting at once. */ +export const HOOK_INBOX_MAX_BYTES = 50 * 1_048_576; +/** + * Requests one hook may have waiting at once. The relay cannot check tokens, + * so this keeps junk sent to one hook id from crowding out the others. + */ +export const HOOK_INBOX_MAX_PER_HOOK = 100; +/** Requests pushed per alarm run; the next run starts right away while more wait. */ +const DELIVERIES_PER_RUN = 20; +const FIRST_RETRY_MS = 30_000; +const MAX_RETRY_MS = 10 * 60_000; + +export interface HeldHook { + readonly id: string; + readonly receivedAt: string; + readonly method: string; + /** Path segments exactly as the sender sent them; the environment decodes them. */ + readonly rawHookId: string; + readonly rawToken: string; + /** Without the leading `?`. */ + readonly query: string; + readonly headers: Readonly>; + readonly body: Uint8Array; +} + +/** `retry` keeps the request and backs off; `delivered` deletes it. */ +export type DeliveryOutcome = "delivered" | "retry"; + +/** 30 s, 1 min, 2 min, ... up to 10 min between attempts while the environment stays away. */ +export const retryDelayMs = (failures: number) => + Math.min(FIRST_RETRY_MS * 2 ** Math.max(0, failures - 1), MAX_RETRY_MS); + +const HeadersJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)); +const encodeHeaders = Schema.encodeSync(HeadersJson); +const decodeHeaders = Schema.decodeUnknownSync(HeadersJson); + +interface HeldHookRow { + readonly id: string; + readonly received_at: string; + readonly method: string; + readonly raw_hook_id: string; + readonly raw_token: string; + readonly query: string; + readonly headers: string; + readonly body: Uint8Array; +} + +interface TargetRow { + readonly base_url: string; + readonly failures: number; +} + +const iso = (epochMillis: number) => DateTime.formatIso(DateTime.makeUnsafe(epochMillis)); + +export const migrate = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql` + CREATE TABLE IF NOT EXISTS held_hooks ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT NOT NULL UNIQUE, + received_at TEXT NOT NULL, + method TEXT NOT NULL, + raw_hook_id TEXT NOT NULL, + raw_token TEXT NOT NULL, + query TEXT NOT NULL, + headers TEXT NOT NULL, + body BLOB NOT NULL + ) + `; + yield* sql`CREATE INDEX IF NOT EXISTS held_hooks_hook ON held_hooks (raw_hook_id)`; + // Where to push, and how many attempts in a row have failed. One row. + yield* sql` + CREATE TABLE IF NOT EXISTS held_hooks_target ( + id INTEGER PRIMARY KEY CHECK (id = 1), + base_url TEXT NOT NULL, + failures INTEGER NOT NULL DEFAULT 0 + ) + `; +}); + +const setTarget = (baseUrl: string, options: { readonly resetFailures: boolean }) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* options.resetFailures + ? sql` + INSERT INTO held_hooks_target (id, base_url) VALUES (1, ${baseUrl}) + ON CONFLICT (id) DO UPDATE SET base_url = excluded.base_url, failures = 0 + ` + : sql` + INSERT INTO held_hooks_target (id, base_url) VALUES (1, ${baseUrl}) + ON CONFLICT (id) DO UPDATE SET base_url = excluded.base_url + `; + }); + +const readTarget = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql`SELECT base_url, failures FROM held_hooks_target WHERE id = 1`; + return rows[0] ?? null; +}); + +const setFailures = (failures: number) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`UPDATE held_hooks_target SET failures = ${failures} WHERE id = 1`; + }); + +const hasPending = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ readonly id: string }>`SELECT id FROM held_hooks LIMIT 1`; + return rows.length > 0; +}); + +/** + * Stores a request for later delivery to `baseUrl`. Returns the time the + * first attempt is due, or null when the inbox is full and nothing was stored. + */ +export const hold = Effect.fn("HookInboxStore.hold")(function* (hook: HeldHook, baseUrl: string) { + const sql = yield* SqlClient.SqlClient; + // One statement, so the caps hold however many requests arrive at once. + const inserted = yield* sql<{ readonly id: string }>` + INSERT INTO held_hooks (id, received_at, method, raw_hook_id, raw_token, query, headers, body) + SELECT ${hook.id}, ${hook.receivedAt}, ${hook.method}, ${hook.rawHookId}, ${hook.rawToken}, + ${hook.query}, ${encodeHeaders(hook.headers)}, ${hook.body} + WHERE (SELECT count(*) FROM held_hooks) < ${HOOK_INBOX_MAX_REQUESTS} + AND (SELECT count(*) FROM held_hooks WHERE raw_hook_id = ${hook.rawHookId}) + < ${HOOK_INBOX_MAX_PER_HOOK} + AND (SELECT coalesce(sum(length(body)), 0) FROM held_hooks) + ${hook.body.byteLength} + <= ${HOOK_INBOX_MAX_BYTES} + ON CONFLICT (id) DO NOTHING + RETURNING id + `; + if (inserted.length === 0) return null; + yield* setTarget(baseUrl, { resetFailures: false }); + const target = yield* readTarget; + return (yield* Clock.currentTimeMillis) + retryDelayMs(Math.max(1, target?.failures ?? 0)); +}); + +/** + * The environment is reachable again at `baseUrl`. Returns whether anything + * is waiting, so the caller can deliver right away. + */ +export const wake = Effect.fn("HookInboxStore.wake")(function* (baseUrl: string) { + if (!(yield* hasPending)) return false; + yield* setTarget(baseUrl, { resetFailures: true }); + return true; +}); + +export const clear = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`DELETE FROM held_hooks`; + yield* sql`DELETE FROM held_hooks_target`; +}); + +/** + * Pushes the oldest held requests to the environment, one at a time, and + * stops at the first one that has to be retried. Drops requests older than + * the TTL. Returns when the next run is due, or null when nothing is left. + */ +export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( + send: (baseUrl: string, hook: HeldHook) => Effect.Effect, +) { + const sql = yield* SqlClient.SqlClient; + const startedAt = yield* Clock.currentTimeMillis; + yield* sql`DELETE FROM held_hooks WHERE received_at < ${iso(startedAt - HOOK_INBOX_TTL_MS)}`; + const target = yield* readTarget; + const batch = yield* sql` + SELECT id, received_at, method, raw_hook_id, raw_token, query, headers, body + FROM held_hooks ORDER BY seq LIMIT ${DELIVERIES_PER_RUN} + `; + if (batch.length === 0 || target === null) { + if (target === null) yield* sql`DELETE FROM held_hooks`; + return null; + } + + let failures = target.failures; + for (const row of batch) { + const outcome = yield* send(target.base_url, { + id: row.id, + receivedAt: row.received_at, + method: row.method, + rawHookId: row.raw_hook_id, + rawToken: row.raw_token, + query: row.query, + headers: decodeHeaders(row.headers), + body: row.body, + }); + if (outcome === "retry") { + failures += 1; + yield* setFailures(failures); + return (yield* Clock.currentTimeMillis) + retryDelayMs(failures); + } + yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`; + if (failures !== 0) { + failures = 0; + yield* setFailures(0); + } + } + return (yield* hasPending) ? yield* Clock.currentTimeMillis : null; +}); diff --git a/infra/relay/src/hooks/HookMailbox.test.ts b/infra/relay/src/hooks/HookMailbox.test.ts deleted file mode 100644 index 9cec3624ecba..000000000000 --- a/infra/relay/src/hooks/HookMailbox.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { PgDialect } from "drizzle-orm/pg-core"; -import type { SQL } from "drizzle-orm"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; - -import * as RelayDb from "../db.ts"; -import { relayHookMailbox } from "../persistence/schema.ts"; -import * as HookMailbox from "./HookMailbox.ts"; - -const dialect = new PgDialect(); -const render = (condition: unknown) => dialect.sqlToQuery(condition as SQL); - -const layerWithDb = (db: RelayDb.RelayDb["Service"]) => - HookMailbox.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db))); - -const hook: HookMailbox.HeldHook = { - id: "delivery-1", - environmentId: "environment-1", - receivedAt: "2026-10-04T10:00:00.000Z", - method: "POST", - rawHookId: "task", - rawToken: "token", - query: "", - headers: { "content-type": "application/json" }, - body: new Uint8Array([1, 2, 3]), -}; - -describe("HookMailbox", () => { - it.effect("stores a request with a 24 hour expiry while there is room", () => { - const inserted: Array> = []; - const db = { - select: () => ({ - from: () => ({ where: () => Effect.succeed([{ value: 3 }]) }), - }), - insert: (table: unknown) => { - expect(table).toBe(relayHookMailbox); - return { - values: (values: Record) => { - inserted.push(values); - return { onConflictDoNothing: () => Effect.void }; - }, - }; - }, - } as unknown as RelayDb.RelayDb["Service"]; - - return Effect.gen(function* () { - const mailbox = yield* HookMailbox.HookMailbox; - expect(yield* mailbox.enqueue(hook)).toBe(true); - expect(inserted[0]?.expiresAt).toBe("2026-10-05T10:00:00.000Z"); - expect([...(inserted[0]?.body as Uint8Array)]).toEqual([1, 2, 3]); - }).pipe(Effect.provide(layerWithDb(db))); - }); - - it.effect("refuses a request once the environment's mailbox is full", () => { - let insertedAny = false; - const db = { - select: () => ({ - from: () => ({ - where: () => Effect.succeed([{ value: HookMailbox.HOOK_MAILBOX_MAX_PER_ENVIRONMENT }]), - }), - }), - insert: () => { - insertedAny = true; - return { values: () => ({ onConflictDoNothing: () => Effect.void }) }; - }, - } as unknown as RelayDb.RelayDb["Service"]; - - return Effect.gen(function* () { - const mailbox = yield* HookMailbox.HookMailbox; - expect(yield* mailbox.enqueue(hook)).toBe(false); - expect(insertedAny).toBe(false); - }).pipe(Effect.provide(layerWithDb(db))); - }); - - it.effect("lists and acks only the calling environment's requests", () => { - const conditions: Array = []; - let limit = 0; - const db = { - select: () => ({ - from: () => ({ - where: (condition: unknown) => { - conditions.push(condition); - return { - orderBy: () => ({ - limit: (value: number) => { - limit = value; - return Effect.succeed([]); - }, - }), - }; - }, - }), - }), - delete: () => ({ - where: (condition: unknown) => { - conditions.push(condition); - return { returning: () => Effect.succeed([{ id: "delivery-1" }]) }; - }, - }), - } as unknown as RelayDb.RelayDb["Service"]; - - return Effect.gen(function* () { - const mailbox = yield* HookMailbox.HookMailbox; - yield* mailbox.listPending({ environmentId: "environment-1", limit: 1_000 }); - expect(limit).toBe(HookMailbox.HOOK_MAILBOX_MAX_PULL); - expect(yield* mailbox.ack({ environmentId: "environment-1", ids: ["delivery-1"] })).toBe(1); - for (const condition of conditions) { - const query = render(condition); - expect(query.sql).toContain('"relay_hook_mailbox"."environment_id" = $1'); - expect(query.params[0]).toBe("environment-1"); - } - }).pipe(Effect.provide(layerWithDb(db))); - }); - - it.effect("acks nothing for an empty id list", () => { - const db = { - delete: () => { - throw new Error("no delete expected"); - }, - } as unknown as RelayDb.RelayDb["Service"]; - - return Effect.gen(function* () { - const mailbox = yield* HookMailbox.HookMailbox; - expect(yield* mailbox.ack({ environmentId: "environment-1", ids: [] })).toBe(0); - }).pipe(Effect.provide(layerWithDb(db))); - }); -}); diff --git a/infra/relay/src/hooks/HookMailbox.ts b/infra/relay/src/hooks/HookMailbox.ts deleted file mode 100644 index 22d117ee0360..000000000000 --- a/infra/relay/src/hooks/HookMailbox.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { and, asc, count, eq, inArray, lt } from "drizzle-orm"; -import * as Context from "effect/Context"; -import * as DateTime from "effect/DateTime"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; -import * as Schema from "effect/Schema"; - -import * as RelayDb from "../db.ts"; -import { relayHookMailbox } from "../persistence/schema.ts"; - -/** How long a held request waits for its environment. */ -export const HOOK_MAILBOX_TTL_MS = 24 * 60 * 60 * 1000; -/** Requests one environment may have waiting at once. */ -export const HOOK_MAILBOX_MAX_PER_ENVIRONMENT = 500; -/** Requests returned per pull; bodies are up to 1 MiB each. */ -export const HOOK_MAILBOX_MAX_PULL = 10; - -export class HookMailboxPersistenceError extends Schema.TaggedError()( - "HookMailboxPersistenceError", - { - operation: Schema.Literals(["enqueue", "list", "ack", "prune", "clear"]), - environmentId: Schema.optional(Schema.String), - cause: Schema.Defect(), - }, -) { - override get message(): string { - return `Hook mailbox '${this.operation}' failed${this.environmentId ? ` for environment '${this.environmentId}'` : ""}`; - } -} - -export interface HeldHook { - readonly id: string; - readonly environmentId: string; - readonly receivedAt: string; - readonly method: string; - readonly rawHookId: string; - readonly rawToken: string; - readonly query: string; - readonly headers: Readonly>; - readonly body: Uint8Array; -} - -export class HookMailbox extends Context.Service< - HookMailbox, - { - /** Stores a request; returns false when the environment's mailbox is full. */ - readonly enqueue: (hook: HeldHook) => Effect.Effect; - /** Oldest first, never another environment's requests. */ - readonly listPending: (input: { - readonly environmentId: string; - readonly limit: number; - }) => Effect.Effect, HookMailboxPersistenceError>; - readonly ack: (input: { - readonly environmentId: string; - readonly ids: ReadonlyArray; - }) => Effect.Effect; - readonly pruneExpired: (input: { - readonly now: string; - }) => Effect.Effect; - /** Deletes everything held for an environment, used when it is unlinked. */ - readonly clearEnvironment: (input: { - readonly environmentId: string; - }) => Effect.Effect; - } ->()("t3code-relay/hooks/HookMailbox") {} - -export const make = Effect.gen(function* () { - const db = yield* RelayDb.RelayDb; - const fail = - (operation: HookMailboxPersistenceError["operation"], environmentId?: string) => - (cause: unknown) => - new HookMailboxPersistenceError({ - operation, - ...(environmentId === undefined ? {} : { environmentId }), - cause, - }); - - return HookMailbox.of({ - enqueue: Effect.fn("relay.hook_mailbox.enqueue")(function* (hook) { - yield* Effect.annotateCurrentSpan({ "relay.environment_id": hook.environmentId }); - const [held] = yield* db - .select({ value: count() }) - .from(relayHookMailbox) - .where(eq(relayHookMailbox.environmentId, hook.environmentId)) - .pipe(Effect.mapError(fail("enqueue", hook.environmentId))); - if ((held?.value ?? 0) >= HOOK_MAILBOX_MAX_PER_ENVIRONMENT) return false; - const expiresAt = DateTime.formatIso( - DateTime.add(DateTime.makeUnsafe(hook.receivedAt), { milliseconds: HOOK_MAILBOX_TTL_MS }), - ); - yield* db - .insert(relayHookMailbox) - .values({ - id: hook.id, - environmentId: hook.environmentId, - receivedAt: hook.receivedAt, - expiresAt, - method: hook.method, - rawHookId: hook.rawHookId, - rawToken: hook.rawToken, - query: hook.query, - headers: { ...hook.headers }, - body: Buffer.from(hook.body), - }) - .onConflictDoNothing() - .pipe(Effect.mapError(fail("enqueue", hook.environmentId))); - return true; - }), - - listPending: Effect.fn("relay.hook_mailbox.list_pending")(function* (input) { - yield* Effect.annotateCurrentSpan({ "relay.environment_id": input.environmentId }); - const rows = yield* db - .select() - .from(relayHookMailbox) - .where(eq(relayHookMailbox.environmentId, input.environmentId)) - .orderBy(asc(relayHookMailbox.receivedAt), asc(relayHookMailbox.id)) - .limit(Math.max(1, Math.min(input.limit, HOOK_MAILBOX_MAX_PULL))) - .pipe(Effect.mapError(fail("list", input.environmentId))); - return rows.map((row) => ({ - id: row.id, - environmentId: row.environmentId, - receivedAt: row.receivedAt, - method: row.method, - rawHookId: row.rawHookId, - rawToken: row.rawToken, - query: row.query, - headers: row.headers, - body: new Uint8Array(row.body), - })); - }), - - ack: Effect.fn("relay.hook_mailbox.ack")(function* (input) { - yield* Effect.annotateCurrentSpan({ "relay.environment_id": input.environmentId }); - if (input.ids.length === 0) return 0; - const deleted = yield* db - .delete(relayHookMailbox) - .where( - and( - eq(relayHookMailbox.environmentId, input.environmentId), - inArray(relayHookMailbox.id, [...input.ids]), - ), - ) - .returning({ id: relayHookMailbox.id }) - .pipe(Effect.mapError(fail("ack", input.environmentId))); - return deleted.length; - }), - - pruneExpired: Effect.fn("relay.hook_mailbox.prune_expired")(function* (input) { - yield* db - .delete(relayHookMailbox) - .where(lt(relayHookMailbox.expiresAt, input.now)) - .pipe(Effect.mapError(fail("prune"))); - }), - - clearEnvironment: Effect.fn("relay.hook_mailbox.clear_environment")(function* (input) { - yield* db - .delete(relayHookMailbox) - .where(eq(relayHookMailbox.environmentId, input.environmentId)) - .pipe(Effect.mapError(fail("clear", input.environmentId))); - }), - }); -}); - -export const layer = Layer.effect(HookMailbox, make); diff --git a/infra/relay/src/hooks/upstream.ts b/infra/relay/src/hooks/upstream.ts new file mode 100644 index 000000000000..af50c8381136 --- /dev/null +++ b/infra/relay/src/hooks/upstream.ts @@ -0,0 +1,67 @@ +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientRequest from "effect/http/HttpClientRequest"; + +import { withoutRedirects } from "../environments/EnvironmentConnector.ts"; + +/** Set by the relay on every forward; the environment uses it as the delivery id. */ +const RELAY_DELIVERY_ID_HEADER = "x-t3-relay-delivery-id"; +export const RELAY_HOOK_UPSTREAM_TIMEOUT_MS = 8_000; +// Cloudflare answers 530 when the tunnel for a hostname has no connected origin. +export const TUNNEL_OFFLINE_STATUS = 530; + +/** A webhook request as the relay sends it on to the environment. */ +export interface UpstreamHook { + readonly id: string; + readonly method: string; + /** Path segments exactly as the sender sent them; the environment decodes them. */ + readonly rawHookId: string; + readonly rawToken: string; + /** Without the leading `?`. */ + readonly query: string; + readonly headers: Readonly>; + readonly body: Uint8Array; +} + +export interface UpstreamResponse { + readonly status: number; + readonly contentType: string | undefined; + readonly body: Uint8Array; +} + +/** + * Sends a webhook request through the environment's tunnel. Fails when the + * environment cannot be reached, and succeeds with None on timeout. + */ +export const sendUpstream = (baseUrl: string, hook: UpstreamHook) => + Effect.gen(function* () { + const httpClient = yield* HttpClient.HttpClient; + const base = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`; + const headers: Record = { + ...hook.headers, + [RELAY_DELIVERY_ID_HEADER]: hook.id, + }; + let request = HttpClientRequest.make(hook.method as "GET" | "POST" | "PUT" | "PATCH")( + `${base}api/hooks/${hook.rawHookId}/${hook.rawToken}${hook.query ? `?${hook.query}` : ""}`, + { headers }, + ); + if (hook.method !== "GET") { + request = HttpClientRequest.bodyUint8Array(request, hook.body, headers["content-type"]); + } + return yield* httpClient.execute(request).pipe( + Effect.flatMap((response) => + response.arrayBuffer.pipe( + Effect.map((bytes): UpstreamResponse => ({ + status: response.status, + contentType: response.headers["content-type"], + body: new Uint8Array(bytes), + })), + ), + ), + withoutRedirects, + // The client span would record url.full, which carries the token. + Effect.provideService(HttpClient.TracerDisabledWhen, () => true), + Effect.timeoutOption(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)), + ); + }); diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index adf02955ca5a..30a599070190 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -61,7 +61,7 @@ import { import * as RelayConfiguration from "../Config.ts"; import * as RelayDb from "../db.ts"; import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts"; -import * as HookMailbox from "../hooks/HookMailbox.ts"; +import * as HookInbox from "../hooks/HookInbox.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts"; @@ -125,7 +125,7 @@ describe("device listing compatibility", () => { Layer.mock(EnvironmentLinks.EnvironmentLinks, {}), Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}), Layer.mock(RelayDb.RelayTransactions, {}), - Layer.mock(HookMailbox.HookMailbox, {}), + Layer.mock(HookInbox.HookInbox, {}), ), ), Layer.provide( @@ -319,11 +319,12 @@ function relayUnlinkTestLayer(input?: { readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"]; readonly reconcileOrigin?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["reconcileOrigin"]; readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]; - readonly clearMailbox?: HookMailbox.HookMailbox["Service"]["clearEnvironment"]; + readonly clearInbox?: HookInbox.HookInbox["Service"]["clear"]; + readonly activeLinks?: number; }) { return Layer.mergeAll( - Layer.mock(HookMailbox.HookMailbox, { - clearEnvironment: input?.clearMailbox ?? (() => Effect.void), + Layer.mock(HookInbox.HookInbox, { + clear: input?.clearInbox ?? (() => Effect.void), }), Layer.succeed( RelayDb.RelayTransactions, @@ -338,7 +339,14 @@ function relayUnlinkTestLayer(input?: { listDeliveryUsersForEnvironment: () => Effect.die("unused listDeliveryUsersForEnvironment"), listForUser: () => Effect.die("unused listForUser"), getForUser: input?.getForUser ?? (() => Effect.succeed(null)), - findActiveManagedForEnvironment: () => Effect.succeed([]), + findActiveManagedForEnvironment: () => + Effect.succeed( + Array.from({ length: input?.activeLinks ?? 0 }, () => ({ + ...linkedEnvironmentRecord, + userId: "user-2", + holdWebhooksWhileOffline: true, + })), + ), setHoldWebhooksWhileOffline: () => Effect.void, revokeForUser: input?.revokeForUser ?? (() => Effect.succeed(false)), }), @@ -906,6 +914,27 @@ describe("relay environment unlink", () => { ); }); + it.effect("drops held webhooks only once no user links the environment", () => { + const cleared: Array = []; + const unlink = (activeLinks: number) => + unlinkEnvironmentRecord({ userId: "user-1", environmentId: "environment-1" }).pipe( + Effect.provide( + relayUnlinkTestLayer({ + activeLinks, + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + revokeForUser: () => Effect.succeed(true), + clearInbox: ({ environmentId }) => Effect.sync(() => void cleared.push(environmentId)), + }), + ), + ); + return Effect.gen(function* () { + yield* unlink(1); + expect(cleared).toEqual([]); + yield* unlink(0); + expect(cleared).toEqual(["environment-1"]); + }); + }); + it.effect("commits database revocation before deprovisioning the managed endpoint", () => { const calls: Array = []; const deprovisionTarget = { @@ -1226,7 +1255,9 @@ describe("relay routing fallback", () => { publisher, signatures, Layer.mock(EnvironmentLinks.EnvironmentLinks, {}), - Layer.mock(HookMailbox.HookMailbox, {}), + Layer.mock(HookInbox.HookInbox, {}), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, {}), + Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings), ]), ), ), diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index aa514f7b2f9f..34d0ad72796f 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -66,7 +66,8 @@ import * as DpopProofs from "../auth/DpopProofs.ts"; import * as RelayTokens from "../auth/RelayTokens.ts"; import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; -import * as HookMailbox from "../hooks/HookMailbox.ts"; +import * as HookForwarder from "../hooks/HookForwarder.ts"; +import * as HookInbox from "../hooks/HookInbox.ts"; import * as LiveActivities from "../agentActivity/LiveActivities.ts"; import * as RelayConfiguration from "../Config.ts"; import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher.ts"; @@ -520,8 +521,8 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron environmentId: input.environmentId, }); if (remaining.length === 0) { - const mailbox = yield* HookMailbox.HookMailbox; - yield* mailbox.clearEnvironment({ environmentId: input.environmentId }); + const inbox = yield* HookInbox.HookInbox; + yield* inbox.clear({ environmentId: input.environmentId }); } // External teardown cannot share the SQL transaction. Run it only after @@ -1138,7 +1139,9 @@ export const serverApi = HttpApiBuilder.group( const publisher = yield* AgentActivityPublisher.AgentActivityPublisher; const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures; const links = yield* EnvironmentLinks.EnvironmentLinks; - const mailbox = yield* HookMailbox.HookMailbox; + const inbox = yield* HookInbox.HookInbox; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const settings = yield* RelayConfiguration.RelayConfiguration; const requireOwnEnvironment = (environmentId: string) => Effect.gen(function* () { const principal = yield* RelayEnvironmentPrincipal; @@ -1368,36 +1371,23 @@ export const serverApi = HttpApiBuilder.group( }, mapRelayCommonApiErrors("not_authorized")), ) .handle( - "listPendingHooks", - Effect.fn("relay.api.server.listPendingHooks")(function* ({ params, query }) { + "wakeHeldHooks", + Effect.fn("relay.api.server.wakeHeldHooks")(function* ({ params }) { yield* requireOwnEnvironment(params.environmentId); - const held = yield* mailbox.listPending({ - environmentId: params.environmentId, - limit: query.limit ?? HookMailbox.HOOK_MAILBOX_MAX_PULL, - }); - return { - deliveries: held.map((hook) => ({ - id: hook.id, - receivedAt: hook.receivedAt, - method: hook.method, - rawHookId: hook.rawHookId, - rawToken: hook.rawToken, - query: hook.query, - headers: hook.headers, - bodyBase64: Buffer.from(hook.body).toString("base64"), - })), - }; - }, mapRelayCommonApiErrors("not_authorized")), - ) - .handle( - "ackPendingHooks", - Effect.fn("relay.api.server.ackPendingHooks")(function* ({ params, payload }) { - yield* requireOwnEnvironment(params.environmentId); - const deleted = yield* mailbox.ack({ + const endpoint = yield* HookForwarder.resolveHookEndpoint(params.environmentId).pipe( + Effect.provideService(EnvironmentLinks.EnvironmentLinks, links), + Effect.provideService( + ManagedEndpointAllocations.ManagedEndpointAllocations, + allocations, + ), + Effect.provideService(RelayConfiguration.RelayConfiguration, settings), + ); + if (endpoint === null) return { pending: false }; + const pending = yield* inbox.wake({ environmentId: params.environmentId, - ids: payload.ids, + baseUrl: endpoint.httpBaseUrl, }); - return { deleted }; + return { pending }; }, mapRelayCommonApiErrors("not_authorized")), ); }), @@ -1442,7 +1432,7 @@ const RelayCommonPersistenceError = Schema.Union([ LiveActivities.LiveActivityDeliveryMarkPersistenceError, DeliveryAttempts.DeliveryAttemptRecordPersistenceError, EnvironmentLinks.EnvironmentLinkEnvironmentLookupPersistenceError, - HookMailbox.HookMailboxPersistenceError, + HookInbox.HookInboxError, ]); type RelayCommonPersistenceError = typeof RelayCommonPersistenceError.Type; const isRelayCommonPersistenceError = Schema.is(RelayCommonPersistenceError); diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index c86d14923fcd..e24bb1a5c4c0 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -6,7 +6,6 @@ import type { } from "@t3tools/contracts/relay"; import { boolean, - bytea, index, integer, jsonb, @@ -198,28 +197,3 @@ export const relayDpopProofs = pgTable( index("idx_relay_dpop_proofs_expires_at").on(table.expiresAt), ], ); - -/** - * Webhook requests held for an environment that opted in, while it was - * offline. Rows are deleted once the environment acks them, after 24 hours, or - * when the environment is unlinked. - */ -export const relayHookMailbox = pgTable( - "relay_hook_mailbox", - { - id: varchar("id", { length: 36 }).primaryKey(), - environmentId: varchar("environment_id", { length: 191 }).notNull(), - receivedAt: varchar("received_at", { length: 64 }).notNull(), - expiresAt: varchar("expires_at", { length: 64 }).notNull(), - method: varchar("method", { length: 16 }).notNull(), - rawHookId: varchar("raw_hook_id", { length: 512 }).notNull(), - rawToken: varchar("raw_token", { length: 512 }).notNull(), - query: text("query").notNull(), - headers: jsonb("headers").notNull().$type>(), - body: bytea("body").notNull(), - }, - (table) => [ - index("idx_relay_hook_mailbox_environment").on(table.environmentId, table.receivedAt), - index("idx_relay_hook_mailbox_expires").on(table.expiresAt), - ], -); diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index 9b13cb278c8a..a811f622f0f4 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -74,7 +74,8 @@ import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts" import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts"; import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts"; import * as HookForwarder from "./hooks/HookForwarder.ts"; -import * as HookMailbox from "./hooks/HookMailbox.ts"; +import * as HookInbox from "./hooks/HookInbox.ts"; +import { HookInboxObject, HookInboxObjectLive } from "./hooks/HookInboxObject.ts"; const webcryptoLayer = Layer.succeed( Crypto.Crypto, @@ -198,6 +199,7 @@ export const ApiLive = Api.make( period: HookForwarder.RELAY_HOOK_RATE_LIMIT.periodSeconds, }, }); + const hookInboxes = yield* HookInboxObject; // // 3. Runtime layers and app construction @@ -229,6 +231,34 @@ export const ApiLive = Api.make( }).pipe(Effect.map(makeRelayTraceLayer)), ); + // Each environment's held webhook requests live in its own Durable Object. + const inboxCall = + (operation: HookInbox.HookInboxError["operation"], environmentId: string) => + (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), + Effect.catchCause((cause) => + Effect.fail( + new HookInbox.HookInboxError({ + operation, + environmentId, + cause: Cause.squash(cause), + }), + ), + ), + ); + const hookInboxLayer = Layer.succeed(HookInbox.HookInbox, { + hold: ({ environmentId, baseUrl, hook }) => + hookInboxes + .getByName(environmentId) + .hold(hook, baseUrl) + .pipe(inboxCall("hold", environmentId)), + wake: ({ environmentId, baseUrl }) => + hookInboxes.getByName(environmentId).wake(baseUrl).pipe(inboxCall("wake", environmentId)), + clear: ({ environmentId }) => + hookInboxes.getByName(environmentId).clear().pipe(inboxCall("clear", environmentId)), + }); + const runtimeLayer = Layer.empty.pipe( Layer.provideMerge(MobileRegistrations.layer), Layer.provideMerge(AgentActivityPublisher.layer), @@ -270,7 +300,7 @@ export const ApiLive = Api.make( Layer.provideMerge( ApnsDeliveryQueue.layerCloudflareQueues(apnsDeliveryQueueSender, alchemyRuntimeContext), ), - Layer.provideMerge(Layer.mergeAll(AgentActivityRows.layer, Devices.layer, HookMailbox.layer)), + Layer.provideMerge(Layer.mergeAll(AgentActivityRows.layer, Devices.layer, hookInboxLayer)), Layer.provideMerge(EnvironmentCredentials.layer), Layer.provideMerge( Layer.mergeAll( @@ -372,14 +402,6 @@ export const ApiLive = Api.make( ), ), ), - // Held webhook requests expire 24 hours after the relay received them. - Effect.andThen( - Effect.all([HookMailbox.HookMailbox, DateTime.now]).pipe( - Effect.flatMap(([mailbox, now]) => - mailbox.pruneExpired({ now: DateTime.formatIso(now) }), - ), - ), - ), Effect.catchCause((cause) => Cause.hasInterrupts(cause) ? Effect.interrupt @@ -435,6 +457,7 @@ export const ApiLive = Api.make( Layer.provideMerge(Cloudflare.Tunnel.ReadWriteTunnelBinding), Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp), Layer.provideMerge(Cloudflare.Workers.RateLimitBinding), + Layer.provideMerge(HookInboxObjectLive), ), ), ), diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 693c78c34c42..55c189979a36 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -1142,34 +1142,11 @@ export const RelayEnvironmentLinkPreferencesRequest = Schema.Struct({ export type RelayEnvironmentLinkPreferencesRequest = typeof RelayEnvironmentLinkPreferencesRequest.Type; -/** A webhook request the relay held because the environment was offline. */ -export const RelayPendingHook = Schema.Struct({ - id: Schema.String, - receivedAt: Schema.String, - method: Schema.String, - /** Path segments exactly as the sender sent them; the environment decodes them. */ - rawHookId: Schema.String, - rawToken: Schema.String, - query: Schema.String, - headers: Schema.Record(Schema.String, Schema.String), - bodyBase64: Schema.String, +export const RelayWakeHeldHooksResponse = Schema.Struct({ + /** True when the relay was holding requests and has started delivering them. */ + pending: Schema.Boolean, }); -export type RelayPendingHook = typeof RelayPendingHook.Type; - -export const RelayPendingHooksResponse = Schema.Struct({ - deliveries: Schema.Array(RelayPendingHook), -}); -export type RelayPendingHooksResponse = typeof RelayPendingHooksResponse.Type; - -export const RelayAckHooksRequest = Schema.Struct({ - ids: Schema.Array(Schema.String), -}); -export type RelayAckHooksRequest = typeof RelayAckHooksRequest.Type; - -export const RelayAckHooksResponse = Schema.Struct({ - deleted: Schema.Number, -}); -export type RelayAckHooksResponse = typeof RelayAckHooksResponse.Type; +export type RelayWakeHeldHooksResponse = typeof RelayWakeHeldHooksResponse.Type; const RelayServerGroup = HttpApiGroup.make("server") .add( @@ -1216,18 +1193,14 @@ const RelayServerGroup = HttpApiGroup.make("server") error: RelayAuthAndInternalErrors, }, ).annotate(OpenApi.Summary, "Update an environment's link preferences"), - HttpApiEndpoint.get("listPendingHooks", "/v1/environments/:environmentId/hooks/pending", { - params: Schema.Struct({ environmentId: EnvironmentId }), - query: Schema.Struct({ limit: Schema.optional(Schema.NumberFromString) }), - success: RelayPendingHooksResponse, - error: RelayAuthAndInternalErrors, - }).annotate(OpenApi.Summary, "List webhook requests held while the environment was offline"), - HttpApiEndpoint.post("ackPendingHooks", "/v1/environments/:environmentId/hooks/ack", { + HttpApiEndpoint.post("wakeHeldHooks", "/v1/environments/:environmentId/hooks/wake", { params: Schema.Struct({ environmentId: EnvironmentId }), - payload: RelayAckHooksRequest, - success: RelayAckHooksResponse, + success: RelayWakeHeldHooksResponse, error: RelayAuthAndInternalErrors, - }).annotate(OpenApi.Summary, "Delete delivered webhook requests"), + }).annotate( + OpenApi.Summary, + "Deliver webhook requests held while the environment was offline now", + ), ) .annotate(OpenApi.Description, "Environment-authenticated activity publication.") .middleware(RelayEnvironmentAuth); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0b81889e809e..403cf940ff9c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,6 +9,9 @@ catalogs: '@effect/openapi-generator': specifier: 4.0.1 version: 4.0.1 + '@effect/sql-sqlite-do': + specifier: 4.0.1 + version: 4.0.1 '@effect/tsgo': specifier: 0.41.0 version: 0.41.0 @@ -837,6 +840,9 @@ importers: '@effect/sql-pg': specifier: 4.0.1 version: 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) + '@effect/sql-sqlite-do': + specifier: 'catalog:' + version: 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@noble/curves': specifier: 'catalog:' version: 1.9.1 @@ -854,10 +860,10 @@ importers: version: link:../../packages/shared alchemy: specifier: 2.0.0-beta.80 - version: 2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(824b7c95c4257cf54626a40f00f69c9b) + version: 2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(d02b4e60a6a3c6b542dfa700209cf5ce) drizzle-orm: specifier: 1.0.0-rc.5-ab785fc - version: 1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d) + version: 1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c) effect: specifier: 4.0.1 version: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f) @@ -2362,8 +2368,8 @@ packages: peerDependencies: effect: 4.0.1 - '@effect/sql-sqlite-do@4.0.0': - resolution: {integrity: sha512-XkRZmQOPblzwjmAJ0RWSOiLlTsKkf1EigZsxUNuIuk4uZ/pnufSqvDpXubCwv8iyRTbOLFMmM+EyOEH5V5XqAw==} + '@effect/sql-sqlite-do@4.0.1': + resolution: {integrity: sha512-tS+qQok3xKsycnnhvIDX+6nvlM4Y2ouvru3voOzxtDQqX96NSOMX2xReLf4Feb6GN3jnzgEEtVGDDWk3YLrpsg==} peerDependencies: effect: 4.0.1 @@ -13051,7 +13057,7 @@ snapshots: effect: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f) optional: true - '@effect/sql-sqlite-do@4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))': + '@effect/sql-sqlite-do@4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))': dependencies: effect: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f) @@ -16625,7 +16631,7 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 - alchemy@2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(824b7c95c4257cf54626a40f00f69c9b): + alchemy@2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(d02b4e60a6a3c6b542dfa700209cf5ce): dependencies: '@alchemy.run/cloudflare-runtime': 2.0.0-beta.80(@distilled.cloud/cloudflare@1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)))(@effect/platform-node@4.0.1(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(redis@6.2.1)(utf-8-validate@6.0.6))(@types/node@24.12.4)(@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0))(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(rolldown@1.2.5)(typescript@7.0.2) '@alchemy.run/floci': 2.0.0-beta.80(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) @@ -16648,7 +16654,7 @@ snapshots: '@distilled.cloud/stripe': 1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@distilled.cloud/zerossl': 1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@effect/sql-d1': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) - '@effect/sql-sqlite-do': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) + '@effect/sql-sqlite-do': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@libsql/client': 0.17.3(bufferutil@4.1.0)(utf-8-validate@6.0.6) '@neon/functions': 0.11.0(hono@4.13.7) '@octokit/rest': 22.0.1 @@ -16668,7 +16674,7 @@ snapshots: '@effect/sql-pg': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@effect/vitest': 4.0.1(patch_hash=359f6fb2f7b3ec145bb72208edb9034f02489791aa2491a55cdbd69bd56ee0d2)(@types/node@24.12.4)(@vitest/ui@5.0.1)(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) drizzle-kit: 1.0.0-rc.5-ab785fc - drizzle-orm: 1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d) + drizzle-orm: 1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c) mongodb: 6.21.0(@aws-sdk/credential-providers@3.1062.0)(socks@2.8.9) pg: 8.23.0 vite: '@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0)' @@ -17825,13 +17831,13 @@ snapshots: get-tsconfig: 4.14.3 jiti: 2.7.0 - drizzle-orm@1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d): + drizzle-orm@1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c): optionalDependencies: '@cloudflare/workers-types': 4.20260604.1 '@effect/sql-d1': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@effect/sql-pg': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@effect/sql-sqlite-bun': 4.0.0-rc.112(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) - '@effect/sql-sqlite-do': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) + '@effect/sql-sqlite-do': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)) '@electric-sql/pglite': 0.3.15 '@libsql/client': 0.17.3(bufferutil@4.1.0)(utf-8-validate@6.0.6) bun-types: 1.3.14 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 460a39ca541e..7f74dc5ba16b 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -35,6 +35,7 @@ catalog: "@effect/platform-node": 4.0.1 "@effect/platform-node-shared": 4.0.1 "@effect/sql-pg": 4.0.1 + "@effect/sql-sqlite-do": 4.0.1 "@effect/tsgo": 0.41.0 "@effect/vitest": 4.0.1 "@legendapp/list": 3.3.5 @@ -75,6 +76,7 @@ minimumReleaseAgeExclude: - "@effect/platform-node-shared@4.0.1" - "@effect/platform-node@4.0.1" - "@effect/sql-pg@4.0.1" + - "@effect/sql-sqlite-do@4.0.1" - "@effect/vitest@4.0.1" - alchemy@2.0.0-beta.80 - effect@4.0.1 From c4897cce48319fda2170d76e151a6c52b3c16036 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:32:48 -0700 Subject: [PATCH 05/35] fix: held webhooks keep their arrival time and invalid age limits are rejected - The relay records when a request arrived, not when the environment failed to answer, so held requests keep their order. - Web and mobile reject an age limit outside 1..1440 whole minutes instead of saving it as "no limit". - If the local opt-in cannot be saved, the relay is put back to the previous value so the two never disagree. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../settings/scheduledTaskDraft.test.ts | 3 +++ .../features/settings/scheduledTaskDraft.ts | 15 ++++++++----- apps/server/src/cloud/http.ts | 21 ++++++++++++++----- .../settings/ScheduledTasksSettings.tsx | 11 ++++++++++ .../scheduledTasksSettings.logic.test.ts | 11 ++++++++++ .../settings/scheduledTasksSettings.logic.ts | 11 +++++++--- infra/relay/src/hooks/HookForwarder.test.ts | 20 ++++++++++++++++++ infra/relay/src/hooks/HookForwarder.ts | 4 +++- 8 files changed, 82 insertions(+), 14 deletions(-) diff --git a/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts b/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts index ea43d60aa5c8..9f83e0c8fa32 100644 --- a/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts +++ b/apps/mobile/src/features/settings/scheduledTaskDraft.test.ts @@ -45,6 +45,9 @@ describe("scheduleDraftForTask", () => { }); expect(draft.maxDeliveryAgeMinutes).toBe("45"); expect(scheduleFromDraft(draft)).toMatchObject({ maxDeliveryAgeMinutes: 45 }); + // An invalid limit is an invalid schedule, never a silently removed one. + expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "1.5" })).toBeNull(); + expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "0" })).toBeNull(); expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "" })).toMatchObject({ maxDeliveryAgeMinutes: null, }); diff --git a/apps/mobile/src/features/settings/scheduledTaskDraft.ts b/apps/mobile/src/features/settings/scheduledTaskDraft.ts index b2e9509436b5..5f6d8185765a 100644 --- a/apps/mobile/src/features/settings/scheduledTaskDraft.ts +++ b/apps/mobile/src/features/settings/scheduledTaskDraft.ts @@ -8,7 +8,7 @@ import type { ScheduledTaskWebhookSignature, } from "@t3tools/contracts"; -import { DEFAULT_SERVER_SETTINGS } from "@t3tools/contracts"; +import { DEFAULT_SERVER_SETTINGS, MAX_WEBHOOK_DELIVERY_AGE_MINUTES } from "@t3tools/contracts"; import { resolveProjectSettings, type LegacyProjectSettingsFields, @@ -89,14 +89,19 @@ export function scheduleDraftForTask(task: Pick): Sch } } -/** Blank or invalid input means "no limit"; the server rejects values past the relay's TTL. */ -function parseMaxDeliveryAge(value: string): number | null { +/** Blank means "no limit"; undefined means the input is not a valid limit. */ +function parseMaxDeliveryAge(value: string): number | null | undefined { + if (value.trim() === "") return null; const minutes = Number(value.trim()); - return value.trim() !== "" && Number.isInteger(minutes) && minutes > 0 ? minutes : null; + return Number.isInteger(minutes) && minutes > 0 && minutes <= MAX_WEBHOOK_DELIVERY_AGE_MINUTES + ? minutes + : undefined; } export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSchedule | null { if (draft.mode === "webhook") { + const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes); + if (maxDeliveryAgeMinutes === undefined) return null; // No secret is sent, so the server keeps the stored one. return { type: "webhook", @@ -108,7 +113,7 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche encoding: draft.signature.encoding, prefix: draft.signature.prefix, }, - maxDeliveryAgeMinutes: parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes), + maxDeliveryAgeMinutes, }; } if (draft.mode === "interval") { diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index dc33f1369854..4cb29c37b67f 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -89,6 +89,7 @@ import { encodeConfirmedOriginJson, PUBLISH_AGENT_ACTIVITY_SECRET, HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, + readHoldWebhooksWhileOffline, readRelayConnection, RELAY_ENVIRONMENT_CREDENTIAL_SECRET, RELAY_ISSUER_SECRET, @@ -1386,12 +1387,22 @@ const cloudPreferencesHandler = Effect.fn("environment.cloud.preferences")( yield* requireEnvironmentScope(AuthRelayWriteScope); if (payload.holdWebhooksWhileOffline !== undefined) { // The relay decides whether to hold a request, so it is told first; the - // local copy is only saved once the relay has the same value. + // local copy is only saved once the relay has the same value, and the + // relay is put back if that save fails. + const previous = yield* readHoldWebhooksWhileOffline(dependencies.secrets); yield* pushHoldWebhooksWhileOffline(dependencies, payload.holdWebhooksWhileOffline); - yield* dependencies.secrets.set( - HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, - stringToBytes(String(payload.holdWebhooksWhileOffline)), - ); + yield* dependencies.secrets + .set( + HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, + stringToBytes(String(payload.holdWebhooksWhileOffline)), + ) + .pipe( + Effect.tapError(() => + previous === payload.holdWebhooksWhileOffline + ? Effect.void + : pushHoldWebhooksWhileOffline(dependencies, previous).pipe(Effect.ignore), + ), + ); } yield* dependencies.secrets.set( PUBLISH_AGENT_ACTIVITY_SECRET, diff --git a/apps/web/src/components/settings/ScheduledTasksSettings.tsx b/apps/web/src/components/settings/ScheduledTasksSettings.tsx index 6c29c5945b73..b1a8153af3bd 100644 --- a/apps/web/src/components/settings/ScheduledTasksSettings.tsx +++ b/apps/web/src/components/settings/ScheduledTasksSettings.tsx @@ -61,6 +61,7 @@ import { DEFAULT_WEBHOOK_PROMPT, WEBHOOK_SIGNATURE_DEFAULTS, matchesScheduledTaskScope, + parseMaxDeliveryAge, scheduleFromDraft, scheduledTaskDefaultModel, taskToDraft, @@ -834,6 +835,16 @@ function ScheduledTaskEditorDialog({ reportFailure("Scheduled task is incomplete", "Add a title, prompt, project, and model."); return; } + if ( + draft.scheduleMode === "webhook" && + parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes) === undefined + ) { + reportFailure( + "Invalid age limit", + `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`, + ); + return; + } const schedule = scheduleFromDraft(draft); if ( schedule.type === "webhook" && diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts index bda8ea0e0563..84e57ba50379 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts @@ -19,6 +19,7 @@ import { deriveProviderInstanceEntries } from "../../providerInstances"; import { scheduledTaskDefaultModel, matchesScheduledTaskScope, + parseMaxDeliveryAge, scheduleFromDraft, taskToDraft, } from "./scheduledTasksSettings.logic"; @@ -313,3 +314,13 @@ describe("scheduled task model defaults", () => { ).toBeNull(); }); }); + +describe("parseMaxDeliveryAge", () => { + it("treats blank as no limit and rejects values the server would not accept", () => { + expect(parseMaxDeliveryAge("")).toBeNull(); + expect(parseMaxDeliveryAge(" 45 ")).toBe(45); + for (const invalid of ["0", "1.5", "-3", "abc", "1441"]) { + expect(parseMaxDeliveryAge(invalid)).toBeUndefined(); + } + }); +}); diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts index 719f814e384b..e0f6445b3bba 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts @@ -1,5 +1,6 @@ import { EnvironmentId, + MAX_WEBHOOK_DELIVERY_AGE_MINUTES, type ProjectId, ScheduledTaskId, type ScheduledTask, @@ -93,9 +94,13 @@ export const WEBHOOK_SIGNATURE_DEFAULTS = { export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}"; /** Blank or invalid input means "no limit"; the server rejects values past the relay's TTL. */ -function parseMaxDeliveryAge(value: string): number | null { +/** Blank means "no limit"; undefined means the input is not a valid limit. */ +export function parseMaxDeliveryAge(value: string): number | null | undefined { + if (value.trim() === "") return null; const minutes = Number(value.trim()); - return value.trim() !== "" && Number.isInteger(minutes) && minutes > 0 ? minutes : null; + return Number.isInteger(minutes) && minutes > 0 && minutes <= MAX_WEBHOOK_DELIVERY_AGE_MINUTES + ? minutes + : undefined; } export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule { @@ -111,7 +116,7 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul ...(secret ? { secret } : {}), } : null, - maxDeliveryAgeMinutes: parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes), + maxDeliveryAgeMinutes: parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes) ?? null, }; } if (draft.scheduleMode === "interval") { diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index 0eedf0ed6cbb..9b1bf003802e 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -3,6 +3,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { describe, expect, it } from "@effect/vitest"; import { RelayApi } from "@t3tools/contracts/relay"; +import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -377,6 +378,25 @@ describe("HookForwarder", () => { }), ); + it.effect("holds a timed-out request with the time it arrived", () => + Effect.gen(function* () { + const reachedUpstream = yield* Deferred.make(); + const harness = makeHarness({ + links: [{ ...managedLink, holdWebhooksWhileOffline: true }], + execute: () => + Deferred.succeed(reachedUpstream, undefined).pipe(Effect.andThen(Effect.never)), + }); + const arrivedAt = DateTime.formatIso(yield* DateTime.now); + const fiber = yield* harness + .send(new Request(hookUrl(), { method: "POST", body: "{}" })) + .pipe(Effect.forkChild); + yield* Deferred.await(reachedUpstream); + yield* TestClock.adjust(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)); + expect((yield* Fiber.join(fiber)).status).toBe(202); + expect(harness.held[0]?.receivedAt).toBe(arrivedAt); + }), + ); + it.effect("answers OPTIONS without a CORS preflight or forwarding", () => Effect.gen(function* () { const harness = makeHarness(); diff --git a/infra/relay/src/hooks/HookForwarder.ts b/infra/relay/src/hooks/HookForwarder.ts index 15b46f18c82d..24551bd93ed2 100644 --- a/infra/relay/src/hooks/HookForwarder.ts +++ b/infra/relay/src/hooks/HookForwarder.ts @@ -219,6 +219,8 @@ const make = Effect.gen(function* () { request: HttpServerRequest.HttpServerRequest, ) { const outcome = (value: string) => Effect.annotateCurrentSpan({ "relay.hook.outcome": value }); + // When the sender called, not when the environment failed to answer. + const receivedAt = DateTime.formatIso(yield* DateTime.now); const parsed = parseHookPath(request.url); if (!parsed) { yield* outcome("invalid_path"); @@ -291,7 +293,7 @@ const make = Effect.gen(function* () { .hold({ environmentId: parsed.environmentId, baseUrl: endpoint.httpBaseUrl, - hook: { ...hook, receivedAt: DateTime.formatIso(yield* DateTime.now) }, + hook: { ...hook, receivedAt }, }) .pipe( Effect.catch((cause) => From a860f909672f2b526828d6c2d818fb5b7275edb4 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:40:58 -0700 Subject: [PATCH 06/35] fix(relay,server): held webhooks keep their receive time and one cap per hook - The relay sends `x-t3-relay-received-at` with the delivery id, and the environment logs that time and checks the per-task max age against it. The relay strips any copy a sender supplied. - The per-hook inbox cap is keyed by the decoded hook id, so `hook` and `%68ook` share one cap. - Upstream responses are read with a 64 KiB cap instead of buffered whole. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/scheduledTasks/webhookRoute.test.ts | 26 +++++++++++ .../server/src/scheduledTasks/webhookRoute.ts | 5 +- infra/relay/src/hooks/HookForwarder.test.ts | 4 +- infra/relay/src/hooks/HookForwarder.ts | 5 +- infra/relay/src/hooks/HookInboxStore.test.ts | 5 +- infra/relay/src/hooks/HookInboxStore.ts | 16 +++++-- infra/relay/src/hooks/upstream.ts | 46 +++++++++++++++++-- 7 files changed, 94 insertions(+), 13 deletions(-) diff --git a/apps/server/src/scheduledTasks/webhookRoute.test.ts b/apps/server/src/scheduledTasks/webhookRoute.test.ts index e6fb5ff7d1ed..27a2a45a0322 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.test.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.test.ts @@ -77,6 +77,32 @@ describe("webhook route", () => { } }); + it("uses the relay's receive time only alongside its delivery id", async () => { + const received: Array = []; + const { handler, dispose } = handlerFor((request) => { + received.push(request); + return Effect.succeed({ + _tag: "accepted", + deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"), + }); + }); + try { + const receivedAt = "2026-10-04T10:00:00.000Z"; + await handler( + post("/api/hooks/id/tok", "{}", { + "x-t3-relay-delivery-id": "relay-1", + "x-t3-relay-received-at": receivedAt, + }), + ); + await handler(post("/api/hooks/id/tok", "{}", { "x-t3-relay-received-at": receivedAt })); + expect(received[0]?.relayDeliveryId).toBe("relay-1"); + expect(received[0]?.receivedAt).toBe(receivedAt); + expect(received[1]?.receivedAt).toBeUndefined(); + } finally { + await dispose(); + } + }); + it("maps service outcomes to status codes", async () => { const cases: ReadonlyArray<[WebhookTriggerResult["_tag"], number]> = [ ["not_found", 404], diff --git a/apps/server/src/scheduledTasks/webhookRoute.ts b/apps/server/src/scheduledTasks/webhookRoute.ts index c85c878ba334..3b7f64bc51b9 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.ts @@ -54,8 +54,10 @@ const handleWebhook = if (typeof value === "string") headers[name.toLowerCase()] = value; } const queryIndex = request.url.indexOf("?"); - // Only the relay sets this; it strips any copy a sender supplied. + // Only the relay sets these; it strips any copy a sender supplied. The + // receive time matters for requests the relay held while we were offline. const relayDeliveryId = headers["x-t3-relay-delivery-id"]; + const relayReceivedAt = relayDeliveryId ? headers["x-t3-relay-received-at"] : undefined; const result = yield* scheduledTasks .triggerWebhook({ @@ -68,6 +70,7 @@ const handleWebhook = body: body.value, bodyText: new TextDecoder().decode(body.value), ...(relayDeliveryId ? { relayDeliveryId } : {}), + ...(relayReceivedAt ? { receivedAt: relayReceivedAt } : {}), }) .pipe( Effect.catch((cause) => diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index 9b1bf003802e..5bea17f90e8d 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -524,7 +524,7 @@ describe("HookForwarder", () => { }); const body = new Uint8Array([0, 255, 10]); const response = yield* harness.send( - new Request(hookUrl("hook-1/tok%2Fen", "?a=1"), { + new Request(hookUrl("%68ook-1/tok%2Fen", "?a=1"), { method: "POST", body, headers: { "x-t3-relay-delivery-id": "forged", "x-sig": "s" }, @@ -537,6 +537,8 @@ describe("HookForwarder", () => { expect(hook?.query).toBe("a=1"); expect([...(hook?.body ?? [])]).toEqual([0, 255, 10]); expect(hook?.headers["x-sig"]).toBe("s"); + // Every spelling of the hook id shares one per-hook cap. + expect(hook?.hookKey).toBe("hook-1"); // The sender cannot choose the delivery id. expect(hook?.headers["x-t3-relay-delivery-id"]).toBeUndefined(); expect(hook?.id).not.toBe("forged"); diff --git a/infra/relay/src/hooks/HookForwarder.ts b/infra/relay/src/hooks/HookForwarder.ts index 24551bd93ed2..d4b717b5524a 100644 --- a/infra/relay/src/hooks/HookForwarder.ts +++ b/infra/relay/src/hooks/HookForwarder.ts @@ -36,6 +36,7 @@ const DROPPED_REQUEST_HEADERS = new Set([ "x-real-ip", // Only the relay may set this; a sender could otherwise collide delivery ids. "x-t3-relay-delivery-id", + "x-t3-relay-received-at", ]); const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-"]; @@ -275,9 +276,11 @@ const make = Effect.gen(function* () { // timeout and is later delivered from the inbox runs only once. const hook = { id: yield* crypto.randomUUIDv4.pipe(Effect.orDie), + receivedAt, method: request.method, rawHookId: parsed.rawHookId, rawToken: parsed.rawToken, + hookKey: parsed.hookId, query: parsed.search.replace(/^\?/, ""), headers: forwardedHeaders(request.headers), body: body.success, @@ -293,7 +296,7 @@ const make = Effect.gen(function* () { .hold({ environmentId: parsed.environmentId, baseUrl: endpoint.httpBaseUrl, - hook: { ...hook, receivedAt }, + hook, }) .pipe( Effect.catch((cause) => diff --git a/infra/relay/src/hooks/HookInboxStore.test.ts b/infra/relay/src/hooks/HookInboxStore.test.ts index 37c85c983db2..2c409e275a50 100644 --- a/infra/relay/src/hooks/HookInboxStore.test.ts +++ b/infra/relay/src/hooks/HookInboxStore.test.ts @@ -16,6 +16,7 @@ const hook = (id: string, overrides: Partial = {}) => receivedAt: DateTime.formatIso(DateTime.makeUnsafe(now)), method: "POST", rawHookId: "hook-1", + hookKey: "hook-1", rawToken: "tok%2Fen", query: "a=1", headers: { "content-type": "application/json", "x-sig": "s" }, @@ -123,7 +124,7 @@ describe("HookInboxStore", () => { } expect(yield* HookInboxStore.hold(yield* hook("one-too-many"), BASE_URL)).toBeNull(); // Another hook still has room. - const other = yield* hook("other", { rawHookId: "hook-2" }); + const other = yield* hook("other", { rawHookId: "hook-2", hookKey: "hook-2" }); expect(yield* HookInboxStore.hold(other, BASE_URL)).not.toBeNull(); }), ), @@ -139,7 +140,7 @@ describe("HookInboxStore", () => { const small = new Uint8Array(11); expect( yield* HookInboxStore.hold( - yield* hook("small", { rawHookId: "x", body: small }), + yield* hook("small", { rawHookId: "x", hookKey: "x", body: small }), BASE_URL, ), ).toBeNull(); diff --git a/infra/relay/src/hooks/HookInboxStore.ts b/infra/relay/src/hooks/HookInboxStore.ts index b5c086facac6..a3ced78dff61 100644 --- a/infra/relay/src/hooks/HookInboxStore.ts +++ b/infra/relay/src/hooks/HookInboxStore.ts @@ -33,6 +33,8 @@ export interface HeldHook { /** Path segments exactly as the sender sent them; the environment decodes them. */ readonly rawHookId: string; readonly rawToken: string; + /** The decoded hook id, so every spelling of one hook shares its cap. */ + readonly hookKey: string; /** Without the leading `?`. */ readonly query: string; readonly headers: Readonly>; @@ -56,6 +58,7 @@ interface HeldHookRow { readonly method: string; readonly raw_hook_id: string; readonly raw_token: string; + readonly hook_key: string; readonly query: string; readonly headers: string; readonly body: Uint8Array; @@ -78,12 +81,13 @@ export const migrate = Effect.gen(function* () { method TEXT NOT NULL, raw_hook_id TEXT NOT NULL, raw_token TEXT NOT NULL, + hook_key TEXT NOT NULL, query TEXT NOT NULL, headers TEXT NOT NULL, body BLOB NOT NULL ) `; - yield* sql`CREATE INDEX IF NOT EXISTS held_hooks_hook ON held_hooks (raw_hook_id)`; + yield* sql`CREATE INDEX IF NOT EXISTS held_hooks_hook ON held_hooks (hook_key)`; // Where to push, and how many attempts in a row have failed. One row. yield* sql` CREATE TABLE IF NOT EXISTS held_hooks_target ( @@ -134,11 +138,12 @@ export const hold = Effect.fn("HookInboxStore.hold")(function* (hook: HeldHook, const sql = yield* SqlClient.SqlClient; // One statement, so the caps hold however many requests arrive at once. const inserted = yield* sql<{ readonly id: string }>` - INSERT INTO held_hooks (id, received_at, method, raw_hook_id, raw_token, query, headers, body) + INSERT INTO held_hooks + (id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body) SELECT ${hook.id}, ${hook.receivedAt}, ${hook.method}, ${hook.rawHookId}, ${hook.rawToken}, - ${hook.query}, ${encodeHeaders(hook.headers)}, ${hook.body} + ${hook.hookKey}, ${hook.query}, ${encodeHeaders(hook.headers)}, ${hook.body} WHERE (SELECT count(*) FROM held_hooks) < ${HOOK_INBOX_MAX_REQUESTS} - AND (SELECT count(*) FROM held_hooks WHERE raw_hook_id = ${hook.rawHookId}) + AND (SELECT count(*) FROM held_hooks WHERE hook_key = ${hook.hookKey}) < ${HOOK_INBOX_MAX_PER_HOOK} AND (SELECT coalesce(sum(length(body)), 0) FROM held_hooks) + ${hook.body.byteLength} <= ${HOOK_INBOX_MAX_BYTES} @@ -180,7 +185,7 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( yield* sql`DELETE FROM held_hooks WHERE received_at < ${iso(startedAt - HOOK_INBOX_TTL_MS)}`; const target = yield* readTarget; const batch = yield* sql` - SELECT id, received_at, method, raw_hook_id, raw_token, query, headers, body + SELECT id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body FROM held_hooks ORDER BY seq LIMIT ${DELIVERIES_PER_RUN} `; if (batch.length === 0 || target === null) { @@ -196,6 +201,7 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( method: row.method, rawHookId: row.raw_hook_id, rawToken: row.raw_token, + hookKey: row.hook_key, query: row.query, headers: decodeHeaders(row.headers), body: row.body, diff --git a/infra/relay/src/hooks/upstream.ts b/infra/relay/src/hooks/upstream.ts index af50c8381136..22053c25e391 100644 --- a/infra/relay/src/hooks/upstream.ts +++ b/infra/relay/src/hooks/upstream.ts @@ -1,12 +1,19 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientRequest from "effect/http/HttpClientRequest"; +import type * as HttpClientResponse from "effect/http/HttpClientResponse"; import { withoutRedirects } from "../environments/EnvironmentConnector.ts"; /** Set by the relay on every forward; the environment uses it as the delivery id. */ const RELAY_DELIVERY_ID_HEADER = "x-t3-relay-delivery-id"; +/** When the relay received the request; the environment trusts it only with the delivery id. */ +const RELAY_RECEIVED_AT_HEADER = "x-t3-relay-received-at"; +/** The environment answers with a small JSON status; anything past this is cut off. */ +const MAX_RESPONSE_BYTES = 64 * 1024; export const RELAY_HOOK_UPSTREAM_TIMEOUT_MS = 8_000; // Cloudflare answers 530 when the tunnel for a hostname has no connected origin. export const TUNNEL_OFFLINE_STATUS = 530; @@ -14,6 +21,7 @@ export const TUNNEL_OFFLINE_STATUS = 530; /** A webhook request as the relay sends it on to the environment. */ export interface UpstreamHook { readonly id: string; + readonly receivedAt: string; readonly method: string; /** Path segments exactly as the sender sent them; the environment decodes them. */ readonly rawHookId: string; @@ -30,6 +38,37 @@ export interface UpstreamResponse { readonly body: Uint8Array; } +class ResponseTooLarge extends Schema.TaggedError()("ResponseTooLarge", {}) {} + +/** Reads a response body, failing once it passes the cap rather than buffering it all. */ +const readCapped = (response: HttpClientResponse.HttpClientResponse) => + Effect.suspend(() => { + const chunks: Array = []; + let total = 0; + return response.stream.pipe( + Stream.runForEach((chunk) => { + total += chunk.length; + if (total > MAX_RESPONSE_BYTES) return Effect.fail(new ResponseTooLarge()); + chunks.push(chunk); + return Effect.void; + }), + Effect.map(() => { + const body = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.length; + } + return body; + }), + // A response without a body, such as a redirect, has no stream at all. + Effect.catchIf( + (error) => error._tag === "HttpClientError" && error.reason._tag === "EmptyBodyError", + () => Effect.succeed(new Uint8Array(0)), + ), + ); + }); + /** * Sends a webhook request through the environment's tunnel. Fails when the * environment cannot be reached, and succeeds with None on timeout. @@ -41,6 +80,7 @@ export const sendUpstream = (baseUrl: string, hook: UpstreamHook) => const headers: Record = { ...hook.headers, [RELAY_DELIVERY_ID_HEADER]: hook.id, + [RELAY_RECEIVED_AT_HEADER]: hook.receivedAt, }; let request = HttpClientRequest.make(hook.method as "GET" | "POST" | "PUT" | "PATCH")( `${base}api/hooks/${hook.rawHookId}/${hook.rawToken}${hook.query ? `?${hook.query}` : ""}`, @@ -51,11 +91,11 @@ export const sendUpstream = (baseUrl: string, hook: UpstreamHook) => } return yield* httpClient.execute(request).pipe( Effect.flatMap((response) => - response.arrayBuffer.pipe( - Effect.map((bytes): UpstreamResponse => ({ + readCapped(response).pipe( + Effect.map((body): UpstreamResponse => ({ status: response.status, contentType: response.headers["content-type"], - body: new Uint8Array(bytes), + body, })), ), ), From fca6d7773077363e5fe6a0f0bd5969e7762279e1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 19:38:10 -0700 Subject: [PATCH 07/35] fix(mobile): a deleted webhook task no longer shows its URL or Rotate Co-Authored-By: Claude Opus 5.5 (1M context) --- .../features/settings/SettingsScheduledTasksRouteScreen.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index 62aaaf252ccd..ad6b1328ca9b 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -913,8 +913,12 @@ function TaskForm({ <> task.id === draft.task?.id) ?? draft.task ?? null + tasks.data + ? (tasks.data.tasks.find((task) => task.id === draft.task?.id) ?? null) + : draft.task } signatureConfigured={draft.schedule.signature !== null} /> From 22f8bcb784cb30dcf05f75807e09dd7644a0c259 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 19:38:25 -0700 Subject: [PATCH 08/35] fix(mobile): Rotate URL is disabled while rotating, saving, or disconnected Co-Authored-By: Claude Opus 5.5 (1M context) --- .../SettingsScheduledTasksRouteScreen.tsx | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index ad6b1328ca9b..89f62fa0f5e6 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -921,6 +921,7 @@ function TaskForm({ : draft.task } signatureConfigured={draft.schedule.signature !== null} + disabled={saving || environmentUnavailable} /> Alert.alert("Rotate URL?", "The current URL stops working immediately.", [ { text: "Cancel", style: "cancel" }, { text: "Rotate", style: "destructive", - onPress: () => + onPress: () => { + setRotating(true); void rotate({ environmentId, input: { id: task.id } }).then((result) => { + setRotating(false); if (result._tag === "Failure" && !isAtomCommandInterrupted(result)) { Alert.alert( "Could not rotate URL", String(squashAtomCommandFailure(result)), ); } - }), + }); + }, }, ]) } - className="min-h-11 justify-center active:opacity-70" + className="min-h-11 justify-center active:opacity-70 disabled:opacity-50" > - Rotate URL + + {rotating ? "Rotating…" : "Rotate URL"} + )} From 07567454f20b118690aec84ea83769aec3b262a9 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 19:38:59 -0700 Subject: [PATCH 09/35] fix(web,mobile): an invalid webhook age limit blocks saving on both clients Web's scheduleFromDraft no longer turns an invalid limit into "no limit"; it returns null like mobile's. Mobile now uses an integer keypad for the field and shows the same "whole minutes from 1 to 1440" error as web instead of the generic incomplete-task message. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../SettingsScheduledTasksRouteScreen.tsx | 20 ++++++++++++++++--- .../features/settings/scheduledTaskDraft.ts | 4 ++-- .../settings/ScheduledTasksSettings.tsx | 8 ++------ .../scheduledTasksSettings.logic.test.ts | 7 +++++-- .../settings/scheduledTasksSettings.logic.ts | 10 ++++++---- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index 89f62fa0f5e6..356d2a91969a 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -4,7 +4,10 @@ import type { ScheduledTask, ScheduledTaskUpsertInput, } from "@t3tools/contracts"; -import { resolveEnvironmentMachineKind } from "@t3tools/contracts"; +import { + MAX_WEBHOOK_DELIVERY_AGE_MINUTES, + resolveEnvironmentMachineKind, +} from "@t3tools/contracts"; import type { MenuAction } from "@react-native-menu/menu"; import { DateTimePicker } from "@expo/ui/community/datetime-picker"; import { @@ -60,6 +63,7 @@ import { useSettingsEnvironmentFilter, type SettingsTarget } from "./settings-en import { editDraft, DEFAULT_WEBHOOK_PROMPT, + parseMaxDeliveryAge, scheduledTaskDefaultModel, scheduleFromDraft, type ScheduledTaskDraft as Draft, @@ -132,7 +136,7 @@ function FormField(props: { readonly label: string; readonly value: string; readonly onChange: (value: string) => void; - readonly keyboardType?: "decimal-pad"; + readonly keyboardType?: "decimal-pad" | "number-pad"; readonly disabled?: boolean; readonly placeholder?: string; readonly borderTop?: boolean; @@ -611,6 +615,16 @@ function TaskForm({ ? { ...draft.schedule, signature: liveTask.schedule.signature } : draft.schedule, ); + if ( + draft.schedule.mode === "webhook" && + parseMaxDeliveryAge(draft.schedule.maxDeliveryAgeMinutes) === undefined + ) { + Alert.alert( + "Invalid age limit", + `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`, + ); + return; + } if ( !draft.title.trim() || !draft.prompt.trim() || @@ -927,7 +941,7 @@ function TaskForm({ label="Skip requests older than (minutes)" value={draft.schedule.maxDeliveryAgeMinutes} placeholder="Run every request" - keyboardType="decimal-pad" + keyboardType="number-pad" disabled={saving} borderTop onChange={(maxDeliveryAgeMinutes) => diff --git a/apps/mobile/src/features/settings/scheduledTaskDraft.ts b/apps/mobile/src/features/settings/scheduledTaskDraft.ts index 5f6d8185765a..07c87b34e592 100644 --- a/apps/mobile/src/features/settings/scheduledTaskDraft.ts +++ b/apps/mobile/src/features/settings/scheduledTaskDraft.ts @@ -89,8 +89,8 @@ export function scheduleDraftForTask(task: Pick): Sch } } -/** Blank means "no limit"; undefined means the input is not a valid limit. */ -function parseMaxDeliveryAge(value: string): number | null | undefined { +/** Blank means "no limit"; undefined means the input is not a valid limit, which blocks saving. */ +export function parseMaxDeliveryAge(value: string): number | null | undefined { if (value.trim() === "") return null; const minutes = Number(value.trim()); return Number.isInteger(minutes) && minutes > 0 && minutes <= MAX_WEBHOOK_DELIVERY_AGE_MINUTES diff --git a/apps/web/src/components/settings/ScheduledTasksSettings.tsx b/apps/web/src/components/settings/ScheduledTasksSettings.tsx index b1a8153af3bd..c5a616c529bf 100644 --- a/apps/web/src/components/settings/ScheduledTasksSettings.tsx +++ b/apps/web/src/components/settings/ScheduledTasksSettings.tsx @@ -61,7 +61,6 @@ import { DEFAULT_WEBHOOK_PROMPT, WEBHOOK_SIGNATURE_DEFAULTS, matchesScheduledTaskScope, - parseMaxDeliveryAge, scheduleFromDraft, scheduledTaskDefaultModel, taskToDraft, @@ -835,17 +834,14 @@ function ScheduledTaskEditorDialog({ reportFailure("Scheduled task is incomplete", "Add a title, prompt, project, and model."); return; } - if ( - draft.scheduleMode === "webhook" && - parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes) === undefined - ) { + const schedule = scheduleFromDraft(draft); + if (schedule === null) { reportFailure( "Invalid age limit", `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`, ); return; } - const schedule = scheduleFromDraft(draft); if ( schedule.type === "webhook" && schedule.signature && diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts index 84e57ba50379..da04ce4b1e35 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts @@ -205,18 +205,21 @@ describe("webhook scheduled tasks", () => { }); }); - it("round-trips the max age and treats a blank or invalid entry as no limit", () => { + it("round-trips the max age, treats blank as no limit, and rejects an invalid limit", () => { const draft = taskToDraft({ ...webhookTask, schedule: { type: "webhook", signature: null, maxDeliveryAgeMinutes: 90 }, }); expect(draft.maxDeliveryAgeMinutes).toBe("90"); expect(scheduleFromDraft(draft)).toMatchObject({ maxDeliveryAgeMinutes: 90 }); - for (const blank of ["", " ", "0", "-5", "1.5", "abc"]) { + for (const blank of ["", " "]) { expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: blank })).toMatchObject({ maxDeliveryAgeMinutes: null, }); } + for (const invalid of ["0", "-5", "1.5", "abc", "1441"]) { + expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: invalid })).toBeNull(); + } }); }); diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts index e0f6445b3bba..566c58fcd338 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts @@ -93,8 +93,7 @@ export const WEBHOOK_SIGNATURE_DEFAULTS = { /** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */ export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}"; -/** Blank or invalid input means "no limit"; the server rejects values past the relay's TTL. */ -/** Blank means "no limit"; undefined means the input is not a valid limit. */ +/** Blank means "no limit"; undefined means the input is not a valid limit, which blocks saving. */ export function parseMaxDeliveryAge(value: string): number | null | undefined { if (value.trim() === "") return null; const minutes = Number(value.trim()); @@ -103,8 +102,11 @@ export function parseMaxDeliveryAge(value: string): number | null | undefined { : undefined; } -export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule { +/** Null when the draft's webhook age limit is invalid; the caller reports it and does not save. */ +export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule | null { if (draft.scheduleMode === "webhook") { + const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes); + if (maxDeliveryAgeMinutes === undefined) return null; const secret = draft.signatureSecret.trim(); return { type: "webhook", @@ -116,7 +118,7 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul ...(secret ? { secret } : {}), } : null, - maxDeliveryAgeMinutes: parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes) ?? null, + maxDeliveryAgeMinutes, }; } if (draft.scheduleMode === "interval") { From 7e569d9028fe22a08bfb30d5f2eee49b3fda9ab0 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 19:39:29 -0700 Subject: [PATCH 10/35] refactor(client-runtime): share the webhook prompt default and age-limit parser Web and mobile each carried a copy of DEFAULT_WEBHOOK_PROMPT and parseMaxDeliveryAge; both now import them from client-runtime. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../SettingsScheduledTasksRouteScreen.tsx | 6 ++++-- .../src/features/settings/scheduledTaskDraft.ts | 15 ++------------- .../settings/ScheduledTasksSettings.tsx | 2 +- .../settings/scheduledTasksSettings.logic.test.ts | 11 ----------- .../settings/scheduledTasksSettings.logic.ts | 14 +------------- packages/client-runtime/package.json | 4 ++++ .../src/scheduledTaskWebhook.test.ts | 13 +++++++++++++ .../client-runtime/src/scheduledTaskWebhook.ts | 13 +++++++++++++ 8 files changed, 38 insertions(+), 40 deletions(-) create mode 100644 packages/client-runtime/src/scheduledTaskWebhook.test.ts create mode 100644 packages/client-runtime/src/scheduledTaskWebhook.ts diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index 356d2a91969a..7de8ff515014 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -15,6 +15,10 @@ import { squashAtomCommandFailure, type AtomCommandResult, } from "@t3tools/client-runtime/state/runtime"; +import { + DEFAULT_WEBHOOK_PROMPT, + parseMaxDeliveryAge, +} from "@t3tools/client-runtime/scheduled-task-webhook"; import { useCallback, useEffect, @@ -62,8 +66,6 @@ import { SettingsSection } from "./components/SettingsSection"; import { useSettingsEnvironmentFilter, type SettingsTarget } from "./settings-environment-filter"; import { editDraft, - DEFAULT_WEBHOOK_PROMPT, - parseMaxDeliveryAge, scheduledTaskDefaultModel, scheduleFromDraft, type ScheduledTaskDraft as Draft, diff --git a/apps/mobile/src/features/settings/scheduledTaskDraft.ts b/apps/mobile/src/features/settings/scheduledTaskDraft.ts index 07c87b34e592..0ba9a437cdb6 100644 --- a/apps/mobile/src/features/settings/scheduledTaskDraft.ts +++ b/apps/mobile/src/features/settings/scheduledTaskDraft.ts @@ -8,7 +8,8 @@ import type { ScheduledTaskWebhookSignature, } from "@t3tools/contracts"; -import { DEFAULT_SERVER_SETTINGS, MAX_WEBHOOK_DELIVERY_AGE_MINUTES } from "@t3tools/contracts"; +import { DEFAULT_SERVER_SETTINGS } from "@t3tools/contracts"; +import { parseMaxDeliveryAge } from "@t3tools/client-runtime/scheduled-task-webhook"; import { resolveProjectSettings, type LegacyProjectSettingsFields, @@ -57,9 +58,6 @@ export const DEFAULT_SCHEDULE: ScheduleDraft = { maxDeliveryAgeMinutes: "", }; -/** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */ -export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}"; - export function scheduleDraftForTask(task: Pick): ScheduleDraft { switch (task.schedule.type) { case "fixed_time": @@ -89,15 +87,6 @@ export function scheduleDraftForTask(task: Pick): Sch } } -/** Blank means "no limit"; undefined means the input is not a valid limit, which blocks saving. */ -export function parseMaxDeliveryAge(value: string): number | null | undefined { - if (value.trim() === "") return null; - const minutes = Number(value.trim()); - return Number.isInteger(minutes) && minutes > 0 && minutes <= MAX_WEBHOOK_DELIVERY_AGE_MINUTES - ? minutes - : undefined; -} - export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSchedule | null { if (draft.mode === "webhook") { const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes); diff --git a/apps/web/src/components/settings/ScheduledTasksSettings.tsx b/apps/web/src/components/settings/ScheduledTasksSettings.tsx index c5a616c529bf..f6741f6129d4 100644 --- a/apps/web/src/components/settings/ScheduledTasksSettings.tsx +++ b/apps/web/src/components/settings/ScheduledTasksSettings.tsx @@ -23,6 +23,7 @@ import type { ScheduledTaskWebhookDeliverySummary, ThreadId, } from "@t3tools/contracts"; +import { DEFAULT_WEBHOOK_PROMPT } from "@t3tools/client-runtime/scheduled-task-webhook"; import { MAX_WEBHOOK_DELIVERY_AGE_MINUTES, MIN_SCHEDULED_TASK_INTERVAL_MS, @@ -58,7 +59,6 @@ import { WorktreeBaseBranchPicker } from "../WorktreeBaseBranchPicker"; import { EnvironmentMachineIcon } from "../EnvironmentMachineIcon"; import { useSettingsScope } from "./SettingsScopeContext"; import { - DEFAULT_WEBHOOK_PROMPT, WEBHOOK_SIGNATURE_DEFAULTS, matchesScheduledTaskScope, scheduleFromDraft, diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts index da04ce4b1e35..5c4b6c84355e 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts @@ -19,7 +19,6 @@ import { deriveProviderInstanceEntries } from "../../providerInstances"; import { scheduledTaskDefaultModel, matchesScheduledTaskScope, - parseMaxDeliveryAge, scheduleFromDraft, taskToDraft, } from "./scheduledTasksSettings.logic"; @@ -317,13 +316,3 @@ describe("scheduled task model defaults", () => { ).toBeNull(); }); }); - -describe("parseMaxDeliveryAge", () => { - it("treats blank as no limit and rejects values the server would not accept", () => { - expect(parseMaxDeliveryAge("")).toBeNull(); - expect(parseMaxDeliveryAge(" 45 ")).toBe(45); - for (const invalid of ["0", "1.5", "-3", "abc", "1441"]) { - expect(parseMaxDeliveryAge(invalid)).toBeUndefined(); - } - }); -}); diff --git a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts index 566c58fcd338..6c738ce0eff0 100644 --- a/apps/web/src/components/settings/scheduledTasksSettings.logic.ts +++ b/apps/web/src/components/settings/scheduledTasksSettings.logic.ts @@ -1,6 +1,5 @@ import { EnvironmentId, - MAX_WEBHOOK_DELIVERY_AGE_MINUTES, type ProjectId, ScheduledTaskId, type ScheduledTask, @@ -10,6 +9,7 @@ import { type ProviderInteractionMode, type ServerSettings, } from "@t3tools/contracts"; +import { parseMaxDeliveryAge } from "@t3tools/client-runtime/scheduled-task-webhook"; import { resolveProjectSettings, @@ -90,18 +90,6 @@ export const WEBHOOK_SIGNATURE_DEFAULTS = { signaturePrefix: "sha256=", } as const; -/** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */ -export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}"; - -/** Blank means "no limit"; undefined means the input is not a valid limit, which blocks saving. */ -export function parseMaxDeliveryAge(value: string): number | null | undefined { - if (value.trim() === "") return null; - const minutes = Number(value.trim()); - return Number.isInteger(minutes) && minutes > 0 && minutes <= MAX_WEBHOOK_DELIVERY_AGE_MINUTES - ? minutes - : undefined; -} - /** Null when the draft's webhook age limit is invalid; the caller reports it and does not save. */ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule | null { if (draft.scheduleMode === "webhook") { diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 7063fed96ca8..6f95926cd1d1 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -35,6 +35,10 @@ "types": "./src/userMessage.ts", "default": "./src/userMessage.ts" }, + "./scheduled-task-webhook": { + "types": "./src/scheduledTaskWebhook.ts", + "default": "./src/scheduledTaskWebhook.ts" + }, "./connection": { "types": "./src/connection/index.ts", "default": "./src/connection/index.ts" diff --git a/packages/client-runtime/src/scheduledTaskWebhook.test.ts b/packages/client-runtime/src/scheduledTaskWebhook.test.ts new file mode 100644 index 000000000000..c2de7d240bc7 --- /dev/null +++ b/packages/client-runtime/src/scheduledTaskWebhook.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { parseMaxDeliveryAge } from "./scheduledTaskWebhook.ts"; + +describe("parseMaxDeliveryAge", () => { + it("treats blank as no limit and rejects values the server would not accept", () => { + expect(parseMaxDeliveryAge("")).toBeNull(); + expect(parseMaxDeliveryAge(" 45 ")).toBe(45); + for (const invalid of ["0", "1.5", "-3", "abc", "1441"]) { + expect(parseMaxDeliveryAge(invalid)).toBeUndefined(); + } + }); +}); diff --git a/packages/client-runtime/src/scheduledTaskWebhook.ts b/packages/client-runtime/src/scheduledTaskWebhook.ts new file mode 100644 index 000000000000..d81fd39cb2e4 --- /dev/null +++ b/packages/client-runtime/src/scheduledTaskWebhook.ts @@ -0,0 +1,13 @@ +import { MAX_WEBHOOK_DELIVERY_AGE_MINUTES } from "@t3tools/contracts"; + +/** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */ +export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}"; + +/** Blank means "no limit"; undefined means the input is not a valid limit, which blocks saving. */ +export function parseMaxDeliveryAge(value: string): number | null | undefined { + if (value.trim() === "") return null; + const minutes = Number(value.trim()); + return Number.isInteger(minutes) && minutes > 0 && minutes <= MAX_WEBHOOK_DELIVERY_AGE_MINUTES + ? minutes + : undefined; +} From 4227c757b1227a051cd9b634de39eb02e1e58906 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 19:39:38 -0700 Subject: [PATCH 11/35] fix(client-runtime): new webhook tasks start with a body-only prompt The default prompt was {{request}}, which includes request headers. A body-only default keeps credentials a sender puts in headers away from the agent unless the user opts in. Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/client-runtime/src/scheduledTaskWebhook.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client-runtime/src/scheduledTaskWebhook.ts b/packages/client-runtime/src/scheduledTaskWebhook.ts index d81fd39cb2e4..785a3bcfe8d1 100644 --- a/packages/client-runtime/src/scheduledTaskWebhook.ts +++ b/packages/client-runtime/src/scheduledTaskWebhook.ts @@ -1,7 +1,7 @@ import { MAX_WEBHOOK_DELIVERY_AGE_MINUTES } from "@t3tools/contracts"; -/** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */ -export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}"; +/** Prompt a new webhook task starts with: just the body, so request headers stay out unless the user adds them. */ +export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{body}}"; /** Blank means "no limit"; undefined means the input is not a valid limit, which blocks saving. */ export function parseMaxDeliveryAge(value: string): number | null | undefined { From 9784b3f0f3c3b2fed6f93564850451d3cceba004 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 19:40:54 -0700 Subject: [PATCH 12/35] fix(web): settings search offers webhook holding only when its row shows The "Hold webhooks while offline" row only renders while the managed tunnel is on, but search offered it whenever T3 Connect was configured, landing on an anchor that wasn't there. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../settings/settingsSearch.test.ts | 19 +++++++++++++++++++ .../src/components/settings/settingsSearch.ts | 7 ++++++- .../useAvailableSettingsSearchItems.ts | 7 +++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/apps/web/src/components/settings/settingsSearch.test.ts b/apps/web/src/components/settings/settingsSearch.test.ts index 76f26b2eeab6..17bc6407d14b 100644 --- a/apps/web/src/components/settings/settingsSearch.test.ts +++ b/apps/web/src/components/settings/settingsSearch.test.ts @@ -170,6 +170,7 @@ describe("searchSettings", () => { "source-control-writer-model", "source-control-writing-style", "t3-connect", + "hold-webhooks-while-offline", "tailscale-https", "wsl-backend", "auto-settle-inactive-threads", @@ -221,6 +222,24 @@ describe("searchSettings", () => { expect(browser).toContain("publish-agent-activity"); }); + it("offers webhook holding only while the managed tunnel is on, like its row", () => { + const availability = { + hasCloudPublicConfig: true, + hasEnvironment: true, + hasProviderSettingsEnvironment: true, + hasMacProviderSettingsEnvironment: false, + canManageLocalBackend: true, + isWslSettingsRowVisible: false, + hasThreadAutoSettlement: false, + }; + const itemIds = (managedTunnelActive: boolean) => + filterAvailableSettingsSearchItems({ ...availability, managedTunnelActive }).map( + (item) => item.id, + ); + expect(itemIds(false)).not.toContain("hold-webhooks-while-offline"); + expect(itemIds(true)).toContain("hold-webhooks-while-offline"); + }); + it("shows automatic settlement settings when the server supports them", () => { const available = filterAvailableSettingsSearchItems({ hasCloudPublicConfig: false, diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 3a36bf9cb705..be1eb15a6c1d 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -60,6 +60,8 @@ export interface SettingsSearchItem { readonly localBackendManagementOnly?: boolean; readonly localEnvironmentOnly?: boolean; readonly wslAvailableOnly?: boolean; + // Its row only renders while this environment's T3 Connect managed tunnel is on. + readonly managedTunnelOnly?: boolean; /** * Sorts after every other match. Keybinding commands mirror rows on other * surfaces, so "model" must still lead with Default model, not Model Picker. @@ -77,6 +79,7 @@ export interface SettingsSearchAvailability { readonly canManageLocalBackend: boolean; readonly isWslSettingsRowVisible: boolean; readonly hasThreadAutoSettlement: boolean; + readonly managedTunnelActive?: boolean; } /** @@ -844,6 +847,7 @@ export const SETTINGS_SEARCH_ITEMS = [ targetId: "connections-environment", searchTerms: ["webhook automations offline queue mailbox t3 connect"], cloudOnly: true, + managedTunnelOnly: true, }, { id: "publish-agent-activity", @@ -1027,7 +1031,8 @@ export function filterAvailableSettingsSearchItems( (!item.localBackendManagementOnly || availability.canManageLocalBackend) && (!item.localEnvironmentOnly || !availability.localEnvironmentDisabled) && (!item.wslAvailableOnly || availability.isWslSettingsRowVisible) && - (!item.requiresThreadAutoSettlement || availability.hasThreadAutoSettlement), + (!item.requiresThreadAutoSettlement || availability.hasThreadAutoSettlement) && + (!item.managedTunnelOnly || availability.managedTunnelActive === true), ); } diff --git a/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts b/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts index a5be950b81ab..3820357e59fa 100644 --- a/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts +++ b/apps/web/src/components/settings/useAvailableSettingsSearchItems.ts @@ -1,6 +1,7 @@ import { useMemo } from "react"; import { AuthAccessWriteScope } from "@t3tools/contracts"; +import { usePrimaryCloudLinkState } from "~/cloud/primaryCloudLinkState"; import { hasCloudPublicConfig } from "~/cloud/publicConfig"; import { isElectron } from "~/env"; import { isLocalEnvironmentDisabled } from "~/localEnvironment"; @@ -23,6 +24,10 @@ export function useAvailableSettingsSearchItems(scopeSearch: SettingsScopeSearch const desktopWsl = useEnvironmentQuery( isElectron && !localEnvironmentDisabled ? desktopWslStateAtom : null, ); + const cloudLinkState = usePrimaryCloudLinkState().data; + // Same fallback as the Connections row: older servers imply a tunnel from `linked`. + const managedTunnelActive = + cloudLinkState?.managedTunnelActive ?? cloudLinkState?.linked ?? false; const canManageLocalBackend = !localEnvironmentDisabled && (isElectron || @@ -59,8 +64,10 @@ export function useAvailableSettingsSearchItems(scopeSearch: SettingsScopeSearch }), hasThreadAutoSettlement: getThreadAutoSettlementSearchAvailability(environments).eligibleEnvironmentIds.length > 0, + managedTunnelActive, }), [ + managedTunnelActive, canManageLocalBackend, desktopWsl.data, desktopWsl.error, From 51c9791dbea000e6cafdc5fc0f791498e7ee4680 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 19:41:04 -0700 Subject: [PATCH 13/35] fix(web): thread automations hide Run now for webhook tasks Matches the settings list on web and mobile, which already omit it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../chat/ThreadAutomationsPanel.tsx | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/apps/web/src/components/chat/ThreadAutomationsPanel.tsx b/apps/web/src/components/chat/ThreadAutomationsPanel.tsx index 13fde2d2e38f..d4434fffbf18 100644 --- a/apps/web/src/components/chat/ThreadAutomationsPanel.tsx +++ b/apps/web/src/components/chat/ThreadAutomationsPanel.tsx @@ -186,28 +186,26 @@ export function ThreadAutomationsPanel(props: { /> Edit automation - - void runNow(task)} - > - - - } - /> - Run now - + {/* A webhook task runs from its URL; there is no request to run it with. */} + {task.schedule.type === "webhook" ? null : ( + + void runNow(task)} + > + + + } + /> + Run now + + )} Date: Sun, 4 Oct 2026 19:41:22 -0700 Subject: [PATCH 14/35] docs(user): webhook URLs need a managed tunnel, and what mobile can't do Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/user/project-settings.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/user/project-settings.md b/docs/user/project-settings.md index 43401e8800a8..c25eec9199e3 100644 --- a/docs/user/project-settings.md +++ b/docs/user/project-settings.md @@ -61,16 +61,18 @@ using its project, model, and workspace settings. Fixed-time schedules use that environment's time zone, which may differ from your phone's. You can edit, pause, resume, run immediately, or delete a task from the list. +Webhook tasks only run when their URL is called, so they can't be run +immediately. Leaving an edited form asks before discarding unsaved changes. ## Webhook automations -In **Settings → Scheduled tasks**, choose **On webhook** as a task's schedule to run it whenever another service -calls its URL, such as GitHub on a new pull request or a CI job that failed. -After you save the task, copy its URL from the editor. If the environment uses a -[T3 Connect](remote-access.md) managed tunnel, the URL is public; otherwise it -works anywhere the environment itself is reachable. **Rotate** replaces the URL and -the old one stops working. +In **Settings → Scheduled tasks**, choose **On webhook** (**Webhook** on mobile) +as a task's schedule to run it whenever another service calls its URL, such as +GitHub on a new pull request or a CI job that failed. A public URL needs a +[T3 Connect](remote-access.md) managed tunnel; after you save the task, copy +its URL from the editor. Without one, the editor shows only the URL's path. +**Rotate** replaces the URL and the old one stops working. The prompt decides what the agent sees. Placeholders pull values out of the request: `{{body.path}}` for a JSON or form field, `{{headers.name}}`, @@ -81,7 +83,9 @@ pull request link. A placeholder with no value is left empty. For GitHub, turn on **Require signature**, keep the header `x-hub-signature-256`, hex encoding and the `sha256=` prefix, and enter the same secret in the repository's webhook settings with content type -`application/json`. Requests without a valid signature are rejected. +`application/json`. Requests without a valid signature are rejected. Set this +up on desktop or web; mobile keeps an existing signature check but can't turn +one on. On desktop and web, pick **Deliveries** from a task's menu to see recent requests and the prompt each one produced. From 7e987504379fe210359130e62c6ecd42650de80f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 19:47:16 -0700 Subject: [PATCH 15/35] fix(server): relay webhook URLs use the tunnel key, not the environment id The relay hook URL is now /v1/hooks///, where the endpoint key is the managed tunnel name's 16-hex suffix. Without a valid tunnel name there is no relay URL and clients show the path. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scheduledTasks/ScheduledTaskService.ts | 36 ++++++++++---- .../ScheduledTaskService.webhook.test.ts | 35 ++++++++++++- .../server/src/scheduledTasks/webhookRoute.ts | 3 +- apps/server/src/server.ts | 49 +++++++++++-------- 4 files changed, 92 insertions(+), 31 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index f7ccf79eb751..4bfc6e5262c5 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -55,16 +55,37 @@ const WEBHOOK_MAX_QUEUED_PER_TASK = 20; /** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */ const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60; -/** Where a webhook task's public URL points. `relayUrl` is null when the environment is not linked to T3 Connect. */ +/** + * Where a webhook task's public URL points: `${relayHookBaseUrl}/${taskId}/${token}`. + * Null when the environment has no managed tunnel on T3 Connect; clients then show the path. + */ interface WebhookOrigin { - readonly environmentId: string; - readonly relayUrl: string | null; + readonly relayHookBaseUrl: string | null; +} + +const ENDPOINT_KEY = /^[0-9a-f]{16}$/; + +/** + * The relay's hook URL prefix for this environment. The relay finds the + * environment by its managed tunnel's key (the tunnel name's last segment), + * so the URL never reveals the environment id. + */ +export function relayHookBaseUrl(input: { + readonly relayUrl: string; + readonly tunnelName: string | undefined; +}): string | null { + const endpointKey = input.tunnelName?.split("-").at(-1); + const relayUrl = input.relayUrl.replace(/\/+$/, ""); + if (relayUrl === "" || endpointKey === undefined || !ENDPOINT_KEY.test(endpointKey)) { + return null; + } + return `${relayUrl}/v1/hooks/${endpointKey}`; } export class ScheduledTaskWebhookOrigin extends Context.Reference>( "t3/scheduledTasks/ScheduledTaskWebhookOrigin", { - defaultValue: () => Effect.succeed({ environmentId: "local", relayUrl: null }), + defaultValue: () => Effect.succeed({ relayHookBaseUrl: null }), }, ) {} @@ -255,13 +276,10 @@ function webhookEndpoint( origin: WebhookOrigin | null, ): ScheduledTask["webhook"] { if (row.webhook_token === null) return undefined; - const relayUrl = origin?.relayUrl?.replace(/\/+$/, "") ?? null; + const base = origin?.relayHookBaseUrl ?? null; return { path: webhookPath(row.task_id, row.webhook_token), - url: - relayUrl === null || origin === null - ? null - : `${relayUrl}/v1/hooks/${encodeURIComponent(origin.environmentId)}/${encodeURIComponent(row.task_id)}/${row.webhook_token}`, + url: base === null ? null : `${base}/${encodeURIComponent(row.task_id)}/${row.webhook_token}`, hasSecret: row.webhook_secret !== null, }; } diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index fc15b222a1b1..ce2712737351 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -64,7 +64,7 @@ const withService = ( readonly service: ScheduledTaskService.ScheduledTaskService["Service"]; readonly launches: Queue.Queue; }) => Effect.Effect, - options: { readonly gate?: Deferred.Deferred } = {}, + options: { readonly gate?: Deferred.Deferred; readonly relayHookBaseUrl?: string } = {}, ) => Effect.gen(function* () { const launches = yield* Queue.unbounded(); @@ -79,6 +79,10 @@ const withService = ( ), }), Layer.mock(ThreadManagementService.ThreadManagementService)({}), + Layer.succeed( + ScheduledTaskService.ScheduledTaskWebhookOrigin, + Effect.succeed({ relayHookBaseUrl: options.relayHookBaseUrl ?? null }), + ), ); return yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; @@ -121,6 +125,35 @@ it.effect("dispatches exactly the rendered prompt and logs the delivery", () => ), ); +it("builds the relay hook URL from the managed tunnel's key, never the environment id", () => { + const relayUrl = "https://relay.example.com/"; + assert.equal( + ScheduledTaskService.relayHookBaseUrl({ + relayUrl, + tunnelName: "t3coderelay-managedendpoint-dev-julius-0123456789abcdef", + }), + "https://relay.example.com/v1/hooks/0123456789abcdef", + ); + for (const tunnelName of [undefined, "t3coderelay-managedendpoint", "x-0123456789ABCDEF"]) { + assert.isNull(ScheduledTaskService.relayHookBaseUrl({ relayUrl, tunnelName })); + } +}); + +it.effect("gives webhook tasks a relay URL when the environment has a managed tunnel", () => + withService( + ({ service }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + const token = task.webhook!.path.split("/").at(-1); + assert.equal( + task.webhook?.url, + `https://relay.example.com/v1/hooks/0123456789abcdef/scheduled-task%3Ahook/${token}`, + ); + }), + { relayHookBaseUrl: "https://relay.example.com/v1/hooks/0123456789abcdef" }, + ), +); + it.effect("answers not found for a wrong token or unknown hook without logging", () => withService(({ service }) => Effect.gen(function* () { diff --git a/apps/server/src/scheduledTasks/webhookRoute.ts b/apps/server/src/scheduledTasks/webhookRoute.ts index 3b7f64bc51b9..1e02a4996286 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.ts @@ -19,7 +19,8 @@ const json = (status: number, body: Record) => * Handles `/api/hooks/:hookId/:token` for every accepted method. The endpoint * is raw so the signature is checked over the exact body bytes; the service * checks the token and signature. It is reachable directly, over the managed - * tunnel, or through the relay's stable `/v1/hooks/...` URL. + * tunnel, or through the relay's stable `/v1/hooks/:endpointKey/:hookId/:token` + * URL, where the endpoint key is this environment's managed tunnel key. */ const handleWebhook = (scheduledTasks: ScheduledTaskService.ScheduledTaskService["Service"]) => diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 8328f35fafbe..3be16c3a2ffb 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -120,8 +120,15 @@ import * as ReplayMarkers from "./auth/replayMarkers.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import { webhookHttpApiLayer } from "./scheduledTasks/webhookRoute.ts"; import * as HeldHooksWaker from "./relay/HeldHooksWaker.ts"; -import { ScheduledTaskWebhookOrigin } from "./scheduledTasks/ScheduledTaskService.ts"; -import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./cloud/config.ts"; +import { + relayHookBaseUrl, + ScheduledTaskWebhookOrigin, +} from "./scheduledTasks/ScheduledTaskService.ts"; +import { + CLOUD_ENDPOINT_RUNTIME_CONFIG, + decodeRuntimeConfig, + RELAY_URL_SECRET, +} from "./cloud/config.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import { connectHttpApiLayer, @@ -453,24 +460,26 @@ const CloudManagedEndpointRuntimeLive = Layer.mergeAll( // to is configured; otherwise clients show the environment-relative path. const ScheduledTaskWebhookOriginLive = Layer.effect( ScheduledTaskWebhookOrigin, - Effect.map( - Effect.all([ServerEnvironment.ServerEnvironment, ServerSecretStore.ServerSecretStore]), - ([environment, secrets]) => - // The reference holds an effect so each read sees the current link state. - Effect.gen(function* () { - const [relayUrl, tunnelConfig] = yield* Effect.all([ - secrets.get(RELAY_URL_SECRET), - secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), - ]).pipe(Effect.orElseSucceed(() => [Option.none(), Option.none()] as const)); - return { - environmentId: yield* environment.getEnvironmentId, - relayUrl: - Option.isSome(relayUrl) && Option.isSome(tunnelConfig) - ? new TextDecoder().decode(relayUrl.value) || null - : null, - }; - }), - ), + Effect.gen(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + // The reference holds an effect so each read sees the current link state. + return Effect.gen(function* () { + const [relayUrl, tunnelConfig] = yield* Effect.all([ + secrets.get(RELAY_URL_SECRET), + secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + ]).pipe(Effect.orElseSucceed(() => [Option.none(), Option.none()] as const)); + if (Option.isNone(relayUrl) || Option.isNone(tunnelConfig)) { + return { relayHookBaseUrl: null }; + } + const config = decodeRuntimeConfig(new TextDecoder().decode(tunnelConfig.value)); + return { + relayHookBaseUrl: relayHookBaseUrl({ + relayUrl: new TextDecoder().decode(relayUrl.value), + tunnelName: Option.isSome(config) ? config.value.tunnelName : undefined, + }), + }; + }); + }), ); const OrchestrationV2RuntimeLayerLive = OrchestrationV2ProductionLayerLive.pipe( From ad90b99e9c73ac90b3889c1a4126c75badc3b054 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 20:04:54 -0700 Subject: [PATCH 16/35] fix(mobile): the webhook trigger is labelled On webhook, as on web Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/features/settings/SettingsScheduledTasksRouteScreen.tsx | 2 +- docs/user/project-settings.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index 7de8ff515014..e8593ff407f0 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -839,7 +839,7 @@ function TaskForm({ options={[ { value: "fixed_time", label: "At a time" }, { value: "interval", label: "Interval" }, - { value: "webhook", label: "Webhook" }, + { value: "webhook", label: "On webhook" }, ]} selected={draft.schedule.mode} onSelect={(mode) => { diff --git a/docs/user/project-settings.md b/docs/user/project-settings.md index c25eec9199e3..d1affc3f0793 100644 --- a/docs/user/project-settings.md +++ b/docs/user/project-settings.md @@ -67,7 +67,7 @@ Leaving an edited form asks before discarding unsaved changes. ## Webhook automations -In **Settings → Scheduled tasks**, choose **On webhook** (**Webhook** on mobile) +In **Settings → Scheduled tasks**, choose **On webhook** as a task's schedule to run it whenever another service calls its URL, such as GitHub on a new pull request or a CI job that failed. A public URL needs a [T3 Connect](remote-access.md) managed tunnel; after you save the task, copy From eeaabef487b5830fe7139c4d60d951e1854506c5 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 20:04:54 -0700 Subject: [PATCH 17/35] perf(web): skip the T3 Connect link-state read in builds without T3 Connect Settings search now reads the link state to decide whether to offer the webhook-hold setting. Builds without T3 Connect have no link, so the read is skipped there instead of polling the environment. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/web/src/cloud/primaryCloudLinkState.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/web/src/cloud/primaryCloudLinkState.ts b/apps/web/src/cloud/primaryCloudLinkState.ts index d46ee641d574..8d9b919db05f 100644 --- a/apps/web/src/cloud/primaryCloudLinkState.ts +++ b/apps/web/src/cloud/primaryCloudLinkState.ts @@ -13,6 +13,7 @@ import { usePrimaryEnvironment } from "../state/environments"; import { runtime } from "../lib/runtime"; import { appAtomRegistry } from "../rpc/atomRegistry"; import { readPrimaryCloudLinkState, type CloudLinkTarget } from "./linkEnvironment"; +import { hasCloudPublicConfig } from "./publicConfig"; const primaryCloudLinkAtomRuntime = Atom.runtime( Layer.effect( @@ -62,9 +63,11 @@ export function usePrimaryCloudLinkState() { : null, [primary], ); - const atom = target - ? primaryCloudLinkStateAtom(targetKey(target)) - : EMPTY_PRIMARY_CLOUD_LINK_STATE_ATOM; + // Builds without T3 Connect have no link to read; skip the request. + const atom = + target && hasCloudPublicConfig() + ? primaryCloudLinkStateAtom(targetKey(target)) + : EMPTY_PRIMARY_CLOUD_LINK_STATE_ATOM; const result = useAtomValue(atom); const refresh = useCallback(() => { refreshPrimaryCloudLinkState(target); From 577fa27f4d16743a498cd71d992bd6c5ffb81662 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 19:49:37 -0700 Subject: [PATCH 18/35] fix(relay): webhook URLs name one managed endpoint, not a claimable environment id The relay routed /v1/hooks//... to whichever active managed link of that environment id Postgres returned first. Environment ids are public (they are in every webhook URL) and any account can link one with a self-made key, so another account could receive an environment's webhooks, flip its hold opt-in, and point its held requests at their own tunnel. Hook URLs now carry the managed endpoint's key: the hash suffix of its tunnel name, which covers user and environment and is unique, so it names exactly one allocation and link. Held requests live in one inbox per endpoint key. The hold opt-in and wake act only on links proven by the caller's environment key, and opting out drops what is held. Also, from review: - 502/503/504 from cloudflared (local server down) are held like 530. - In the inbox, a 429 or 500 from the environment keeps the request but lets other hooks' requests through; a held request that cannot be decoded is dropped instead of stalling the inbox. - An overall budget per endpoint, so minting tokens or hook ids cannot buy unlimited lookups, forwards, or inbox space. - Hook routes get a 25 s deadline, so the hold step is not cut off. - Unlinking clears the endpoint's inbox after teardown, best effort, instead of failing the unlink before the tunnel is removed. - Upstream failures are annotated on the span and undelivered retries are logged. Co-Authored-By: Claude Opus 5.5 (1M context) --- infra/relay/src/deploymentConfig.ts | 13 ++ .../environments/EnvironmentConnector.test.ts | 1 + .../src/environments/EnvironmentLinks.ts | 19 ++- .../ManagedEndpointAllocations.ts | 28 +++++ .../ManagedEndpointProvider.test.ts | 1 + .../ManagedEndpointReaper.test.ts | 1 + infra/relay/src/hooks/HookForwarder.test.ts | 97 ++++++++++++--- infra/relay/src/hooks/HookForwarder.ts | 114 ++++++++++++----- infra/relay/src/hooks/HookInbox.ts | 14 +-- infra/relay/src/hooks/HookInboxObject.ts | 23 +++- infra/relay/src/hooks/HookInboxStore.test.ts | 41 +++++- infra/relay/src/hooks/HookInboxStore.ts | 52 ++++++-- infra/relay/src/http/Api.test.ts | 56 ++++++--- infra/relay/src/http/Api.ts | 117 +++++++++++++----- infra/relay/src/worker.ts | 42 ++++--- packages/contracts/src/relay.ts | 8 +- 16 files changed, 484 insertions(+), 143 deletions(-) diff --git a/infra/relay/src/deploymentConfig.ts b/infra/relay/src/deploymentConfig.ts index 961c6f1b2f4a..e6ad6c4a8ee4 100644 --- a/infra/relay/src/deploymentConfig.ts +++ b/infra/relay/src/deploymentConfig.ts @@ -124,3 +124,16 @@ export function managedEndpointTunnelNamePrefix(stage: string): string { export function managedEndpointTunnelName(stage: string, hash: string): string { return `${managedEndpointTunnelNamePrefix(stage)}${stableSuffix(hash)}`; } + +/** + * A managed endpoint's public key in webhook URLs: the hash suffix its tunnel + * name ends with. The hash covers user and environment, so one key names + * exactly one link, unlike the environment id, which any account can claim. + */ +export const MANAGED_ENDPOINT_KEY_PATTERN = new RegExp( + `^[0-9a-f]{${MANAGED_ENDPOINT_HASH_LENGTH}}$`, +); + +export function managedEndpointTunnelNameForKey(stage: string, endpointKey: string): string { + return `${managedEndpointTunnelNamePrefix(stage)}${endpointKey}`; +} diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index 7746cbeaae22..a0da2c17d1d9 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -194,6 +194,7 @@ function makeAllocations( }, ): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] { return { + getByTunnelName: () => Effect.die("unused getByTunnelName"), get: () => Effect.succeed(allocation), reserve: () => Effect.die("unused"), recordTunnel: () => Effect.die("unused"), diff --git a/infra/relay/src/environments/EnvironmentLinks.ts b/infra/relay/src/environments/EnvironmentLinks.ts index c2e68fa13b1b..8b2dd3c17176 100644 --- a/infra/relay/src/environments/EnvironmentLinks.ts +++ b/infra/relay/src/environments/EnvironmentLinks.ts @@ -126,9 +126,15 @@ export class EnvironmentLinks extends Context.Service< readonly userId: string; readonly environmentId: string; }) => Effect.Effect; - /** Active relay-managed links for an environment, across all users (webhook forwarding). */ + /** + * Active relay-managed links for an environment, narrowed to one user or to + * links proven by one environment key. The environment id alone is public + * and any account can link it, so callers acting on it must narrow. + */ readonly findActiveManagedForEnvironment: (input: { readonly environmentId: string; + readonly userId?: string; + readonly environmentPublicKey?: string; }) => Effect.Effect< ReadonlyArray< RelayLinkedEnvironmentRecord & { @@ -138,9 +144,10 @@ export class EnvironmentLinks extends Context.Service< >, EnvironmentLinkEnvironmentLookupPersistenceError >; - /** Sets the webhook-hold opt-in on every active link of an environment. */ + /** Sets the webhook-hold opt-in on the active links proven by one environment key. */ readonly setHoldWebhooksWhileOffline: (input: { readonly environmentId: string; + readonly environmentPublicKey: string; readonly holdWebhooksWhileOffline: boolean; }) => Effect.Effect; readonly revokeForUser: (input: { @@ -384,10 +391,13 @@ const make = Effect.gen(function* () { isNull(relayEnvironmentLinks.revokedAt), eq(relayEnvironmentLinks.endpointProviderKind, "cloudflare_tunnel"), eq(relayEnvironmentLinks.managedTunnelsEnabled, true), + input.userId === undefined ? undefined : eq(relayEnvironmentLinks.userId, input.userId), + input.environmentPublicKey === undefined + ? undefined + : eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey), ), ) - // One row per user who linked this environment; every row is checked - // until one has a ready endpoint, so none may be cut off. + // At most one row per user who linked this environment. .pipe( Effect.map((rows) => rows.map((row) => ({ @@ -426,6 +436,7 @@ const make = Effect.gen(function* () { .where( and( eq(relayEnvironmentLinks.environmentId, input.environmentId), + eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey), isNull(relayEnvironmentLinks.revokedAt), ), ) diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 1b4d3744963b..e2d8475a1b2a 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -64,6 +64,7 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro "claim-deprovision", "remove", "remove-claimed", + "get-by-tunnel-name", ]), stage: Schema.Literals(["database-request", "resolve-reservation"]), userId: Schema.String, @@ -132,6 +133,10 @@ export class ManagedEndpointAllocations extends Context.Service< readonly get: ( input: ManagedEndpointAllocationKey, ) => Effect.Effect; + /** The allocation that owns a tunnel name; tunnel names are unique. */ + readonly getByTunnelName: ( + tunnelName: string, + ) => Effect.Effect; readonly reserve: ( input: ReserveManagedEndpointAllocationInput, ) => Effect.Effect; @@ -230,6 +235,29 @@ export const make = Effect.gen(function* () { ), ); }), + getByTunnelName: Effect.fn("relay.managed_endpoint_allocations.get_by_tunnel_name")(function* ( + tunnelName: string, + ) { + return yield* db + .select(allocationSelection) + .from(relayManagedEndpointAllocations) + .where(eq(relayManagedEndpointAllocations.tunnelName, tunnelName)) + .limit(1) + .pipe( + Effect.map((rows) => rows[0] ?? null), + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "get-by-tunnel-name", + stage: "database-request", + userId: "", + environmentId: "", + tunnelName, + cause, + }), + ), + ); + }), reserve: Effect.fn("relay.managed_endpoint_allocations.reserve")(function* ( input: ReserveManagedEndpointAllocationInput, ) { diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index becaf11488c8..94675125ded6 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -204,6 +204,7 @@ function makeAllocations(calls: AllocationCall[] = []) { } }; return ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + getByTunnelName: () => Effect.die("unused getByTunnelName"), get: (input) => Effect.sync(() => { calls.push({ operation: "get", input }); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index 9a2d62e36067..6f4d38d6a06d 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -166,6 +166,7 @@ function harness(input?: { }), }); const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + getByTunnelName: () => Effect.die("unused"), get: () => Effect.die("unused"), reserve: () => Effect.die("unused"), recordTunnel: () => Effect.die("unused"), diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index 5bea17f90e8d..f3acc8a7fcf6 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -47,16 +47,17 @@ const settings: RelayConfiguration.RelayConfiguration["Service"] = { cloudMintPrivateKey: Redacted.make("cloud-mint-private-key"), cloudMintPublicKey: "cloud-mint-public-key", managedEndpointBaseDomain: "example.test", - managedEndpointNamespace: undefined, + managedEndpointNamespace: "dev", }; const environmentId = "env-hook"; +const endpointKey = "0123456789abcdef"; const readyAllocation: ManagedEndpointAllocations.ManagedEndpointAllocation = { userId: "user_1", environmentId, hostname: "env.example.test", tunnelId: "tunnel-id", - tunnelName: "tunnel-name", + tunnelName: `t3coderelay-managedendpoint-dev-${endpointKey}`, dnsRecordId: "dns-record-id", readyAt: "2026-05-25T00:00:00.000Z", origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, @@ -85,6 +86,7 @@ interface Harness { readonly links?: ReadonlyArray; readonly allocation?: ManagedEndpointAllocations.ManagedEndpointAllocation | null; readonly allow?: (key: string) => boolean; + readonly allowEndpoint?: (endpointKey: string) => boolean; /** Inbox capacity; hold reports full once this many requests are held. */ readonly inboxCapacity?: number; } @@ -109,12 +111,19 @@ function makeHarness(options: Harness = {}) { Layer.mock(EnvironmentLinks.EnvironmentLinks, { findActiveManagedForEnvironment: (input) => Effect.succeed( - input.environmentId === environmentId ? (options.links ?? [managedLink]) : [], + (options.links ?? [managedLink]).filter( + (link) => + link.environmentId === input.environmentId && + (input.userId === undefined || link.userId === input.userId), + ), ), }), Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, { - get: () => - Effect.succeed(options.allocation === undefined ? readyAllocation : options.allocation), + getByTunnelName: (tunnelName) => { + const allocation = + options.allocation === undefined ? readyAllocation : options.allocation; + return Effect.succeed(allocation?.tunnelName === tunnelName ? allocation : null); + }, }), Layer.succeed( HttpClient.HttpClient, @@ -133,11 +142,13 @@ function makeHarness(options: Harness = {}) { }), NodeCrypto.layer, Layer.succeed(HookForwarder.HookRateLimiter, { - allow: (key) => + allowHook: (key) => Effect.sync(() => { rateLimitKeys.push(key); return options.allow ? options.allow(key) : true; }), + allowEndpoint: (key) => + Effect.sync(() => (options.allowEndpoint ? options.allowEndpoint(key) : true)), }), ), ), @@ -166,7 +177,7 @@ function makeHarness(options: Harness = {}) { } const hookUrl = (path = "hook-1/secret-token", query = "") => - `https://relay.test/v1/hooks/${environmentId}/${path}${query}`; + `https://relay.test/v1/hooks/${endpointKey}/${path}${query}`; const readBody = (response: HttpServerResponse.HttpServerResponse) => Effect.promise(() => HttpServerResponse.toWeb(response).arrayBuffer()).pipe( @@ -302,14 +313,17 @@ describe("HookForwarder", () => { }), ); - it.effect("returns 404 for unknown environments and unready endpoints", () => + it.effect("returns 404 for unknown endpoints and unready endpoints", () => Effect.gen(function* () { const unknown = makeHarness(); - const response = yield* unknown.send( - new Request("https://relay.test/v1/hooks/other-env/hook-1/token", { method: "POST" }), - ); - expect(response.status).toBe(404); - expect(yield* readJson(response)).toEqual({ error: "hook_not_found" }); + for (const key of ["fedcba9876543210", environmentId]) { + const response = yield* unknown.send( + new Request(`https://relay.test/v1/hooks/${key}/hook-1/token`, { method: "POST" }), + ); + expect(response.status).toBe(404); + expect(yield* readJson(response)).toEqual({ error: "hook_not_found" }); + } + expect(unknown.sent).toHaveLength(0); const unready = makeHarness({ allocation: { ...readyAllocation, readyAt: null } }); const unreadyResponse = yield* unready.send(new Request(hookUrl(), { method: "POST" })); @@ -318,6 +332,42 @@ describe("HookForwarder", () => { }), ); + it.effect("forwards only to the link that owns the endpoint key", () => + Effect.gen(function* () { + // Another account linked the same environment id under its own key; its + // link must not receive this endpoint's hooks, whatever order rows come in. + const intruder = { + ...managedLink, + userId: "user_attacker", + environmentPublicKey: "attacker-key", + endpoint: { ...managedLink.endpoint, httpBaseUrl: "https://attacker.example.test/" }, + }; + const harness = makeHarness({ links: [intruder, managedLink] }); + const response = yield* harness.send(new Request(hookUrl(), { method: "POST", body: "{}" })); + expect(response.status).toBe(200); + expect(harness.sent.map((request) => new URL(request.url).host)).toEqual([ + "env.example.test", + ]); + + // Without the owner's link, the key resolves to nothing at all. + const orphaned = makeHarness({ links: [intruder] }); + const orphanedResponse = yield* orphaned.send( + new Request(hookUrl(), { method: "POST", body: "{}" }), + ); + expect(orphanedResponse.status).toBe(404); + expect(orphaned.sent).toHaveLength(0); + }), + ); + + it.effect("returns 429 when the endpoint's overall budget is spent", () => + Effect.gen(function* () { + const harness = makeHarness({ allowEndpoint: () => false }); + const response = yield* harness.send(new Request(hookUrl(), { method: "POST" })); + expect(response.status).toBe(429); + expect(harness.sent).toHaveLength(0); + }), + ); + it.effect("maps tunnel-offline and network failures to 503", () => Effect.gen(function* () { const offline = makeHarness({ @@ -491,9 +541,7 @@ describe("HookForwarder", () => { expect(serialized).not.toContain("also-secret"); expect(serialized).not.toContain("header-secret"); const server = spans.find((span) => span.kind === "server"); - expect(server?.attributes.get("url.path")).toBe( - `/v1/hooks/${environmentId}/hook-1/`, - ); + expect(server?.attributes.get("url.path")).toBe(`/v1/hooks/${endpointKey}/hook-1/`); }), ); @@ -505,6 +553,23 @@ describe("HookForwarder", () => { }), ); + it.effect("holds when cloudflared answers that the local server is down", () => + Effect.gen(function* () { + for (const status of [502, 503, 504, 530]) { + const harness = makeHarness({ + links: [{ ...managedLink, holdWebhooksWhileOffline: true }], + execute: (request) => + Effect.succeed(HttpClientResponse.fromWeb(request, new Response("", { status }))), + }); + const response = yield* harness.send( + new Request(hookUrl(), { method: "POST", body: "{}" }), + ); + expect(response.status).toBe(202); + expect(harness.held).toHaveLength(1); + } + }), + ); + it.effect("stays a plain proxy when the environment has not opted in", () => Effect.gen(function* () { const harness = makeHarness({ execute: offline }); diff --git a/infra/relay/src/hooks/HookForwarder.ts b/infra/relay/src/hooks/HookForwarder.ts index d4b717b5524a..cd1cfbd973ff 100644 --- a/infra/relay/src/hooks/HookForwarder.ts +++ b/infra/relay/src/hooks/HookForwarder.ts @@ -14,6 +14,10 @@ import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import { RelayApi } from "@t3tools/contracts/relay"; import * as RelayConfiguration from "../Config.ts"; +import { + MANAGED_ENDPOINT_KEY_PATTERN, + managedEndpointTunnelNameForKey, +} from "../deploymentConfig.ts"; import { validateManagedEndpoint } from "../environments/EnvironmentConnector.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; @@ -23,6 +27,22 @@ import { sendUpstream, TUNNEL_OFFLINE_STATUS } from "./upstream.ts"; export const RELAY_HOOK_PATH_PREFIX = "/v1/hooks/"; export const RELAY_HOOK_MAX_BODY_BYTES = 1_048_576; export const RELAY_HOOK_RATE_LIMIT = { limit: 60, periodSeconds: 60 } as const; +/** + * Hook budgets are per URL, and a sender who knows an endpoint key can mint + * new URLs for free, so every endpoint also has one overall budget. + */ +export const RELAY_HOOK_ENDPOINT_RATE_LIMIT = { limit: 600, periodSeconds: 60 } as const; +/** + * Upstream statuses that mean the environment did not take the request: the + * tunnel has no origin (530) or cloudflared cannot reach the local server + * (502, 503, 504) while it restarts. + */ +export const ENVIRONMENT_UNREACHABLE_STATUSES: ReadonlySet = new Set([ + 502, + 503, + 504, + TUNNEL_OFFLINE_STATUS, +]); const DROPPED_REQUEST_HEADERS = new Set([ "host", @@ -54,13 +74,13 @@ export const redactRelayHookUrl = (url: string): string => { /** * Request budget for public hook forwarding, keyed by a hash of the hook URL - * (environment, hook and token). Requests with a wrong token get their own + * (endpoint, hook and token). Requests with a wrong token get their own * budget, so they cannot use up a real sender's; the environment rejects them. * Built from decoded segments, because the environment decodes them too: two * spellings of one token (`token`, `%74oken`) must share one budget. */ const hookBudgetKey = (hook: { - readonly environmentId: string; + readonly endpointKey: string; readonly hookId: string; readonly token: string; }) => @@ -69,7 +89,7 @@ const hookBudgetKey = (hook: { "SHA-256", // Length-prefixed, so no segment contents can make two keys collide. new TextEncoder().encode( - [hook.environmentId, hook.hookId, hook.token] + [hook.endpointKey, hook.hookId, hook.token] .map((part) => `${part.length}:${part}`) .join(""), ), @@ -82,7 +102,12 @@ const hookBudgetKey = (hook: { export class HookRateLimiter extends Context.Service< HookRateLimiter, - { readonly allow: (key: string) => Effect.Effect } + { + /** One hook URL's budget, keyed by `hookBudgetKey`. */ + readonly allowHook: (key: string) => Effect.Effect; + /** One endpoint's overall budget, keyed by its endpoint key. */ + readonly allowEndpoint: (endpointKey: string) => Effect.Effect; + } >()("t3code-relay/hooks/HookForwarder/HookRateLimiter") {} export class HookForwarder extends Context.Service< @@ -106,13 +131,15 @@ function parseHookPath(url: string) { const path = queryIndex === -1 ? url : url.slice(0, queryIndex); const search = queryIndex === -1 ? "" : url.slice(queryIndex); const segments = path.split("/"); - // ["", "v1", "hooks", environmentId, hookId, token] + // ["", "v1", "hooks", endpointKey, hookId, token] if (segments.length !== 6) return null; - const [, , , rawEnvironmentId, rawHookId, rawToken] = segments; - if (!rawEnvironmentId || !rawHookId || !rawToken) return null; + const [, , , endpointKey, rawHookId, rawToken] = segments; + if (!endpointKey || !MANAGED_ENDPOINT_KEY_PATTERN.test(endpointKey) || !rawHookId || !rawToken) { + return null; + } try { return { - environmentId: decodeURIComponent(rawEnvironmentId), + endpointKey, hookId: decodeURIComponent(rawHookId), token: decodeURIComponent(rawToken), // Forward the encoded segments byte-for-byte; the environment decodes them. @@ -183,30 +210,47 @@ const readCappedBody = (request: HttpServerRequest.HttpServerRequest) => }); /** - * The environment's ready managed endpoint, across every user that linked it, - * with whether it opted in to holding webhooks while offline. + * The ready managed endpoint a webhook URL's endpoint key names, with whether + * its link opted in to holding webhooks while offline. The key is the tunnel + * name's hash of user and environment, so it names exactly one allocation and + * at most one active link; nobody else can link their way onto it. */ export const resolveHookEndpoint = Effect.fn("relay.hooks.resolve_endpoint")(function* ( - environmentId: string, + endpointKey: string, ) { const links = yield* EnvironmentLinks.EnvironmentLinks; const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; const settings = yield* RelayConfiguration.RelayConfiguration; - const candidates = yield* links.findActiveManagedForEnvironment({ environmentId }); - for (const link of candidates) { - const allocation = yield* allocations.get({ userId: link.userId, environmentId }); - const result = validateManagedEndpoint({ - link, - allocation, - baseDomain: settings.managedEndpointBaseDomain, - }); - if (Result.isSuccess(result)) { - return { ...result.success, holdWhileOffline: link.holdWebhooksWhileOffline }; - } - } - return null; + if (!settings.managedEndpointNamespace) return null; + const allocation = yield* allocations.getByTunnelName( + managedEndpointTunnelNameForKey(settings.managedEndpointNamespace, endpointKey), + ); + if (allocation === null) return null; + const [link] = yield* links.findActiveManagedForEnvironment({ + environmentId: allocation.environmentId, + userId: allocation.userId, + }); + if (!link) return null; + const result = validateManagedEndpoint({ + link, + allocation, + baseDomain: settings.managedEndpointBaseDomain, + }); + if (Result.isFailure(result)) return null; + return { + ...result.success, + environmentId: allocation.environmentId, + holdWhileOffline: link.holdWebhooksWhileOffline, + }; }); +/** The endpoint key of an environment's own managed endpoint, for authenticated callers. */ +export const endpointKeyForTunnelName = (namespace: string, tunnelName: string): string | null => { + const prefix = managedEndpointTunnelNameForKey(namespace, ""); + const key = tunnelName.startsWith(prefix) ? tunnelName.slice(prefix.length) : ""; + return MANAGED_ENDPOINT_KEY_PATTERN.test(key) ? key : null; +}; + const make = Effect.gen(function* () { const links = yield* EnvironmentLinks.EnvironmentLinks; const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; @@ -228,10 +272,15 @@ const make = Effect.gen(function* () { return hookNotFound(); } yield* Effect.annotateCurrentSpan({ - "relay.environment_id": parsed.environmentId, + "relay.hook.endpoint_key": parsed.endpointKey, "relay.hook_id": parsed.hookId, }); - if (!(yield* rateLimiter.allow(yield* hookBudgetKey(parsed)))) { + // A coarse budget per endpoint first, so minting new hook ids or tokens + // cannot buy unlimited lookups and forwards, or fill the inbox. + if ( + !(yield* rateLimiter.allowEndpoint(parsed.endpointKey)) || + !(yield* rateLimiter.allowHook(yield* hookBudgetKey(parsed))) + ) { yield* outcome("rate_limited"); return errorResponse(429, "rate_limited", { "retry-after": String(RELAY_HOOK_RATE_LIMIT.periodSeconds), @@ -243,13 +292,13 @@ const make = Effect.gen(function* () { return errorResponse(413, "payload_too_large"); } - const endpoint = yield* resolveHookEndpoint(parsed.environmentId).pipe( + const endpoint = yield* resolveHookEndpoint(parsed.endpointKey).pipe( Effect.provideService(EnvironmentLinks.EnvironmentLinks, links), Effect.provideService(ManagedEndpointAllocations.ManagedEndpointAllocations, allocations), Effect.provideService(RelayConfiguration.RelayConfiguration, settings), Effect.catch((error) => Effect.logWarning("Failed to resolve hook endpoint", { - environmentId: parsed.environmentId, + endpointKey: parsed.endpointKey, errorTag: error._tag, }).pipe(Effect.as(null)), ), @@ -258,6 +307,7 @@ const make = Effect.gen(function* () { yield* outcome("not_found"); return hookNotFound(); } + yield* Effect.annotateCurrentSpan({ "relay.environment_id": endpoint.environmentId }); const body = request.method === "GET" @@ -294,14 +344,14 @@ const make = Effect.gen(function* () { } const stored = yield* inbox .hold({ - environmentId: parsed.environmentId, + endpointKey: parsed.endpointKey, baseUrl: endpoint.httpBaseUrl, hook, }) .pipe( Effect.catch((cause) => Effect.logWarning("Could not hold webhook request", { - environmentId: parsed.environmentId, + environmentId: endpoint.environmentId, errorTag: cause._tag, }).pipe(Effect.as(null)), ), @@ -323,13 +373,15 @@ const make = Effect.gen(function* () { Effect.result, ); if (Result.isFailure(upstream)) { + yield* Effect.annotateCurrentSpan({ "relay.hook.upstream_error": upstream.failure._tag }); return yield* holdOrFail(503, "environment_unavailable"); } if (Option.isNone(upstream.success)) { return yield* holdOrFail(504, "environment_timeout"); } const response = upstream.success.value; - if (response.status === TUNNEL_OFFLINE_STATUS) { + if (ENVIRONMENT_UNREACHABLE_STATUSES.has(response.status)) { + yield* Effect.annotateCurrentSpan({ "relay.hook.upstream_status": response.status }); return yield* holdOrFail(503, "environment_unavailable"); } yield* Effect.annotateCurrentSpan({ diff --git a/infra/relay/src/hooks/HookInbox.ts b/infra/relay/src/hooks/HookInbox.ts index 61f332612dfe..cc3d1517df4c 100644 --- a/infra/relay/src/hooks/HookInbox.ts +++ b/infra/relay/src/hooks/HookInbox.ts @@ -6,35 +6,35 @@ import type { HeldHook } from "./HookInboxStore.ts"; export class HookInboxError extends Schema.TaggedError()("HookInboxError", { operation: Schema.Literals(["hold", "wake", "clear"]), - environmentId: Schema.String, + endpointKey: Schema.String, cause: Schema.Defect(), }) { override get message(): string { - return `Hook inbox '${this.operation}' failed for environment '${this.environmentId}'`; + return `Hook inbox '${this.operation}' failed for endpoint '${this.endpointKey}'`; } } /** * Webhook requests held for environments that opted in, while they are - * offline. Each environment's requests live in its own `HookInboxObject`, - * which delivers them itself once the environment is back. + * offline. Each managed endpoint's requests live in its own `HookInboxObject`, + * named by endpoint key, which delivers them itself once the environment is back. */ export class HookInbox extends Context.Service< HookInbox, { /** False when the environment's inbox is full and nothing was stored. */ readonly hold: (input: { - readonly environmentId: string; + readonly endpointKey: string; readonly baseUrl: string; readonly hook: HeldHook; }) => Effect.Effect; /** Delivers what is waiting now; true when anything was waiting. */ readonly wake: (input: { - readonly environmentId: string; + readonly endpointKey: string; readonly baseUrl: string; }) => Effect.Effect; readonly clear: (input: { - readonly environmentId: string; + readonly endpointKey: string; }) => Effect.Effect; } >()("t3code-relay/hooks/HookInbox") {} diff --git a/infra/relay/src/hooks/HookInboxObject.ts b/infra/relay/src/hooks/HookInboxObject.ts index afe1f5c461b1..41786d5d4846 100644 --- a/infra/relay/src/hooks/HookInboxObject.ts +++ b/infra/relay/src/hooks/HookInboxObject.ts @@ -10,8 +10,14 @@ import * as FetchHttpClient from "effect/http/FetchHttpClient"; import * as HookInboxStore from "./HookInboxStore.ts"; import { sendUpstream, TUNNEL_OFFLINE_STATUS } from "./upstream.ts"; -/** Statuses that mean the environment did not get to the request; it is tried again later. */ -const RETRY_STATUSES = new Set([429, 502, 503, 504, TUNNEL_OFFLINE_STATUS]); +/** Statuses that mean the environment is not there; the whole inbox waits and backs off. */ +const UNREACHABLE_STATUSES = new Set([502, 503, 504, TUNNEL_OFFLINE_STATUS]); +/** + * Statuses that mean the environment is there but did not take this request + * yet: its task's queue is full (429) or it failed while handling it (500). + * The environment drops a delivery id it has already run, so retrying is safe. + */ +const BUSY_STATUSES = new Set([429, 500]); /** When a run itself fails, the next one is tried after this long. */ const RUN_FAILURE_RETRY_MS = 60_000; @@ -26,7 +32,7 @@ export interface HookInboxObjectShape { } /** - * One per environment, addressed by environment id. Holds webhook requests + * One per managed endpoint, addressed by endpoint key. Holds webhook requests * the environment could not take, in SQLite, and pushes them back through * its tunnel from the alarm, oldest first, backing off while it stays away. */ @@ -41,9 +47,16 @@ const deliver = (baseUrl: string, hook: HookInboxStore.HeldHook) => Effect.map((result): HookInboxStore.DeliveryOutcome => { // Unreachable or timed out: a timeout may still have run it, and the // environment drops a delivery id it has already seen. - if (Result.isFailure(result) || Option.isNone(result.success)) return "retry"; - return RETRY_STATUSES.has(result.success.value.status) ? "retry" : "delivered"; + if (Result.isFailure(result) || Option.isNone(result.success)) return "unreachable"; + const status = result.success.value.status; + if (UNREACHABLE_STATUSES.has(status)) return "unreachable"; + return BUSY_STATUSES.has(status) ? "busy" : "delivered"; }), + Effect.tap((outcome) => + outcome === "delivered" + ? Effect.void + : Effect.logInfo("Held webhook not delivered yet", { outcome, deliveryId: hook.id }), + ), Effect.provide(FetchHttpClient.layer), ); diff --git a/infra/relay/src/hooks/HookInboxStore.test.ts b/infra/relay/src/hooks/HookInboxStore.test.ts index 2c409e275a50..2c2dfce20804 100644 --- a/infra/relay/src/hooks/HookInboxStore.test.ts +++ b/infra/relay/src/hooks/HookInboxStore.test.ts @@ -4,6 +4,7 @@ import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; import * as HookInboxStore from "./HookInboxStore.ts"; @@ -64,7 +65,7 @@ describe("HookInboxStore", () => { Effect.gen(function* () { yield* HookInboxStore.hold(yield* hook("first"), BASE_URL); yield* HookInboxStore.hold(yield* hook("second"), BASE_URL); - const offline = deliverer(() => "retry"); + const offline = deliverer(() => "unreachable"); const now = yield* Clock.currentTimeMillis; // Stops at the first failure, so order is kept. expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 30_000); @@ -79,6 +80,42 @@ describe("HookInboxStore", () => { ), ); + it.effect("lets other hooks through while one hook's environment is busy", () => + withInbox( + Effect.gen(function* () { + const stuck = { rawHookId: "stuck", hookKey: "stuck" }; + yield* HookInboxStore.hold(yield* hook("stuck-1", stuck), BASE_URL); + yield* HookInboxStore.hold(yield* hook("other-1"), BASE_URL); + yield* HookInboxStore.hold(yield* hook("stuck-2", stuck), BASE_URL); + const busy = deliverer((held) => (held.hookKey === "stuck" ? "busy" : "delivered")); + const now = yield* Clock.currentTimeMillis; + // The other hook is delivered; the busy hook keeps its order and runs again soon. + expect(yield* HookInboxStore.deliverDue(busy.send)).toBe(now); + expect(busy.sent.map((entry) => entry.hook.id)).toEqual(["stuck-1", "other-1"]); + // Only the busy hook is left, so the next run waits rather than spinning. + expect(yield* HookInboxStore.deliverDue(busy.send)).toBe(now + 30_000); + + const drained = deliverer(() => "delivered"); + expect(yield* HookInboxStore.deliverDue(drained.send)).toBeNull(); + expect(drained.sent.map((entry) => entry.hook.id)).toEqual(["stuck-1", "stuck-2"]); + }), + ), + ); + + it.effect("drops a held request it cannot read instead of stalling on it", () => + withInbox( + Effect.gen(function* () { + yield* HookInboxStore.hold(yield* hook("broken"), BASE_URL); + yield* HookInboxStore.hold(yield* hook("fine"), BASE_URL); + const sql = yield* SqlClient.SqlClient; + yield* sql`UPDATE held_hooks SET headers = 'not json' WHERE id = 'broken'`; + const { sent, send } = deliverer(() => "delivered"); + expect(yield* HookInboxStore.deliverDue(send)).toBeNull(); + expect(sent.map((entry) => entry.hook.id)).toEqual(["fine"]); + }), + ), + ); + it("caps the wait between attempts at 10 minutes", () => { expect(HookInboxStore.retryDelayMs(1)).toBe(30_000); expect(HookInboxStore.retryDelayMs(20)).toBe(10 * 60_000); @@ -89,7 +126,7 @@ describe("HookInboxStore", () => { Effect.gen(function* () { expect(yield* HookInboxStore.wake(BASE_URL)).toBe(false); yield* HookInboxStore.hold(yield* hook("first"), "https://old.example.test/"); - const offline = deliverer(() => "retry"); + const offline = deliverer(() => "unreachable"); yield* HookInboxStore.deliverDue(offline.send); yield* HookInboxStore.deliverDue(offline.send); diff --git a/infra/relay/src/hooks/HookInboxStore.ts b/infra/relay/src/hooks/HookInboxStore.ts index a3ced78dff61..90e884cd4ae2 100644 --- a/infra/relay/src/hooks/HookInboxStore.ts +++ b/infra/relay/src/hooks/HookInboxStore.ts @@ -1,6 +1,7 @@ import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; @@ -23,6 +24,8 @@ export const HOOK_INBOX_MAX_BYTES = 50 * 1_048_576; export const HOOK_INBOX_MAX_PER_HOOK = 100; /** Requests pushed per alarm run; the next run starts right away while more wait. */ const DELIVERIES_PER_RUN = 20; +/** Requests read per run, so requests behind a busy hook's backlog are still reached. */ +const ROWS_READ_PER_RUN = 200; const FIRST_RETRY_MS = 30_000; const MAX_RETRY_MS = 10 * 60_000; @@ -41,8 +44,13 @@ export interface HeldHook { readonly body: Uint8Array; } -/** `retry` keeps the request and backs off; `delivered` deletes it. */ -export type DeliveryOutcome = "delivered" | "retry"; +/** + * `delivered` deletes the request. `unreachable` keeps it and backs off the + * whole inbox, since nothing else will get through either. `busy` keeps it + * and its hook's later requests for the next run, but lets other hooks' + * requests go ahead, so one stuck task cannot hold up the rest. + */ +export type DeliveryOutcome = "delivered" | "busy" | "unreachable"; /** 30 s, 1 min, 2 min, ... up to 10 min between attempts while the environment stays away. */ export const retryDelayMs = (failures: number) => @@ -50,7 +58,7 @@ export const retryDelayMs = (failures: number) => const HeadersJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)); const encodeHeaders = Schema.encodeSync(HeadersJson); -const decodeHeaders = Schema.decodeUnknownSync(HeadersJson); +const decodeHeaders = Schema.decodeUnknownOption(HeadersJson); interface HeldHookRow { readonly id: string; @@ -173,9 +181,10 @@ export const clear = Effect.gen(function* () { }); /** - * Pushes the oldest held requests to the environment, one at a time, and - * stops at the first one that has to be retried. Drops requests older than - * the TTL. Returns when the next run is due, or null when nothing is left. + * Pushes the oldest held requests to the environment, one at a time, in + * order per hook. Stops at the first sign the environment is unreachable; + * skips past a hook whose environment answered busy. Drops requests older + * than the TTL. Returns when the next run is due, or null when nothing is left. */ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( send: (baseUrl: string, hook: HeldHook) => Effect.Effect, @@ -186,7 +195,7 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( const target = yield* readTarget; const batch = yield* sql` SELECT id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body - FROM held_hooks ORDER BY seq LIMIT ${DELIVERIES_PER_RUN} + FROM held_hooks ORDER BY seq LIMIT ${ROWS_READ_PER_RUN} `; if (batch.length === 0 || target === null) { if (target === null) yield* sql`DELETE FROM held_hooks`; @@ -194,7 +203,20 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( } let failures = target.failures; + let sent = 0; + let delivered = 0; + const busyHooks = new Set(); for (const row of batch) { + if (sent === DELIVERIES_PER_RUN) break; + // Later requests to a busy hook wait their turn, so its order is kept. + if (busyHooks.has(row.hook_key)) continue; + const headers = decodeHeaders(row.headers); + if (Option.isNone(headers)) { + // Unreadable: it can never be delivered, and must not block the rest. + yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`; + continue; + } + sent += 1; const outcome = yield* send(target.base_url, { id: row.id, receivedAt: row.received_at, @@ -203,19 +225,27 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( rawToken: row.raw_token, hookKey: row.hook_key, query: row.query, - headers: decodeHeaders(row.headers), + headers: headers.value, body: row.body, }); - if (outcome === "retry") { + if (outcome === "unreachable") { failures += 1; yield* setFailures(failures); return (yield* Clock.currentTimeMillis) + retryDelayMs(failures); } - yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`; if (failures !== 0) { failures = 0; yield* setFailures(0); } + if (outcome === "busy") { + busyHooks.add(row.hook_key); + continue; + } + yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`; + delivered += 1; } - return (yield* hasPending) ? yield* Clock.currentTimeMillis : null; + if (!(yield* hasPending)) return null; + // Everything left this run was busy: give those tasks a moment to drain. + const now = yield* Clock.currentTimeMillis; + return delivered === 0 && busyHooks.size > 0 ? now + FIRST_RETRY_MS : now; }); diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 30a599070190..4f1cd17e094c 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -320,7 +320,6 @@ function relayUnlinkTestLayer(input?: { readonly reconcileOrigin?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["reconcileOrigin"]; readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]; readonly clearInbox?: HookInbox.HookInbox["Service"]["clear"]; - readonly activeLinks?: number; }) { return Layer.mergeAll( Layer.mock(HookInbox.HookInbox, { @@ -339,15 +338,8 @@ function relayUnlinkTestLayer(input?: { listDeliveryUsersForEnvironment: () => Effect.die("unused listDeliveryUsersForEnvironment"), listForUser: () => Effect.die("unused listForUser"), getForUser: input?.getForUser ?? (() => Effect.succeed(null)), - findActiveManagedForEnvironment: () => - Effect.succeed( - Array.from({ length: input?.activeLinks ?? 0 }, () => ({ - ...linkedEnvironmentRecord, - userId: "user-2", - holdWebhooksWhileOffline: true, - })), - ), - setHoldWebhooksWhileOffline: () => Effect.void, + findActiveManagedForEnvironment: () => Effect.die("unused findActiveManagedForEnvironment"), + setHoldWebhooksWhileOffline: () => Effect.die("unused setHoldWebhooksWhileOffline"), revokeForUser: input?.revokeForUser ?? (() => Effect.succeed(false)), }), ), @@ -914,24 +906,50 @@ describe("relay environment unlink", () => { ); }); - it.effect("drops held webhooks only once no user links the environment", () => { + it.effect("drops the unlinked endpoint's held webhooks, even if clearing fails", () => { const cleared: Array = []; - const unlink = (activeLinks: number) => - unlinkEnvironmentRecord({ userId: "user-1", environmentId: "environment-1" }).pipe( + const endpointKey = "0123456789abcdef"; + const unlink = (clearFails: boolean) => + unlinkEnvironmentRecord({ + userId: "user-1", + environmentId: "environment-1", + managedEndpointNamespace: "dev", + }).pipe( Effect.provide( relayUnlinkTestLayer({ - activeLinks, getForUser: () => Effect.succeed(linkedEnvironmentRecord), revokeForUser: () => Effect.succeed(true), - clearInbox: ({ environmentId }) => Effect.sync(() => void cleared.push(environmentId)), + prepareDeprovision: () => + Effect.succeed({ + userId: "user-1", + environmentId: "environment-1", + hostname: "dev-0123456789abcdef.example.test", + tunnelId: "tunnel-1", + tunnelName: `t3coderelay-managedendpoint-dev-${endpointKey}`, + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + origin: null, + updatedAt: "2026-07-28T00:00:00.000Z", + generation: 1, + }), + clearInbox: (input) => + clearFails + ? Effect.fail( + new HookInbox.HookInboxError({ + operation: "clear", + endpointKey: input.endpointKey, + cause: new Error("unavailable"), + }), + ) + : Effect.sync(() => void cleared.push(input.endpointKey)), }), ), ); return Effect.gen(function* () { - yield* unlink(1); - expect(cleared).toEqual([]); - yield* unlink(0); - expect(cleared).toEqual(["environment-1"]); + expect(yield* unlink(false)).toBe(true); + expect(cleared).toEqual([endpointKey]); + // The link is already revoked; a failed clear must not fail the unlink. + expect(yield* unlink(true)).toBe(true); }); }); diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 34d0ad72796f..96af0c3263fd 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -190,6 +190,12 @@ export const relayDocsRedirectRoute = HttpRouter.add( // contains the exact child span that stalled, and the response still carries // the traceparent back to the client. export const RELAY_REQUEST_DEADLINE_MS = 9_000; +/** + * Webhook forwarding reads a body of up to 1 MiB, waits up to the upstream + * timeout, and may then hold the request, so it needs more room than an API + * call; cutting it off before the hold would drop a request it should keep. + */ +export const RELAY_HOOK_REQUEST_DEADLINE_MS = 25_000; const relayRequestDeadline = ( httpEffect: Effect.Effect< @@ -197,9 +203,10 @@ const relayRequestDeadline = ( E, HttpServerRequest.HttpServerRequest | R >, + deadlineMs = RELAY_REQUEST_DEADLINE_MS, ) => httpEffect.pipe( - Effect.timeoutOption(Duration.millis(RELAY_REQUEST_DEADLINE_MS)), + Effect.timeoutOption(Duration.millis(deadlineMs)), Effect.flatMap( Option.match({ onNone: () => @@ -208,7 +215,7 @@ const relayRequestDeadline = ( yield* Effect.logError("relay request exceeded deadline", { "http.method": request.method, "http.url": request.url, - "relay.request.deadline_ms": RELAY_REQUEST_DEADLINE_MS, + "relay.request.deadline_ms": deadlineMs, }); yield* Effect.annotateCurrentSpan({ "relay.request.deadline_exceeded": true, @@ -247,6 +254,7 @@ export const traceRelayHttpRequest = ( Effect.andThen( relayRequestDeadline( httpEffect.pipe(Effect.provideService(HttpServerRequest.HttpServerRequest, request)), + RELAY_HOOK_REQUEST_DEADLINE_MS, ), ), ), @@ -496,7 +504,12 @@ export const revokeEnvironmentLinkRecord = Effect.fn( }); export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnvironmentRecord")( - function* (input: { readonly userId: string; readonly environmentId: string }) { + function* (input: { + readonly userId: string; + readonly environmentId: string; + /** The stage's tunnel-name namespace, to find this link's held webhook requests. */ + readonly managedEndpointNamespace?: string | undefined; + }) { const links = yield* EnvironmentLinks.EnvironmentLinks; const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider; const deprovisionTarget = yield* managedEndpointProvider.prepareDeprovision({ @@ -515,16 +528,6 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron environmentId: link.environmentId, environmentPublicKey: link.environmentPublicKey, }); - // Held webhook requests belong to the environment, not one user's link; - // drop them once no user has it linked any more. - const remaining = yield* links.findActiveManagedForEnvironment({ - environmentId: input.environmentId, - }); - if (remaining.length === 0) { - const inbox = yield* HookInbox.HookInbox; - yield* inbox.clear({ environmentId: input.environmentId }); - } - // External teardown cannot share the SQL transaction. Run it only after // revocation commits so a database failure leaves a fully usable active // link. Still run teardown when the link is already revoked, allowing a @@ -534,10 +537,31 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron environmentId: input.environmentId, target: deprovisionTarget, }); + // Requests held for this link's endpoint go with it. Best effort: the link + // is already gone, and its inbox drops anything left after its TTL. + const endpointKey = + deprovisionTarget && input.managedEndpointNamespace + ? HookForwarder.endpointKeyForTunnelName( + input.managedEndpointNamespace, + deprovisionTarget.tunnelName, + ) + : null; + if (endpointKey !== null) { + const inbox = yield* HookInbox.HookInbox; + yield* inbox.clear({ endpointKey }).pipe( + Effect.catch((error) => + Effect.logWarning("Could not clear held webhook requests", { + environmentId: input.environmentId, + errorTag: error._tag, + }), + ), + ); + } if (!deprovisioned) { - const retryTarget = yield* managedEndpointProvider.prepareDeprovision(input); - if (retryTarget !== null && (yield* links.getForUser(input)) === null) { - yield* managedEndpointProvider.deprovision({ ...input, target: retryTarget }); + const key = { userId: input.userId, environmentId: input.environmentId }; + const retryTarget = yield* managedEndpointProvider.prepareDeprovision(key); + if (retryTarget !== null && (yield* links.getForUser(key)) === null) { + yield* managedEndpointProvider.deprovision({ ...key, target: retryTarget }); } } return unlinked; @@ -937,6 +961,7 @@ export const clientApi = HttpApiBuilder.group( const unlinked = yield* unlinkEnvironmentRecord({ userId, environmentId: params.environmentId, + managedEndpointNamespace: config.managedEndpointNamespace, }).pipe( Effect.catchTags({ SqlError: () => relayInternalErrorResponse("internal_error"), @@ -1149,6 +1174,29 @@ export const serverApi = HttpApiBuilder.group( return yield* new HttpApiError.Unauthorized({}); } }); + /** + * Endpoint keys of the managed links the calling environment key proved. + * The environment id alone would also match other accounts' links of it. + */ + const ownEndpointKeys = (environmentId: string) => + Effect.gen(function* () { + const principal = yield* RelayEnvironmentPrincipal; + const namespace = settings.managedEndpointNamespace; + if (!namespace) return []; + const ownLinks = yield* links.findActiveManagedForEnvironment({ + environmentId, + environmentPublicKey: principal.environmentPublicKey, + }); + const keys: Array = []; + for (const link of ownLinks) { + const allocation = yield* allocations.get({ userId: link.userId, environmentId }); + const key = allocation + ? HookForwarder.endpointKeyForTunnelName(namespace, allocation.tunnelName) + : null; + if (key !== null) keys.push(key); + } + return keys; + }); const activityHandlers = handlers.handle( "publishAgentActivity", Effect.fn("relay.api.server.publishAgentActivity")( @@ -1363,10 +1411,19 @@ export const serverApi = HttpApiBuilder.group( "updateLinkPreferences", Effect.fn("relay.api.server.updateLinkPreferences")(function* ({ params, payload }) { yield* requireOwnEnvironment(params.environmentId); + const principal = yield* RelayEnvironmentPrincipal; yield* links.setHoldWebhooksWhileOffline({ environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, holdWebhooksWhileOffline: payload.holdWebhooksWhileOffline, }); + // Opting out also drops what is already held, rather than delivering + // it later to an environment that said it does not want it. + if (!payload.holdWebhooksWhileOffline) { + for (const endpointKey of yield* ownEndpointKeys(params.environmentId)) { + yield* inbox.clear({ endpointKey }); + } + } return payload; }, mapRelayCommonApiErrors("not_authorized")), ) @@ -1374,19 +1431,21 @@ export const serverApi = HttpApiBuilder.group( "wakeHeldHooks", Effect.fn("relay.api.server.wakeHeldHooks")(function* ({ params }) { yield* requireOwnEnvironment(params.environmentId); - const endpoint = yield* HookForwarder.resolveHookEndpoint(params.environmentId).pipe( - Effect.provideService(EnvironmentLinks.EnvironmentLinks, links), - Effect.provideService( - ManagedEndpointAllocations.ManagedEndpointAllocations, - allocations, - ), - Effect.provideService(RelayConfiguration.RelayConfiguration, settings), - ); - if (endpoint === null) return { pending: false }; - const pending = yield* inbox.wake({ - environmentId: params.environmentId, - baseUrl: endpoint.httpBaseUrl, - }); + let pending = false; + for (const endpointKey of yield* ownEndpointKeys(params.environmentId)) { + const endpoint = yield* HookForwarder.resolveHookEndpoint(endpointKey).pipe( + Effect.provideService(EnvironmentLinks.EnvironmentLinks, links), + Effect.provideService( + ManagedEndpointAllocations.ManagedEndpointAllocations, + allocations, + ), + Effect.provideService(RelayConfiguration.RelayConfiguration, settings), + ); + if (endpoint === null) continue; + if (yield* inbox.wake({ endpointKey, baseUrl: endpoint.httpBaseUrl })) { + pending = true; + } + } return { pending }; }, mapRelayCommonApiErrors("not_authorized")), ); diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index a811f622f0f4..fd0f33904f57 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -192,6 +192,8 @@ export const ApiLive = Api.make( const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone); const managedEndpointZoneName = yield* managedEndpointZone.name; const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig; + // Keys are endpoint keys or hashes over them, which already differ per + // stage, so stages sharing an account cannot collide in these namespaces. const hookRateLimit = yield* Cloudflare.RateLimit("HOOK_RATE_LIMIT", { namespaceId: 1001, simple: { @@ -199,6 +201,13 @@ export const ApiLive = Api.make( period: HookForwarder.RELAY_HOOK_RATE_LIMIT.periodSeconds, }, }); + const hookEndpointRateLimit = yield* Cloudflare.RateLimit("HOOK_ENDPOINT_RATE_LIMIT", { + namespaceId: 1002, + simple: { + limit: HookForwarder.RELAY_HOOK_ENDPOINT_RATE_LIMIT.limit, + period: HookForwarder.RELAY_HOOK_ENDPOINT_RATE_LIMIT.periodSeconds, + }, + }); const hookInboxes = yield* HookInboxObject; // @@ -231,9 +240,9 @@ export const ApiLive = Api.make( }).pipe(Effect.map(makeRelayTraceLayer)), ); - // Each environment's held webhook requests live in its own Durable Object. + // Each managed endpoint's held webhook requests live in its own Durable Object. const inboxCall = - (operation: HookInbox.HookInboxError["operation"], environmentId: string) => + (operation: HookInbox.HookInboxError["operation"], endpointKey: string) => (effect: Effect.Effect) => effect.pipe( Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), @@ -241,22 +250,19 @@ export const ApiLive = Api.make( Effect.fail( new HookInbox.HookInboxError({ operation, - environmentId, + endpointKey, cause: Cause.squash(cause), }), ), ), ); const hookInboxLayer = Layer.succeed(HookInbox.HookInbox, { - hold: ({ environmentId, baseUrl, hook }) => - hookInboxes - .getByName(environmentId) - .hold(hook, baseUrl) - .pipe(inboxCall("hold", environmentId)), - wake: ({ environmentId, baseUrl }) => - hookInboxes.getByName(environmentId).wake(baseUrl).pipe(inboxCall("wake", environmentId)), - clear: ({ environmentId }) => - hookInboxes.getByName(environmentId).clear().pipe(inboxCall("clear", environmentId)), + hold: ({ endpointKey, baseUrl, hook }) => + hookInboxes.getByName(endpointKey).hold(hook, baseUrl).pipe(inboxCall("hold", endpointKey)), + wake: ({ endpointKey, baseUrl }) => + hookInboxes.getByName(endpointKey).wake(baseUrl).pipe(inboxCall("wake", endpointKey)), + clear: ({ endpointKey }) => + hookInboxes.getByName(endpointKey).clear().pipe(inboxCall("clear", endpointKey)), }); const runtimeLayer = Layer.empty.pipe( @@ -322,9 +328,10 @@ export const ApiLive = Api.make( ); // Fails open: a limiter outage must not drop webhooks the environment would accept. - const hookRateLimiterLayer = Layer.succeed(HookForwarder.HookRateLimiter, { - allow: (key) => - hookRateLimit.limit({ key }).pipe( + const allowWith = + (limiter: typeof hookRateLimit) => + (key: string): Effect.Effect => + limiter.limit({ key }).pipe( Effect.map((result) => result.success), Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), Effect.catch((error) => @@ -332,7 +339,10 @@ export const ApiLive = Api.make( Effect.as(true), ), ), - ), + ); + const hookRateLimiterLayer = Layer.succeed(HookForwarder.HookRateLimiter, { + allowHook: allowWith(hookRateLimit), + allowEndpoint: allowWith(hookEndpointRateLimit), }); const appLayer = Layer.merge( diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 55c189979a36..db4dd5a86477 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -1209,14 +1209,16 @@ const RelayServerGroup = HttpApiGroup.make("server") * Public, stateless webhook forwarding to an environment's managed tunnel. * Unauthenticated: the token in the path is the environment's credential, and * the relay only routes and forwards. Raw, so the body reaches the - * environment byte for byte and signatures still verify there. + * environment byte for byte and signatures still verify there. `endpointKey` + * names one managed endpoint (its tunnel's hash of user and environment), not + * the environment id, which any account can link. */ const RelayHookParams = Schema.Struct({ - environmentId: Schema.String, + endpointKey: Schema.String, hookId: Schema.String, token: Schema.String, }); -const RELAY_HOOK_PATH = "/v1/hooks/:environmentId/:hookId/:token"; +const RELAY_HOOK_PATH = "/v1/hooks/:endpointKey/:hookId/:token"; const relayHookEndpoint = { params: RelayHookParams } as const; const RelayHooksGroup = HttpApiGroup.make("hooks") .add( From f4ed06221e4c7b61521fb3e10b5fef6f312395e5 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 20:06:06 -0700 Subject: [PATCH 19/35] fix(server): a webhook request dropped mid-flight no longer loses its delivery The relay-delivery claim, log write, queue slot and fork now run as one uninterruptible step. Before, a sender hanging up between the claim and the fork left a claimed delivery that never ran (its retry answered accepted), or a queue slot that was never released. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scheduledTasks/ScheduledTaskService.ts | 248 +++++++++--------- .../ScheduledTaskService.webhook.test.ts | 32 +++ 2 files changed, 161 insertions(+), 119 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 4bfc6e5262c5..0d5d09ffca67 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -1347,7 +1347,8 @@ export const layer = Layer.effect( return { _tag: "not_found" as const }; } const task = yield* decodeRow(row); - if (task.schedule.type !== "webhook") return { _tag: "not_found" as const }; + const schedule = task.schedule; + if (schedule.type !== "webhook") return { _tag: "not_found" as const }; const now = yield* localNow; const receivedAt = @@ -1373,127 +1374,136 @@ export const layer = Layer.effect( DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id = ${request.relayDeliveryId} `.pipe(Effect.ignore, Effect.as(result)); - if (request.relayDeliveryId !== undefined) { - const claimed = yield* sql<{ relay_delivery_id: string }>` - INSERT INTO scheduled_task_webhook_relay_deliveries - (relay_delivery_id, task_id, seen_at) - VALUES (${request.relayDeliveryId}, ${task.id}, ${iso(now)}) - ON CONFLICT (relay_delivery_id) DO NOTHING - RETURNING relay_delivery_id - `.pipe( - Effect.mapError((cause) => - taskError("Could not record webhook delivery.", { taskId: task.id, cause }), - ), - ); - if (claimed.length === 0) return { _tag: "accepted" as const, deliveryId }; - // Older claims can no longer be replayed by the relay. - yield* sql` - DELETE FROM scheduled_task_webhook_relay_deliveries - WHERE seen_at < ${iso(DateTime.subtract(now, { hours: 48 }))} - `.pipe(Effect.ignore); - } - const log = ( - outcome: ScheduledTaskWebhookDeliveryOutcome, - details: { - readonly signatureVerified?: boolean; - readonly missing?: ReadonlyArray; - readonly renderedPrompt?: string; - } = {}, - ) => - recordDelivery({ - id: deliveryId, - taskId: task.id, - receivedAt: iso(receivedAt), - request, - outcome, - signatureVerified: details.signatureVerified ?? false, - missing: details.missing ?? [], - renderedPrompt: details.renderedPrompt ?? null, - }); + // From the claim until the run is forked nothing may interrupt: a + // request dropped in between (the relay hangs up after its timeout) + // would leave a claimed delivery that never runs, or a queue slot + // that is never released. Everything in here is local and quick. + return yield* Effect.uninterruptible( + Effect.gen(function* () { + if (request.relayDeliveryId !== undefined) { + const claimed = yield* sql<{ relay_delivery_id: string }>` + INSERT INTO scheduled_task_webhook_relay_deliveries + (relay_delivery_id, task_id, seen_at) + VALUES (${request.relayDeliveryId}, ${task.id}, ${iso(now)}) + ON CONFLICT (relay_delivery_id) DO NOTHING + RETURNING relay_delivery_id + `.pipe( + Effect.mapError((cause) => + taskError("Could not record webhook delivery.", { taskId: task.id, cause }), + ), + ); + if (claimed.length === 0) return { _tag: "accepted" as const, deliveryId }; + // Older claims can no longer be replayed by the relay. + yield* sql` + DELETE FROM scheduled_task_webhook_relay_deliveries + WHERE seen_at < ${iso(DateTime.subtract(now, { hours: 48 }))} + `.pipe(Effect.ignore); + } + const log = ( + outcome: ScheduledTaskWebhookDeliveryOutcome, + details: { + readonly signatureVerified?: boolean; + readonly missing?: ReadonlyArray; + readonly renderedPrompt?: string; + } = {}, + ) => + recordDelivery({ + id: deliveryId, + taskId: task.id, + receivedAt: iso(receivedAt), + request, + outcome, + signatureVerified: details.signatureVerified ?? false, + missing: details.missing ?? [], + renderedPrompt: details.renderedPrompt ?? null, + }); + + // Only the first rejected request in a window is logged, so a flood + // cannot write rows or push the real deliveries out of the log. + const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(now)); + if (slot !== "allowed") { + if (slot === "first_rejected") yield* log("rate_limited"); + return yield* releaseClaim({ _tag: "rate_limited" as const }); + } + if (!task.enabled) { + yield* log("disabled"); + return { _tag: "disabled" as const }; + } + const signature = schedule.signature; + if (signature !== null) { + const verified = + row.webhook_secret !== null && + verifyWebhookSignature({ + signature, + secret: row.webhook_secret, + headers: request.headers, + body: request.body, + }); + if (!verified) { + yield* log("rejected_signature"); + return { _tag: "rejected_signature" as const }; + } + } + const maxAgeMinutes = schedule.maxDeliveryAgeMinutes ?? null; + if ( + maxAgeMinutes !== null && + DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt) > + maxAgeMinutes * 60_000 + ) { + yield* log("expired", { signatureVerified: signature !== null }); + return { _tag: "expired" as const }; + } - // Only the first rejected request in a window is logged, so a flood - // cannot write rows or push the real deliveries out of the log. - const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(now)); - if (slot !== "allowed") { - if (slot === "first_rejected") yield* log("rate_limited"); - return yield* releaseClaim({ _tag: "rate_limited" as const }); - } - if (!task.enabled) { - yield* log("disabled"); - return { _tag: "disabled" as const }; - } - const signature = task.schedule.signature; - if (signature !== null) { - const verified = - row.webhook_secret !== null && - verifyWebhookSignature({ - signature, - secret: row.webhook_secret, - headers: request.headers, - body: request.body, + const rendered = renderWebhookPrompt(task.prompt, request); + // Bound the deliveries one task holds, so steady traffic to a stuck + // task cannot pile up parked fibers. A refused request is not logged, + // so it cannot push real deliveries out of the log. + const queueKey = `${task.id}\u0000${task.createdAt}`; + const queued = yield* Ref.modify(webhookQueued, (counts) => { + const count = counts.get(queueKey) ?? 0; + return count >= WEBHOOK_MAX_QUEUED_PER_TASK + ? ([false, counts] as const) + : ([true, new Map(counts).set(queueKey, count + 1)] as const); }); - if (!verified) { - yield* log("rejected_signature"); - return { _tag: "rejected_signature" as const }; - } - } - const maxAgeMinutes = task.schedule.maxDeliveryAgeMinutes ?? null; - if ( - maxAgeMinutes !== null && - DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt) > maxAgeMinutes * 60_000 - ) { - yield* log("expired", { signatureVerified: signature !== null }); - return { _tag: "expired" as const }; - } - - const rendered = renderWebhookPrompt(task.prompt, request); - // Bound the deliveries one task holds, so steady traffic to a stuck - // task cannot pile up parked fibers. A refused request is not logged, - // so it cannot push real deliveries out of the log. - const queueKey = `${task.id}\u0000${task.createdAt}`; - const queued = yield* Ref.modify(webhookQueued, (counts) => { - const count = counts.get(queueKey) ?? 0; - return count >= WEBHOOK_MAX_QUEUED_PER_TASK - ? ([false, counts] as const) - : ([true, new Map(counts).set(queueKey, count + 1)] as const); - }); - if (!queued) return yield* releaseClaim({ _tag: "rate_limited" as const }); - // Entries leave the map when their count reaches zero, so a deleted - // task's key does not linger once its last delivery finishes. - const release = Ref.update(webhookQueued, (counts) => { - const next = new Map(counts); - const count = (next.get(queueKey) ?? 1) - 1; - if (count <= 0) next.delete(queueKey); - else next.set(queueKey, count); - return next; - }); - yield* log("accepted", { - signatureVerified: signature !== null, - missing: rendered.missing, - renderedPrompt: rendered.prompt, - }).pipe(Effect.onError(() => release)); - const permit = yield* webhookPermit(task.id); - yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe( - Effect.flatMap((completed) => - completed.lastRunStatus === "failed" - ? markDeliveryFailed(deliveryId, "The run failed to start.") - : Effect.void, - ), - Effect.catchTag("WebhookDeliverySkipped", (skipped) => - markDeliveryFailed(deliveryId, skipped.reason), - ), - // The log is readable over RPC, so it gets a fixed reason; the - // cause, which can carry request data, stays in the server log. - Effect.catchCause((cause) => - Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe( - Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), - ), - ), - permit.withPermits(1), - Effect.ensuring(release), - Effect.forkIn(serviceScope), + if (!queued) return yield* releaseClaim({ _tag: "rate_limited" as const }); + // Entries leave the map when their count reaches zero, so a deleted + // task's key does not linger once its last delivery finishes. + const release = Ref.update(webhookQueued, (counts) => { + const next = new Map(counts); + const count = (next.get(queueKey) ?? 1) - 1; + if (count <= 0) next.delete(queueKey); + else next.set(queueKey, count); + return next; + }); + yield* log("accepted", { + signatureVerified: signature !== null, + missing: rendered.missing, + renderedPrompt: rendered.prompt, + }).pipe(Effect.onError(() => release)); + const permit = yield* webhookPermit(task.id); + yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe( + Effect.flatMap((completed) => + completed.lastRunStatus === "failed" + ? markDeliveryFailed(deliveryId, "The run failed to start.") + : Effect.void, + ), + Effect.catchTag("WebhookDeliverySkipped", (skipped) => + markDeliveryFailed(deliveryId, skipped.reason), + ), + // The log is readable over RPC, so it gets a fixed reason; the + // cause, which can carry request data, stays in the server log. + Effect.catchCause((cause) => + Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe( + Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), + ), + ), + permit.withPermits(1), + Effect.ensuring(release), + Effect.forkIn(serviceScope), + ); + return { _tag: "accepted" as const, deliveryId }; + }), ); - return { _tag: "accepted" as const, deliveryId }; }); return ScheduledTaskService.of({ diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index ce2712737351..dc5ba2b04d84 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -6,8 +6,10 @@ import { ScheduledTaskUpsertInput } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; +import * as EffectScheduler from "effect/Scheduler"; import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; @@ -508,6 +510,36 @@ it.effect("a held request already delivered directly runs only once", () => ), ); +it.effect("a held request whose sender hung up mid-request still runs", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + // The relay times out and the request fiber is interrupted. A small + // operation budget makes the request yield often, so stepping the + // interrupt one yield later each time lands it at every point in the + // request (one takes about 40 yields), then the relay retries. + for (let step = 0; step < 60; step++) { + const request = requestFor(task, { relayDeliveryId: `hung-up-${step}` }); + const fiber = yield* service + .triggerWebhook(request) + .pipe(Effect.provideService(EffectScheduler.MaxOpsBeforeYield, 8), Effect.forkChild); + for (let yields = 0; yields < step; yields++) yield* Effect.yieldNow; + yield* Fiber.interrupt(fiber); + const retried = yield* service.triggerWebhook(request); + assert.equal(retried._tag, "accepted"); + const deliveryId = retried._tag === "accepted" ? retried.deliveryId : undefined; + // Logged and run exactly once, whether or not the first attempt got through. + yield* service.getWebhookDelivery({ id: task.id, deliveryId: deliveryId! }); + const launch = yield* Queue.take(launches); + assert.include(launch.commandId, `hung-up-${step}`); + // Keep each step in a fresh rate-limit window. + yield* TestClock.adjust("61 seconds"); + } + assert.equal(yield* Queue.size(launches), 0); + }), + ), +); + it.effect("a held request runs once even after the log has trimmed it", () => withService(({ service, launches }) => Effect.gen(function* () { From 5086c1f930d9f6984d77cf63a19f2e72419e6e08 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 20:11:04 -0700 Subject: [PATCH 20/35] fix(server): a relay receive time in the future counts as now The tunnel is reachable directly, so a sender can set the relay's received-at header. A future time pinned the delivery at the top of the log and slipped past the task's max delivery age. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scheduledTasks/ScheduledTaskService.ts | 8 +++++- .../ScheduledTaskService.webhook.test.ts | 26 ++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 0d5d09ffca67..7096e7764cfa 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -1351,10 +1351,16 @@ export const layer = Layer.effect( if (schedule.type !== "webhook") return { _tag: "not_found" as const }; const now = yield* localNow; + // Anyone can reach the tunnel directly and set the relay's header, so + // a receive time is never later than now: a future one would pin the + // delivery in the log and slip past the task's max age. const receivedAt = request.receivedAt === undefined ? now - : Option.getOrElse(DateTime.make(request.receivedAt), () => now); + : DateTime.min( + Option.getOrElse(DateTime.make(request.receivedAt), () => now), + now, + ); const deliveryId = ScheduledTaskWebhookDeliveryId.make( request.relayDeliveryId === undefined ? `delivery:${yield* crypto.randomUUIDv4.pipe( diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index dc5ba2b04d84..028bee4df747 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -583,11 +583,31 @@ it.effect("logs a held request at the time the relay received it", () => withService(({ service }) => Effect.gen(function* () { const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); - yield* service.triggerWebhook( - requestFor(task, { relayDeliveryId: "relay-2", receivedAt: "2026-10-04T10:00:00.000Z" }), + const receivedAt = DateTime.formatIso(DateTime.subtract(yield* DateTime.now, { minutes: 5 })); + yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "relay-2", receivedAt })); + const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; + assert.equal(delivery?.receivedAt, receivedAt); + }), + ), +); + +it.effect("a receive time in the future counts as now", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ schedule: { type: "webhook", maxDeliveryAgeMinutes: 30 } }), + ); + const now = yield* DateTime.now; + const result = yield* service.triggerWebhook( + requestFor(task, { + relayDeliveryId: "future", + receivedAt: DateTime.formatIso(DateTime.add(now, { days: 365 })), + }), ); + assert.equal(result._tag, "accepted"); + yield* Queue.take(launches); const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; - assert.equal(delivery?.receivedAt, "2026-10-04T10:00:00.000Z"); + assert.equal(delivery?.receivedAt, DateTime.formatIso(now)); }), ), ); From 9d1f9849cd0f51075e9cc5e801096190918cd370 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 20:11:51 -0700 Subject: [PATCH 21/35] fix(server): webhook prompts redact credential headers like the delivery log {{request}}, {{headers}} and {{query}} now redact credential-named headers and query parameters with the same rule as the delivery log, which moves into webhookTemplate.ts so there is one definition. Naming a header or parameter explicitly still gives its raw value. The logged query string is redacted too. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scheduledTasks/ScheduledTaskService.ts | 23 +++----- .../ScheduledTaskService.webhook.test.ts | 4 +- .../scheduledTasks/webhookTemplate.test.ts | 25 +++++++++ .../src/scheduledTasks/webhookTemplate.ts | 54 ++++++++++++++++--- 4 files changed, 84 insertions(+), 22 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 7096e7764cfa..392c128cf4cd 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -41,7 +41,12 @@ import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts"; -import { renderWebhookPrompt, type WebhookRequest } from "./webhookTemplate.ts"; +import { + redactHeaders, + redactQuery, + renderWebhookPrompt, + type WebhookRequest, +} from "./webhookTemplate.ts"; import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts"; /** Path prefix of the environment route that receives webhook requests. */ @@ -254,19 +259,6 @@ function errorMessage(error: unknown): string { return String(error); } -/** Headers kept out of the delivery log because they commonly carry credentials. */ -const REDACTED_HEADER = - /^(authorization|proxy-authorization|cookie|set-cookie)$|token|secret|signature|key|password|auth/i; - -function redactHeaders(headers: Readonly>): Record { - return Object.fromEntries( - Object.entries(headers).map(([name, value]) => [ - name, - REDACTED_HEADER.test(name) ? "[redacted]" : value, - ]), - ); -} - function webhookPath(taskId: string, token: string): string { return `${WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(taskId)}/${token}`; } @@ -1239,7 +1231,8 @@ export const layer = Layer.effect( ) SELECT ${input.id}, ${input.taskId}, ${input.receivedAt}, ${input.request.method}, - ${input.request.query}, ${encodeHeadersJson(redactHeaders(input.request.headers))}, + ${redactQuery(input.request.query)}, + ${encodeHeadersJson(redactHeaders(input.request.headers))}, ${loggedBody}, ${input.request.body.byteLength}, ${truncated ? 1 : 0}, ${input.outcome}, ${input.signatureVerified ? 1 : 0}, ${encodeMissingFieldsJson(input.missing)}, diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index 028bee4df747..cc1e4fbfdb8d 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -335,12 +335,13 @@ it.effect("keeps the newest 50 deliveries when they share a timestamp", () => ), ); -it.effect("keeps credential headers out of the delivery log", () => +it.effect("keeps credential headers and query values out of the delivery log", () => withService(({ service }) => Effect.gen(function* () { const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); yield* service.triggerWebhook( requestFor(task, { + query: "page=2&api_key=k", headers: { "content-type": "application/json", authorization: "Bearer sender-token", @@ -354,6 +355,7 @@ it.effect("keeps credential headers out of the delivery log", () => id: task.id, deliveryId: summary!.id, }); + assert.equal(delivery.query, "page=2&api_key=[redacted]"); assert.equal(delivery.headers.authorization, "[redacted]"); assert.equal(delivery.headers["x-webhook-key"], "[redacted]"); assert.equal(delivery.headers["x-github-event"], "push"); diff --git a/apps/server/src/scheduledTasks/webhookTemplate.test.ts b/apps/server/src/scheduledTasks/webhookTemplate.test.ts index c87fa8b064a2..8b6f11e91f9f 100644 --- a/apps/server/src/scheduledTasks/webhookTemplate.test.ts +++ b/apps/server/src/scheduledTasks/webhookTemplate.test.ts @@ -69,6 +69,31 @@ describe("renderWebhookPrompt", () => { assert.deepEqual(plain.missing, ["body.field"]); }); + it("redacts credentials in whole-request placeholders but not named ones", () => { + const request: WebhookRequest = { + ...githubPullRequest, + query: "source=github&access_token=q-secret", + headers: { + ...githubPullRequest.headers, + authorization: "Bearer h-secret", + "x-hub-signature-256": "sha256=abc", + }, + }; + const whole = renderWebhookPrompt("{{request}}|{{headers}}|{{query}}", request).prompt; + for (const secret of ["q-secret", "h-secret", "sha256=abc"]) { + assert.notInclude(whole, secret); + } + assert.include(whole, "?source=github&access_token=[redacted]\n"); + assert.include(whole, "authorization: [redacted]"); + assert.include(whole, "x-github-event: pull_request"); + + const named = renderWebhookPrompt( + "{{headers.authorization}} {{query.access_token}}", + request, + ).prompt; + assert.equal(named, "Bearer h-secret q-secret"); + }); + it("does not resolve inherited object properties", () => { const rendered = renderWebhookPrompt( "{{body.constructor}}{{body.__proto__}}", diff --git a/apps/server/src/scheduledTasks/webhookTemplate.ts b/apps/server/src/scheduledTasks/webhookTemplate.ts index e365a32858d6..74c0f08ae778 100644 --- a/apps/server/src/scheduledTasks/webhookTemplate.ts +++ b/apps/server/src/scheduledTasks/webhookTemplate.ts @@ -14,6 +14,10 @@ * * Strings and numbers render as text, objects and arrays as JSON. A path with * no value renders empty and is reported in `missing`. + * + * Credential-looking headers and query parameters are redacted wherever the + * whole set renders (`{{request}}`, `{{headers}}`, `{{query}}`), as in the + * delivery log. Naming one (`{{headers.authorization}}`) gives its raw value. */ export interface WebhookRequest { @@ -31,6 +35,40 @@ export interface RenderedWebhookPrompt { readonly missing: ReadonlyArray; } +/** Header and query parameter names that commonly carry credentials. */ +const CREDENTIAL_NAME = + /^(authorization|proxy-authorization|cookie|set-cookie)$|token|secret|signature|key|password|auth/i; +const REDACTED = "[redacted]"; + +export function redactHeaders(headers: Readonly>): Record { + return Object.fromEntries( + Object.entries(headers).map(([name, value]) => [ + name, + CREDENTIAL_NAME.test(name) ? REDACTED : value, + ]), + ); +} + +/** Redacts credential-named values in a raw query string, keeping the rest as sent. */ +export function redactQuery(query: string): string { + if (query === "") return query; + return query + .split("&") + .map((part) => { + const separator = part.indexOf("="); + if (separator === -1) return part; + const name = part.slice(0, separator); + let decoded = name; + try { + decoded = decodeURIComponent(name.replaceAll("+", " ")); + } catch { + // A malformed escape is matched as sent. + } + return CREDENTIAL_NAME.test(decoded) ? `${name}=${REDACTED}` : part; + }) + .join("&"); +} + const PLACEHOLDER = /\{\{\s*([^{}]*?)\s*\}\}/g; function parseBody(request: WebhookRequest): unknown { @@ -64,9 +102,12 @@ function stringify(value: unknown): string { } function formatWebhookRequest(request: WebhookRequest): string { - const headerLines = Object.entries(request.headers).map(([name, value]) => `${name}: ${value}`); + const headerLines = Object.entries(redactHeaders(request.headers)).map( + ([name, value]) => `${name}: ${value}`, + ); + const query = redactQuery(request.query); return [ - `${request.method} ${request.path}${request.query ? `?${request.query}` : ""}`, + `${request.method} ${request.path}${query ? `?${query}` : ""}`, ...headerLines, "", request.bodyText, @@ -90,12 +131,13 @@ export function renderWebhookPrompt( return segments.length === 0 ? request.bodyText : lookup(parsedBody(), segments); case "headers": return segments.length === 0 - ? request.headers + ? redactHeaders(request.headers) : request.headers[segments.join(".").toLowerCase()]; case "query": { - const params = new URLSearchParams(request.query); - if (segments.length === 0) return Object.fromEntries(params); - return params.get(segments.join(".")) ?? undefined; + if (segments.length === 0) { + return Object.fromEntries(new URLSearchParams(redactQuery(request.query))); + } + return new URLSearchParams(request.query).get(segments.join(".")) ?? undefined; } default: return undefined; From 51cdc7bf51cdc20b22e4a49ae0f90738cdb76be0 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 20:13:00 -0700 Subject: [PATCH 22/35] fix(server): webhook prompts past the provider input limit are not dispatched A rendered prompt can embed a body of up to 1 MiB, past what any provider accepts. Such a delivery is now logged as dispatch_failed with "The filled-in prompt is too long." and no run starts. The logged prompt is capped at 64 KiB like the logged body. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scheduledTasks/ScheduledTaskService.ts | 22 +++++++++++++++-- .../ScheduledTaskService.webhook.test.ts | 24 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 392c128cf4cd..d3b31dfbce41 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -1,6 +1,7 @@ import { CommandId, MessageId, + PROVIDER_SEND_TURN_MAX_INPUT_CHARS, ScheduledTask, ScheduledTaskError, ScheduledTaskId, @@ -55,6 +56,8 @@ export const WEBHOOK_ROUTE_PREFIX = "/api/hooks"; const WEBHOOK_DELIVERY_RETENTION = 50; /** Body text kept in the delivery log. Larger bodies are cut and flagged. */ const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024; +/** Rendered prompt text kept in the delivery log. */ +const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024; /** Deliveries one task may hold at once, running or waiting their turn. */ const WEBHOOK_MAX_QUEUED_PER_TASK = 20; /** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */ @@ -1213,6 +1216,7 @@ export const layer = Layer.effect( readonly signatureVerified: boolean; readonly missing: ReadonlyArray; readonly renderedPrompt: string | null; + readonly error?: string; }) => { const truncated = input.request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT; const loggedBody = truncated @@ -1236,7 +1240,8 @@ export const layer = Layer.effect( ${loggedBody}, ${input.request.body.byteLength}, ${truncated ? 1 : 0}, ${input.outcome}, ${input.signatureVerified ? 1 : 0}, ${encodeMissingFieldsJson(input.missing)}, - ${input.renderedPrompt}, NULL + ${input.renderedPrompt?.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT) ?? null}, + ${input.error ?? null} WHERE EXISTS (SELECT 1 FROM scheduled_tasks WHERE task_id = ${input.taskId}) -- A held request retried after a rate limit reuses its relay -- delivery id; the newer attempt replaces the logged one. @@ -1245,7 +1250,7 @@ export const layer = Layer.effect( signature_verified = excluded.signature_verified, missing_fields_json = excluded.missing_fields_json, rendered_prompt = excluded.rendered_prompt, - error = NULL + error = excluded.error `; yield* sql` DELETE FROM scheduled_task_webhook_deliveries @@ -1404,6 +1409,7 @@ export const layer = Layer.effect( readonly signatureVerified?: boolean; readonly missing?: ReadonlyArray; readonly renderedPrompt?: string; + readonly error?: string; } = {}, ) => recordDelivery({ @@ -1415,6 +1421,7 @@ export const layer = Layer.effect( signatureVerified: details.signatureVerified ?? false, missing: details.missing ?? [], renderedPrompt: details.renderedPrompt ?? null, + ...(details.error === undefined ? {} : { error: details.error }), }); // Only the first rejected request in a window is logged, so a flood @@ -1454,6 +1461,17 @@ export const layer = Layer.effect( } const rendered = renderWebhookPrompt(task.prompt, request); + // A provider refuses a turn this long, so it is not started. The + // delivery is not retryable, so the claim is kept. + if (rendered.prompt.length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) { + yield* log("dispatch_failed", { + signatureVerified: signature !== null, + missing: rendered.missing, + renderedPrompt: rendered.prompt, + error: "The filled-in prompt is too long.", + }); + return { _tag: "accepted" as const, deliveryId }; + } // Bound the deliveries one task holds, so steady traffic to a stuck // task cannot pile up parked fibers. A refused request is not logged, // so it cannot push real deliveries out of the log. diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index cc1e4fbfdb8d..929dd411dd29 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -493,6 +493,30 @@ it.effect("logs a body's first 64 KiB by bytes, not characters", () => ), ); +it.effect("does not start a run when the filled-in prompt is too long", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput({ prompt: "{{body}}" })); + const text = "x".repeat(200_000); + const body = new TextEncoder().encode(text); + const result = yield* service.triggerWebhook(requestFor(task, { body, bodyText: text })); + assert.equal(result._tag, "accepted"); + assert.equal(yield* Queue.size(launches), 0); + const { delivery } = yield* service.getWebhookDelivery({ + id: task.id, + deliveryId: result._tag === "accepted" ? result.deliveryId : ("" as never), + }); + assert.equal(delivery.outcome, "dispatch_failed"); + assert.equal(delivery.error, "The filled-in prompt is too long."); + assert.equal(delivery.renderedPrompt?.length, 64 * 1024); + // The queue slot was never taken: a normal delivery still runs. + const ok = yield* service.triggerWebhook(requestFor(task)); + assert.equal(ok._tag, "accepted"); + yield* Queue.take(launches); + }), + ), +); + it.effect("a held request already delivered directly runs only once", () => withService(({ service, launches }) => Effect.gen(function* () { From f69736791ac83db6eeee5c2f734a14386aa9cf25 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 20:16:45 -0700 Subject: [PATCH 23/35] fix(server): webhook route answers defects with the fixed 500 body A defect in the service escaped the route's error mapping and reached the generic HTTP error path unlogged. It now gets the same logged {"error":"internal_error"} 500 as a typed failure. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/scheduledTasks/webhookRoute.test.ts | 22 +++++++++++-------- .../server/src/scheduledTasks/webhookRoute.ts | 3 ++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/apps/server/src/scheduledTasks/webhookRoute.test.ts b/apps/server/src/scheduledTasks/webhookRoute.test.ts index 27a2a45a0322..921c2d319968 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.test.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.test.ts @@ -158,16 +158,20 @@ describe("webhook route", () => { } }); - it("hides service failures behind a 500", async () => { - const { handler, dispose } = handlerFor(() => + it("hides service failures and defects behind a fixed 500", async () => { + const failures = [ Effect.fail(new ScheduledTaskError({ message: "database locked" })), - ); - try { - const response = await handler(post("/api/hooks/id/tok", "{}")); - expect(response.status).toBe(500); - expect(await response.text()).not.toContain("database"); - } finally { - await dispose(); + Effect.die(new Error("database exploded")), + ]; + for (const failure of failures) { + const { handler, dispose } = handlerFor(() => failure); + try { + const response = await handler(post("/api/hooks/id/tok", "{}")); + expect(response.status).toBe(500); + expect(await response.json()).toEqual({ error: "internal_error" }); + } finally { + await dispose(); + } } }); }); diff --git a/apps/server/src/scheduledTasks/webhookRoute.ts b/apps/server/src/scheduledTasks/webhookRoute.ts index 1e02a4996286..b752bacb4bf7 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.ts @@ -74,7 +74,8 @@ const handleWebhook = ...(relayReceivedAt ? { receivedAt: relayReceivedAt } : {}), }) .pipe( - Effect.catch((cause) => + // Defects too, so the sender only ever sees the fixed error body. + Effect.catchCause((cause) => Effect.logWarning("Webhook delivery failed").pipe( Effect.annotateLogs({ hookId: params.hookId }), Effect.andThen(Effect.logDebug("Webhook delivery failure cause", { cause })), From 1eeb161ed554b9b33354460bf933e538a1852390 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 20:17:08 -0700 Subject: [PATCH 24/35] fix(server): one tunnel reconnect wakes held webhooks once cloudflared logs several registered connections per reconnect, and each one asked T3 Connect to deliver held requests. The signal is now debounced so a reconnect results in a single wake. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/relay/HeldHooksWaker.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/server/src/relay/HeldHooksWaker.ts b/apps/server/src/relay/HeldHooksWaker.ts index c240f9cd52e3..498c81f72d4c 100644 --- a/apps/server/src/relay/HeldHooksWaker.ts +++ b/apps/server/src/relay/HeldHooksWaker.ts @@ -41,6 +41,9 @@ export const layer = Layer.effectDiscard( ), ); yield* runtime.tunnelConnected.pipe( + // cloudflared registers several connections per (re)connect within a + // few seconds; they are one wake. + Stream.debounce("3 seconds"), Stream.runForEach(() => wake), Effect.forkScoped, ); From 794b942e276a8e9f36e6cd6e4e679ed6134245ee Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 21:00:09 -0700 Subject: [PATCH 25/35] fix(relay): held webhooks retry every 10 s for 3 minutes after a wake The environment wakes its inbox when cloudflared registers a tunnel connection, but Cloudflare can take a few minutes to route the hostname to it. The first push landed in that window, and the inbox then backed off 30 s, 1 min, 2 min, so an end-to-end test saw a held webhook arrive 3.5 minutes after the environment was back. Retry delays now come from one Schedule: every 10 s for 18 attempts, then 30 s doubling up to 10 min. A wake resets it to the start. Pushes that do not deliver log why (status or error), so a cold start can be told apart from a timeout. Co-Authored-By: Claude Opus 5.5 (1M context) --- infra/relay/src/hooks/HookInboxObject.ts | 25 +++++++--- infra/relay/src/hooks/HookInboxStore.test.ts | 32 ++++++++---- infra/relay/src/hooks/HookInboxStore.ts | 51 +++++++++++++++++--- 3 files changed, 84 insertions(+), 24 deletions(-) diff --git a/infra/relay/src/hooks/HookInboxObject.ts b/infra/relay/src/hooks/HookInboxObject.ts index 41786d5d4846..de82625dcd5b 100644 --- a/infra/relay/src/hooks/HookInboxObject.ts +++ b/infra/relay/src/hooks/HookInboxObject.ts @@ -44,19 +44,32 @@ export class HookInboxObject extends Cloudflare.DurableObject< const deliver = (baseUrl: string, hook: HookInboxStore.HeldHook) => sendUpstream(baseUrl, hook).pipe( Effect.result, - Effect.map((result): HookInboxStore.DeliveryOutcome => { + Effect.map((result) => { // Unreachable or timed out: a timeout may still have run it, and the // environment drops a delivery id it has already seen. - if (Result.isFailure(result) || Option.isNone(result.success)) return "unreachable"; + if (Result.isFailure(result)) { + return { outcome: "unreachable" as const, reason: result.failure._tag }; + } + if (Option.isNone(result.success)) + return { outcome: "unreachable" as const, reason: "timeout" }; const status = result.success.value.status; - if (UNREACHABLE_STATUSES.has(status)) return "unreachable"; - return BUSY_STATUSES.has(status) ? "busy" : "delivered"; + const outcome: HookInboxStore.DeliveryOutcome = UNREACHABLE_STATUSES.has(status) + ? "unreachable" + : BUSY_STATUSES.has(status) + ? "busy" + : "delivered"; + return { outcome, reason: `status ${status}` }; }), - Effect.tap((outcome) => + Effect.tap(({ outcome, reason }) => outcome === "delivered" ? Effect.void - : Effect.logInfo("Held webhook not delivered yet", { outcome, deliveryId: hook.id }), + : Effect.logInfo("Held webhook not delivered yet", { + outcome, + reason, + deliveryId: hook.id, + }), ), + Effect.map(({ outcome }) => outcome), Effect.provide(FetchHttpClient.layer), ); diff --git a/infra/relay/src/hooks/HookInboxStore.test.ts b/infra/relay/src/hooks/HookInboxStore.test.ts index 2c2dfce20804..1ca488dcbebe 100644 --- a/infra/relay/src/hooks/HookInboxStore.test.ts +++ b/infra/relay/src/hooks/HookInboxStore.test.ts @@ -68,10 +68,9 @@ describe("HookInboxStore", () => { const offline = deliverer(() => "unreachable"); const now = yield* Clock.currentTimeMillis; // Stops at the first failure, so order is kept. - expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 30_000); + expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000); expect(offline.sent.map((entry) => entry.hook.id)).toEqual(["first"]); - expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 60_000); - expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 120_000); + expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000); const online = deliverer(() => "delivered"); expect(yield* HookInboxStore.deliverDue(online.send)).toBeNull(); @@ -116,10 +115,15 @@ describe("HookInboxStore", () => { ), ); - it("caps the wait between attempts at 10 minutes", () => { - expect(HookInboxStore.retryDelayMs(1)).toBe(30_000); - expect(HookInboxStore.retryDelayMs(20)).toBe(10 * 60_000); - }); + it.effect("retries every 10 s for 3 minutes, then backs off to 10 minutes", () => + Effect.gen(function* () { + const delays = yield* Effect.forEach( + [1, 18, 19, 20, 21, 23, 24, 40], + HookInboxStore.retryDelayMs, + ); + expect(delays).toEqual([10_000, 10_000, 30_000, 60_000, 120_000, 480_000, 600_000, 600_000]); + }), + ); it.effect("waking resets the backoff and reports whether anything waits", () => withInbox( @@ -130,10 +134,18 @@ describe("HookInboxStore", () => { yield* HookInboxStore.deliverDue(offline.send); yield* HookInboxStore.deliverDue(offline.send); - expect(yield* HookInboxStore.wake(BASE_URL)).toBe(true); + // Past the fast phase: the environment was away a while. + for (let failure = 0; failure < 20; failure++) { + yield* HookInboxStore.deliverDue(offline.send); + } const now = yield* Clock.currentTimeMillis; - // Back to the first step, and sent to where the environment is now. - expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 30_000); + expect(yield* HookInboxStore.deliverDue(offline.send)).toBeGreaterThan(now + 60_000); + + // A wake means its tunnel just connected, which Cloudflare may not + // route to for a few minutes: back to retrying every 10 s, sent to + // where the environment is now. + expect(yield* HookInboxStore.wake(BASE_URL)).toBe(true); + expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000); expect(offline.sent.at(-1)?.baseUrl).toBe(BASE_URL); }), ), diff --git a/infra/relay/src/hooks/HookInboxStore.ts b/infra/relay/src/hooks/HookInboxStore.ts index 90e884cd4ae2..b1adb6b1c21c 100644 --- a/infra/relay/src/hooks/HookInboxStore.ts +++ b/infra/relay/src/hooks/HookInboxStore.ts @@ -1,7 +1,10 @@ import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; +import * as Pull from "effect/Pull"; +import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; @@ -26,8 +29,28 @@ export const HOOK_INBOX_MAX_PER_HOOK = 100; const DELIVERIES_PER_RUN = 20; /** Requests read per run, so requests behind a busy hook's backlog are still reached. */ const ROWS_READ_PER_RUN = 200; -const FIRST_RETRY_MS = 30_000; -const MAX_RETRY_MS = 10 * 60_000; +/** Wait before trying a hook whose environment answered busy again. */ +const BUSY_RETRY_MS = 30_000; +const MAX_RETRY_DELAY = Duration.minutes(10); + +/** + * Delays between delivery attempts while the environment is unreachable, + * indexed by consecutive failures since it was last reached or woke us. + * A wake means its tunnel just connected, but Cloudflare can take a few + * minutes to route the hostname to it, so the first 3 minutes retry every + * 10 s. After that the environment is likely gone again: 30 s, 1 min, 2 min, + * ... up to 10 min. + */ +export const retrySchedule = Schedule.spaced("10 seconds").pipe( + Schedule.upTo({ times: 18 }), + Schedule.concat( + Schedule.exponential("30 seconds").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(Duration.fromInputUnsafe(duration), MAX_RETRY_DELAY)), + ), + ), + ), +); export interface HeldHook { readonly id: string; @@ -52,9 +75,21 @@ export interface HeldHook { */ export type DeliveryOutcome = "delivered" | "busy" | "unreachable"; -/** 30 s, 1 min, 2 min, ... up to 10 min between attempts while the environment stays away. */ -export const retryDelayMs = (failures: number) => - Math.min(FIRST_RETRY_MS * 2 ** Math.max(0, failures - 1), MAX_RETRY_MS); +/** The `retrySchedule` delay after `failures` consecutive unreachable attempts. */ +export const retryDelayMs = Effect.fn("HookInboxStore.retryDelayMs")(function* (failures: number) { + const step = yield* Schedule.toStep(retrySchedule); + let delay = MAX_RETRY_DELAY; + for (let attempt = 0; attempt < Math.max(1, failures); attempt++) { + const next = yield* step(0, undefined).pipe( + Effect.map(([, duration]) => Option.some(duration)), + // The schedule never ends; stay at the cap if it ever does. + Pull.catchDone(() => Effect.succeedNone), + ); + if (Option.isNone(next)) break; + delay = next.value; + } + return Duration.toMillis(delay); +}); const HeadersJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)); const encodeHeaders = Schema.encodeSync(HeadersJson); @@ -161,7 +196,7 @@ export const hold = Effect.fn("HookInboxStore.hold")(function* (hook: HeldHook, if (inserted.length === 0) return null; yield* setTarget(baseUrl, { resetFailures: false }); const target = yield* readTarget; - return (yield* Clock.currentTimeMillis) + retryDelayMs(Math.max(1, target?.failures ?? 0)); + return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(target?.failures ?? 0)); }); /** @@ -231,7 +266,7 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( if (outcome === "unreachable") { failures += 1; yield* setFailures(failures); - return (yield* Clock.currentTimeMillis) + retryDelayMs(failures); + return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures)); } if (failures !== 0) { failures = 0; @@ -247,5 +282,5 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( if (!(yield* hasPending)) return null; // Everything left this run was busy: give those tasks a moment to drain. const now = yield* Clock.currentTimeMillis; - return delivered === 0 && busyHooks.size > 0 ? now + FIRST_RETRY_MS : now; + return delivered === 0 && busyHooks.size > 0 ? now + BUSY_RETRY_MS : now; }); From 659081c520fec809f6e9e32dc2677477a480a5ac Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 23:00:16 -0700 Subject: [PATCH 26/35] feat(relay,server): webhooks and held deliveries can be monitored Webhooks are a production path without a way to see how they are doing. The relay traced forwards, but held-request inbox runs never left the Durable Object, and the environment recorded deliveries only in a log the user sees. Environment: - ScheduledTaskService.triggerWebhook spans with the outcome, source (relay or direct), body size and how long a held request waited, and each run a delivery starts is its own runWebhookDelivery trace. - Metrics: t3_webhook_deliveries_total by outcome and source, including queue_full, prompt_too_long, duplicate and body_too_large; t3_webhook_runs_total by started/skipped/failed; t3_webhook_held_delay; t3_webhook_delivery_duration. Relay: - Durable Object calls and alarms export to the relay's Axiom dataset. - relay.inbox.hold/wake/deliver spans: which cap refused a hold, each run's result, sends, deliveries, consecutive failures, longest wait and remaining backlog. - Forwards record which budget limited a request, when the rate limiter failed open, the hold opt-in, and body size. docs/operations covers the spans, metrics and example APL queries. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/observability/Metrics.ts | 24 +++++ apps/server/src/relay/HeldHooksWaker.ts | 1 + .../scheduledTasks/ScheduledTaskService.ts | 98 +++++++++++++++++-- .../ScheduledTaskService.webhook.test.ts | 64 ++++++++++++ .../server/src/scheduledTasks/webhookRoute.ts | 11 ++- docs/operations/observability.md | 13 +++ docs/operations/relay-observability.md | 36 +++++++ infra/relay/src/hooks/HookForwarder.ts | 19 +++- infra/relay/src/hooks/HookInboxObject.ts | 21 +++- infra/relay/src/hooks/HookInboxStore.ts | 70 ++++++++++++- infra/relay/src/worker.ts | 18 ++++ 11 files changed, 353 insertions(+), 22 deletions(-) diff --git a/apps/server/src/observability/Metrics.ts b/apps/server/src/observability/Metrics.ts index b75ace399ccc..f133eda3f8b0 100644 --- a/apps/server/src/observability/Metrics.ts +++ b/apps/server/src/observability/Metrics.ts @@ -70,6 +70,30 @@ export const terminalRestartsTotal = Metric.counter("t3_terminal_restarts_total" description: "Total terminal restart requests handled.", }); +/** + * One per webhook request that reached a task, by `outcome` (accepted, + * not_found, rejected_signature, disabled, rate_limited, queue_full, expired, + * prompt_too_long, error) and `source` (relay or direct). + */ +export const webhookDeliveriesTotal = Metric.counter("t3_webhook_deliveries_total", { + description: "Webhook requests handled, by outcome and source.", +}); + +export const webhookDeliveryDuration = Metric.timer("t3_webhook_delivery_duration", { + description: "Time to verify, log, and enqueue one webhook request.", +}); + +/** How long a relay-held request waited before this environment got it. */ +export const webhookHeldDelay = Metric.timer("t3_webhook_held_delay", { + description: + "Time between the relay receiving a webhook request and the environment handling it.", +}); + +/** Runs started by webhook deliveries, by `outcome` (started, skipped, failed). */ +export const webhookRunsTotal = Metric.counter("t3_webhook_runs_total", { + description: "Runs started from webhook deliveries, by outcome.", +}); + export const metricAttributes = ( attributes: Readonly>, ): ReadonlyArray<[string, string]> => Object.entries(compactMetricAttributes(attributes)); diff --git a/apps/server/src/relay/HeldHooksWaker.ts b/apps/server/src/relay/HeldHooksWaker.ts index 498c81f72d4c..9e0dae4ce624 100644 --- a/apps/server/src/relay/HeldHooksWaker.ts +++ b/apps/server/src/relay/HeldHooksWaker.ts @@ -22,6 +22,7 @@ const wakeHeldHooks = Effect.fn("HeldHooksWaker.wake")(function* () { const environmentId = yield* (yield* ServerEnvironment.ServerEnvironment).getEnvironmentId; const client = yield* makeRelayEnvironmentClient(connection); const { pending } = yield* client.server.wakeHeldHooks({ params: { environmentId } }); + yield* Effect.annotateCurrentSpan({ "relay.inbox.pending": pending }); return pending; }); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index d3b31dfbce41..a8abe2613003 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -28,6 +28,8 @@ import * as Crypto from "effect/Crypto"; import * as Data from "effect/Data"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Duration from "effect/Duration"; +import * as Metric from "effect/Metric"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; @@ -39,6 +41,7 @@ import * as Stream from "effect/Stream"; import * as SqlClient from "effect/sql/SqlClient"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; +import * as Metrics from "../observability/Metrics.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts"; @@ -1325,10 +1328,55 @@ export const layer = Layer.effect( // run; scoped to the service so shutdown interrupts it. const serviceScope = yield* Effect.scope; + /** + * Records one handled request on the span and in metrics. `outcome` is + * finer than the result tag: it separates a full queue and an oversized + * prompt from the rest, which is what an operator needs to act on. + */ + const observeDelivery = ( + request: WebhookTriggerRequest, + outcome: string, + receivedAt: DateTime.DateTime | undefined, + now: DateTime.DateTime | undefined, + ) => + Effect.gen(function* () { + const source = request.relayDeliveryId === undefined ? "direct" : "relay"; + yield* Effect.annotateCurrentSpan({ + "scheduled_task.webhook.outcome": outcome, + "scheduled_task.webhook.source": source, + }); + yield* Metrics.increment(Metrics.webhookDeliveriesTotal, { outcome, source }); + if (request.receivedAt !== undefined && receivedAt !== undefined && now !== undefined) { + const heldMs = Math.max( + 0, + DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt), + ); + yield* Effect.annotateCurrentSpan({ "scheduled_task.webhook.held_ms": heldMs }); + yield* Metric.update(Metrics.webhookHeldDelay, Duration.millis(heldMs)); + } + }); + const triggerWebhook: ScheduledTaskService["Service"]["triggerWebhook"] = (request) => + triggerWebhookUnobserved(request).pipe( + Effect.tapError(() => observeDelivery(request, "error", undefined, undefined)), + Metrics.withMetrics({ timer: Metrics.webhookDeliveryDuration }), + Effect.withSpan("ScheduledTaskService.triggerWebhook", { + attributes: { + "scheduled_task.webhook.method": request.method, + "scheduled_task.webhook.body_bytes": request.body.byteLength, + "scheduled_task.webhook.source": + request.relayDeliveryId === undefined ? "direct" : "relay", + }, + }), + ); + + const triggerWebhookUnobserved = (request: WebhookTriggerRequest) => Effect.gen(function* () { const taskId = decodeTaskId(request.hookId); - if (Option.isNone(taskId)) return { _tag: "not_found" as const }; + const notFound = observeDelivery(request, "not_found", undefined, undefined).pipe( + Effect.as({ _tag: "not_found" as const }), + ); + if (Option.isNone(taskId)) return yield* notFound; const rows = yield* getRows(taskId.value).pipe( Effect.mapError((cause) => taskError("Could not load schedule task.", { taskId: taskId.value, cause }), @@ -1342,11 +1390,12 @@ export const layer = Layer.effect( row.webhook_token === null || !constantTimeEquals(request.token, row.webhook_token) ) { - return { _tag: "not_found" as const }; + return yield* notFound; } const task = yield* decodeRow(row); + yield* Effect.annotateCurrentSpan({ "scheduled_task.id": task.id }); const schedule = task.schedule; - if (schedule.type !== "webhook") return { _tag: "not_found" as const }; + if (schedule.type !== "webhook") return yield* notFound; const now = yield* localNow; // Anyone can reach the tunnel directly and set the relay's header, so @@ -1359,6 +1408,7 @@ export const layer = Layer.effect( Option.getOrElse(DateTime.make(request.receivedAt), () => now), now, ); + const observe = (outcome: string) => observeDelivery(request, outcome, receivedAt, now); const deliveryId = ScheduledTaskWebhookDeliveryId.make( request.relayDeliveryId === undefined ? `delivery:${yield* crypto.randomUUIDv4.pipe( @@ -1396,7 +1446,11 @@ export const layer = Layer.effect( taskError("Could not record webhook delivery.", { taskId: task.id, cause }), ), ); - if (claimed.length === 0) return { _tag: "accepted" as const, deliveryId }; + if (claimed.length === 0) { + // A held request this environment already ran: accepted, not run twice. + yield* observe("duplicate"); + return { _tag: "accepted" as const, deliveryId }; + } // Older claims can no longer be replayed by the relay. yield* sql` DELETE FROM scheduled_task_webhook_relay_deliveries @@ -1429,10 +1483,12 @@ export const layer = Layer.effect( const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(now)); if (slot !== "allowed") { if (slot === "first_rejected") yield* log("rate_limited"); + yield* observe("rate_limited"); return yield* releaseClaim({ _tag: "rate_limited" as const }); } if (!task.enabled) { yield* log("disabled"); + yield* observe("disabled"); return { _tag: "disabled" as const }; } const signature = schedule.signature; @@ -1447,6 +1503,7 @@ export const layer = Layer.effect( }); if (!verified) { yield* log("rejected_signature"); + yield* observe("rejected_signature"); return { _tag: "rejected_signature" as const }; } } @@ -1457,6 +1514,7 @@ export const layer = Layer.effect( maxAgeMinutes * 60_000 ) { yield* log("expired", { signatureVerified: signature !== null }); + yield* observe("expired"); return { _tag: "expired" as const }; } @@ -1470,6 +1528,7 @@ export const layer = Layer.effect( renderedPrompt: rendered.prompt, error: "The filled-in prompt is too long.", }); + yield* observe("prompt_too_long"); return { _tag: "accepted" as const, deliveryId }; } // Bound the deliveries one task holds, so steady traffic to a stuck @@ -1482,7 +1541,11 @@ export const layer = Layer.effect( ? ([false, counts] as const) : ([true, new Map(counts).set(queueKey, count + 1)] as const); }); - if (!queued) return yield* releaseClaim({ _tag: "rate_limited" as const }); + if (!queued) { + // The task is busy with WEBHOOK_MAX_QUEUED_PER_TASK deliveries already. + yield* observe("queue_full"); + return yield* releaseClaim({ _tag: "rate_limited" as const }); + } // Entries leave the map when their count reaches zero, so a deleted // task's key does not linger once its last delivery finishes. const release = Ref.update(webhookQueued, (counts) => { @@ -1497,24 +1560,43 @@ export const layer = Layer.effect( missing: rendered.missing, renderedPrompt: rendered.prompt, }).pipe(Effect.onError(() => release)); + yield* observe("accepted"); const permit = yield* webhookPermit(task.id); + const runOutcome = (outcome: "started" | "skipped" | "failed") => + Effect.all([ + Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }), + Metrics.increment(Metrics.webhookRunsTotal, { outcome }), + ]); yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe( Effect.flatMap((completed) => completed.lastRunStatus === "failed" - ? markDeliveryFailed(deliveryId, "The run failed to start.") - : Effect.void, + ? runOutcome("failed").pipe( + Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), + ) + : runOutcome("started"), ), Effect.catchTag("WebhookDeliverySkipped", (skipped) => - markDeliveryFailed(deliveryId, skipped.reason), + runOutcome("skipped").pipe( + Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)), + ), ), // The log is readable over RPC, so it gets a fixed reason; the // cause, which can carry request data, stays in the server log. Effect.catchCause((cause) => Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe( + Effect.andThen(runOutcome("failed")), Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), ), ), permit.withPermits(1), + // Its own trace: the request that triggered it has already been answered. + Effect.withSpan("ScheduledTaskService.runWebhookDelivery", { + root: true, + attributes: { + "scheduled_task.id": task.id, + "scheduled_task.webhook.delivery_id": deliveryId, + }, + }), Effect.ensuring(release), Effect.forkIn(serviceScope), ); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index 929dd411dd29..7b91c1a712f1 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -8,6 +8,7 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Metric from "effect/Metric"; import * as Queue from "effect/Queue"; import * as EffectScheduler from "effect/Scheduler"; import * as Schema from "effect/Schema"; @@ -690,3 +691,66 @@ it.effect("deleting a task removes its delivery log", () => }), ), ); + +const signatureFor = (secret: string) => + `sha256=${NodeCrypto.createHmac("sha256", secret).update(pullRequestBody).digest("hex")}`; + +/** Count recorded by `t3_webhook_deliveries_total` for one outcome and source. */ +const deliveriesCounted = (outcome: string, source: "relay" | "direct") => + Metric.snapshot.pipe( + Effect.map((snapshots) => { + const found = snapshots.find( + (snapshot) => + snapshot.id === "t3_webhook_deliveries_total" && + snapshot.attributes?.outcome === outcome && + snapshot.attributes?.source === source, + ); + return found?.type === "Counter" ? Number(found.state.count) : 0; + }), + ); + +it.effect("counts each handled request by what happened to it", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + signature: { + header: "x-hub-signature-256", + encoding: "hex", + prefix: "sha256=", + secret: "github-secret", + }, + }, + }), + ); + const before = { + accepted: yield* deliveriesCounted("accepted", "direct"), + rejected: yield* deliveriesCounted("rejected_signature", "direct"), + notFound: yield* deliveriesCounted("not_found", "direct"), + relayAccepted: yield* deliveriesCounted("accepted", "relay"), + duplicate: yield* deliveriesCounted("duplicate", "relay"), + }; + const signed = { + "content-type": "application/json", + "x-hub-signature-256": signatureFor("github-secret"), + }; + yield* service.triggerWebhook(requestFor(task, { headers: signed })); + yield* Queue.take(launches); + yield* service.triggerWebhook(requestFor(task)); + yield* service.triggerWebhook(requestFor(task, { token: "wrong" })); + // A request the relay held, then the same request again. + const relayed = requestFor(task, { headers: signed, relayDeliveryId: "relay-1" }); + yield* service.triggerWebhook(relayed); + yield* Queue.take(launches); + yield* service.triggerWebhook(relayed); + + assert.equal((yield* deliveriesCounted("accepted", "direct")) - before.accepted, 1); + assert.equal((yield* deliveriesCounted("rejected_signature", "direct")) - before.rejected, 1); + assert.equal((yield* deliveriesCounted("not_found", "direct")) - before.notFound, 1); + assert.equal((yield* deliveriesCounted("accepted", "relay")) - before.relayAccepted, 1); + assert.equal((yield* deliveriesCounted("duplicate", "relay")) - before.duplicate, 1); + }), + ), +); diff --git a/apps/server/src/scheduledTasks/webhookRoute.ts b/apps/server/src/scheduledTasks/webhookRoute.ts index b752bacb4bf7..f7365e8dd1fd 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.ts @@ -7,6 +7,7 @@ import type * as HttpServerRequest from "effect/http/HttpServerRequest"; import * as HttpServerResponse from "effect/http/HttpServerResponse"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; +import * as Metrics from "../observability/Metrics.ts"; import * as ScheduledTaskService from "./ScheduledTaskService.ts"; /** Largest request body a webhook accepts. The relay enforces the same cap. */ @@ -45,9 +46,15 @@ const handleWebhook = ), Effect.option, ); - if (Option.isNone(body)) return json(413, { error: "body_too_large_or_unreadable" }); + // Refused before a task is looked up, so the service never sees them. + const tooLarge = (error: string) => + Metrics.increment(Metrics.webhookDeliveriesTotal, { + outcome: "body_too_large", + source: request.headers["x-t3-relay-delivery-id"] ? "relay" : "direct", + }).pipe(Effect.as(json(413, { error }))); + if (Option.isNone(body)) return yield* tooLarge("body_too_large_or_unreadable"); if (body.value.byteLength > WEBHOOK_MAX_BODY_BYTES) { - return json(413, { error: "body_too_large" }); + return yield* tooLarge("body_too_large"); } const headers: Record = {}; diff --git a/docs/operations/observability.md b/docs/operations/observability.md index b83a9015abb2..a5a209ea06c2 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -386,6 +386,19 @@ Counters tell you volume and failure rate: - `t3_provider_turns_total` - `t3_git_commands_total` +Webhooks have their own families: + +- `t3_webhook_deliveries_total` by `outcome` and `source` (`relay` or `direct`). Beyond what the + sender sees, `queue_full` means a task already had its limit of deliveries waiting, + `prompt_too_long` means the filled-in prompt passed the provider limit, and `duplicate` means + the relay delivered a request this environment had already run. +- `t3_webhook_runs_total` by `outcome` (`started`, `skipped`, `failed`) for the runs those + deliveries start, which happen after the sender has its answer. +- `t3_webhook_held_delay` for how long requests the relay held waited before arriving. + +`ScheduledTaskService.triggerWebhook` spans carry the same outcome per request, and each run +started from a delivery is its own `ScheduledTaskService.runWebhookDelivery` trace. + Use metrics when the question is: - "is this always slow?" diff --git a/docs/operations/relay-observability.md b/docs/operations/relay-observability.md index c7f84d4ef825..9e809f56ee11 100644 --- a/docs/operations/relay-observability.md +++ b/docs/operations/relay-observability.md @@ -57,3 +57,39 @@ failure means that a signed proof was too old or too far in the future for the r window. It can point to a date or time problem on either device, but it can also result from a delayed request. The client uses this category, and the absence of a category from an older relay, to decide whether clock skew is confirmed or only one possible cause. + +## Webhooks + +A public webhook request is one `relay.hooks.forward` span. Its `relay.hook.outcome` says what +happened: `forwarded`, `held`, `rate_limited`, `inbox_full`, `not_found`, `payload_too_large`, +`environment_unavailable`, or `environment_timeout`. `relay.hook.endpoint_key` identifies the +managed endpoint, and with it the environment. On a forward, `relay.hook.upstream_status` or +`relay.hook.upstream_error` records the environment's answer. `relay.hook.rate_limit` says which +budget ran out: `endpoint` or `hook`. `relay.hook.rate_limiter_failed_open` is set when the +Cloudflare rate limiter was unavailable and the request went through unlimited. + +Held requests are handled in the endpoint's `HookInboxObject`, and each call into it is its own +trace, not a child of the forward span: `relay.inbox.hold`, `relay.inbox.wake`, and +`relay.inbox.deliver` for each alarm run. Join them to forwards on the inbox, which is named by +endpoint key. A hold the inbox refused carries `relay.inbox.refused` (`max_per_hook`, +`max_requests`, `max_bytes`, or `already_held`). A delivery run carries `relay.inbox.run_result` +(`drained`, `more_pending`, `busy`, `unreachable`, or `failed`), how many requests it sent and +delivered, the consecutive failures behind the retry delay, the longest time a delivered request +waited, and the backlog left. + +Questions these answer: + +```apl +// Webhook outcomes per hour +['t3-code-relay-traces-prod'] +| where name == 'relay.hooks.forward' +| summarize count() by bin(_time, 1h), outcome = tostring(['attributes.custom']['relay.hook.outcome']) + +// Inboxes stuck behind an unreachable environment +['t3-code-relay-traces-prod'] +| where name == 'relay.inbox.deliver' +| extend c = ['attributes.custom'] +| where tostring(c['relay.inbox.run_result']) == 'unreachable' +| summarize runs = count(), failures = max(toint(c['relay.inbox.consecutive_failures'])), + backlog = max(toint(c['relay.inbox.held_count'])) by inbox = tostring(c['relay.inbox.id']) +``` diff --git a/infra/relay/src/hooks/HookForwarder.ts b/infra/relay/src/hooks/HookForwarder.ts index cd1cfbd973ff..437e64a0998d 100644 --- a/infra/relay/src/hooks/HookForwarder.ts +++ b/infra/relay/src/hooks/HookForwarder.ts @@ -277,10 +277,14 @@ const make = Effect.gen(function* () { }); // A coarse budget per endpoint first, so minting new hook ids or tokens // cannot buy unlimited lookups and forwards, or fill the inbox. - if ( - !(yield* rateLimiter.allowEndpoint(parsed.endpointKey)) || - !(yield* rateLimiter.allowHook(yield* hookBudgetKey(parsed))) - ) { + const endpointAllowed = yield* rateLimiter.allowEndpoint(parsed.endpointKey); + const hookAllowed = + endpointAllowed && (yield* rateLimiter.allowHook(yield* hookBudgetKey(parsed))); + if (!endpointAllowed || !hookAllowed) { + // Which budget ran out: the whole endpoint's, or this one hook URL's. + yield* Effect.annotateCurrentSpan({ + "relay.hook.rate_limit": endpointAllowed ? "hook" : "endpoint", + }); yield* outcome("rate_limited"); return errorResponse(429, "rate_limited", { "retry-after": String(RELAY_HOOK_RATE_LIMIT.periodSeconds), @@ -307,7 +311,10 @@ const make = Effect.gen(function* () { yield* outcome("not_found"); return hookNotFound(); } - yield* Effect.annotateCurrentSpan({ "relay.environment_id": endpoint.environmentId }); + yield* Effect.annotateCurrentSpan({ + "relay.environment_id": endpoint.environmentId, + "relay.hook.hold_while_offline": endpoint.holdWhileOffline, + }); const body = request.method === "GET" @@ -324,6 +331,7 @@ const make = Effect.gen(function* () { // One id per request, so a request that reached the environment before a // timeout and is later delivered from the inbox runs only once. + yield* Effect.annotateCurrentSpan({ "relay.hook.body_bytes": body.success.byteLength }); const hook = { id: yield* crypto.randomUUIDv4.pipe(Effect.orDie), receivedAt, @@ -361,6 +369,7 @@ const make = Effect.gen(function* () { return errorResponse(status, error); } if (!stored) { + // The inbox span carries which cap refused it (relay.inbox.refused). yield* outcome("inbox_full"); return errorResponse(503, "inbox_full"); } diff --git a/infra/relay/src/hooks/HookInboxObject.ts b/infra/relay/src/hooks/HookInboxObject.ts index de82625dcd5b..015224182f3a 100644 --- a/infra/relay/src/hooks/HookInboxObject.ts +++ b/infra/relay/src/hooks/HookInboxObject.ts @@ -41,6 +41,16 @@ export class HookInboxObject extends Cloudflare.DurableObject< HookInboxObjectShape >()("HookInboxObject") {} +/** + * Each call into an inbox is its own trace: the alarm has no parent, and a + * hold arrives over Durable Object RPC without the forwarding span. The + * object's name is the endpoint key, which the forward spans carry too. + */ +const withInboxSpan = + (name: string, inboxId: string) => + (effect: Effect.Effect) => + effect.pipe(Effect.withSpan(name, { root: true, attributes: { "relay.inbox.id": inboxId } })); + const deliver = (baseUrl: string, hook: HookInboxStore.HeldHook) => sendUpstream(baseUrl, hook).pipe( Effect.result, @@ -81,6 +91,7 @@ export const HookInboxObjectLive = HookInboxObject.make( // @effect-diagnostics-next-line returnEffectInGen:off return Effect.gen(function* () { const sql = SqliteClient.layer({ storage: state.raw.storage }); + const inboxId = state.raw.id.toString(); const run = (effect: Effect.Effect) => effect.pipe(Effect.provide(sql), Effect.orDie); yield* run(HookInboxStore.migrate); @@ -96,21 +107,23 @@ export const HookInboxObjectLive = HookInboxObject.make( hold: (hook: HookInboxStore.HeldHook, baseUrl: string) => Effect.gen(function* () { const dueAt = yield* run(HookInboxStore.hold(hook, baseUrl)); + yield* Effect.annotateCurrentSpan({ "relay.inbox.stored": dueAt !== null }); if (dueAt === null) return false; yield* scheduleBy(dueAt); return true; - }), + }).pipe(withInboxSpan("relay.inbox.hold", inboxId)), wake: (baseUrl: string) => Effect.gen(function* () { const pending = yield* run(HookInboxStore.wake(baseUrl)); + yield* Effect.annotateCurrentSpan({ "relay.inbox.pending": pending }); if (pending) yield* state.storage.setAlarm(yield* Clock.currentTimeMillis); return pending; - }), + }).pipe(withInboxSpan("relay.inbox.wake", inboxId)), clear: () => Effect.gen(function* () { yield* run(HookInboxStore.clear); yield* state.storage.deleteAlarm(); - }), + }).pipe(withInboxSpan("relay.inbox.clear", inboxId)), alarm: () => run(HookInboxStore.deliverDue(deliver)).pipe( Effect.flatMap((nextAt) => @@ -118,10 +131,12 @@ export const HookInboxObjectLive = HookInboxObject.make( ), Effect.catchCause((cause) => Effect.logWarning("Held webhook delivery run failed", { cause }).pipe( + Effect.andThen(Effect.annotateCurrentSpan({ "relay.inbox.run_result": "failed" })), Effect.andThen(Clock.currentTimeMillis), Effect.flatMap((now) => state.storage.setAlarm(now + RUN_FAILURE_RETRY_MS)), ), ), + withInboxSpan("relay.inbox.deliver", inboxId), ), }; }); diff --git a/infra/relay/src/hooks/HookInboxStore.ts b/infra/relay/src/hooks/HookInboxStore.ts index b1adb6b1c21c..0979723f4434 100644 --- a/infra/relay/src/hooks/HookInboxStore.ts +++ b/infra/relay/src/hooks/HookInboxStore.ts @@ -193,12 +193,42 @@ export const hold = Effect.fn("HookInboxStore.hold")(function* (hook: HeldHook, ON CONFLICT (id) DO NOTHING RETURNING id `; - if (inserted.length === 0) return null; + if (inserted.length === 0) { + yield* Effect.annotateCurrentSpan({ "relay.inbox.refused": yield* refusalReason(hook) }); + return null; + } yield* setTarget(baseUrl, { resetFailures: false }); const target = yield* readTarget; + const backlog = yield* backlogSize; + yield* Effect.annotateCurrentSpan({ + "relay.inbox.held_count": backlog.count, + "relay.inbox.held_bytes": backlog.bytes, + }); return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(target?.failures ?? 0)); }); +const backlogSize = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ readonly count: number; readonly bytes: number }>` + SELECT count(*) AS count, coalesce(sum(length(body)), 0) AS bytes FROM held_hooks + `; + return rows[0] ?? { count: 0, bytes: 0 }; +}); + +/** Which cap refused a request, or that it was already held, for traces. */ +const refusalReason = (hook: HeldHook) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const existing = yield* sql<{ readonly id: string }>` + SELECT id FROM held_hooks WHERE id = ${hook.id} + `; + if (existing.length > 0) return "already_held"; + const backlog = yield* backlogSize; + if (backlog.count >= HOOK_INBOX_MAX_REQUESTS) return "max_requests"; + if (backlog.bytes + hook.body.byteLength > HOOK_INBOX_MAX_BYTES) return "max_bytes"; + return "max_per_hook"; + }); + /** * The environment is reachable again at `baseUrl`. Returns whether anything * is waiting, so the caller can deliver right away. @@ -226,7 +256,10 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( ) { const sql = yield* SqlClient.SqlClient; const startedAt = yield* Clock.currentTimeMillis; - yield* sql`DELETE FROM held_hooks WHERE received_at < ${iso(startedAt - HOOK_INBOX_TTL_MS)}`; + const expired = yield* sql<{ readonly id: string }>` + DELETE FROM held_hooks WHERE received_at < ${iso(startedAt - HOOK_INBOX_TTL_MS)} + RETURNING id + `; const target = yield* readTarget; const batch = yield* sql` SELECT id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body @@ -234,13 +267,33 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( `; if (batch.length === 0 || target === null) { if (target === null) yield* sql`DELETE FROM held_hooks`; + yield* Effect.annotateCurrentSpan({ "relay.inbox.expired": expired.length }); return null; } let failures = target.failures; let sent = 0; let delivered = 0; + let unreadable = 0; + let longestWaitMs = 0; const busyHooks = new Set(); + /** What this run did, for the alarm's span; never request contents. */ + const annotateRun = (result: string) => + Effect.gen(function* () { + const backlog = yield* backlogSize; + yield* Effect.annotateCurrentSpan({ + "relay.inbox.run_result": result, + "relay.inbox.sent": sent, + "relay.inbox.delivered": delivered, + "relay.inbox.busy_hooks": busyHooks.size, + "relay.inbox.expired": expired.length, + "relay.inbox.unreadable": unreadable, + "relay.inbox.consecutive_failures": failures, + "relay.inbox.longest_wait_ms": longestWaitMs, + "relay.inbox.held_count": backlog.count, + "relay.inbox.held_bytes": backlog.bytes, + }); + }); for (const row of batch) { if (sent === DELIVERIES_PER_RUN) break; // Later requests to a busy hook wait their turn, so its order is kept. @@ -249,6 +302,7 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( if (Option.isNone(headers)) { // Unreadable: it can never be delivered, and must not block the rest. yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`; + unreadable += 1; continue; } sent += 1; @@ -266,6 +320,7 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( if (outcome === "unreachable") { failures += 1; yield* setFailures(failures); + yield* annotateRun("unreachable"); return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures)); } if (failures !== 0) { @@ -278,9 +333,16 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( } yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`; delivered += 1; + const receivedAtMs = DateTime.toEpochMillis(DateTime.makeUnsafe(row.received_at)); + longestWaitMs = Math.max(longestWaitMs, startedAt - receivedAtMs); + } + if (!(yield* hasPending)) { + yield* annotateRun("drained"); + return null; } - if (!(yield* hasPending)) return null; // Everything left this run was busy: give those tasks a moment to drain. const now = yield* Clock.currentTimeMillis; - return delivered === 0 && busyHooks.size > 0 ? now + BUSY_RETRY_MS : now; + const idle = delivered === 0 && busyHooks.size > 0; + yield* annotateRun(idle ? "busy" : "more_pending"); + return idle ? now + BUSY_RETRY_MS : now; }); diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index fd0f33904f57..cf2ef78489f1 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -1,4 +1,5 @@ import * as Alchemy from "alchemy"; +import * as Axiom from "alchemy/Axiom"; import * as Cloudflare from "alchemy/Cloudflare"; import * as Drizzle from "alchemy/Drizzle/Postgres"; import * as Config from "effect/Config"; @@ -336,6 +337,10 @@ export const ApiLive = Api.make( Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), Effect.catch((error) => Effect.logWarning("Hook rate limiter unavailable", { error: error.message }).pipe( + // Visible on the forward span, so an outage that disables limits shows up. + Effect.andThen( + Effect.annotateCurrentSpan({ "relay.hook.rate_limiter_failed_open": true }), + ), Effect.as(true), ), ), @@ -468,6 +473,19 @@ export const ApiLive = Api.make( Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp), Layer.provideMerge(Cloudflare.Workers.RateLimitBinding), Layer.provideMerge(HookInboxObjectLive), + // Exports spans from events the HTTP tracer does not wrap, notably + // HookInboxObject calls and alarms, to the same Axiom dataset. + Layer.provideMerge( + Layer.unwrap( + Effect.map(RelayObservability, (observability) => + Axiom.Telemetry({ + serviceName: "t3code-relay", + token: observability.workerIngestToken, + traces: observability.traces, + }), + ), + ), + ), ), ), ), From 844ae83ac7ead581eebc0237b40157cd347bf69a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 23:57:21 -0700 Subject: [PATCH 27/35] fix(relay): held webhooks can't be jammed, inherited, or delayed by a wake - Requests other than GET, POST, PUT and PATCH are refused, and only methods with a body get one upstream. - Forwarded responses are served sandboxed with nosniff, since they come from the relay's own origin. - The hold opt-in carries over only from links proven by the same key. - A backoff after a run no longer overwrites an earlier alarm set by a wake, and failures are counted in place, capped, and reset once the inbox drains. - Inbox spans carry the endpoint key, so they line up with forward spans. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/operations/relay-observability.md | 2 +- .../src/environments/EnvironmentLinks.test.ts | 39 +++++++++++++++++++ .../src/environments/EnvironmentLinks.ts | 5 ++- infra/relay/src/hooks/HookForwarder.test.ts | 13 +++++++ infra/relay/src/hooks/HookForwarder.ts | 29 ++++++++++---- infra/relay/src/hooks/HookInboxObject.ts | 14 ++++--- infra/relay/src/hooks/HookInboxStore.test.ts | 18 +++++++++ infra/relay/src/hooks/HookInboxStore.ts | 38 +++++++++++++----- infra/relay/src/hooks/upstream.ts | 3 +- 9 files changed, 137 insertions(+), 24 deletions(-) diff --git a/docs/operations/relay-observability.md b/docs/operations/relay-observability.md index 9e809f56ee11..9c2746d10089 100644 --- a/docs/operations/relay-observability.md +++ b/docs/operations/relay-observability.md @@ -91,5 +91,5 @@ Questions these answer: | extend c = ['attributes.custom'] | where tostring(c['relay.inbox.run_result']) == 'unreachable' | summarize runs = count(), failures = max(toint(c['relay.inbox.consecutive_failures'])), - backlog = max(toint(c['relay.inbox.held_count'])) by inbox = tostring(c['relay.inbox.id']) + backlog = max(toint(c['relay.inbox.held_count'])) by endpoint = tostring(c['relay.hook.endpoint_key']) ``` diff --git a/infra/relay/src/environments/EnvironmentLinks.test.ts b/infra/relay/src/environments/EnvironmentLinks.test.ts index 0a856bcaf7f5..2757b2fc00f0 100644 --- a/infra/relay/src/environments/EnvironmentLinks.test.ts +++ b/infra/relay/src/environments/EnvironmentLinks.test.ts @@ -122,6 +122,45 @@ describe("EnvironmentLinks", () => { ); }); + it.effect("carries the webhook-hold opt-in over only from links with the same key", () => { + const inserted: Array> = []; + const fakeDb = { + insert: () => ({ + values: (values: Record) => { + inserted.push(values); + return { onConflictDoUpdate: () => Effect.void }; + }, + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const links = yield* EnvironmentLinks.EnvironmentLinks; + yield* links.upsert({ + userId: "user-1", + request: { + notificationsEnabled: false, + liveActivitiesEnabled: false, + managedTunnelsEnabled: true, + } as never, + proof: { + environmentId: "env-1", + environmentPublicKey: "public-key-1", + descriptor: { label: "Laptop" }, + } as never, + endpoint: { httpBaseUrl: "https://a.example", wsBaseUrl: "wss://a.example" } as never, + }); + const query = new PgDialect().sqlToQuery(inserted[0]?.holdWebhooksWhileOffline as never); + // An environment id is public: another account linking it with its own + // key must not switch the opt-in on for this one. + expect(query.sql).toContain("environment_public_key"); + expect(query.params).toEqual(["env-1", "public-key-1"]); + }).pipe( + Effect.provide( + EnvironmentLinks.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, fakeDb))), + ), + ); + }); + it.effect("revokes only the active link owned by the requesting user", () => { const updateValues: Array> = []; const whereConditions: Array = []; diff --git a/infra/relay/src/environments/EnvironmentLinks.ts b/infra/relay/src/environments/EnvironmentLinks.ts index 8b2dd3c17176..afa15b917ff8 100644 --- a/infra/relay/src/environments/EnvironmentLinks.ts +++ b/infra/relay/src/environments/EnvironmentLinks.ts @@ -184,10 +184,13 @@ const make = Effect.gen(function* () { const { request, proof } = input; const environmentId = proof.environmentId; // The webhook-hold opt-in belongs to the environment: a new or re-made - // link carries it over from the environment's other active links. + // link carries it over from the environment's other active links. Only + // links proven by the same key count; an environment id is public, so + // anyone can link one and switch the opt-in on for their own link. const inheritedHoldWebhooks = sql`EXISTS ( SELECT 1 FROM ${relayEnvironmentLinks} AS other WHERE other.environment_id = ${environmentId} + AND other.environment_public_key = ${proof.environmentPublicKey} AND other.revoked_at IS NULL AND other.hold_webhooks_while_offline )`; diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index f3acc8a7fcf6..925b08c86b67 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -258,11 +258,24 @@ describe("HookForwarder", () => { expect(response.headers["content-type"]).toBe("application/json"); expect(response.headers["set-cookie"]).toBeUndefined(); expect(response.headers["access-control-allow-origin"]).toBeUndefined(); + // Served from the relay's origin, so it may never render or run there. + expect(response.headers["x-content-type-options"]).toBe("nosniff"); + expect(response.headers["content-security-policy"]).toBe("sandbox; default-src 'none'"); expect(new TextDecoder().decode(yield* readBody(response))).toBe('{"error":"bad_signature"}'); expect(harness.sent[0]?.method).toBe("GET"); }), ); + it.effect("never forwards or holds HEAD, which can carry no body", () => + Effect.gen(function* () { + const harness = makeHarness({ execute: () => Effect.die("must not be sent") }); + const response = yield* harness.send(new Request(hookUrl(), { method: "HEAD" })); + expect(response.status).toBeGreaterThanOrEqual(400); + expect(response.status).toBeLessThan(500); + expect(harness.held).toHaveLength(0); + }), + ); + it.effect("does not follow or relay upstream redirects", () => Effect.gen(function* () { const harness = makeHarness({ diff --git a/infra/relay/src/hooks/HookForwarder.ts b/infra/relay/src/hooks/HookForwarder.ts index 437e64a0998d..cbb9d5edee96 100644 --- a/infra/relay/src/hooks/HookForwarder.ts +++ b/infra/relay/src/hooks/HookForwarder.ts @@ -126,6 +126,18 @@ const errorResponse = (status: number, error: string, headers?: Record errorResponse(404, "hook_not_found"); +/** Methods a webhook can arrive with; HEAD reaches the GET route and is refused. */ +const FORWARDED_METHODS = new Set(["GET", "POST", "PUT", "PATCH"]); + +/** + * Whatever the environment answers is served from the relay's own origin, so + * a body must never render or run there. + */ +const SANDBOXED_RESPONSE_HEADERS = { + "x-content-type-options": "nosniff", + "content-security-policy": "sandbox; default-src 'none'", +} as const; + function parseHookPath(url: string) { const queryIndex = url.indexOf("?"); const path = queryIndex === -1 ? url : url.slice(0, queryIndex); @@ -264,6 +276,10 @@ const make = Effect.gen(function* () { request: HttpServerRequest.HttpServerRequest, ) { const outcome = (value: string) => Effect.annotateCurrentSpan({ "relay.hook.outcome": value }); + if (!FORWARDED_METHODS.has(request.method)) { + yield* outcome("method_not_allowed"); + return errorResponse(405, "method_not_allowed", { allow: "GET, POST, PUT, PATCH" }); + } // When the sender called, not when the environment failed to answer. const receivedAt = DateTime.formatIso(yield* DateTime.now); const parsed = parseHookPath(request.url); @@ -399,17 +415,16 @@ const make = Effect.gen(function* () { }); // Only content-type is passed through: no location (redirects are never // followed or relayed), no cookies, no upstream infrastructure headers. - const contentTypeHeaders = response.contentType - ? { "content-type": response.contentType } - : undefined; + const headers = { + ...SANDBOXED_RESPONSE_HEADERS, + ...(response.contentType ? { "content-type": response.contentType } : {}), + }; if (response.body.length === 0) { - return HttpServerResponse.empty({ - status: response.status, - ...(contentTypeHeaders ? { headers: contentTypeHeaders } : {}), - }); + return HttpServerResponse.empty({ status: response.status, headers }); } return HttpServerResponse.uint8Array(response.body, { status: response.status, + headers, ...(response.contentType ? { contentType: response.contentType } : {}), }); }); diff --git a/infra/relay/src/hooks/HookInboxObject.ts b/infra/relay/src/hooks/HookInboxObject.ts index 015224182f3a..0e6c86af5aa6 100644 --- a/infra/relay/src/hooks/HookInboxObject.ts +++ b/infra/relay/src/hooks/HookInboxObject.ts @@ -49,7 +49,9 @@ export class HookInboxObject extends Cloudflare.DurableObject< const withInboxSpan = (name: string, inboxId: string) => (effect: Effect.Effect) => - effect.pipe(Effect.withSpan(name, { root: true, attributes: { "relay.inbox.id": inboxId } })); + effect.pipe( + Effect.withSpan(name, { root: true, attributes: { "relay.hook.endpoint_key": inboxId } }), + ); const deliver = (baseUrl: string, hook: HookInboxStore.HeldHook) => sendUpstream(baseUrl, hook).pipe( @@ -91,7 +93,9 @@ export const HookInboxObjectLive = HookInboxObject.make( // @effect-diagnostics-next-line returnEffectInGen:off return Effect.gen(function* () { const sql = SqliteClient.layer({ storage: state.raw.storage }); - const inboxId = state.raw.id.toString(); + // Inboxes are opened by endpoint key, so spans line up with the + // forward spans that held their requests. + const inboxId = state.raw.id.name ?? state.raw.id.toString(); const run = (effect: Effect.Effect) => effect.pipe(Effect.provide(sql), Effect.orDie); yield* run(HookInboxStore.migrate); @@ -126,9 +130,9 @@ export const HookInboxObjectLive = HookInboxObject.make( }).pipe(withInboxSpan("relay.inbox.clear", inboxId)), alarm: () => run(HookInboxStore.deliverDue(deliver)).pipe( - Effect.flatMap((nextAt) => - nextAt === null ? Effect.void : state.storage.setAlarm(nextAt), - ), + // A wake during this run may already have asked for an earlier + // run; a backoff must not push it out. + Effect.flatMap((nextAt) => (nextAt === null ? Effect.void : scheduleBy(nextAt))), Effect.catchCause((cause) => Effect.logWarning("Held webhook delivery run failed", { cause }).pipe( Effect.andThen(Effect.annotateCurrentSpan({ "relay.inbox.run_result": "failed" })), diff --git a/infra/relay/src/hooks/HookInboxStore.test.ts b/infra/relay/src/hooks/HookInboxStore.test.ts index 1ca488dcbebe..bdc44d7708f3 100644 --- a/infra/relay/src/hooks/HookInboxStore.test.ts +++ b/infra/relay/src/hooks/HookInboxStore.test.ts @@ -151,6 +151,24 @@ describe("HookInboxStore", () => { ), ); + it.effect("starts the schedule over once the inbox drains", () => + withInbox( + Effect.gen(function* () { + yield* HookInboxStore.hold(yield* hook("old"), BASE_URL); + const offline = deliverer(() => "unreachable"); + for (let failure = 0; failure < 30; failure++) { + yield* HookInboxStore.deliverDue(offline.send); + } + // Nothing got through before the request expired. + yield* TestClock.adjust(Duration.hours(25)); + expect(yield* HookInboxStore.deliverDue(offline.send)).toBeNull(); + + const now = yield* Clock.currentTimeMillis; + expect(yield* HookInboxStore.hold(yield* hook("new"), BASE_URL)).toBe(now + 10_000); + }), + ), + ); + it.effect("drops requests older than 24 hours", () => withInbox( Effect.gen(function* () { diff --git a/infra/relay/src/hooks/HookInboxStore.ts b/infra/relay/src/hooks/HookInboxStore.ts index 0979723f4434..a98712395d73 100644 --- a/infra/relay/src/hooks/HookInboxStore.ts +++ b/infra/relay/src/hooks/HookInboxStore.ts @@ -32,6 +32,8 @@ const ROWS_READ_PER_RUN = 200; /** Wait before trying a hook whose environment answered busy again. */ const BUSY_RETRY_MS = 30_000; const MAX_RETRY_DELAY = Duration.minutes(10); +/** Past this many failures the schedule is at its cap, so stepping further changes nothing. */ +const MAX_COUNTED_FAILURES = 25; /** * Delays between delivery attempts while the environment is unreachable, @@ -79,7 +81,11 @@ export type DeliveryOutcome = "delivered" | "busy" | "unreachable"; export const retryDelayMs = Effect.fn("HookInboxStore.retryDelayMs")(function* (failures: number) { const step = yield* Schedule.toStep(retrySchedule); let delay = MAX_RETRY_DELAY; - for (let attempt = 0; attempt < Math.max(1, failures); attempt++) { + for ( + let attempt = 0; + attempt < Math.min(Math.max(1, failures), MAX_COUNTED_FAILURES); + attempt++ + ) { const next = yield* step(0, undefined).pipe( Effect.map(([, duration]) => Option.some(duration)), // The schedule never ends; stay at the cap if it ever does. @@ -161,11 +167,23 @@ const readTarget = Effect.gen(function* () { return rows[0] ?? null; }); -const setFailures = (failures: number) => - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - yield* sql`UPDATE held_hooks_target SET failures = ${failures} WHERE id = 1`; - }); +const resetFailures = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`UPDATE held_hooks_target SET failures = 0 WHERE id = 1`; +}); + +/** + * Counts one more unreachable attempt. Incremented in place, so a wake that + * reset the count while this run's request was in flight is not overwritten. + */ +const countFailure = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ readonly failures: number }>` + UPDATE held_hooks_target SET failures = min(failures + 1, ${MAX_COUNTED_FAILURES}) + WHERE id = 1 RETURNING failures + `; + return rows[0]?.failures ?? 1; +}); const hasPending = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -267,6 +285,8 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( `; if (batch.length === 0 || target === null) { if (target === null) yield* sql`DELETE FROM held_hooks`; + // Empty, so the next request held starts the schedule from the top. + else yield* resetFailures; yield* Effect.annotateCurrentSpan({ "relay.inbox.expired": expired.length }); return null; } @@ -318,14 +338,13 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( body: row.body, }); if (outcome === "unreachable") { - failures += 1; - yield* setFailures(failures); + failures = yield* countFailure; yield* annotateRun("unreachable"); return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures)); } if (failures !== 0) { failures = 0; - yield* setFailures(0); + yield* resetFailures; } if (outcome === "busy") { busyHooks.add(row.hook_key); @@ -337,6 +356,7 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( longestWaitMs = Math.max(longestWaitMs, startedAt - receivedAtMs); } if (!(yield* hasPending)) { + yield* resetFailures; yield* annotateRun("drained"); return null; } diff --git a/infra/relay/src/hooks/upstream.ts b/infra/relay/src/hooks/upstream.ts index 22053c25e391..4c696c966e1c 100644 --- a/infra/relay/src/hooks/upstream.ts +++ b/infra/relay/src/hooks/upstream.ts @@ -4,6 +4,7 @@ import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientRequest from "effect/http/HttpClientRequest"; +import * as HttpMethod from "effect/http/HttpMethod"; import type * as HttpClientResponse from "effect/http/HttpClientResponse"; import { withoutRedirects } from "../environments/EnvironmentConnector.ts"; @@ -86,7 +87,7 @@ export const sendUpstream = (baseUrl: string, hook: UpstreamHook) => `${base}api/hooks/${hook.rawHookId}/${hook.rawToken}${hook.query ? `?${hook.query}` : ""}`, { headers }, ); - if (hook.method !== "GET") { + if (HttpMethod.hasBody(request.method)) { request = HttpClientRequest.bodyUint8Array(request, hook.body, headers["content-type"]); } return yield* httpClient.execute(request).pipe( From f120f7167192da16604f9feed78a702eeee2bd01 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 5 Oct 2026 00:01:27 -0700 Subject: [PATCH 28/35] docs(relay): inbox run details live on the store spans Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/operations/relay-observability.md | 28 +++++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/operations/relay-observability.md b/docs/operations/relay-observability.md index 9c2746d10089..9d5a0f6a88a4 100644 --- a/docs/operations/relay-observability.md +++ b/docs/operations/relay-observability.md @@ -70,12 +70,14 @@ Cloudflare rate limiter was unavailable and the request went through unlimited. Held requests are handled in the endpoint's `HookInboxObject`, and each call into it is its own trace, not a child of the forward span: `relay.inbox.hold`, `relay.inbox.wake`, and -`relay.inbox.deliver` for each alarm run. Join them to forwards on the inbox, which is named by -endpoint key. A hold the inbox refused carries `relay.inbox.refused` (`max_per_hook`, -`max_requests`, `max_bytes`, or `already_held`). A delivery run carries `relay.inbox.run_result` -(`drained`, `more_pending`, `busy`, `unreachable`, or `failed`), how many requests it sent and -delivered, the consecutive failures behind the retry delay, the longest time a delivered request -waited, and the backlog left. +`relay.inbox.deliver` for each alarm run. Each carries `relay.hook.endpoint_key`, so they join to +forwards. The details are on the child spans: `HookInboxStore.hold` carries +`relay.inbox.refused` when the inbox refused a request (`max_per_hook`, `max_requests`, +`max_bytes`, or `already_held`). `HookInboxStore.deliverDue` carries `relay.inbox.run_result` +(`drained`, `more_pending`, `busy`, or `unreachable`), how many requests it sent and delivered, +the consecutive failures behind the retry delay, the longest time a delivered request waited, and +the backlog left. A run that errored outright has `relay.inbox.run_result = failed` on the root +`relay.inbox.deliver` span instead. Questions these answer: @@ -85,11 +87,17 @@ Questions these answer: | where name == 'relay.hooks.forward' | summarize count() by bin(_time, 1h), outcome = tostring(['attributes.custom']['relay.hook.outcome']) -// Inboxes stuck behind an unreachable environment +// Inboxes stuck behind an unreachable environment, by endpoint ['t3-code-relay-traces-prod'] -| where name == 'relay.inbox.deliver' +| where name == 'HookInboxStore.deliverDue' | extend c = ['attributes.custom'] | where tostring(c['relay.inbox.run_result']) == 'unreachable' -| summarize runs = count(), failures = max(toint(c['relay.inbox.consecutive_failures'])), - backlog = max(toint(c['relay.inbox.held_count'])) by endpoint = tostring(c['relay.hook.endpoint_key']) +| project trace_id, failures = toint(c['relay.inbox.consecutive_failures']), + backlog = toint(c['relay.inbox.held_count']) +| join kind=inner ( + ['t3-code-relay-traces-prod'] + | where name == 'relay.inbox.deliver' + | project trace_id, endpoint = tostring(['attributes.custom']['relay.hook.endpoint_key']) + ) on trace_id +| summarize runs = count(), failures = max(failures), backlog = max(backlog) by endpoint ``` From 6eea59a61da99fca73f32c10ace0107d156735fb Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 5 Oct 2026 00:12:52 -0700 Subject: [PATCH 29/35] fix(server): a webhook prompt's length is counted as the provider counts it Providers trim a turn's input before checking its length, so whitespace a webhook body puts around the prompt no longer turns a valid delivery away. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/scheduledTasks/ScheduledTaskService.ts | 5 +++-- .../ScheduledTaskService.webhook.test.ts | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index a8abe2613003..a955ce50c970 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -1520,8 +1520,9 @@ export const layer = Layer.effect( const rendered = renderWebhookPrompt(task.prompt, request); // A provider refuses a turn this long, so it is not started. The - // delivery is not retryable, so the claim is kept. - if (rendered.prompt.length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) { + // delivery is not retryable, so the claim is kept. Providers trim + // the prompt before checking, so whitespace around it is free. + if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) { yield* log("dispatch_failed", { signatureVerified: signature !== null, missing: rendered.missing, diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index 7b91c1a712f1..58c32ea42a81 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -494,6 +494,21 @@ it.effect("logs a body's first 64 KiB by bytes, not characters", () => ), ); +it.effect("counts the prompt as the provider does, without surrounding whitespace", () => + withService(({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput({ prompt: "{{body.text}}" })); + // Over the limit as sent, within it once the padding is trimmed. + const text = `{"text":"${" ".repeat(1_000)}${"x".repeat(119_990)}${"\\n".repeat(1_000)}"}`; + const result = yield* service.triggerWebhook( + requestFor(task, { body: new TextEncoder().encode(text), bodyText: text }), + ); + assert.equal(result._tag, "accepted"); + yield* Queue.take(launches); + }), + ), +); + it.effect("does not start a run when the filled-in prompt is too long", () => withService(({ service, launches }) => Effect.gen(function* () { From 48984782e1a1bd5a776f36c62a6a6c5384337623 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 5 Oct 2026 00:34:13 -0700 Subject: [PATCH 30/35] fix(relay): held requests behind busy hooks' backlogs are reached in the same run A delivery run read one fixed batch of the oldest held requests. When busy hooks filled that batch, other hooks' requests behind it waited for the busy retry. A run now keeps reading past busy hooks until it has tried every other held request. Co-Authored-By: Claude Opus 5.5 (1M context) --- infra/relay/src/hooks/HookInboxStore.test.ts | 21 ++++ infra/relay/src/hooks/HookInboxStore.ts | 100 +++++++++++-------- 2 files changed, 77 insertions(+), 44 deletions(-) diff --git a/infra/relay/src/hooks/HookInboxStore.test.ts b/infra/relay/src/hooks/HookInboxStore.test.ts index bdc44d7708f3..121ce84a686c 100644 --- a/infra/relay/src/hooks/HookInboxStore.test.ts +++ b/infra/relay/src/hooks/HookInboxStore.test.ts @@ -101,6 +101,27 @@ describe("HookInboxStore", () => { ), ); + it.effect("reaches a hook queued behind busy hooks' full backlogs in the same run", () => + withInbox( + Effect.gen(function* () { + for (const name of ["stuck-a", "stuck-b"]) { + const stuck = { rawHookId: name, hookKey: name }; + for (let index = 0; index < HookInboxStore.HOOK_INBOX_MAX_PER_HOOK; index++) { + yield* HookInboxStore.hold(yield* hook(`${name}-${index}`, stuck), BASE_URL); + } + } + yield* HookInboxStore.hold(yield* hook("other-1"), BASE_URL); + const busy = deliverer((held) => (held.hookKey.startsWith("stuck") ? "busy" : "delivered")); + yield* HookInboxStore.deliverDue(busy.send); + expect(busy.sent.map((entry) => entry.hook.id)).toEqual([ + "stuck-a-0", + "stuck-b-0", + "other-1", + ]); + }), + ), + ); + it.effect("drops a held request it cannot read instead of stalling on it", () => withInbox( Effect.gen(function* () { diff --git a/infra/relay/src/hooks/HookInboxStore.ts b/infra/relay/src/hooks/HookInboxStore.ts index a98712395d73..64049f6b86ab 100644 --- a/infra/relay/src/hooks/HookInboxStore.ts +++ b/infra/relay/src/hooks/HookInboxStore.ts @@ -27,8 +27,8 @@ export const HOOK_INBOX_MAX_BYTES = 50 * 1_048_576; export const HOOK_INBOX_MAX_PER_HOOK = 100; /** Requests pushed per alarm run; the next run starts right away while more wait. */ const DELIVERIES_PER_RUN = 20; -/** Requests read per run, so requests behind a busy hook's backlog are still reached. */ -const ROWS_READ_PER_RUN = 200; +/** Requests read at a time; a run keeps reading past busy hooks' backlogs. */ +const ROWS_READ_PER_PAGE = 50; /** Wait before trying a hook whose environment answered busy again. */ const BUSY_RETRY_MS = 30_000; const MAX_RETRY_DELAY = Duration.minutes(10); @@ -102,6 +102,7 @@ const encodeHeaders = Schema.encodeSync(HeadersJson); const decodeHeaders = Schema.decodeUnknownOption(HeadersJson); interface HeldHookRow { + readonly seq: number; readonly id: string; readonly received_at: string; readonly method: string; @@ -279,10 +280,16 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( RETURNING id `; const target = yield* readTarget; - const batch = yield* sql` - SELECT id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body - FROM held_hooks ORDER BY seq LIMIT ${ROWS_READ_PER_RUN} - `; + /** Oldest held requests after `afterSeq`, skipping hooks already found busy. */ + const readPage = (afterSeq: number, skip: ReadonlySet) => + sql` + SELECT seq, id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body + FROM held_hooks + WHERE seq > ${afterSeq} + ${skip.size === 0 ? sql`` : sql`AND hook_key NOT IN ${sql.in([...skip])}`} + ORDER BY seq LIMIT ${ROWS_READ_PER_PAGE} + `; + let batch = yield* readPage(0, new Set()); if (batch.length === 0 || target === null) { if (target === null) yield* sql`DELETE FROM held_hooks`; // Empty, so the next request held starts the schedule from the top. @@ -314,46 +321,51 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* ( "relay.inbox.held_bytes": backlog.bytes, }); }); - for (const row of batch) { - if (sent === DELIVERIES_PER_RUN) break; - // Later requests to a busy hook wait their turn, so its order is kept. - if (busyHooks.has(row.hook_key)) continue; - const headers = decodeHeaders(row.headers); - if (Option.isNone(headers)) { - // Unreadable: it can never be delivered, and must not block the rest. + // Pages run out only once every non-busy request has been tried, so a + // backlog behind busy hooks never hides another hook's requests. + pages: while (batch.length > 0) { + for (const row of batch) { + if (sent === DELIVERIES_PER_RUN) break pages; + // Later requests to a busy hook wait their turn, so its order is kept. + if (busyHooks.has(row.hook_key)) continue; + const headers = decodeHeaders(row.headers); + if (Option.isNone(headers)) { + // Unreadable: it can never be delivered, and must not block the rest. + yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`; + unreadable += 1; + continue; + } + sent += 1; + const outcome = yield* send(target.base_url, { + id: row.id, + receivedAt: row.received_at, + method: row.method, + rawHookId: row.raw_hook_id, + rawToken: row.raw_token, + hookKey: row.hook_key, + query: row.query, + headers: headers.value, + body: row.body, + }); + if (outcome === "unreachable") { + failures = yield* countFailure; + yield* annotateRun("unreachable"); + return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures)); + } + if (failures !== 0) { + failures = 0; + yield* resetFailures; + } + if (outcome === "busy") { + busyHooks.add(row.hook_key); + continue; + } yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`; - unreadable += 1; - continue; - } - sent += 1; - const outcome = yield* send(target.base_url, { - id: row.id, - receivedAt: row.received_at, - method: row.method, - rawHookId: row.raw_hook_id, - rawToken: row.raw_token, - hookKey: row.hook_key, - query: row.query, - headers: headers.value, - body: row.body, - }); - if (outcome === "unreachable") { - failures = yield* countFailure; - yield* annotateRun("unreachable"); - return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures)); - } - if (failures !== 0) { - failures = 0; - yield* resetFailures; - } - if (outcome === "busy") { - busyHooks.add(row.hook_key); - continue; + delivered += 1; + const receivedAtMs = DateTime.toEpochMillis(DateTime.makeUnsafe(row.received_at)); + longestWaitMs = Math.max(longestWaitMs, startedAt - receivedAtMs); } - yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`; - delivered += 1; - const receivedAtMs = DateTime.toEpochMillis(DateTime.makeUnsafe(row.received_at)); - longestWaitMs = Math.max(longestWaitMs, startedAt - receivedAtMs); + batch = yield* readPage(batch.at(-1)!.seq, busyHooks); } if (!(yield* hasPending)) { yield* resetFailures; From 6d33885c694715c1dd8d041fa48f710df4d842a2 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 5 Oct 2026 10:58:11 -0700 Subject: [PATCH 31/35] feat(relay,server): the relay records what the environment did with each webhook The environment answers every webhook with x-t3-hook-outcome, which the relay records as relay.hook.upstream_outcome. A 202 alone could not tell a started run from a duplicate or an oversized prompt. The relay also sends its own traceparent with each forward and drops any a sender supplied. For relay deliveries, the environment's triggerWebhook span joins that trace and goes to the T3 Connect trace export, so a request reads as one trace from relay to run. Direct requests never join a sender's trace. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scheduledTasks/ScheduledTaskService.ts | 50 +++++++++++--- .../ScheduledTaskService.webhook.test.ts | 3 +- .../src/scheduledTasks/webhookRoute.test.ts | 53 +++++++++++--- .../server/src/scheduledTasks/webhookRoute.ts | 38 +++++++--- docs/operations/observability.md | 4 +- docs/operations/relay-observability.md | 8 ++- infra/relay/src/hooks/HookForwarder.test.ts | 69 +++++++++++++++++++ infra/relay/src/hooks/HookForwarder.ts | 9 ++- infra/relay/src/hooks/HookInboxObject.ts | 8 ++- infra/relay/src/hooks/upstream.ts | 25 +++++-- infra/relay/src/http/Api.ts | 15 +++- 11 files changed, 239 insertions(+), 43 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index a955ce50c970..f784484ad19b 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -122,12 +122,37 @@ export interface WebhookTriggerRequest extends WebhookRequest { } /** What the HTTP route should answer. `not_found` covers unknown hooks and wrong tokens alike. */ +/** + * What happened to one webhook request, as recorded in metrics and spans. + * Sent back to the relay as `x-t3-hook-outcome`; never request contents. + */ +export type WebhookDeliveryOutcome = + | "accepted" + | "duplicate" + | "prompt_too_long" + | "queue_full" + | "rate_limited" + | "disabled" + | "rejected_signature" + | "expired" + | "not_found" + | "error"; + export type WebhookTriggerResult = - | { readonly _tag: "accepted"; readonly deliveryId: ScheduledTaskWebhookDeliveryId } + | { + readonly _tag: "accepted"; + readonly deliveryId: ScheduledTaskWebhookDeliveryId; + /** A 202 covers more than a started run; this says which. */ + readonly outcome: "accepted" | "duplicate" | "prompt_too_long"; + } | { readonly _tag: "not_found" } | { readonly _tag: "rejected_signature" } | { readonly _tag: "disabled" } - | { readonly _tag: "rate_limited" } + | { + readonly _tag: "rate_limited"; + /** Too many requests to this hook, or too many runs already waiting. */ + readonly outcome: "rate_limited" | "queue_full"; + } | { readonly _tag: "expired" }; const decodeTask = Schema.decodeUnknownEffect(ScheduledTask); @@ -1335,7 +1360,7 @@ export const layer = Layer.effect( */ const observeDelivery = ( request: WebhookTriggerRequest, - outcome: string, + outcome: WebhookDeliveryOutcome, receivedAt: DateTime.DateTime | undefined, now: DateTime.DateTime | undefined, ) => @@ -1408,7 +1433,8 @@ export const layer = Layer.effect( Option.getOrElse(DateTime.make(request.receivedAt), () => now), now, ); - const observe = (outcome: string) => observeDelivery(request, outcome, receivedAt, now); + const observe = (outcome: WebhookDeliveryOutcome) => + observeDelivery(request, outcome, receivedAt, now); const deliveryId = ScheduledTaskWebhookDeliveryId.make( request.relayDeliveryId === undefined ? `delivery:${yield* crypto.randomUUIDv4.pipe( @@ -1449,7 +1475,7 @@ export const layer = Layer.effect( if (claimed.length === 0) { // A held request this environment already ran: accepted, not run twice. yield* observe("duplicate"); - return { _tag: "accepted" as const, deliveryId }; + return { _tag: "accepted" as const, deliveryId, outcome: "duplicate" as const }; } // Older claims can no longer be replayed by the relay. yield* sql` @@ -1484,7 +1510,10 @@ export const layer = Layer.effect( if (slot !== "allowed") { if (slot === "first_rejected") yield* log("rate_limited"); yield* observe("rate_limited"); - return yield* releaseClaim({ _tag: "rate_limited" as const }); + return yield* releaseClaim({ + _tag: "rate_limited" as const, + outcome: "rate_limited" as const, + }); } if (!task.enabled) { yield* log("disabled"); @@ -1530,7 +1559,7 @@ export const layer = Layer.effect( error: "The filled-in prompt is too long.", }); yield* observe("prompt_too_long"); - return { _tag: "accepted" as const, deliveryId }; + return { _tag: "accepted" as const, deliveryId, outcome: "prompt_too_long" as const }; } // Bound the deliveries one task holds, so steady traffic to a stuck // task cannot pile up parked fibers. A refused request is not logged, @@ -1545,7 +1574,10 @@ export const layer = Layer.effect( if (!queued) { // The task is busy with WEBHOOK_MAX_QUEUED_PER_TASK deliveries already. yield* observe("queue_full"); - return yield* releaseClaim({ _tag: "rate_limited" as const }); + return yield* releaseClaim({ + _tag: "rate_limited" as const, + outcome: "queue_full" as const, + }); } // Entries leave the map when their count reaches zero, so a deleted // task's key does not linger once its last delivery finishes. @@ -1601,7 +1633,7 @@ export const layer = Layer.effect( Effect.ensuring(release), Effect.forkIn(serviceScope), ); - return { _tag: "accepted" as const, deliveryId }; + return { _tag: "accepted" as const, deliveryId, outcome: "accepted" as const }; }), ); }); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index 58c32ea42a81..5fdaf45dcf3c 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -544,7 +544,8 @@ it.effect("a held request already delivered directly runs only once", () => requestFor(task, { relayDeliveryId: "relay-1", receivedAt: "2026-10-04T10:00:00.000Z" }), ); assert.equal(direct._tag, "accepted"); - assert.deepEqual(replayed, direct); + // Same delivery, answered the same way, but recorded as a duplicate. + assert.deepEqual(replayed, { ...direct, outcome: "duplicate" } as typeof replayed); yield* Queue.take(launches); const logged = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries; assert.equal(logged.length, 1); diff --git a/apps/server/src/scheduledTasks/webhookRoute.test.ts b/apps/server/src/scheduledTasks/webhookRoute.test.ts index 921c2d319968..523fb56bc5cf 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.test.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.test.ts @@ -56,6 +56,7 @@ describe("webhook route", () => { return Effect.succeed({ _tag: "accepted", deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"), + outcome: "accepted", }); }); try { @@ -84,6 +85,7 @@ describe("webhook route", () => { return Effect.succeed({ _tag: "accepted", deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"), + outcome: "accepted", }); }); try { @@ -103,25 +105,54 @@ describe("webhook route", () => { } }); - it("maps service outcomes to status codes", async () => { - const cases: ReadonlyArray<[WebhookTriggerResult["_tag"], number]> = [ - ["not_found", 404], - ["rejected_signature", 401], - ["disabled", 409], - ["rate_limited", 429], + it("maps service outcomes to status codes and names each outcome for the relay", async () => { + const deliveryId = ScheduledTaskWebhookDeliveryId.make("delivery:1"); + const cases: ReadonlyArray<[WebhookTriggerResult, number, string]> = [ + [{ _tag: "accepted", deliveryId, outcome: "accepted" }, 202, "accepted"], + // Same status as a started run; only the header tells them apart. + [{ _tag: "accepted", deliveryId, outcome: "duplicate" }, 202, "duplicate"], + [{ _tag: "accepted", deliveryId, outcome: "prompt_too_long" }, 202, "prompt_too_long"], + [{ _tag: "not_found" }, 404, "not_found"], + [{ _tag: "rejected_signature" }, 401, "rejected_signature"], + [{ _tag: "disabled" }, 409, "disabled"], + [{ _tag: "rate_limited", outcome: "rate_limited" }, 429, "rate_limited"], + [{ _tag: "rate_limited", outcome: "queue_full" }, 429, "queue_full"], + [{ _tag: "expired" }, 410, "expired"], ]; - for (const [tag, status] of cases) { - const { handler, dispose } = handlerFor(() => - Effect.succeed({ _tag: tag } as WebhookTriggerResult), - ); + for (const [result, status, outcome] of cases) { + const { handler, dispose } = handlerFor(() => Effect.succeed(result)); try { - expect((await handler(post("/api/hooks/id/tok", "{}"))).status).toBe(status); + const response = await handler(post("/api/hooks/id/tok", "{}")); + expect(response.status).toBe(status); + expect(response.headers.get("x-t3-hook-outcome")).toBe(outcome); } finally { await dispose(); } } }); + it("joins the relay's trace only for requests the relay forwarded", async () => { + const parents: Array = []; + const { handler, dispose } = handlerFor(() => + Effect.gen(function* () { + const span = yield* Effect.currentParentSpan.pipe(Effect.option); + parents.push(span._tag === "Some" ? span.value.traceId : undefined); + return { _tag: "not_found" } as const; + }), + ); + const traceparent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; + try { + await handler( + post("/api/hooks/id/tok", "{}", { traceparent, "x-t3-relay-delivery-id": "relay-1" }), + ); + // A sender calling the URL directly cannot attach to our traces. + await handler(post("/api/hooks/id/tok", "{}", { traceparent })); + expect(parents).toEqual(["0af7651916cd43dd8448eb211c80319c", undefined]); + } finally { + await dispose(); + } + }); + it("rejects oversized bodies and malformed paths before reaching the service", async () => { let calls = 0; const { handler, dispose } = handlerFor(() => { diff --git a/apps/server/src/scheduledTasks/webhookRoute.ts b/apps/server/src/scheduledTasks/webhookRoute.ts index f7365e8dd1fd..c41777152af6 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.ts @@ -2,9 +2,12 @@ import { EnvironmentHttpApi } from "@t3tools/contracts"; import * as ByteSize from "effect/ByteSize"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; +import * as Tracer from "effect/Tracer"; import * as HttpIncomingMessage from "effect/http/HttpIncomingMessage"; import type * as HttpServerRequest from "effect/http/HttpServerRequest"; import * as HttpServerResponse from "effect/http/HttpServerResponse"; +import * as HttpTraceContext from "effect/http/HttpTraceContext"; +import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import * as Metrics from "../observability/Metrics.ts"; @@ -13,8 +16,11 @@ import * as ScheduledTaskService from "./ScheduledTaskService.ts"; /** Largest request body a webhook accepts. The relay enforces the same cap. */ export const WEBHOOK_MAX_BODY_BYTES = 1024 * 1024; -const json = (status: number, body: Record) => - HttpServerResponse.jsonUnsafe(body, { status }); +/** Response header naming what happened to the request; the relay records it. */ +const WEBHOOK_OUTCOME_HEADER = "x-t3-hook-outcome"; + +const json = (status: number, body: Record, outcome: string) => + HttpServerResponse.jsonUnsafe(body, { status, headers: { [WEBHOOK_OUTCOME_HEADER]: outcome } }); /** * Handles `/api/hooks/:hookId/:token` for every accepted method. The endpoint @@ -35,7 +41,7 @@ const handleWebhook = Effect.gen(function* () { const contentLength = Number(request.headers["content-length"] ?? "0"); if (!Number.isFinite(contentLength) || contentLength > WEBHOOK_MAX_BODY_BYTES) { - return json(413, { error: "body_too_large" }); + return json(413, { error: "body_too_large" }, "body_too_large"); } // Chunked requests carry no content-length, so the reader itself is capped. const body = yield* request.arrayBuffer.pipe( @@ -51,7 +57,7 @@ const handleWebhook = Metrics.increment(Metrics.webhookDeliveriesTotal, { outcome: "body_too_large", source: request.headers["x-t3-relay-delivery-id"] ? "relay" : "direct", - }).pipe(Effect.as(json(413, { error }))); + }).pipe(Effect.as(json(413, { error }, "body_too_large"))); if (Option.isNone(body)) return yield* tooLarge("body_too_large_or_unreadable"); if (body.value.byteLength > WEBHOOK_MAX_BODY_BYTES) { return yield* tooLarge("body_too_large"); @@ -67,6 +73,12 @@ const handleWebhook = const relayDeliveryId = headers["x-t3-relay-delivery-id"]; const relayReceivedAt = relayDeliveryId ? headers["x-t3-relay-received-at"] : undefined; + // A relay delivery joins the relay's trace, and goes to the T3 Connect + // tracer with it. A sender's own traceparent is never trusted: anyone + // calling the URL directly could otherwise attach to our traces. + const relayParent = relayDeliveryId + ? HttpTraceContext.fromHeaders(request.headers) + : Option.none(); const result = yield* scheduledTasks .triggerWebhook({ hookId: params.hookId, @@ -81,6 +93,10 @@ const handleWebhook = ...(relayReceivedAt ? { receivedAt: relayReceivedAt } : {}), }) .pipe( + Option.isSome(relayParent) + ? (effect) => + effect.pipe(Effect.withParentSpan(relayParent.value), withRelayClientTracing) + : (effect) => effect, // Defects too, so the sender only ever sees the fixed error body. Effect.catchCause((cause) => Effect.logWarning("Webhook delivery failed").pipe( @@ -93,19 +109,19 @@ const handleWebhook = switch (result._tag) { case "accepted": - return json(202, { deliveryId: result.deliveryId }); + return json(202, { deliveryId: result.deliveryId }, result.outcome); case "not_found": - return json(404, { error: "hook_not_found" }); + return json(404, { error: "hook_not_found" }, "not_found"); case "rejected_signature": - return json(401, { error: "invalid_signature" }); + return json(401, { error: "invalid_signature" }, "rejected_signature"); case "disabled": - return json(409, { error: "hook_disabled" }); + return json(409, { error: "hook_disabled" }, "disabled"); case "rate_limited": - return json(429, { error: "rate_limited" }); + return json(429, { error: "rate_limited" }, result.outcome); case "expired": - return json(410, { error: "delivery_too_old" }); + return json(410, { error: "delivery_too_old" }, "expired"); case "error": - return json(500, { error: "internal_error" }); + return json(500, { error: "internal_error" }, "error"); } }); diff --git a/docs/operations/observability.md b/docs/operations/observability.md index a5a209ea06c2..7a39ac3dd601 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -397,7 +397,9 @@ Webhooks have their own families: - `t3_webhook_held_delay` for how long requests the relay held waited before arriving. `ScheduledTaskService.triggerWebhook` spans carry the same outcome per request, and each run -started from a delivery is its own `ScheduledTaskService.runWebhookDelivery` trace. +started from a delivery is its own `ScheduledTaskService.runWebhookDelivery` trace. For a request +the relay forwarded, the span also goes to the T3 Connect trace export as a child of the relay's +span; requests that reach the environment directly never join a sender's trace. Use metrics when the question is: diff --git a/docs/operations/relay-observability.md b/docs/operations/relay-observability.md index 9d5a0f6a88a4..1c4400108bd6 100644 --- a/docs/operations/relay-observability.md +++ b/docs/operations/relay-observability.md @@ -64,7 +64,13 @@ A public webhook request is one `relay.hooks.forward` span. Its `relay.hook.outc happened: `forwarded`, `held`, `rate_limited`, `inbox_full`, `not_found`, `payload_too_large`, `environment_unavailable`, or `environment_timeout`. `relay.hook.endpoint_key` identifies the managed endpoint, and with it the environment. On a forward, `relay.hook.upstream_status` or -`relay.hook.upstream_error` records the environment's answer. `relay.hook.rate_limit` says which +`relay.hook.upstream_error` records the environment's answer, and +`relay.hook.upstream_outcome` what it did with the request (`accepted`, `duplicate`, +`prompt_too_long`, `queue_full`, `rejected_signature`, `expired`, `disabled`, ...), from its +`x-t3-hook-outcome` response header. A held request's delivery records the same on its +`relay.inbox.deliver` span. The relay sends its own `traceparent` with each forward and drops any a +sender supplied, so on environments that export to T3 Connect, the environment's +`ScheduledTaskService.triggerWebhook` span lands in the same trace. `relay.hook.rate_limit` says which budget ran out: `endpoint` or `hook`. `relay.hook.rate_limiter_failed_open` is set when the Cloudflare rate limiter was unavailable and the request went through unlimited. diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index 925b08c86b67..bdad796b697f 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -558,6 +558,75 @@ describe("HookForwarder", () => { }), ); + it.effect("joins the environment to its trace and records what the environment did", () => + Effect.gen(function* () { + const spans: Array = []; + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, + }); + const respondWith = (outcome: string) => + makeHarness({ + execute: (request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response('{"deliveryId":"d"}', { + status: 202, + headers: { "x-t3-hook-outcome": outcome }, + }), + ), + ), + }); + const forward = (harness: ReturnType) => + Effect.gen(function* () { + const handler = yield* harness.httpEffect; + return yield* traceRelayHttpRequestWith( + handler, + Layer.succeed(Tracer.Tracer, tracer), + ).pipe( + Effect.provideService( + HttpServerRequest.HttpServerRequest, + HttpServerRequest.fromWeb( + new Request(hookUrl(), { + method: "POST", + // A sender's own trace context never reaches the environment. + headers: { + traceparent: "00-11111111111111111111111111111111-2222222222222222-01", + "x-b3-traceid": "33333333333333333333333333333333", + }, + body: "{}", + }), + ), + ), + ); + }); + + const duplicate = respondWith("duplicate"); + expect((yield* forward(duplicate)).status).toBe(202); + yield* Effect.yieldNow; + const forwardSpan = spans.find((span) => span.name === "relay.hooks.forward"); + expect(forwardSpan?.attributes.get("relay.hook.upstream_outcome")).toBe("duplicate"); + const sent = duplicate.sent[0]!; + expect(sent.headers.traceparent).toContain(forwardSpan!.traceId); + expect(sent.headers.traceparent).not.toContain("1111111111"); + expect(sent.headers["x-b3-traceid"]).toBeUndefined(); + + // Only a plain outcome name is recorded. + spans.length = 0; + yield* forward(respondWith("")); + yield* Effect.yieldNow; + expect( + spans + .find((span) => span.name === "relay.hooks.forward") + ?.attributes.has("relay.hook.upstream_outcome"), + ).toBe(false); + }), + ); + describe("holding requests while the environment is offline", () => { const offline = (request: HttpClientRequest.HttpClientRequest) => Effect.fail( diff --git a/infra/relay/src/hooks/HookForwarder.ts b/infra/relay/src/hooks/HookForwarder.ts index cbb9d5edee96..fe4d5271d229 100644 --- a/infra/relay/src/hooks/HookForwarder.ts +++ b/infra/relay/src/hooks/HookForwarder.ts @@ -57,8 +57,12 @@ const DROPPED_REQUEST_HEADERS = new Set([ // Only the relay may set this; a sender could otherwise collide delivery ids. "x-t3-relay-delivery-id", "x-t3-relay-received-at", + // The environment trusts trace context only from the relay, which sets its own. + "traceparent", + "tracestate", + "b3", ]); -const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-"]; +const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-", "x-b3-"]; export const isRelayHookPath = (url: string): boolean => url.startsWith(RELAY_HOOK_PATH_PREFIX); @@ -412,6 +416,9 @@ const make = Effect.gen(function* () { yield* Effect.annotateCurrentSpan({ "relay.hook.outcome": "forwarded", "relay.hook.upstream_status": response.status, + ...(response.outcome === undefined + ? {} + : { "relay.hook.upstream_outcome": response.outcome }), }); // Only content-type is passed through: no location (redirects are never // followed or relayed), no cookies, no upstream infrastructure headers. diff --git a/infra/relay/src/hooks/HookInboxObject.ts b/infra/relay/src/hooks/HookInboxObject.ts index 0e6c86af5aa6..1ec020f1e260 100644 --- a/infra/relay/src/hooks/HookInboxObject.ts +++ b/infra/relay/src/hooks/HookInboxObject.ts @@ -65,13 +65,19 @@ const deliver = (baseUrl: string, hook: HookInboxStore.HeldHook) => if (Option.isNone(result.success)) return { outcome: "unreachable" as const, reason: "timeout" }; const status = result.success.value.status; + const upstreamOutcome = result.success.value.outcome; const outcome: HookInboxStore.DeliveryOutcome = UNREACHABLE_STATUSES.has(status) ? "unreachable" : BUSY_STATUSES.has(status) ? "busy" : "delivered"; - return { outcome, reason: `status ${status}` }; + return { outcome, reason: `status ${status}`, upstreamOutcome }; }), + Effect.tap(({ upstreamOutcome }) => + upstreamOutcome === undefined + ? Effect.void + : Effect.annotateCurrentSpan({ "relay.hook.upstream_outcome": upstreamOutcome }), + ), Effect.tap(({ outcome, reason }) => outcome === "delivered" ? Effect.void diff --git a/infra/relay/src/hooks/upstream.ts b/infra/relay/src/hooks/upstream.ts index 4c696c966e1c..185a59f286d9 100644 --- a/infra/relay/src/hooks/upstream.ts +++ b/infra/relay/src/hooks/upstream.ts @@ -1,10 +1,12 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientRequest from "effect/http/HttpClientRequest"; import * as HttpMethod from "effect/http/HttpMethod"; +import * as HttpTraceContext from "effect/http/HttpTraceContext"; import type * as HttpClientResponse from "effect/http/HttpClientResponse"; import { withoutRedirects } from "../environments/EnvironmentConnector.ts"; @@ -36,9 +38,14 @@ export interface UpstreamHook { export interface UpstreamResponse { readonly status: number; readonly contentType: string | undefined; + /** What the environment did with the request (`x-t3-hook-outcome`). */ + readonly outcome: string | undefined; readonly body: Uint8Array; } +/** Outcome names are short identifiers; anything else is not recorded. */ +const OUTCOME_PATTERN = /^[a-z_]{1,32}$/; + class ResponseTooLarge extends Schema.TaggedError()("ResponseTooLarge", {}) {} /** Reads a response body, failing once it passes the cap rather than buffering it all. */ @@ -78,8 +85,12 @@ export const sendUpstream = (baseUrl: string, hook: UpstreamHook) => Effect.gen(function* () { const httpClient = yield* HttpClient.HttpClient; const base = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`; + // The environment's span joins this trace. Set by hand: the client span + // that would propagate it is off, because it records the token in url.full. + const parent = yield* Effect.currentSpan.pipe(Effect.option); const headers: Record = { ...hook.headers, + ...(Option.isSome(parent) ? HttpTraceContext.toHeaders(parent.value) : {}), [RELAY_DELIVERY_ID_HEADER]: hook.id, [RELAY_RECEIVED_AT_HEADER]: hook.receivedAt, }; @@ -93,11 +104,15 @@ export const sendUpstream = (baseUrl: string, hook: UpstreamHook) => return yield* httpClient.execute(request).pipe( Effect.flatMap((response) => readCapped(response).pipe( - Effect.map((body): UpstreamResponse => ({ - status: response.status, - contentType: response.headers["content-type"], - body, - })), + Effect.map((body): UpstreamResponse => { + const outcome = response.headers["x-t3-hook-outcome"]; + return { + status: response.status, + contentType: response.headers["content-type"], + outcome: outcome !== undefined && OUTCOME_PATTERN.test(outcome) ? outcome : undefined, + body, + }; + }), ), ), withoutRedirects, diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 96af0c3263fd..84ff434c5b8c 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -230,6 +230,9 @@ const relayRequestDeadline = ( ), ); +/** Trace context headers in every format the tracer reads (W3C and B3). */ +const SENDER_TRACE_HEADER = /^(traceparent|tracestate|b3|x-b3-.*)$/i; + export const traceRelayHttpRequest = ( httpEffect: Effect.Effect< HttpServerResponse.HttpServerResponse, @@ -247,8 +250,16 @@ export const traceRelayHttpRequest = ( return yield* HttpMiddleware.tracer(traced).pipe(Effect.ensuring(Effect.yieldNow)); } // Hook URLs carry a secret token: the tracer and deadline log see a redacted - // request, while the route itself still receives the original. - const redacted = request.modify({ url: redactRelayHookUrl(request.url) }); + // request, while the route itself still receives the original. A webhook + // sender's trace context is dropped, so it cannot pick the trace our relay + // and environment spans land in. + const redacted = request.modify({ + url: redactRelayHookUrl(request.url), + headers: Headers.removeMany( + request.headers, + Object.keys(request.headers).filter((name) => SENDER_TRACE_HEADER.test(name)), + ), + }); return yield* HttpMiddleware.tracer( appendRelayTraceContextResponseHeader.pipe( Effect.andThen( From 00775e0ca8419fb4933b195037b75bf5f4b67813 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 5 Oct 2026 11:07:11 -0700 Subject: [PATCH 32/35] fix(relay): the worker's own request span never records a webhook token The worker runtime traces every request itself, outside the relay's redacting tracer. For webhook paths that span held the raw URL, token included, and adopted any traceparent the sender sent. It is now off for hook paths, which keep only the relay's redacted span. Co-Authored-By: Claude Opus 5.5 (1M context) --- infra/relay/src/hooks/HookForwarder.test.ts | 44 +++++++++++++++++++++ infra/relay/src/http/Api.ts | 2 + infra/relay/src/worker.ts | 13 ++++++ 3 files changed, 59 insertions(+) diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index bdad796b697f..011c31b149fa 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -20,6 +20,7 @@ import * as Etag from "effect/http/Etag"; import * as HttpRouter from "effect/http/HttpRouter"; import * as HttpApi from "effect/http-api/HttpApi"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; +import * as HttpMiddleware from "effect/http/HttpMiddleware"; import * as HttpServerRequest from "effect/http/HttpServerRequest"; import * as HttpServerResponse from "effect/http/HttpServerResponse"; @@ -558,6 +559,49 @@ describe("HookForwarder", () => { }), ); + it.effect("records one redacted server span even inside the worker's own HTTP tracer", () => + Effect.gen(function* () { + const spans: Array = []; + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, + }); + const harness = makeHarness(); + const handler = yield* harness.httpEffect; + // As the worker runtime runs it: its own tracer around ours, off for hook + // paths. This checks the predicate; whether alchemy applies it per event + // is only visible on a deployed worker (see worker.ts). + yield* HttpMiddleware.tracer( + traceRelayHttpRequestWith(handler, Layer.succeed(Tracer.Tracer, tracer)), + ).pipe( + Effect.provideService(HttpMiddleware.TracerDisabledWhen, (request) => + HookForwarder.isRelayHookPath(request.url), + ), + Effect.withTracer(tracer), + Effect.provideService( + HttpServerRequest.HttpServerRequest, + HttpServerRequest.fromWeb( + new Request(hookUrl("hook-1/super-secret-token"), { + method: "POST", + headers: { traceparent: "00-11111111111111111111111111111111-2222222222222222-01" }, + body: "{}", + }), + ), + ), + ); + yield* Effect.yieldNow; + const servers = spans.filter((span) => span.kind === "server"); + expect(servers).toHaveLength(1); + expect(servers[0]?.attributes.get("url.path")).toBe( + `/v1/hooks/${endpointKey}/hook-1/`, + ); + expect(spans.every((span) => span.traceId !== "11111111111111111111111111111111")).toBe(true); + }), + ); + it.effect("joins the environment to its trace and records what the environment did", () => Effect.gen(function* () { const spans: Array = []; diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 84ff434c5b8c..7b308653d510 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -271,6 +271,8 @@ export const traceRelayHttpRequest = ( ), ).pipe( Effect.provideService(HttpServerRequest.HttpServerRequest, redacted), + // The worker disables its own span for hook paths; this one is ours. + Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false), Effect.ensuring(Effect.yieldNow), ); }); diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index cf2ef78489f1..25cdd257f166 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -12,6 +12,7 @@ import * as Option from "effect/Option"; import * as Redacted from "effect/Redacted"; import * as Stream from "effect/Stream"; import * as Etag from "effect/http/Etag"; +import * as HttpMiddleware from "effect/http/HttpMiddleware"; import * as HttpPlatform from "effect/http/HttpPlatform"; import * as HttpRouter from "effect/http/HttpRouter"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; @@ -473,6 +474,18 @@ export const ApiLive = Api.make( Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp), Layer.provideMerge(Cloudflare.Workers.RateLimitBinding), Layer.provideMerge(HookInboxObjectLive), + // The worker runtime opens its own HTTP span around ours. For webhook + // paths it would record the raw URL, token included, and adopt the + // sender's traceparent, so only our redacted span covers those. + // Registered as telemetry: request-time context is assembled per + // event, and only these layers are built into it. + Layer.provideMerge( + Alchemy.Telemetry.layer( + Layer.succeed(HttpMiddleware.TracerDisabledWhen)((request) => + HookForwarder.isRelayHookPath(request.url), + ), + ), + ), // Exports spans from events the HTTP tracer does not wrap, notably // HookInboxObject calls and alarms, to the same Axiom dataset. Layer.provideMerge( From bbe3a6bcc14ee418f86517f45351f77c0a109a7a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 5 Oct 2026 12:00:07 -0700 Subject: [PATCH 33/35] fix(relay): a failed delivery run cannot push back a wake's earlier alarm Co-Authored-By: Claude Opus 5.5 (1M context) --- infra/relay/src/hooks/HookInboxObject.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/relay/src/hooks/HookInboxObject.ts b/infra/relay/src/hooks/HookInboxObject.ts index 1ec020f1e260..82897e24d5c8 100644 --- a/infra/relay/src/hooks/HookInboxObject.ts +++ b/infra/relay/src/hooks/HookInboxObject.ts @@ -143,7 +143,7 @@ export const HookInboxObjectLive = HookInboxObject.make( Effect.logWarning("Held webhook delivery run failed", { cause }).pipe( Effect.andThen(Effect.annotateCurrentSpan({ "relay.inbox.run_result": "failed" })), Effect.andThen(Clock.currentTimeMillis), - Effect.flatMap((now) => state.storage.setAlarm(now + RUN_FAILURE_RETRY_MS)), + Effect.flatMap((now) => scheduleBy(now + RUN_FAILURE_RETRY_MS)), ), ), withInboxSpan("relay.inbox.deliver", inboxId), From 614183b2cfffdd335d7959c1805b4dd031a0e6f0 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 5 Oct 2026 12:09:52 -0700 Subject: [PATCH 34/35] feat(web): a webhook task says whether T3 Connect forwards or holds its requests Under the public URL, the task dialog says whether requests are forwarded live or held for up to 24 hours while the environment is offline, with a link to the setting in Connections. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../settings/ScheduledTasksSettings.tsx | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/apps/web/src/components/settings/ScheduledTasksSettings.tsx b/apps/web/src/components/settings/ScheduledTasksSettings.tsx index f6741f6129d4..d82b6f163694 100644 --- a/apps/web/src/components/settings/ScheduledTasksSettings.tsx +++ b/apps/web/src/components/settings/ScheduledTasksSettings.tsx @@ -43,8 +43,10 @@ import { deriveProviderInstanceEntries, sortProviderInstanceEntries, } from "../../providerInstances"; +import { usePrimaryCloudLinkState } from "../../cloud/primaryCloudLinkState"; import { requestConfirmDialog } from "../../confirmDialog"; import { webhookAddress } from "@t3tools/client-runtime/webhook-address"; +import { Link } from "@tanstack/react-router"; import { useCopyToClipboard } from "../../hooks/useCopyToClipboard"; import { useEnvironment, @@ -730,10 +732,31 @@ function WebhookEndpointField({
{note !== null ?

{note}

: null} + {endpoint.url !== null ? : null}
); } +/** + * Whether T3 Connect forwards requests live or holds them while the + * environment is offline. The setting is per environment and only readable + * for this machine's own environment, so other environments show nothing. + */ +function WebhookDeliveryMode({ environmentId }: { readonly environmentId: EnvironmentId }) { + const cloudLink = usePrimaryCloudLinkState(); + if (cloudLink.target?.environmentId !== environmentId || cloudLink.data === null) return null; + return ( +

+ {cloudLink.data.holdWebhooksWhileOffline + ? "Held for up to 24 hours while this environment is offline. " + : "Forwarded live. Requests fail while this environment is offline. "} + + Change in Connections + +

+ ); +} + function ScheduledTaskEditorDialog({ initialEnvironmentId, task, From 734a2fe95450c67366fe81f372eee09ad5ff4f1c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 5 Oct 2026 12:35:14 -0700 Subject: [PATCH 35/35] feat(relay,server): the environment trusts relay delivery headers only from the relay The relay's delivery id, receive time and trace context arrived as plain headers, and the webhook URL can also be called directly, so anyone with a hook token could claim to be the relay. The relay now signs each forward with its mint key (x-t3-relay-delivery), naming the environment, delivery id, receive time and hook. The environment checks it against the mint public key it already holds from linking, and treats a request without a valid proof as direct. A held request carries the proof signed when it was received, so the inbox never needs the key. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/scheduledTasks/relayDeliveryProof.ts | 74 +++++++ .../src/scheduledTasks/webhookRoute.test.ts | 209 ++++++++++++++---- .../server/src/scheduledTasks/webhookRoute.ts | 26 ++- docs/operations/relay-observability.md | 3 +- infra/relay/src/hooks/HookForwarder.test.ts | 46 +++- infra/relay/src/hooks/HookForwarder.ts | 59 ++++- infra/relay/src/hooks/upstream.ts | 2 + packages/contracts/src/relay.ts | 15 ++ packages/shared/src/relayJwt.ts | 3 + 9 files changed, 377 insertions(+), 60 deletions(-) create mode 100644 apps/server/src/scheduledTasks/relayDeliveryProof.ts diff --git a/apps/server/src/scheduledTasks/relayDeliveryProof.ts b/apps/server/src/scheduledTasks/relayDeliveryProof.ts new file mode 100644 index 000000000000..9c277e6998d8 --- /dev/null +++ b/apps/server/src/scheduledTasks/relayDeliveryProof.ts @@ -0,0 +1,74 @@ +/** + * Whether a webhook request really came through this environment's relay. + * + * The relay's delivery id, receive time and trace context arrive as plain + * headers, and the webhook URL can also be called directly, so they are only + * trusted alongside a proof the relay signed with its mint key. A request + * without a valid proof is handled as a direct request. + */ +import { RelayHookDeliveryProofPayload } from "@t3tools/contracts/relay"; +import { + normalizeRelayIssuer, + RELAY_HOOK_DELIVERY_HEADER, + RELAY_HOOK_DELIVERY_TYP, + verifyRelayJwt, +} from "@t3tools/shared/relayJwt"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import { CLOUD_MINT_PUBLIC_KEY, RELAY_ISSUER_SECRET, RELAY_URL_SECRET } from "../cloud/config.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; + +/** Covers the relay's 24-hour hold plus a margin. */ +const MAX_PROOF_AGE_SECONDS = 25 * 60 * 60; + +const decodePayload = Schema.decodeUnknownOption(RelayHookDeliveryProofPayload); +const text = (bytes: Option.Option) => + Option.map(bytes, (value) => new TextDecoder().decode(value)); + +/** The relay's own delivery id and receive time, when the request proves it came from the relay. */ +export const makeRelayDeliveryVerifier = Effect.gen(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + const environment = yield* ServerEnvironment.ServerEnvironment; + return (input: { readonly headers: Readonly>; readonly hookId: string }) => + Effect.gen(function* () { + const proof = input.headers[RELAY_HOOK_DELIVERY_HEADER]; + const deliveryId = input.headers["x-t3-relay-delivery-id"]; + const receivedAt = input.headers["x-t3-relay-received-at"]; + if (proof === undefined || deliveryId === undefined || receivedAt === undefined) { + return Option.none(); + } + const publicKey = text(yield* secrets.get(CLOUD_MINT_PUBLIC_KEY)); + const issuer = Option.orElse(text(yield* secrets.get(RELAY_ISSUER_SECRET)), () => + Option.none(), + ); + const relayUrl = text(yield* secrets.get(RELAY_URL_SECRET)); + const relayIssuer = Option.isSome(issuer) ? issuer : relayUrl; + // Not linked to T3 Connect: no relay can be delivering to us. + if (Option.isNone(publicKey) || Option.isNone(relayIssuer)) return Option.none(); + const environmentId = yield* environment.getEnvironmentId; + const payload = yield* verifyRelayJwt({ + publicKey: publicKey.value, + token: proof, + typ: RELAY_HOOK_DELIVERY_TYP, + issuer: normalizeRelayIssuer(relayIssuer.value), + audience: `t3-env:${environmentId}`, + nowEpochSeconds: Math.floor((yield* Clock.currentTimeMillis) / 1_000), + maxTokenAge: MAX_PROOF_AGE_SECONDS, + }).pipe(Effect.map(decodePayload), Effect.orElseSucceed(Option.none)); + // The proof must name exactly this delivery, so it cannot be lifted onto another one. + if ( + Option.isNone(payload) || + payload.value.environmentId !== environmentId || + payload.value.deliveryId !== deliveryId || + payload.value.receivedAt !== receivedAt || + payload.value.hookId !== input.hookId + ) { + return Option.none(); + } + return Option.some({ deliveryId, receivedAt }); + }).pipe(Effect.orElseSucceed(Option.none), Effect.withSpan("webhook.verifyRelayDelivery")); +}); diff --git a/apps/server/src/scheduledTasks/webhookRoute.test.ts b/apps/server/src/scheduledTasks/webhookRoute.test.ts index 523fb56bc5cf..511b4f5ad875 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.test.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "@effect/vitest"; +import * as NodeCrypto from "node:crypto"; +import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Etag from "effect/http/Etag"; import * as HttpPlatform from "effect/http/HttpPlatform"; @@ -8,8 +11,10 @@ import * as HttpRouter from "effect/http/HttpRouter"; import * as HttpApi from "effect/http-api/HttpApi"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; +import { RELAY_HOOK_DELIVERY_TYP, signRelayJwt } from "@t3tools/shared/relayJwt"; import { EnvironmentHttpApi, + EnvironmentId, ScheduledTaskWebhookDeliveryId, ScheduledTaskError, } from "@t3tools/contracts"; @@ -18,19 +23,76 @@ import { type WebhookTriggerRequest, type WebhookTriggerResult, } from "./ScheduledTaskService.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import { CLOUD_MINT_PUBLIC_KEY, RELAY_ISSUER_SECRET } from "../cloud/config.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import { WEBHOOK_MAX_BODY_BYTES, webhookHttpApiLayer } from "./webhookRoute.ts"; class WebhookTestApi extends HttpApi.make("environment").add(EnvironmentHttpApi.groups.webhooks) {} +const environmentId = EnvironmentId.make("env-1"); +const relayIssuer = "https://relay.example.test"; +const mintKeys = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, +}); +/** The secrets a T3 Connect-linked environment holds, keyed by name. */ +const linkedSecrets: ReadonlyMap = new Map([ + [CLOUD_MINT_PUBLIC_KEY, mintKeys.publicKey], + [RELAY_ISSUER_SECRET, relayIssuer], +]); + +/** A proof as the relay signs it for one delivery. */ +const relayProof = Effect.fn("relayProof")(function* (claims: { + readonly deliveryId: string; + readonly receivedAt: string; + readonly hookId: string; + readonly privateKey?: string; +}) { + const now = Math.floor((yield* Clock.currentTimeMillis) / 1_000); + return yield* signRelayJwt({ + privateKey: claims.privateKey ?? mintKeys.privateKey, + typ: RELAY_HOOK_DELIVERY_TYP, + payload: { + iss: relayIssuer, + aud: `t3-env:${environmentId}`, + sub: environmentId, + jti: `proof:${claims.deliveryId}`, + iat: now, + exp: now + 3_600, + environmentId, + deliveryId: claims.deliveryId, + receivedAt: claims.receivedAt, + hookId: claims.hookId, + }, + }); +}); + const handlerFor = ( trigger: ( request: WebhookTriggerRequest, ) => Effect.Effect, + secrets: ReadonlyMap = linkedSecrets, ) => HttpRouter.toWebHandler( HttpApiBuilder.layer(WebhookTestApi).pipe( Layer.provide(webhookHttpApiLayer), Layer.provide(Layer.mock(ScheduledTaskService)({ triggerWebhook: trigger })), + Layer.provide( + Layer.mock(ServerSecretStore.ServerSecretStore)({ + get: (name) => + Effect.succeed( + Option.map(Option.fromUndefinedOr(secrets.get(name)), (value) => + new TextEncoder().encode(value), + ), + ), + }), + ), + Layer.provide( + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getEnvironmentId: Effect.succeed(environmentId), + }), + ), Layer.provide( HttpPlatform.layer.pipe( Layer.provideMerge(NodeServices.layer), @@ -78,32 +140,86 @@ describe("webhook route", () => { } }); - it("uses the relay's receive time only alongside its delivery id", async () => { - const received: Array = []; - const { handler, dispose } = handlerFor((request) => { - received.push(request); - return Effect.succeed({ - _tag: "accepted", - deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"), - outcome: "accepted", + // Live clock: the server checks proofs against real time. + it.live("trusts the relay's delivery id and receive time only with its signed proof", () => + Effect.gen(function* () { + const received: Array = []; + const { handler, dispose } = handlerFor((request) => { + received.push(request); + return Effect.succeed({ + _tag: "accepted", + deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"), + outcome: "accepted", + }); }); - }); - try { + const send = (path: string, headers: Record) => + Effect.promise(() => handler(post(path, "{}", headers))); const receivedAt = "2026-10-04T10:00:00.000Z"; - await handler( - post("/api/hooks/id/tok", "{}", { - "x-t3-relay-delivery-id": "relay-1", - "x-t3-relay-received-at": receivedAt, - }), - ); - await handler(post("/api/hooks/id/tok", "{}", { "x-t3-relay-received-at": receivedAt })); - expect(received[0]?.relayDeliveryId).toBe("relay-1"); + const relayHeaders = { + "x-t3-relay-delivery-id": "relay-1", + "x-t3-relay-received-at": receivedAt, + }; + const forged = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" }); + const forgedProof = yield* relayProof({ + deliveryId: "relay-1", + receivedAt, + hookId: "id", + privateKey: forged.privateKey, + }); + // 0: signed by the relay for exactly this delivery. + yield* send("/api/hooks/id/tok", { ...relayHeaders, "x-t3-relay-delivery": proof }); + // 1: a direct caller claiming to be the relay, without a proof. + yield* send("/api/hooks/id/tok", relayHeaders); + // 2: a proof signed with any other key. + yield* send("/api/hooks/id/tok", { ...relayHeaders, "x-t3-relay-delivery": forgedProof }); + // 3: a real proof lifted onto another delivery id. + yield* send("/api/hooks/id/tok", { + ...relayHeaders, + "x-t3-relay-delivery-id": "relay-2", + "x-t3-relay-delivery": proof, + }); + // 4: a real proof for another hook. + yield* send("/api/hooks/other/tok", { ...relayHeaders, "x-t3-relay-delivery": proof }); + yield* Effect.promise(() => dispose()); + expect(received.map((request) => request.relayDeliveryId)).toEqual([ + "relay-1", + undefined, + undefined, + undefined, + undefined, + ]); expect(received[0]?.receivedAt).toBe(receivedAt); - expect(received[1]?.receivedAt).toBeUndefined(); - } finally { - await dispose(); - } - }); + expect(received.slice(1).every((request) => request.receivedAt === undefined)).toBe(true); + }), + ); + + // Live clock: the server checks proofs against real time. + it.live("trusts no relay headers on an environment not linked to T3 Connect", () => + Effect.gen(function* () { + const received: Array = []; + const { handler, dispose } = handlerFor((request) => { + received.push(request); + return Effect.succeed({ _tag: "not_found" }); + }, new Map()); + const receivedAt = "2026-10-04T10:00:00.000Z"; + const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" }); + yield* Effect.promise(() => + handler( + post("/api/hooks/id/tok", "{}", { + "x-t3-relay-delivery-id": "relay-1", + "x-t3-relay-received-at": receivedAt, + "x-t3-relay-delivery": proof, + }), + ), + ); + yield* Effect.promise(() => dispose()); + expect(received[0]?.relayDeliveryId).toBeUndefined(); + }), + ); it("maps service outcomes to status codes and names each outcome for the relay", async () => { const deliveryId = ScheduledTaskWebhookDeliveryId.make("delivery:1"); @@ -131,27 +247,34 @@ describe("webhook route", () => { } }); - it("joins the relay's trace only for requests the relay forwarded", async () => { - const parents: Array = []; - const { handler, dispose } = handlerFor(() => - Effect.gen(function* () { - const span = yield* Effect.currentParentSpan.pipe(Effect.option); - parents.push(span._tag === "Some" ? span.value.traceId : undefined); - return { _tag: "not_found" } as const; - }), - ); - const traceparent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; - try { - await handler( - post("/api/hooks/id/tok", "{}", { traceparent, "x-t3-relay-delivery-id": "relay-1" }), + // Live clock: the server checks proofs against real time. + it.live("joins the relay's trace only for requests the relay forwarded", () => + Effect.gen(function* () { + const parents: Array = []; + const { handler, dispose } = handlerFor(() => + Effect.gen(function* () { + const span = yield* Effect.currentParentSpan.pipe(Effect.option); + parents.push(span._tag === "Some" ? span.value.traceId : undefined); + return { _tag: "not_found" } as const; + }), ); - // A sender calling the URL directly cannot attach to our traces. - await handler(post("/api/hooks/id/tok", "{}", { traceparent })); - expect(parents).toEqual(["0af7651916cd43dd8448eb211c80319c", undefined]); - } finally { - await dispose(); - } - }); + const send = (headers: Record) => + Effect.promise(() => handler(post("/api/hooks/id/tok", "{}", headers))); + const traceparent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; + const receivedAt = "2026-10-04T10:00:00.000Z"; + const relayHeaders = { + "x-t3-relay-delivery-id": "relay-1", + "x-t3-relay-received-at": receivedAt, + }; + const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" }); + yield* send({ ...relayHeaders, traceparent, "x-t3-relay-delivery": proof }); + // Claiming to be the relay without its proof cannot attach to our traces. + yield* send({ ...relayHeaders, traceparent }); + yield* send({ traceparent }); + yield* Effect.promise(() => dispose()); + expect(parents).toEqual(["0af7651916cd43dd8448eb211c80319c", undefined, undefined]); + }), + ); it("rejects oversized bodies and malformed paths before reaching the service", async () => { let calls = 0; diff --git a/apps/server/src/scheduledTasks/webhookRoute.ts b/apps/server/src/scheduledTasks/webhookRoute.ts index c41777152af6..eee969f2d34c 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.ts @@ -11,6 +11,7 @@ import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import * as Metrics from "../observability/Metrics.ts"; +import { makeRelayDeliveryVerifier } from "./relayDeliveryProof.ts"; import * as ScheduledTaskService from "./ScheduledTaskService.ts"; /** Largest request body a webhook accepts. The relay enforces the same cap. */ @@ -30,7 +31,10 @@ const json = (status: number, body: Record, outcome: string) => * URL, where the endpoint key is this environment's managed tunnel key. */ const handleWebhook = - (scheduledTasks: ScheduledTaskService.ScheduledTaskService["Service"]) => + ( + scheduledTasks: ScheduledTaskService.ScheduledTaskService["Service"], + verifyRelayDelivery: Effect.Success, + ) => ({ params, request, @@ -68,15 +72,16 @@ const handleWebhook = if (typeof value === "string") headers[name.toLowerCase()] = value; } const queryIndex = request.url.indexOf("?"); - // Only the relay sets these; it strips any copy a sender supplied. The - // receive time matters for requests the relay held while we were offline. - const relayDeliveryId = headers["x-t3-relay-delivery-id"]; - const relayReceivedAt = relayDeliveryId ? headers["x-t3-relay-received-at"] : undefined; + // The relay's delivery id and receive time count only with its signed + // proof; the URL can also be called directly. The receive time matters + // for requests the relay held while we were offline. + const relay = yield* verifyRelayDelivery({ headers, hookId: params.hookId }); + const relayDeliveryId = Option.isSome(relay) ? relay.value.deliveryId : undefined; + const relayReceivedAt = Option.isSome(relay) ? relay.value.receivedAt : undefined; // A relay delivery joins the relay's trace, and goes to the T3 Connect - // tracer with it. A sender's own traceparent is never trusted: anyone - // calling the URL directly could otherwise attach to our traces. - const relayParent = relayDeliveryId + // tracer with it. Anyone else's traceparent is never trusted. + const relayParent = Option.isSome(relay) ? HttpTraceContext.fromHeaders(request.headers) : Option.none(); const result = yield* scheduledTasks @@ -129,7 +134,10 @@ export const webhookHttpApiLayer = HttpApiBuilder.group( EnvironmentHttpApi, "webhooks", Effect.fnUntraced(function* (handlers) { - const handler = handleWebhook(yield* ScheduledTaskService.ScheduledTaskService); + const handler = handleWebhook( + yield* ScheduledTaskService.ScheduledTaskService, + yield* makeRelayDeliveryVerifier, + ); return handlers .handleRaw("webhookPost", handler) .handleRaw("webhookPut", handler) diff --git a/docs/operations/relay-observability.md b/docs/operations/relay-observability.md index 1c4400108bd6..dd8537abb132 100644 --- a/docs/operations/relay-observability.md +++ b/docs/operations/relay-observability.md @@ -69,7 +69,8 @@ managed endpoint, and with it the environment. On a forward, `relay.hook.upstrea `prompt_too_long`, `queue_full`, `rejected_signature`, `expired`, `disabled`, ...), from its `x-t3-hook-outcome` response header. A held request's delivery records the same on its `relay.inbox.deliver` span. The relay sends its own `traceparent` with each forward and drops any a -sender supplied, so on environments that export to T3 Connect, the environment's +sender supplied, and signs each forward (`x-t3-relay-delivery`) so the environment trusts the +relay's delivery id, receive time and trace context only from the relay, so on environments that export to T3 Connect, the environment's `ScheduledTaskService.triggerWebhook` span lands in the same trace. `relay.hook.rate_limit` says which budget ran out: `endpoint` or `hook`. `relay.hook.rate_limiter_failed_open` is set when the Cloudflare rate limiter was unavailable and the request went through unlimited. diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index 011c31b149fa..aab7c991af1e 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -1,11 +1,13 @@ import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeCrypto from "node:crypto"; +import * as EffectNodeCrypto from "@effect/platform-node/NodeCrypto"; import { describe, expect, it } from "@effect/vitest"; import { RelayApi } from "@t3tools/contracts/relay"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; +import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; @@ -34,10 +36,16 @@ import { traceRelayHttpRequestWith, } from "../http/Api.ts"; import * as HookForwarder from "./HookForwarder.ts"; +import { RELAY_HOOK_DELIVERY_TYP, verifyRelayJwt } from "@t3tools/shared/relayJwt"; import * as HookInbox from "./HookInbox.ts"; import type { HeldHook } from "./HookInboxStore.ts"; import { RELAY_HOOK_UPSTREAM_TIMEOUT_MS } from "./upstream.ts"; +const mintKeys = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, +}); + const settings: RelayConfiguration.RelayConfiguration["Service"] = { relayIssuer: "https://relay.example.test", apns: null, @@ -45,8 +53,8 @@ const settings: RelayConfiguration.RelayConfiguration["Service"] = { clerkPublishableKey: "pk_test_test", clerkJwtAudience: "t3-code-relay", apnsDeliveryJobSigningSecret: Redacted.make("apns-delivery-secret"), - cloudMintPrivateKey: Redacted.make("cloud-mint-private-key"), - cloudMintPublicKey: "cloud-mint-public-key", + cloudMintPrivateKey: Redacted.make(mintKeys.privateKey), + cloudMintPublicKey: mintKeys.publicKey, managedEndpointBaseDomain: "example.test", managedEndpointNamespace: "dev", }; @@ -141,7 +149,7 @@ function makeHarness(options: Harness = {}) { return true; }), }), - NodeCrypto.layer, + EffectNodeCrypto.layer, Layer.succeed(HookForwarder.HookRateLimiter, { allowHook: (key) => Effect.sync(() => { @@ -240,6 +248,36 @@ describe("HookForwarder", () => { }), ); + it.effect("signs each forward for the environment, replacing any copy a sender sent", () => + Effect.gen(function* () { + const harness = makeHarness(); + yield* harness.send( + new Request(hookUrl("hook-1/tok"), { + method: "POST", + headers: { "x-t3-relay-delivery": "forged", "x-t3-relay-delivery-id": "forged" }, + body: "{}", + }), + ); + const sent = harness.sent[0]!; + const payload = yield* verifyRelayJwt({ + publicKey: mintKeys.publicKey, + token: sent.headers["x-t3-relay-delivery"]!, + typ: RELAY_HOOK_DELIVERY_TYP, + issuer: settings.relayIssuer, + audience: `t3-env:${environmentId}`, + nowEpochSeconds: Math.floor((yield* Clock.currentTimeMillis) / 1_000), + }); + // The proof names exactly the delivery the environment receives. + expect(payload).toMatchObject({ + environmentId, + hookId: "hook-1", + deliveryId: sent.headers["x-t3-relay-delivery-id"], + receivedAt: sent.headers["x-t3-relay-received-at"], + }); + expect(sent.headers["x-t3-relay-delivery-id"]).not.toBe("forged"); + }), + ); + it.effect("passes upstream status, body and content-type through", () => Effect.gen(function* () { const harness = makeHarness({ diff --git a/infra/relay/src/hooks/HookForwarder.ts b/infra/relay/src/hooks/HookForwarder.ts index fe4d5271d229..f5031389dfc7 100644 --- a/infra/relay/src/hooks/HookForwarder.ts +++ b/infra/relay/src/hooks/HookForwarder.ts @@ -1,9 +1,11 @@ +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; @@ -11,7 +13,14 @@ import * as HttpClient from "effect/http/HttpClient"; import type * as HttpServerRequest from "effect/http/HttpServerRequest"; import * as HttpServerResponse from "effect/http/HttpServerResponse"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; -import { RelayApi } from "@t3tools/contracts/relay"; +import { EnvironmentId } from "@t3tools/contracts"; +import { RelayApi, type RelayHookDeliveryProofPayload } from "@t3tools/contracts/relay"; +import { + normalizeRelayIssuer, + RELAY_HOOK_DELIVERY_HEADER, + RELAY_HOOK_DELIVERY_TYP, + signRelayJwt, +} from "@t3tools/shared/relayJwt"; import * as RelayConfiguration from "../Config.ts"; import { @@ -57,6 +66,7 @@ const DROPPED_REQUEST_HEADERS = new Set([ // Only the relay may set this; a sender could otherwise collide delivery ids. "x-t3-relay-delivery-id", "x-t3-relay-received-at", + RELAY_HOOK_DELIVERY_HEADER, // The environment trusts trace context only from the relay, which sets its own. "traceparent", "tracestate", @@ -130,6 +140,37 @@ const errorResponse = (status: number, error: string, headers?: Record errorResponse(404, "hook_not_found"); +/** Longer than the inbox holds a request, so a held delivery's proof still verifies. */ +const DELIVERY_PROOF_LIFETIME_SECONDS = 25 * 60 * 60; + +const signDeliveryProof = (input: { + readonly settings: RelayConfiguration.RelayConfiguration["Service"]; + readonly environmentId: string; + readonly deliveryId: string; + readonly receivedAt: string; + readonly hookId: string; + readonly jti: string; +}) => + Effect.gen(function* () { + const now = Math.floor((yield* Clock.currentTimeMillis) / 1_000); + return yield* signRelayJwt({ + privateKey: Redacted.value(input.settings.cloudMintPrivateKey), + typ: RELAY_HOOK_DELIVERY_TYP, + payload: { + iss: normalizeRelayIssuer(input.settings.relayIssuer), + aud: `t3-env:${input.environmentId}`, + sub: input.environmentId, + jti: input.jti, + iat: now, + exp: now + DELIVERY_PROOF_LIFETIME_SECONDS, + environmentId: EnvironmentId.make(input.environmentId), + deliveryId: input.deliveryId, + receivedAt: input.receivedAt, + hookId: input.hookId, + } satisfies RelayHookDeliveryProofPayload, + }); + }).pipe(Effect.orDie); + /** Methods a webhook can arrive with; HEAD reaches the GET route and is refused. */ const FORWARDED_METHODS = new Set(["GET", "POST", "PUT", "PATCH"]); @@ -352,15 +393,27 @@ const make = Effect.gen(function* () { // One id per request, so a request that reached the environment before a // timeout and is later delivered from the inbox runs only once. yield* Effect.annotateCurrentSpan({ "relay.hook.body_bytes": body.success.byteLength }); + const deliveryId = yield* crypto.randomUUIDv4.pipe(Effect.orDie); + // Proves to the environment that this delivery id, receive time and + // trace context came from the relay. Signed once here and stored with a + // held request, so the inbox never needs the signing key. + const proof = yield* signDeliveryProof({ + settings, + environmentId: endpoint.environmentId, + deliveryId, + receivedAt, + hookId: parsed.hookId, + jti: yield* crypto.randomUUIDv4.pipe(Effect.orDie), + }); const hook = { - id: yield* crypto.randomUUIDv4.pipe(Effect.orDie), + id: deliveryId, receivedAt, method: request.method, rawHookId: parsed.rawHookId, rawToken: parsed.rawToken, hookKey: parsed.hookId, query: parsed.search.replace(/^\?/, ""), - headers: forwardedHeaders(request.headers), + headers: { ...forwardedHeaders(request.headers), [RELAY_HOOK_DELIVERY_HEADER]: proof }, body: body.success, }; // Held only for environments that opted in; otherwise the relay is a plain proxy. diff --git a/infra/relay/src/hooks/upstream.ts b/infra/relay/src/hooks/upstream.ts index 185a59f286d9..9871317f99bf 100644 --- a/infra/relay/src/hooks/upstream.ts +++ b/infra/relay/src/hooks/upstream.ts @@ -88,6 +88,8 @@ export const sendUpstream = (baseUrl: string, hook: UpstreamHook) => // The environment's span joins this trace. Set by hand: the client span // that would propagate it is off, because it records the token in url.full. const parent = yield* Effect.currentSpan.pipe(Effect.option); + // `hook.headers` carries the signed delivery proof, set once when the + // relay received the request, so a held request sends the same proof. const headers: Record = { ...hook.headers, ...(Option.isSome(parent) ? HttpTraceContext.toHeaders(parent.value) : {}), diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index db4dd5a86477..f5cb83b4f8dc 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -864,6 +864,21 @@ export const RelayCloudEnvironmentHealthProofPayload = Schema.Struct({ export type RelayCloudEnvironmentHealthProofPayload = typeof RelayCloudEnvironmentHealthProofPayload.Type; +/** + * Sent with every webhook the relay forwards, signed with the relay's mint key + * (`x-t3-relay-delivery`). The environment trusts the relay's delivery id, + * receive time and trace context only when this verifies, since its webhook + * URL can also be called directly. + */ +export const RelayHookDeliveryProofPayload = Schema.Struct({ + ...RelaySignedJwtRegisteredClaims, + environmentId: EnvironmentId, + deliveryId: TrimmedNonEmptyString, + receivedAt: TrimmedNonEmptyString, + hookId: TrimmedNonEmptyString, +}); +export type RelayHookDeliveryProofPayload = typeof RelayHookDeliveryProofPayload.Type; + export const RelayCloudEnvironmentHealthProof = TrimmedNonEmptyString; export type RelayCloudEnvironmentHealthProof = typeof RelayCloudEnvironmentHealthProof.Type; diff --git a/packages/shared/src/relayJwt.ts b/packages/shared/src/relayJwt.ts index cabe340d7009..0b06c00d4b5d 100644 --- a/packages/shared/src/relayJwt.ts +++ b/packages/shared/src/relayJwt.ts @@ -11,6 +11,9 @@ export const RELAY_MINT_RESPONSE_TYP = "t3-env-mint+jwt"; export const RELAY_HEALTH_RESPONSE_TYP = "t3-env-health+jwt"; export const RELAY_ACTIVITY_PUBLISH_TYP = "t3-env-activity+jwt"; export const RELAY_MANAGED_TUNNEL_RECOVERY_TYP = "t3-env-managed-tunnel-recovery+jwt"; +export const RELAY_HOOK_DELIVERY_TYP = "t3-relay-hook-delivery+jwt"; +/** Header carrying the signed proof that a webhook request came from the relay. */ +export const RELAY_HOOK_DELIVERY_HEADER = "x-t3-relay-delivery"; export class RelayJwtError extends Schema.TaggedError()("RelayJwtError", { operation: Schema.Literals(["sign", "verify"]),