From 3a652f3991468b879439656aa4b4095aaf9555a5 Mon Sep 17 00:00:00 2001 From: Kiet Ho Date: Sat, 19 Sep 2026 12:25:46 -0700 Subject: [PATCH 1/3] fix(db,ci): make migration 0119 online-safe and bound production migrations The 17 September outage was migration 0119 rewriting ingest.webhook_events (enum to text, a full rewrite of the largest table in production) while holding ACCESS EXCLUSIVE locks on auth.users, auth.organizations and integration_connections from its earlier statements. drizzle applies every pending migration in one transaction, so those locks were held for the whole rewrite and every authenticated request queued behind them until the pooler ran out of connections. 0119 rolled back and is still pending, so the next production deploy would have repeated it. 0119 is regenerated without the column type change. Every writer of webhook_events.provider passes linear, github, notion or slack, all already in the integration_provider enum, so the column stays the enum and recordWebhookDelivery goes back to the enum type, which makes a non-enum provider a compile error rather than a production insert failure. The journal timestamp is kept so a dev database that applied the old 0119 skips this one instead of failing on an existing table. The production migrate job gets a concurrency group, since three migrators ran at once during the incident, and lock_timeout=5s / statement_timeout=60s via PGOPTIONS, so a migration that cannot get its locks fails and rolls back instead of stalling traffic. The preview migrate step gets the same timeouts so the preview database check runs what production will run. Rehearsed on a Neon branch of production (122 migrations applied, 540 GB webhook_events, 2 CU): without timeouts the migrator hangs on DROP TABLE CASCADE behind a single held read lock on auth.users; with them it exits 1 in 7s and rolls back cleanly; unblocked it applies in 2.3s. --- .github/workflows/deploy-preview.yml | 1 + .github/workflows/deploy-production.yml | 6 ++++++ apps/api/src/lib/ingest/recordWebhookDelivery.ts | 4 ++-- .../db/drizzle/0119_plugin_oauth_clients_and_connectors.sql | 1 - packages/db/drizzle/meta/0119_snapshot.json | 5 +++-- packages/db/src/schema/ingest.ts | 4 +++- 6 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-preview.yml b/.github/workflows/deploy-preview.yml index 0fc50677015..8ae87b2fa6d 100644 --- a/.github/workflows/deploy-preview.yml +++ b/.github/workflows/deploy-preview.yml @@ -57,6 +57,7 @@ jobs: env: DATABASE_URL: ${{ steps.create-branch.outputs.db_url_pooled }} DATABASE_URL_UNPOOLED: ${{ steps.create-branch.outputs.db_url }} + PGOPTIONS: -c lock_timeout=5s -c statement_timeout=60s run: | bun drizzle-kit check bun drizzle-kit migrate diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 789a3f72d19..17853c4de7d 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -13,6 +13,9 @@ jobs: name: Deploy Database Migrations runs-on: ubuntu-latest environment: production + concurrency: + group: production-database-migrations + cancel-in-progress: false steps: - name: Checkout repository @@ -38,6 +41,9 @@ jobs: env: DATABASE_URL: ${{ secrets.DATABASE_URL_UNPOOLED }} DATABASE_URL_UNPOOLED: ${{ secrets.DATABASE_URL_UNPOOLED }} + # drizzle applies every pending migration in one transaction, so a + # migration that queues for a lock blocks all traffic behind it. + PGOPTIONS: -c lock_timeout=5s -c statement_timeout=60s run: bun drizzle-kit migrate deploy-api: diff --git a/apps/api/src/lib/ingest/recordWebhookDelivery.ts b/apps/api/src/lib/ingest/recordWebhookDelivery.ts index 759e57fa800..1ed3b6f8af9 100644 --- a/apps/api/src/lib/ingest/recordWebhookDelivery.ts +++ b/apps/api/src/lib/ingest/recordWebhookDelivery.ts @@ -1,9 +1,9 @@ import { db } from "@superset/db/client"; +import type { integrationProvider } from "@superset/db/schema"; import { DrizzleQueryError, sql } from "drizzle-orm"; import { databaseErrorMessage } from "@/lib/databaseErrorMessage"; -/** The connector slug a delivery arrived for; free text since 0114. */ -type Provider = string; +type Provider = (typeof integrationProvider.enumValues)[number]; export interface RecordedDelivery { id: string; diff --git a/packages/db/drizzle/0119_plugin_oauth_clients_and_connectors.sql b/packages/db/drizzle/0119_plugin_oauth_clients_and_connectors.sql index 483721748c5..d2f441e911f 100644 --- a/packages/db/drizzle/0119_plugin_oauth_clients_and_connectors.sql +++ b/packages/db/drizzle/0119_plugin_oauth_clients_and_connectors.sql @@ -42,7 +42,6 @@ ALTER TABLE "plugin_connections" DISABLE ROW LEVEL SECURITY;--> statement-breakp DROP TABLE "plugin_connections" CASCADE;--> statement-breakpoint ALTER TABLE "automation_events" DROP CONSTRAINT "automation_events_integration_connection_id_integration_connections_id_fk"; --> statement-breakpoint -ALTER TABLE "ingest"."webhook_events" ALTER COLUMN "provider" SET DATA TYPE text;--> statement-breakpoint ALTER TABLE "connections" ADD CONSTRAINT "connections_organization_id_organizations_id_fk" FOREIGN KEY ("organization_id") REFERENCES "auth"."organizations"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint ALTER TABLE "connections" ADD CONSTRAINT "connections_connected_by_user_id_users_id_fk" FOREIGN KEY ("connected_by_user_id") REFERENCES "auth"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint CREATE UNIQUE INDEX "plugin_oauth_clients_issuer_redirect_unique" ON "plugin_oauth_clients" USING btree ("issuer","redirect_uri");--> statement-breakpoint diff --git a/packages/db/drizzle/meta/0119_snapshot.json b/packages/db/drizzle/meta/0119_snapshot.json index 98e3fced17e..fe2d0b4bb6c 100644 --- a/packages/db/drizzle/meta/0119_snapshot.json +++ b/packages/db/drizzle/meta/0119_snapshot.json @@ -1,5 +1,5 @@ { - "id": "aedc55ed-1344-43c3-a89d-d002ea861018", + "id": "e2dbc465-8d48-4ac6-957e-cf51815f7afe", "prevId": "52dcd5f1-52d3-461e-8609-5e6452d3ecd0", "version": "7", "dialect": "postgresql", @@ -2564,7 +2564,8 @@ }, "provider": { "name": "provider", - "type": "text", + "type": "integration_provider", + "typeSchema": "public", "primaryKey": false, "notNull": true }, diff --git a/packages/db/src/schema/ingest.ts b/packages/db/src/schema/ingest.ts index 529c779ed69..3fb7505f6db 100644 --- a/packages/db/src/schema/ingest.ts +++ b/packages/db/src/schema/ingest.ts @@ -7,6 +7,8 @@ import { uniqueIndex, uuid, } from "drizzle-orm/pg-core"; +import { integrationProvider } from "./schema"; + export const ingestSchema = pgSchema("ingest"); /** @@ -28,7 +30,7 @@ export const webhookEvents = ingestSchema.table( id: uuid().primaryKey().defaultRandom(), // Source - provider: text().notNull(), + provider: integrationProvider().notNull(), eventId: text("event_id").notNull(), eventType: text("event_type"), From deb1c3b172e21ad1bc4edfb447d46d76a711bd1b Mon Sep 17 00:00:00 2001 From: Kiet Ho Date: Sat, 19 Sep 2026 12:48:57 -0700 Subject: [PATCH 2/3] fix(db,ci): order 0119's locks like the application, retry lost lock races Rehearsing 0119 under load on a Neon branch of production reproduced the 08:53 deadlock from 17 September on three attempts out of three. The generated order drops plugin_connections first, which takes ACCESS EXCLUSIVE on auth.organizations, and only then alters automation_events. An in-flight insert into automation_events holds that table while waiting on auth.organizations for its foreign-key check, so the two wait on each other until Postgres kills the migrator. It fails safely, at the cost of a one second stall on auth reads per attempt, but it would keep the migration from ever landing under real traffic. The automation_events foreign-key drop now runs first, so the migration takes locks child table first like every insert does. The end schema and snapshot are unchanged. With the same load it applied first try on four runs out of four, in under two seconds, with no failed transactions and no stall beyond the branch's normal latency. Rarer orderings can still collide, so the production migrate step retries up to five times, only when the output names a deadlock or a lock timeout. Both roll back completely. Anything else, a statement timeout included, still fails at once. --- .github/workflows/deploy-production.yml | 10 +++++++++- .../0119_plugin_oauth_clients_and_connectors.sql | 8 ++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 17853c4de7d..898f245e618 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -44,7 +44,15 @@ jobs: # drizzle applies every pending migration in one transaction, so a # migration that queues for a lock blocks all traffic behind it. PGOPTIONS: -c lock_timeout=5s -c statement_timeout=60s - run: bun drizzle-kit migrate + # Losing a lock race rolls back cleanly, so only that is retried. + run: | + for attempt in 1 2 3 4 5; do + if bun drizzle-kit migrate 2>&1 | tee migrate.log; then exit 0; fi + grep -qE "deadlock detected|lock timeout" migrate.log || exit 1 + echo "::warning::Migration lost a lock race (attempt $attempt), retrying" + sleep $((attempt * 5)) + done + exit 1 deploy-api: name: Deploy API to Vercel diff --git a/packages/db/drizzle/0119_plugin_oauth_clients_and_connectors.sql b/packages/db/drizzle/0119_plugin_oauth_clients_and_connectors.sql index d2f441e911f..cd0e3b737fb 100644 --- a/packages/db/drizzle/0119_plugin_oauth_clients_and_connectors.sql +++ b/packages/db/drizzle/0119_plugin_oauth_clients_and_connectors.sql @@ -38,10 +38,14 @@ CREATE TABLE "connections" ( CONSTRAINT "connections_user_identity_present" CHECK (owner_kind <> 'user' OR external_user_id IS NOT NULL) ); --> statement-breakpoint -ALTER TABLE "plugin_connections" DISABLE ROW LEVEL SECURITY;--> statement-breakpoint -DROP TABLE "plugin_connections" CASCADE;--> statement-breakpoint +-- Hand-ordered: this must run before the DROP TABLE below. That statement takes +-- ACCESS EXCLUSIVE on auth.organizations, and an in-flight insert into +-- automation_events holds automation_events while waiting on auth.organizations +-- for its foreign-key check, so the generated order deadlocks under traffic. ALTER TABLE "automation_events" DROP CONSTRAINT "automation_events_integration_connection_id_integration_connections_id_fk"; --> statement-breakpoint +ALTER TABLE "plugin_connections" DISABLE ROW LEVEL SECURITY;--> statement-breakpoint +DROP TABLE "plugin_connections" CASCADE;--> statement-breakpoint ALTER TABLE "connections" ADD CONSTRAINT "connections_organization_id_organizations_id_fk" FOREIGN KEY ("organization_id") REFERENCES "auth"."organizations"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint ALTER TABLE "connections" ADD CONSTRAINT "connections_connected_by_user_id_users_id_fk" FOREIGN KEY ("connected_by_user_id") REFERENCES "auth"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint CREATE UNIQUE INDEX "plugin_oauth_clients_issuer_redirect_unique" ON "plugin_oauth_clients" USING btree ("issuer","redirect_uri");--> statement-breakpoint From d9218aaf1cf2f654f4ce302e5b6219fae7bcd364 Mon Sep 17 00:00:00 2001 From: Kiet Ho Date: Sat, 19 Sep 2026 12:55:39 -0700 Subject: [PATCH 3/3] ci(db): give the production migration eight tries at its locks Sampling lock holders on production once a second for three minutes: automation_events is read-locked 70% of the time by the dispatch sweep and the payload prune job, each holding for 30 to 60 seconds, and a five second lock wait succeeds from 44% of start times. auth.users, auth.organizations and plugin_installs were obtainable from every start time. Five tries lands 0119 about 94% of the time, eight about 99%. A lost try waits on automation_events while holding nothing else, since that statement now runs first, so it costs a five second delay on automation_events writes and nothing on the auth tables. --- .github/workflows/deploy-production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 898f245e618..5f3ecedbcb0 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -46,7 +46,7 @@ jobs: PGOPTIONS: -c lock_timeout=5s -c statement_timeout=60s # Losing a lock race rolls back cleanly, so only that is retried. run: | - for attempt in 1 2 3 4 5; do + for attempt in 1 2 3 4 5 6 7 8; do if bun drizzle-kit migrate 2>&1 | tee migrate.log; then exit 0; fi grep -qE "deadlock detected|lock timeout" migrate.log || exit 1 echo "::warning::Migration lost a lock race (attempt $attempt), retrying"