Repository navigation
fix(integrations): backfill existing connections into the table #7317 reads #7675
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| name: Backfill Connections | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| apply: | ||
| description: Write the rows. Leave unchecked for a dry run that only reports. | ||
| type: boolean | ||
| default: false | ||
|
|
||
| jobs: | ||
| backfill: | ||
| name: Backfill Connections | ||
| runs-on: ubuntu-latest | ||
| environment: production | ||
| concurrency: | ||
| group: backfill-connections | ||
| cancel-in-progress: false | ||
|
|
||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | ||
|
|
||
| - name: Setup Bun | ||
| id: setup-bun | ||
| uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 | ||
| with: | ||
| bun-version-file: .bun-version | ||
|
|
||
| - name: Cache dependencies | ||
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | ||
| with: | ||
| path: ~/.bun/install/cache | ||
| key: ${{ runner.os }}-bun-${{ steps.setup-bun.outputs.bun-revision }}-${{ hashFiles('bun.lock') }} | ||
|
|
||
| - name: Install dependencies | ||
| run: bun install --frozen --ignore-scripts | ||
|
|
||
| - name: Backfill connections | ||
| env: | ||
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | ||
| DATABASE_URL_UNPOOLED: ${{ secrets.DATABASE_URL_UNPOOLED }} | ||
| # Must be the key the API decrypts with, which is why this runs here. | ||
| SECRETS_ENCRYPTION_KEY: ${{ secrets.SECRETS_ENCRYPTION_KEY }} | ||
| APPLY: ${{ inputs.apply }} | ||
| run: | | ||
| if [ "$APPLY" = "true" ]; then | ||
| bun run packages/trpc/scripts/backfill-connections.ts --apply | ||
| else | ||
| bun run packages/trpc/scripts/backfill-connections.ts | ||
| fi |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,173 @@ | ||
| import { db } from "@superset/db/client"; | ||
| import { | ||
| connections, | ||
| integrationConnections, | ||
| userIdentities, | ||
| } from "@superset/db/schema"; | ||
| import { inArray } from "drizzle-orm"; | ||
| import { | ||
| type ExternalUserSource, | ||
| type LegacyIdentity, | ||
| legacyConnectionRow, | ||
| } from "../src/lib/connectors/legacy"; | ||
| import { secretsKeyConfigured } from "../src/lib/secret-box"; | ||
| import { decryptSecret, encryptSecret } from "../src/router/plugins/crypto"; | ||
|
|
||
| /** | ||
| * Copies `integration_connections` into `connections`, the table every | ||
| * integration has read from since #7317. That change shipped without moving the | ||
| * existing rows, so each connection made before it reads as disconnected. | ||
| * | ||
| * Reports what it would do unless `--apply` is passed. Safe to run again: a row | ||
| * already copied is skipped by id, and one whose owner has reconnected since is | ||
| * left alone so the lookup never finds two live rows. | ||
| * | ||
| * Tokens are sealed with SECRETS_ENCRYPTION_KEY, which must be the key the API | ||
| * decrypts with. Run it where that key lives (the Backfill Connections | ||
| * workflow), not from a laptop. | ||
| * | ||
| * Usage: bun run packages/trpc/scripts/backfill-connections.ts [--apply] | ||
| */ | ||
|
|
||
| const BATCH_SIZE = 100; | ||
|
|
||
| type Outcome = "copy" | "already-copied" | "reconnected"; | ||
|
|
||
| function liveKey(row: { | ||
| organizationId: string; | ||
| connector: string; | ||
| ownerKind: string; | ||
| connectedByUserId: string; | ||
| }): string { | ||
| return row.ownerKind === "org" | ||
| ? `${row.organizationId}:${row.connector}` | ||
| : `${row.organizationId}:${row.connector}:${row.connectedByUserId}`; | ||
| } | ||
|
|
||
| async function main() { | ||
| const apply = process.argv.includes("--apply"); | ||
|
|
||
| if (!secretsKeyConfigured()) | ||
| throw new Error( | ||
| "SECRETS_ENCRYPTION_KEY is not set. Tokens sealed without the API's key cannot be read by it.", | ||
| ); | ||
| if ((await decryptSecret(await encryptSecret("probe"))) !== "probe") | ||
| throw new Error("The encryption key does not round-trip."); | ||
|
|
||
| const legacyRows = await db.select().from(integrationConnections); | ||
| const existing = await db | ||
| .select({ | ||
| id: connections.id, | ||
| organizationId: connections.organizationId, | ||
| connector: connections.connector, | ||
| ownerKind: connections.ownerKind, | ||
| connectedByUserId: connections.connectedByUserId, | ||
| disconnectedAt: connections.disconnectedAt, | ||
| }) | ||
| .from(connections); | ||
| const identityRows = await db | ||
| .select() | ||
| .from(userIdentities) | ||
| .where( | ||
| inArray(userIdentities.provider, [ | ||
| ...new Set(legacyRows.map((row) => row.provider as string)), | ||
| ]), | ||
| ); | ||
|
|
||
| const copiedIds = new Set(existing.map((row) => row.id)); | ||
| const liveKeys = new Set( | ||
| existing.filter((row) => !row.disconnectedAt).map(liveKey), | ||
| ); | ||
| const identitiesByOwner = new Map<string, LegacyIdentity[]>(); | ||
| for (const identity of identityRows) { | ||
| const key = `${identity.organizationId}:${identity.provider}:${identity.userId}`; | ||
| const list = identitiesByOwner.get(key) ?? []; | ||
| list.push(identity); | ||
| identitiesByOwner.set(key, list); | ||
| } | ||
|
|
||
| const outcomes = new Map<string, Record<Outcome, number>>(); | ||
| const sources = new Map<string, Record<ExternalUserSource, number>>(); | ||
| const toInsert: Awaited<ReturnType<typeof legacyConnectionRow>>["row"][] = []; | ||
| const plaintextById = new Map<string, string>(); | ||
|
|
||
| for (const legacy of legacyRows) { | ||
| const { row, externalUserSource } = await legacyConnectionRow( | ||
| legacy, | ||
| identitiesByOwner.get( | ||
| `${legacy.organizationId}:${legacy.provider}:${legacy.connectedByUserId}`, | ||
| ) ?? [], | ||
| encryptSecret, | ||
| ); | ||
|
|
||
| const outcome: Outcome = copiedIds.has(legacy.id) | ||
| ? "already-copied" | ||
| : !row.disconnectedAt && liveKeys.has(liveKey(row)) | ||
| ? "reconnected" | ||
| : "copy"; | ||
|
|
||
| const tally = outcomes.get(legacy.provider) ?? { | ||
| copy: 0, | ||
| "already-copied": 0, | ||
| reconnected: 0, | ||
| }; | ||
| tally[outcome] += 1; | ||
| outcomes.set(legacy.provider, tally); | ||
|
|
||
| if (outcome !== "copy") continue; | ||
| toInsert.push(row); | ||
| plaintextById.set(legacy.id, legacy.accessToken); | ||
| if (!row.disconnectedAt) liveKeys.add(liveKey(row)); | ||
|
|
||
| const source = sources.get(legacy.provider) ?? { | ||
| identity: 0, | ||
| "superset-user": 0, | ||
| "not-required": 0, | ||
| }; | ||
| source[externalUserSource] += 1; | ||
| sources.set(legacy.provider, source); | ||
| } | ||
|
|
||
| console.log(apply ? "Applying.\n" : "Dry run. Pass --apply to write.\n"); | ||
| for (const [provider, tally] of outcomes) { | ||
| const source = sources.get(provider); | ||
| console.log( | ||
| `${provider}: ${tally.copy} to copy, ${tally["already-copied"]} already copied, ${tally.reconnected} reconnected since` + | ||
| (source | ||
| ? ` | external user id from identity ${source.identity}, from Superset user ${source["superset-user"]}, not required ${source["not-required"]}` | ||
| : ""), | ||
| ); | ||
| } | ||
|
|
||
| if (!apply) return; | ||
|
|
||
| let inserted = 0; | ||
| for (let i = 0; i < toInsert.length; i += BATCH_SIZE) { | ||
| const written = await db | ||
| .insert(connections) | ||
| .values(toInsert.slice(i, i + BATCH_SIZE)) | ||
| .onConflictDoNothing() | ||
| .returning({ id: connections.id }); | ||
|
Comment on lines
+146
to
+150
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🗄️ Data Integrity & Integration | 🟠 Major | 🏗️ Heavy lift Roll back inserts when read-back validation fails. Each batch commits before the script validates token decryption. If validation fails, the script exits with an error but leaves the invalid rows in Run the inserts and read-back validation in one transaction. Throw inside the transaction when a token is unreadable or an expected insert conflicts. 🤖 Prompt for AI Agents |
||
| inserted += written.length; | ||
| } | ||
|
|
||
| let unreadable = 0; | ||
| const ids = [...plaintextById.keys()]; | ||
| for (let i = 0; i < ids.length; i += BATCH_SIZE) { | ||
| const written = await db | ||
| .select({ id: connections.id, accessToken: connections.accessToken }) | ||
| .from(connections) | ||
| .where(inArray(connections.id, ids.slice(i, i + BATCH_SIZE))); | ||
| for (const row of written) { | ||
| const opened = await decryptSecret(row.accessToken).catch(() => null); | ||
| if (opened !== plaintextById.get(row.id)) unreadable += 1; | ||
| } | ||
| } | ||
|
|
||
| console.log( | ||
| `\n${inserted} of ${toInsert.length} rows written, ${toInsert.length - inserted} skipped on conflict, ${unreadable} whose token does not read back.`, | ||
| ); | ||
| if (unreadable > 0 || inserted !== toInsert.length) process.exitCode = 1; | ||
| } | ||
|
|
||
| await main(); | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🩺 Stability & Availability | 🟠 Major | ⚡ Quick win
Skip existing IDs before mapping the legacy row.
legacyConnectionRowruns before thealready-copiedcheck. An already-copied legacy row with a missingexternalOrgIdor undefined connector aborts the complete backfill even though the script must skip that row.Check
copiedIdsbefore callinglegacyConnectionRow.🤖 Prompt for AI Agents