diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index b48a01cc6d03..9896fc324b36 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -1118,7 +1118,7 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); - it.effect("does no Cloudflare work when the registered origin is unchanged", () => { + it.effect("only confirms the tunnel exists when the registered origin is unchanged", () => { const tunnelCalls: TunnelCall[] = []; const layer = layerProvider(makePersistentTunnelClient(tunnelCalls)); @@ -1137,7 +1137,31 @@ describe("ManagedEndpointProvider", () => { endpoint: provisioned.endpoint, }), ).toBe("ready"); - expect(tunnelCalls).toEqual([]); + expect(tunnelCalls).toEqual([{ operation: "get", input: "tunnel-id" }]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("asks for recovery when the recorded tunnel was deleted", () => { + const tunnelCalls: TunnelCall[] = []; + const layer = layerProvider(makePersistentTunnelClient(tunnelCalls)); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const; + const provisioned = yield* provider.provision({ ...key, origin }); + // A shutdown release deletes the tunnel but keeps the recorded id; the + // host was killed before it dropped its stored config. + expect(yield* provider.release(key)).toBe(true); + + expect( + yield* provider.reconcileOrigin({ + ...key, + tunnelId: provisioned.runtime.tunnelId!, + origin, + endpoint: provisioned.endpoint, + }), + ).toBe("recovery_required"); }).pipe(Effect.provide(layer)); }); @@ -1163,6 +1187,7 @@ describe("ManagedEndpointProvider", () => { }), ).toBe("ready"); expect(tunnelCalls).toEqual([ + { operation: "get", input: "tunnel-id" }, { operation: "putConfiguration", input: { diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index aba8b426ab5f..eb6b7b7b73a9 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -55,6 +55,7 @@ const ManagedEndpointProvisioningStage = Schema.Literals([ "mark-allocation-ready", "load-allocation", "verify-endpoint", + "verify-tunnel", "sync-origin", ]); @@ -208,6 +209,7 @@ export interface ManagedEndpointTunnel { readonly status?: string | null; readonly createdAt?: string | null; readonly connsInactiveAt?: string | null; + readonly deletedAt?: string | null; } export interface ManagedEndpointTunnelListRequest { @@ -581,6 +583,30 @@ export const make = Effect.gen(function* () { hostname: allocation.hostname, }); } + // A release keeps the recorded tunnel id, so the record alone cannot + // tell a live tunnel from one deleted by a shutdown whose host was + // killed before it dropped its config. Ask Cloudflare, or the host + // starts a connector that can never connect. + const recorded = yield* tunnels.get(input.tunnelId).pipe( + Effect.asSome, + Effect.catchTags({ + ManagedEndpointTunnelClientError: (cause) => + isManagedEndpointNotFound(cause.cause) + ? Effect.succeedNone + : Effect.fail( + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "verify-tunnel", + tunnelId: input.tunnelId, + cause, + }), + ), + }), + ); + if (Option.isNone(recorded) || recorded.value.deletedAt) { + return "recovery_required"; + } if ( allocation.origin?.localHttpHost === input.origin.localHttpHost && allocation.origin.localHttpPort === input.origin.localHttpPort