fix(relay): a host restarting onto a deleted tunnel gets a new one

infra/relay/src/environments/ManagedEndpointProvider.test.ts

@@ -1118,7 +1118,7 @@ describe("ManagedEndpointProvider", () => {
     }).pipe(Effect.provide(layer));
   });
 
-  it.effect("does no Cloudflare work when the registered origin is unchanged", () => {
+  it.effect("only confirms the tunnel exists when the registered origin is unchanged", () => {
     const tunnelCalls: TunnelCall[] = [];
     const layer = layerProvider(makePersistentTunnelClient(tunnelCalls));
 
@@ -1137,7 +1137,31 @@ describe("ManagedEndpointProvider", () => {
           endpoint: provisioned.endpoint,
         }),
       ).toBe("ready");
-      expect(tunnelCalls).toEqual([]);
+      expect(tunnelCalls).toEqual([{ operation: "get", input: "tunnel-id" }]);
+    }).pipe(Effect.provide(layer));
+  });
+
+  it.effect("asks for recovery when the recorded tunnel was deleted", () => {
+    const tunnelCalls: TunnelCall[] = [];
+    const layer = layerProvider(makePersistentTunnelClient(tunnelCalls));
+
+    return Effect.gen(function* () {
+      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
+      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
+      const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const;
+      const provisioned = yield* provider.provision({ ...key, origin });
+      // A shutdown release deletes the tunnel but keeps the recorded id; the
+      // host was killed before it dropped its stored config.
+      expect(yield* provider.release(key)).toBe(true);
+
+      expect(
+        yield* provider.reconcileOrigin({
+          ...key,
+          tunnelId: provisioned.runtime.tunnelId!,
+          origin,
+          endpoint: provisioned.endpoint,
+        }),
+      ).toBe("recovery_required");
     }).pipe(Effect.provide(layer));
   });
 
@@ -1163,6 +1187,7 @@ describe("ManagedEndpointProvider", () => {
         }),
       ).toBe("ready");
       expect(tunnelCalls).toEqual([
+        { operation: "get", input: "tunnel-id" },
         {
           operation: "putConfiguration",
           input: {

infra/relay/src/environments/ManagedEndpointProvider.ts

@@ -55,6 +55,7 @@ const ManagedEndpointProvisioningStage = Schema.Literals([
   "mark-allocation-ready",
   "load-allocation",
   "verify-endpoint",
+  "verify-tunnel",
   "sync-origin",
 ]);
 
@@ -208,6 +209,7 @@ export interface ManagedEndpointTunnel {
   readonly status?: string | null;
   readonly createdAt?: string | null;
   readonly connsInactiveAt?: string | null;
+  readonly deletedAt?: string | null;
 }
 
 export interface ManagedEndpointTunnelListRequest {
@@ -581,6 +583,30 @@ export const make = Effect.gen(function* () {
           hostname: allocation.hostname,
         });
       }
+      // A release keeps the recorded tunnel id, so the record alone cannot
+      // tell a live tunnel from one deleted by a shutdown whose host was
+      // killed before it dropped its config. Ask Cloudflare, or the host
+      // starts a connector that can never connect.
+      const recorded = yield* tunnels.get(input.tunnelId).pipe(
+        Effect.asSome,
+        Effect.catchTags({
+          ManagedEndpointTunnelClientError: (cause) =>
+            isManagedEndpointNotFound(cause.cause)
+              ? Effect.succeedNone
+              : Effect.fail(
+                  new ManagedEndpointProvisioningFailed({
+                    userId: input.userId,
+                    environmentId: input.environmentId,
+                    stage: "verify-tunnel",
+                    tunnelId: input.tunnelId,
+                    cause,
+                  }),
+                ),
+        }),
+      );
+      if (Option.isNone(recorded) || recorded.value.deletedAt) {
+        return "recovery_required";
+      }
       if (
         allocation.origin?.localHttpHost === input.origin.localHttpHost &&
         allocation.origin.localHttpPort === input.origin.localHttpPort