fix(server): a deregistered T3 Connect host shows it was removed and offers a relink

apps/server/src/cli/connect.ts

@@ -31,6 +31,7 @@ import * as CliState from "../cloud/CliState.ts";
 import * as CliTokenManager from "../cloud/CliTokenManager.ts";
 import { filterRelayResponse } from "../cloud/relayResponse.ts";
 import {
+  CLOUD_LINK_REVOKED,
   CLOUD_LINKED_USER_ID,
   isAgentActivityPublishingEnabledValue,
   PUBLISH_AGENT_ACTIVITY_SECRET,
@@ -140,6 +141,8 @@ interface CloudCliStatus {
   readonly desired: boolean;
   readonly authenticated: boolean;
   readonly linked: boolean;
+  // The relay rejected the stored link, usually after a Deregister.
+  readonly revoked: boolean;
   readonly cloudUserId: string | null;
   readonly relayUrl: string | null;
   readonly publishAgentActivity: boolean;
@@ -176,18 +179,22 @@ function formatCloudStatus(status: CloudCliStatus, options?: { readonly json?: b
     return JSON.stringify(status, null, 2);
   }
 
-  const provisioned = status.linked
-    ? "provisioned"
-    : status.desired && status.authenticated
-      ? "pending server startup"
-      : "not provisioned";
+  const provisioned = status.revoked
+    ? "removed from T3 Connect"
+    : status.linked
+      ? "provisioned"
+      : status.desired && status.authenticated
+        ? "pending server startup"
+        : "not provisioned";
   const nextStep = !status.authenticated
     ? "Run `t3 connect link` to authorize and enable T3 Connect."
-    : !status.desired
-      ? "Run `t3 connect link` to enable T3 Connect."
-      : !status.linked
-        ? "Start T3 to provision the environment link and launch its managed tunnel."
-        : undefined;
+    : status.revoked
+      ? "Run `t3 connect link` to link this environment again."
+      : !status.desired
+        ? "Run `t3 connect link` to enable T3 Connect."
+        : !status.linked
+          ? "Start T3 to provision the environment link and launch its managed tunnel."
+          : undefined;
 
   return [
     "T3 Connect",
@@ -477,6 +484,11 @@ const linkEnvironmentForConnect = Effect.fn("cloud.cli.link_environment")(functi
 
   const identity = yield* authorizeCli(options);
   yield* CliState.setCliDesiredCloudLink(true, publishOnly ? "publish_only" : "managed");
+  if (yield* CliState.dropRevokedCloudLink) {
+    yield* Console.log(
+      "Removed the link T3 Connect no longer accepts. The next start links this environment again.",
+    );
+  }
   if (publishOnly) {
     const secrets = yield* ServerSecretStore.ServerSecretStore;
     yield* secrets.set(PUBLISH_AGENT_ACTIVITY_SECRET, stringToBytes("true"));
@@ -543,22 +555,31 @@ const connectStatusCommand = Command.make("status", {
         const secrets = yield* ServerSecretStore.ServerSecretStore;
         const relayClient = yield* RelayClient.RelayClient;
         const tokens = yield* CliTokenManager.CloudCliTokenManager;
-        const [desired, authenticated, cloudUserId, relayUrl, publishAgentActivity, executable] =
-          yield* Effect.all(
-            [
-              CliState.readCliDesiredCloudLink,
-              tokens.hasCredential,
-              secrets.get(CLOUD_LINKED_USER_ID),
-              secrets.get(RELAY_URL_SECRET),
-              secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET),
-              relayClient.resolve,
-            ],
-            { concurrency: "unbounded" },
-          );
+        const [
+          desired,
+          authenticated,
+          cloudUserId,
+          relayUrl,
+          publishAgentActivity,
+          revoked,
+          executable,
+        ] = yield* Effect.all(
+          [
+            CliState.readCliDesiredCloudLink,
+            tokens.hasCredential,
+            secrets.get(CLOUD_LINKED_USER_ID),
+            secrets.get(RELAY_URL_SECRET),
+            secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET),
+            secrets.get(CLOUD_LINK_REVOKED),
+            relayClient.resolve,
+          ],
+          { concurrency: "unbounded" },
+        );
         const status: CloudCliStatus = {
           desired,
           authenticated,
           linked: Option.isSome(cloudUserId),
+          revoked: Option.isSome(cloudUserId) && Option.isSome(revoked),
           cloudUserId: Option.isSome(cloudUserId) ? bytesToString(cloudUserId.value) : null,
           relayUrl: Option.isSome(relayUrl) ? bytesToString(relayUrl.value) : null,
           publishAgentActivity: isAgentActivityPublishingEnabledValue(

apps/server/src/cloud/CliState.test.ts

@@ -10,6 +10,7 @@ import * as CliState from "./CliState.ts";
 import {
   CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
   CLOUD_ENDPOINT_RUNTIME_CONFIG,
+  CLOUD_LINK_REVOKED,
   CLOUD_LINKED_USER_ID,
   CLOUD_MINT_PUBLIC_KEY,
   PUBLISH_AGENT_ACTIVITY_SECRET,
@@ -59,6 +60,37 @@ it.layer(NodeServices.layer)("CliState", (it) => {
     }).pipe(Effect.provide(layerTest())),
   );
 
+  it.effect("drops only a revoked link, keeping the sign-in and preferences", () =>
+    Effect.gen(function* () {
+      const secrets = yield* ServerSecretStore.ServerSecretStore;
+      const dropped = [
+        CLOUD_LINKED_USER_ID,
+        RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
+        CLOUD_MINT_PUBLIC_KEY,
+        CLOUD_ENDPOINT_RUNTIME_CONFIG,
+        CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
+      ];
+      const kept = [RELAY_URL_SECRET, PUBLISH_AGENT_ACTIVITY_SECRET];
+      for (const name of [...dropped, ...kept]) {
+        yield* secrets.set(name, new TextEncoder().encode(name));
+      }
+      yield* CliState.setCliDesiredCloudLink(true);
+
+      assert.isFalse(yield* CliState.dropRevokedCloudLink);
+      assert.isTrue(Option.isSome(yield* secrets.get(CLOUD_LINKED_USER_ID)));
+
+      yield* secrets.set(CLOUD_LINK_REVOKED, new TextEncoder().encode("true"));
+      assert.isTrue(yield* CliState.dropRevokedCloudLink);
+      for (const name of [...dropped, CLOUD_LINK_REVOKED]) {
+        assert.isTrue(Option.isNone(yield* secrets.get(name)));
+      }
+      for (const name of kept) {
+        assert.isTrue(Option.isSome(yield* secrets.get(name)));
+      }
+      assert.isTrue(yield* CliState.readCliDesiredCloudLink);
+    }).pipe(Effect.provide(layerTest())),
+  );
+
   it.effect("round-trips the desired link mode and defaults legacy links to managed", () =>
     Effect.gen(function* () {
       const secrets = yield* ServerSecretStore.ServerSecretStore;

apps/server/src/cloud/CliState.ts

@@ -5,6 +5,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
 import {
   CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
   CLOUD_ENDPOINT_RUNTIME_CONFIG,
+  CLOUD_LINK_REVOKED,
   CLOUD_LINKED_USER_ID,
   CLOUD_MINT_PUBLIC_KEY,
   PUBLISH_AGENT_ACTIVITY_SECRET,
@@ -57,6 +58,31 @@ export const setCliDesiredCloudLink = Effect.fn("cloud.cli_state.set_desired")(f
   }
 });
 
+/**
+ * Drops a link the relay rejected, keeping the sign-in and preferences, so the
+ * next start links this environment again. `t3 connect link` calls it: the
+ * server never relinks a revoked link on its own. Returns whether it dropped one.
+ */
+export const dropRevokedCloudLink = Effect.gen(function* () {
+  const secrets = yield* ServerSecretStore.ServerSecretStore;
+  if (Option.isNone(yield* secrets.get(CLOUD_LINK_REVOKED))) {
+    return false;
+  }
+  yield* Effect.all(
+    [
+      secrets.remove(CLOUD_LINKED_USER_ID),
+      secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
+      secrets.remove(CLOUD_MINT_PUBLIC_KEY),
+      secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
+      secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
+    ],
+    { concurrency: "unbounded" },
+  );
+  // Last, so a failed removal above leaves the mark for the next attempt.
+  yield* secrets.remove(CLOUD_LINK_REVOKED);
+  return true;
+});
+
 export const clearPersistedCloudLink = Effect.gen(function* () {
   const secrets = yield* ServerSecretStore.ServerSecretStore;
   yield* Effect.all(
@@ -70,6 +96,7 @@ export const clearPersistedCloudLink = Effect.gen(function* () {
       secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
       secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
       secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
+      secrets.remove(CLOUD_LINK_REVOKED),
     ],
     { concurrency: "unbounded" },
   );

apps/server/src/cloud/CloudLink.lifecycle.test.ts

@@ -34,6 +34,7 @@ import { RelayManagedEndpointRecoveryRegistrationRequest } from "@t3tools/contra
 import {
   CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
   CLOUD_ENDPOINT_RUNTIME_CONFIG,
+  CLOUD_LINK_REVOKED,
   CLOUD_LINKED_USER_ID,
   decodeConfirmedOrigin,
   decodeRuntimeConfig,
@@ -694,6 +695,115 @@ describe("releaseManagedTunnelOnShutdown", () => {
     );
   });
 
+  it.effect("marks a link the relay rejects as revoked without dropping or replacing it", () => {
+    const { store, values } = makeMemorySecretStore([
+      [
+        CLOUD_ENDPOINT_RUNTIME_CONFIG,
+        '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
+      ],
+      [RELAY_URL_SECRET, "https://relay.example.test"],
+      [CLOUD_LINKED_USER_ID, "user-123"],
+      [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
+      [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"],
+    ]);
+    const applyConfigCalls: Array<unknown> = [];
+    const requests: Array<HttpClientRequest.HttpClientRequest> = [];
+
+    return Effect.gen(function* () {
+      const link = yield* CloudLink.CloudLink;
+      expect(yield* link.registerManagedTunnelRecovery("http://127.0.0.1:3773")).toEqual({
+        status: "revoked",
+      });
+      expect((yield* link.linkState()).linkRevoked).toBe(true);
+      // One registration request, no relink, and nothing started or removed.
+      expect(requests).toHaveLength(1);
+      expect(applyConfigCalls).toEqual([]);
+      expect(values.has(RELAY_ENVIRONMENT_CREDENTIAL_SECRET)).toBe(true);
+      expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
+    }).pipe(
+      provideReleaseHarness({
+        store,
+        applyConfigCalls,
+        requests,
+        respond: () =>
+          Response.json(
+            {
+              _tag: "RelayAuthInvalidError",
+              code: "auth_invalid",
+              reason: "not_authorized",
+              traceId: "trace-revoked",
+            },
+            { status: 401 },
+          ),
+      }),
+    );
+  });
+
+  it.effect("does not mark a link that was replaced while its rejection was in flight", () => {
+    const { store, values } = makeMemorySecretStore([
+      [
+        CLOUD_ENDPOINT_RUNTIME_CONFIG,
+        '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
+      ],
+      [RELAY_URL_SECRET, "https://relay.example.test"],
+      [CLOUD_LINKED_USER_ID, "user-123"],
+      [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "old-credential"],
+    ]);
+
+    return Effect.gen(function* () {
+      const link = yield* CloudLink.CloudLink;
+      expect(yield* link.registerManagedTunnelRecovery("http://127.0.0.1:3773")).toEqual({
+        status: "superseded",
+      });
+      expect(values.has(CLOUD_LINK_REVOKED)).toBe(false);
+    }).pipe(
+      provideReleaseHarness({
+        store,
+        applyConfigCalls: [],
+        requests: [],
+        respond: () => {
+          values.set(
+            RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
+            new TextEncoder().encode("replacement-credential"),
+          );
+          return Response.json(
+            {
+              _tag: "RelayAuthInvalidError",
+              code: "auth_invalid",
+              reason: "not_authorized",
+              traceId: "trace-stale",
+            },
+            { status: 401 },
+          );
+        },
+      }),
+    );
+  });
+
+  it.effect("clears the revoked mark when the environment is linked again", () => {
+    const { store } = makeMemorySecretStore([
+      [RELAY_URL_SECRET, "https://relay.example.test"],
+      [CLOUD_LINKED_USER_ID, "user-123"],
+      [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "revoked-credential"],
+      [CLOUD_LINK_REVOKED, "true"],
+    ]);
+    const cloudMintPublicKey =
+      "-----BEGIN PUBLIC KEY-----\nMCowBQYDK2VwAyEAqZ2u2LjMoAyG2QTHXmITJ9w6RzZtVUAv394XCcPV0zE=\n-----END PUBLIC KEY-----";
+
+    return Effect.gen(function* () {
+      const link = yield* CloudLink.CloudLink;
+      expect((yield* link.linkState()).linkRevoked).toBe(true);
+      yield* link.applyRelayConfig({
+        relayUrl: "https://relay.example.test",
+        cloudUserId: "user-123",
+        environmentCredential: "fresh-credential",
+        cloudMintPublicKey,
+        endpointRuntime: null,
+      });
+      expect(yield* link.linkState()).toMatchObject({ linked: true, linkRevoked: false });
+    }).pipe(provideReleaseHarness({ store, applyConfigCalls: [], requests: [] }));
+  });
+
   it.effect("reconciles a changed port after a relay outage outlasts the startup fallback", () => {
     const config = {
       providerKind: "cloudflare_tunnel" as const,
@@ -944,7 +1054,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
     }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
   });
 
-  it.effect("recovers a web-linked tunnel with its environment credential", () => {
+  it.effect("recovers a web-linked tunnel and clears its revoked mark", () => {
     const oldConfig =
       '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}';
     const nextConfig = {
@@ -957,6 +1067,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
       [RELAY_URL_SECRET, "https://relay.example.test"],
       [CLOUD_LINKED_USER_ID, "user-123"],
       [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
+      [CLOUD_LINK_REVOKED, "true"],
     ]);
     const applyConfigCalls: Array<unknown> = [];
     const requests: Array<HttpClientRequest.HttpClientRequest> = [];
@@ -974,6 +1085,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
           decodeRuntimeConfig(new TextDecoder().decode(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG))),
         ),
       ).toEqual(nextConfig);
+      expect(values.has(CLOUD_LINK_REVOKED)).toBe(false);
     }).pipe(
       provideReleaseHarness({
         store,
@@ -992,6 +1104,44 @@ describe("releaseManagedTunnelOnShutdown", () => {
     );
   });
 
+  it.effect("marks the link revoked when the relay rejects a running host's recovery", () => {
+    const { store, values } = makeMemorySecretStore([
+      [
+        CLOUD_ENDPOINT_RUNTIME_CONFIG,
+        '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}',
+      ],
+      [RELAY_URL_SECRET, "https://relay.example.test"],
+      [CLOUD_LINKED_USER_ID, "user-123"],
+      [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
+    ]);
+    const applyConfigCalls: Array<unknown> = [];
+
+    return Effect.gen(function* () {
+      const link = yield* CloudLink.CloudLink;
+      const error = yield* Effect.flip(link.recoverManagedTunnel("http://127.0.0.1:3773"));
+      expect(error).toMatchObject({ _tag: "RelayRequestError", rejection: "unauthorized" });
+      expect((yield* link.linkState()).linkRevoked).toBe(true);
+      expect(applyConfigCalls).toEqual([]);
+      expect(values.has(RELAY_ENVIRONMENT_CREDENTIAL_SECRET)).toBe(true);
+    }).pipe(
+      provideReleaseHarness({
+        store,
+        applyConfigCalls,
+        requests: [],
+        respond: () =>
+          Response.json(
+            {
+              _tag: "RelayAuthInvalidError",
+              code: "auth_invalid",
+              reason: "not_authorized",
+              traceId: "trace-recovery-revoked",
+            },
+            { status: 401 },
+          ),
+      }),
+    );
+  });
+
   it.effect("allows managed tunnel provisioning to take longer than ten seconds", () =>
     Effect.gen(function* () {
       const oldConfig =