fix(server): T3 Connect recovers by itself once a wrong clock is fixed

apps/server/src/cli/connect.ts

@@ -6,6 +6,7 @@ import {
 } from "@t3tools/contracts";
 import { RelayOkResponse } from "@t3tools/contracts/relay";
 import * as HostProcess from "@t3tools/shared/HostProcess";
+import { describeClockSkew } from "@t3tools/shared/clockSkew";
 import * as RelayClient from "@t3tools/shared/relayClient";
 import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
 import * as Cause from "effect/Cause";
@@ -29,6 +30,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
 import * as BootService from "../cloud/bootService.ts";
 import * as CliState from "../cloud/CliState.ts";
 import * as CliTokenManager from "../cloud/CliTokenManager.ts";
+import { checkRelayClock } from "../cloud/relayClock.ts";
 import { filterRelayResponse } from "../cloud/relayResponse.ts";
 import {
   CLOUD_LINK_REVOKED,
@@ -147,6 +149,7 @@ interface CloudCliStatus {
   readonly relayUrl: string | null;
   readonly publishAgentActivity: boolean;
   readonly relayClient: RelayClient.RelayClientStatus;
+  readonly clockSkewSeconds: number | null;
 }
 
 function formatRelayClientStatus(executable: RelayClient.RelayClientStatus): ReadonlyArray<string> {
@@ -174,8 +177,11 @@ function formatRelayClientStatus(executable: RelayClient.RelayClientStatus): Rea
   }
 }
 
-function formatCloudStatus(status: CloudCliStatus, options?: { readonly json?: boolean }): string {
-  if (options?.json) {
+function formatCloudStatus(
+  status: CloudCliStatus,
+  options: { readonly json?: boolean; readonly platform: NodeJS.Platform },
+): string {
+  if (options.json) {
     return JSON.stringify(status, null, 2);
   }
 
@@ -204,6 +210,9 @@ function formatCloudStatus(status: CloudCliStatus, options?: { readonly json?: b
     `  Relay: ${status.relayUrl ?? "not provisioned"}`,
     `  Publish agent activity: ${status.publishAgentActivity ? "enabled" : "disabled"}`,
     ...formatRelayClientStatus(status.relayClient),
+    ...(status.clockSkewSeconds === null
+      ? []
+      : ["", describeClockSkew(status.clockSkewSeconds, options.platform)]),
     "",
     "This is saved setup, not a live connection check. Check the background service with `t3 service status`.",
     ...(nextStep ? ["", `Next: ${nextStep}`] : []),
@@ -575,6 +584,16 @@ const connectStatusCommand = Command.make("status", {
           ],
           { concurrency: "unbounded" },
         );
+        // Before the first link succeeds only the configured relay is known,
+        // and a wrong clock is one reason that link keeps failing.
+        const clockRelayUrl = Option.isSome(relayUrl)
+          ? Option.some(bytesToString(relayUrl.value))
+          : desired
+            ? yield* relayUrlConfig.pipe(Effect.option)
+            : Option.none();
+        const clockSkewSeconds = Option.isSome(clockRelayUrl)
+          ? yield* checkRelayClock(clockRelayUrl.value)
+          : null;
         const status: CloudCliStatus = {
           desired,
           authenticated,
@@ -586,8 +605,11 @@ const connectStatusCommand = Command.make("status", {
             Option.isSome(publishAgentActivity) ? bytesToString(publishAgentActivity.value) : null,
           ),
           relayClient: executable,
+          clockSkewSeconds,
         };
-        yield* Console.log(formatCloudStatus(status, { json: flags.json }));
+        yield* Console.log(
+          formatCloudStatus(status, { json: flags.json, platform: yield* HostProcess.Platform }),
+        );
       }),
       {
         quietLogs: flags.json,

apps/server/src/cloud/CloudLink.lifecycle.test.ts

@@ -716,7 +716,8 @@ describe("releaseManagedTunnelOnShutdown", () => {
       });
       expect((yield* link.linkState()).linkRevoked).toBe(true);
       // One registration request, no relink, and nothing started or removed.
-      expect(requests).toHaveLength(1);
+      // linkState also sends a HEAD to read the relay's clock.
+      expect(requests.filter((request) => request.method !== "HEAD")).toHaveLength(1);
       expect(applyConfigCalls).toEqual([]);
       expect(values.has(RELAY_ENVIRONMENT_CREDENTIAL_SECRET)).toBe(true);
       expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
@@ -739,6 +740,49 @@ describe("releaseManagedTunnelOnShutdown", () => {
     );
   });
 
+  it.effect(
+    "does not mark a link revoked when the rejection came while the clock was wrong",
+    () => {
+      const { store, values } = makeMemorySecretStore([
+        [
+          CLOUD_ENDPOINT_RUNTIME_CONFIG,
+          '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
+        ],
+        [RELAY_URL_SECRET, "https://relay.example.test"],
+        [CLOUD_LINKED_USER_ID, "user-123"],
+        [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
+        [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"],
+      ]);
+
+      return Effect.gen(function* () {
+        const link = yield* CloudLink.CloudLink;
+        const error = yield* link
+          .registerManagedTunnelRecovery("http://127.0.0.1:3773")
+          .pipe(Effect.flip);
+        expect(error).toMatchObject({ _tag: "RelayRequestError", clockSkewSeconds: -420 });
+        expect((yield* link.linkState()).linkRevoked).toBe(false);
+        expect(values.has(CLOUD_LINK_REVOKED)).toBe(false);
+      }).pipe(
+        provideReleaseHarness({
+          store,
+          applyConfigCalls: [],
+          requests: [],
+          respond: () =>
+            Response.json(
+              {
+                _tag: "RelayAuthInvalidError",
+                code: "auth_invalid",
+                reason: "not_authorized",
+                traceId: "trace-clock",
+              },
+              // The test clock starts at the epoch, so this relay is 7 minutes ahead.
+              { status: 401, headers: { date: "Thu, 01 Jan 1970 00:07:00 GMT" } },
+            ),
+        }),
+      );
+    },
+  );
+
   it.effect("does not mark a link that was replaced while its rejection was in flight", () => {
     const { store, values } = makeMemorySecretStore([
       [

apps/server/src/cloud/CloudLink.test.ts

@@ -5,7 +5,7 @@ import * as Fiber from "effect/Fiber";
 import * as Layer from "effect/Layer";
 import * as Option from "effect/Option";
 import * as FetchHttpClient from "effect/http/FetchHttpClient";
-import { HttpClient, HttpServer } from "effect/http";
+import { HttpClient, HttpClientResponse, HttpServer } from "effect/http";
 import * as NetAddress from "effect/net/NetAddress";
 import * as NodeServices from "@effect/platform-node/NodeServices";
 
@@ -34,6 +34,8 @@ const withService = <A, E>(
     readonly relayFails?: boolean;
     readonly failActivityRead?: boolean;
     readonly failHoldRead?: boolean;
+    /** The `Date` the relay answers a clock check with. */
+    readonly relayDate?: string;
     /** The first relay call reports itself, then waits for this before answering. */
     readonly holdFirstRelayCall?: {
       readonly started: () => void;
@@ -42,6 +44,7 @@ const withService = <A, E>(
   },
   body: (input: {
     readonly preferences: { readonly update: CloudLink.CloudLink["Service"]["updatePreferences"] };
+    readonly linkState: CloudLink.CloudLink["Service"]["linkState"];
     readonly stored: Map<string, Uint8Array>;
     readonly relayCalls: Array<boolean>;
   }) => Effect.Effect<A, E, never>,
@@ -105,13 +108,27 @@ const withService = <A, E>(
       Layer.mock(ServerConfig.ServerConfig)({} as ServerConfig.ServerConfig["Service"]),
       Layer.succeed(
         HttpClient.HttpClient,
-        HttpClient.make(() => Effect.die("unused")),
+        HttpClient.make((request) =>
+          options.relayDate === undefined
+            ? Effect.die("unused")
+            : Effect.succeed(
+                HttpClientResponse.fromWeb(
+                  request,
+                  new Response(null, { status: 404, headers: { date: options.relayDate } }),
+                ),
+              ),
+        ),
       ),
       NodeServices.layer,
     );
     return yield* Effect.gen(function* () {
       const link = yield* CloudLink.CloudLink;
-      return yield* body({ preferences: { update: link.updatePreferences }, stored, relayCalls });
+      return yield* body({
+        preferences: { update: link.updatePreferences },
+        linkState: link.linkState,
+        stored,
+        relayCalls,
+      });
     }).pipe(
       Effect.provide(CloudLink.layer.pipe(Layer.provide(layerDependencies))),
       Effect.provideService(FetchHttpClient.Fetch, fetch),
@@ -272,3 +289,22 @@ it.effect("an update the client walks away from still finishes", () => {
     }),
   );
 });
+
+// The test clock starts at the epoch.
+it.effect("reports how far off this machine's clock is when it breaks T3 Connect", () =>
+  withService({ relayDate: "Thu, 01 Jan 1970 00:07:00 GMT" }, ({ linkState }) =>
+    Effect.gen(function* () {
+      const state = yield* linkState();
+      assert.equal(state.clockSkewSeconds, -420);
+    }),
+  ),
+);
+
+it.effect("leaves the clock out of the link state when it is close enough", () =>
+  withService({ relayDate: "Thu, 01 Jan 1970 00:00:30 GMT" }, ({ linkState }) =>
+    Effect.gen(function* () {
+      const state = yield* linkState();
+      assert.isUndefined(state.clockSkewSeconds);
+    }),
+  ),
+);

apps/server/src/cloud/CloudLink.ts

@@ -102,6 +102,7 @@ import {
 import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts";
 import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
 import { relayUrlConfig } from "./publicConfig.ts";
+import { checkRelayClock } from "./relayClock.ts";
 import {
   filterRelayResponse,
   relayRequestError,
@@ -470,6 +471,7 @@ export class CloudLink extends Context.Service<
       | EnvironmentAuth.ServerAuthInternalError
       | RelayRequestError
     >;
+    /** The saved link, plus how far this machine's clock is off when that breaks it. */
     readonly linkState: () => Effect.Effect<
       EnvironmentCloudLinkStateResult,
       CloudLinkInternalError
@@ -1024,10 +1026,10 @@ const make = Effect.gen(function* () {
   /**
    * Marks the link revoked after the relay answered 401 to `credential`, usually
    * because the environment was deregistered. Retrying cannot fix that, and
-   * relinking is the user's call, so only mark it. Nothing stored is removed:
-   * the relay also answers 401 for a skewed clock, and the next accepted
-   * registration clears the mark. Returns false, without marking, when a relink
-   * replaced the credential while the rejected request was in flight.
+   * relinking is the user's call, so only mark it. Nothing stored is removed,
+   * and the next accepted registration clears the mark. Returns false, without
+   * marking, when a relink replaced the credential while the rejected request
+   * was in flight.
    */
   const markLinkRevoked = (credential: string, cause: RelayRequestError) =>
     endpointRuntime.withLinkStateLock(
@@ -1045,6 +1047,11 @@ const make = Effect.gen(function* () {
       }),
     );
 
+  // The relay also answers 401 while this machine's clock is wrong. That
+  // refusal is retried instead, since fixing the clock is enough to recover.
+  const isLinkRevocation = (error: RelayRequestError) =>
+    error.rejection === "unauthorized" && error.clockSkewSeconds === undefined;
+
   const registerManagedTunnelRecovery = Effect.fn(
     "environment.cloud.registerManagedCloudTunnelRecovery",
   )(function* (localOrigin: string, options?: { readonly retryRuntimeFailures?: boolean }) {
@@ -1101,7 +1108,7 @@ const make = Effect.gen(function* () {
     }).pipe(
       Effect.catchTags({
         RelayRequestError: (error) =>
-          error.rejection === "unauthorized"
+          isLinkRevocation(error)
             ? markLinkRevoked(credential, error).pipe(
                 Effect.map((marked) => (marked ? ("revoked" as const) : ("superseded" as const))),
               )
@@ -1191,7 +1198,7 @@ const make = Effect.gen(function* () {
       // A running host learns of a Deregister here, when its dead tunnel asks
       // for a replacement. The rejection still fails the recovery.
       Effect.tapError((error) =>
-        error.rejection === "unauthorized" ? markLinkRevoked(credential, error) : Effect.void,
+        isLinkRevocation(error) ? markLinkRevoked(credential, error) : Effect.void,
       ),
     );
     if (recovered.endpointRuntime.providerKind !== "cloudflare_tunnel") {
@@ -1398,7 +1405,13 @@ const make = Effect.gen(function* () {
 
   const linkState = Effect.fn("environment.cloud.linkState")(
     function* () {
-      return yield* readCloudLinkState();
+      const state = yield* readCloudLinkState();
+      // Only a configured link has a relay whose clock matters.
+      if (state.relayUrl === null) return state;
+      const clockSkewSeconds = yield* checkRelayClock(state.relayUrl).pipe(
+        Effect.provideService(HttpClient.HttpClient, httpClient),
+      );
+      return clockSkewSeconds === null ? state : { ...state, clockSkewSeconds };
     },
     Effect.catchIf(ServerSecretStore.isSecretStoreError, internalError("read-relay-config")),
   );

apps/server/src/cloud/relayClock.ts

@@ -0,0 +1,19 @@
+import { clockSkewFromDateHeader } from "@t3tools/shared/clockSkew";
+import * as Clock from "effect/Clock";
+import * as Effect from "effect/Effect";
+import { HttpClient, HttpClientRequest } from "effect/http";
+
+/**
+ * Seconds this machine's clock is off from the relay's when that is enough to
+ * break T3 Connect, `null` when it is close enough or the relay cannot be
+ * reached. Any relay response carries its time, so the request needs no auth.
+ */
+export const checkRelayClock = (relayUrl: string) =>
+  Effect.gen(function* () {
+    const httpClient = yield* HttpClient.HttpClient;
+    const response = yield* httpClient.execute(HttpClientRequest.head(`${relayUrl}/`));
+    return clockSkewFromDateHeader(response.headers.date, yield* Clock.currentTimeMillis);
+  }).pipe(
+    Effect.timeout("2 seconds"),
+    Effect.orElseSucceed(() => null),
+  );