From 8a64ee2773d0aa679979adc89c787ec3b61e7378 Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Wed, 14 Oct 2026 01:52:24 -0400 Subject: [PATCH 01/13] fix(server): T3 Connect recovers by itself once a wrong clock is fixed When the host clock is more than a minute off, the relay refuses the host's signed proofs. Startup treated those refusals as permanent, so the link stayed down until a restart, and the error pointed at keys or credentials. A relay refusal whose Date header shows the clock is off is now reported as a clock problem with steps to fix it, and startup keeps retrying until the clock is right. Settings > Connections and `t3 connect status` say when the clock is off. Tunnel recovery's cooldown uses monotonic time so a clock corrected backward cannot stall it. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cli/connect.ts | 28 ++++++- apps/server/src/cloud/CloudLink.test.ts | 42 +++++++++- apps/server/src/cloud/CloudLink.ts | 10 ++- apps/server/src/cloud/relayClock.ts | 19 +++++ apps/server/src/cloud/relayResponse.test.ts | 79 +++++++++++++++---- apps/server/src/cloud/relayResponse.ts | 49 ++++++++++-- apps/server/src/server.ts | 62 +++++++++++++-- .../settings/ConnectionsSettings.tsx | 26 +++++- docs/user/remote-access.md | 4 +- .../src/relay/errorPresentation.ts | 10 ++- packages/contracts/src/environmentHttp.ts | 3 + packages/shared/package.json | 4 + packages/shared/src/clockSkew.test.ts | 45 +++++++++++ packages/shared/src/clockSkew.ts | 62 +++++++++++++++ 14 files changed, 403 insertions(+), 40 deletions(-) create mode 100644 apps/server/src/cloud/relayClock.ts create mode 100644 packages/shared/src/clockSkew.test.ts create mode 100644 packages/shared/src/clockSkew.ts diff --git a/apps/server/src/cli/connect.ts b/apps/server/src/cli/connect.ts index 9f812e615e00..9292d16d958a 100644 --- a/apps/server/src/cli/connect.ts +++ b/apps/server/src/cli/connect.ts @@ -6,6 +6,7 @@ import { } from "@t3tools/contracts"; import { RelayOkResponse } from "@t3tools/contracts/relay"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { describeClockSkew } from "@t3tools/shared/clockSkew"; import * as RelayClient from "@t3tools/shared/relayClient"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import * as Cause from "effect/Cause"; @@ -29,6 +30,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as BootService from "../cloud/bootService.ts"; import * as CliState from "../cloud/CliState.ts"; import * as CliTokenManager from "../cloud/CliTokenManager.ts"; +import { checkRelayClock } from "../cloud/relayClock.ts"; import { filterRelayResponse } from "../cloud/relayResponse.ts"; import { CLOUD_LINK_REVOKED, @@ -147,6 +149,7 @@ interface CloudCliStatus { readonly relayUrl: string | null; readonly publishAgentActivity: boolean; readonly relayClient: RelayClient.RelayClientStatus; + readonly clockSkewSeconds: number | null; } function formatRelayClientStatus(executable: RelayClient.RelayClientStatus): ReadonlyArray { @@ -174,8 +177,11 @@ function formatRelayClientStatus(executable: RelayClient.RelayClientStatus): Rea } } -function formatCloudStatus(status: CloudCliStatus, options?: { readonly json?: boolean }): string { - if (options?.json) { +function formatCloudStatus( + status: CloudCliStatus, + options: { readonly json?: boolean; readonly platform: NodeJS.Platform }, +): string { + if (options.json) { return JSON.stringify(status, null, 2); } @@ -204,6 +210,9 @@ function formatCloudStatus(status: CloudCliStatus, options?: { readonly json?: b ` Relay: ${status.relayUrl ?? "not provisioned"}`, ` Publish agent activity: ${status.publishAgentActivity ? "enabled" : "disabled"}`, ...formatRelayClientStatus(status.relayClient), + ...(status.clockSkewSeconds === null + ? [] + : ["", describeClockSkew(status.clockSkewSeconds, options.platform)]), "", "This is saved setup, not a live connection check. Check the background service with `t3 service status`.", ...(nextStep ? ["", `Next: ${nextStep}`] : []), @@ -575,6 +584,16 @@ const connectStatusCommand = Command.make("status", { ], { concurrency: "unbounded" }, ); + // Before the first link succeeds only the configured relay is known, + // and a wrong clock is one reason that link keeps failing. + const clockRelayUrl = Option.isSome(relayUrl) + ? Option.some(bytesToString(relayUrl.value)) + : desired + ? yield* relayUrlConfig.pipe(Effect.option) + : Option.none(); + const clockSkewSeconds = Option.isSome(clockRelayUrl) + ? yield* checkRelayClock(clockRelayUrl.value) + : null; const status: CloudCliStatus = { desired, authenticated, @@ -586,8 +605,11 @@ const connectStatusCommand = Command.make("status", { Option.isSome(publishAgentActivity) ? bytesToString(publishAgentActivity.value) : null, ), relayClient: executable, + clockSkewSeconds, }; - yield* Console.log(formatCloudStatus(status, { json: flags.json })); + yield* Console.log( + formatCloudStatus(status, { json: flags.json, platform: yield* HostProcess.Platform }), + ); }), { quietLogs: flags.json, diff --git a/apps/server/src/cloud/CloudLink.test.ts b/apps/server/src/cloud/CloudLink.test.ts index fd196a1b33ae..8948976234a3 100644 --- a/apps/server/src/cloud/CloudLink.test.ts +++ b/apps/server/src/cloud/CloudLink.test.ts @@ -5,7 +5,7 @@ import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as FetchHttpClient from "effect/http/FetchHttpClient"; -import { HttpClient, HttpServer } from "effect/http"; +import { HttpClient, HttpClientResponse, HttpServer } from "effect/http"; import * as NetAddress from "effect/net/NetAddress"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -34,6 +34,8 @@ const withService = ( readonly relayFails?: boolean; readonly failActivityRead?: boolean; readonly failHoldRead?: boolean; + /** The `Date` the relay answers a clock check with. */ + readonly relayDate?: string; /** The first relay call reports itself, then waits for this before answering. */ readonly holdFirstRelayCall?: { readonly started: () => void; @@ -42,6 +44,7 @@ const withService = ( }, body: (input: { readonly preferences: { readonly update: CloudLink.CloudLink["Service"]["updatePreferences"] }; + readonly linkState: CloudLink.CloudLink["Service"]["linkState"]; readonly stored: Map; readonly relayCalls: Array; }) => Effect.Effect, @@ -105,13 +108,27 @@ const withService = ( Layer.mock(ServerConfig.ServerConfig)({} as ServerConfig.ServerConfig["Service"]), Layer.succeed( HttpClient.HttpClient, - HttpClient.make(() => Effect.die("unused")), + HttpClient.make((request) => + options.relayDate === undefined + ? Effect.die("unused") + : Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response(null, { status: 404, headers: { date: options.relayDate } }), + ), + ), + ), ), NodeServices.layer, ); return yield* Effect.gen(function* () { const link = yield* CloudLink.CloudLink; - return yield* body({ preferences: { update: link.updatePreferences }, stored, relayCalls }); + return yield* body({ + preferences: { update: link.updatePreferences }, + linkState: link.linkState, + stored, + relayCalls, + }); }).pipe( Effect.provide(CloudLink.layer.pipe(Layer.provide(layerDependencies))), Effect.provideService(FetchHttpClient.Fetch, fetch), @@ -272,3 +289,22 @@ it.effect("an update the client walks away from still finishes", () => { }), ); }); + +// The test clock starts at the epoch. +it.effect("reports how far off this machine's clock is when it breaks T3 Connect", () => + withService({ relayDate: "Thu, 01 Jan 1970 00:07:00 GMT" }, ({ linkState }) => + Effect.gen(function* () { + const state = yield* linkState(); + assert.equal(state.clockSkewSeconds, -420); + }), + ), +); + +it.effect("leaves the clock out of the link state when it is close enough", () => + withService({ relayDate: "Thu, 01 Jan 1970 00:00:30 GMT" }, ({ linkState }) => + Effect.gen(function* () { + const state = yield* linkState(); + assert.isUndefined(state.clockSkewSeconds); + }), + ), +); diff --git a/apps/server/src/cloud/CloudLink.ts b/apps/server/src/cloud/CloudLink.ts index f11b94115325..516adcfe6f70 100644 --- a/apps/server/src/cloud/CloudLink.ts +++ b/apps/server/src/cloud/CloudLink.ts @@ -102,6 +102,7 @@ import { import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { relayUrlConfig } from "./publicConfig.ts"; +import { checkRelayClock } from "./relayClock.ts"; import { filterRelayResponse, relayRequestError, @@ -470,6 +471,7 @@ export class CloudLink extends Context.Service< | EnvironmentAuth.ServerAuthInternalError | RelayRequestError >; + /** The saved link, plus how far this machine's clock is off when that breaks it. */ readonly linkState: () => Effect.Effect< EnvironmentCloudLinkStateResult, CloudLinkInternalError @@ -1398,7 +1400,13 @@ const make = Effect.gen(function* () { const linkState = Effect.fn("environment.cloud.linkState")( function* () { - return yield* readCloudLinkState(); + const state = yield* readCloudLinkState(); + // Only a configured link has a relay whose clock matters. + if (state.relayUrl === null) return state; + const clockSkewSeconds = yield* checkRelayClock(state.relayUrl).pipe( + Effect.provideService(HttpClient.HttpClient, httpClient), + ); + return clockSkewSeconds === null ? state : { ...state, clockSkewSeconds }; }, Effect.catchIf(ServerSecretStore.isSecretStoreError, internalError("read-relay-config")), ); diff --git a/apps/server/src/cloud/relayClock.ts b/apps/server/src/cloud/relayClock.ts new file mode 100644 index 000000000000..2a4f0ae4b5ba --- /dev/null +++ b/apps/server/src/cloud/relayClock.ts @@ -0,0 +1,19 @@ +import { clockSkewFromDateHeader } from "@t3tools/shared/clockSkew"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import { HttpClient, HttpClientRequest } from "effect/http"; + +/** + * Seconds this machine's clock is off from the relay's when that is enough to + * break T3 Connect, `null` when it is close enough or the relay cannot be + * reached. Any relay response carries its time, so the request needs no auth. + */ +export const checkRelayClock = (relayUrl: string) => + Effect.gen(function* () { + const httpClient = yield* HttpClient.HttpClient; + const response = yield* httpClient.execute(HttpClientRequest.head(`${relayUrl}/`)); + return clockSkewFromDateHeader(response.headers.date, yield* Clock.currentTimeMillis); + }).pipe( + Effect.timeout("2 seconds"), + Effect.orElseSucceed(() => null), + ); diff --git a/apps/server/src/cloud/relayResponse.test.ts b/apps/server/src/cloud/relayResponse.test.ts index 8221899ca3a6..99094d2109d8 100644 --- a/apps/server/src/cloud/relayResponse.test.ts +++ b/apps/server/src/cloud/relayResponse.test.ts @@ -5,10 +5,14 @@ import * as HttpClientError from "effect/http/HttpClientError"; import { filterRelayResponse, + isRelayClockSkewError, relayRequestError, shouldRetryRelayRequest, } from "./relayResponse.ts"; +// The test clock starts at the epoch, so this relay is seven minutes ahead of it. +const RELAY_SEVEN_MINUTES_AHEAD = { date: "Thu, 01 Jan 1970 00:07:00 GMT" }; + const response = ( status: number, body: string | Record, @@ -76,6 +80,50 @@ it.effect("makes revoked authorization actionable and non-retryable", () => }), ); +it.effect("blames a refused link proof on this machine's clock when it is far off", () => + Effect.gen(function* () { + const error = yield* filterRelayResponse( + response( + 400, + { + _tag: "RelayEnvironmentLinkProofInvalidError", + code: "environment_link_proof_invalid", + reason: "invalid_signature_or_scope", + traceId: "trace-clock", + }, + RELAY_SEVEN_MINUTES_AHEAD, + ), + ).pipe(Effect.flip); + + expect(error).toMatchObject({ rejection: "rejected", clockSkewSeconds: -420 }); + expect(isRelayClockSkewError(error)).toBe(true); + expect(shouldRetryRelayRequest(error)).toBe(true); + expect(error.message).toContain("clock is 7 minutes behind"); + expect(error.message).toContain("Trace ID: trace-clock"); + }), +); + +it.effect("keeps a refusal about the request itself permanent despite a skewed clock", () => + Effect.gen(function* () { + const error = yield* filterRelayResponse( + response( + 403, + { + _tag: "RelayEnvironmentLinkLimitExceededError", + code: "environment_link_limit_exceeded", + maxTunnels: 3, + traceId: "trace-limit", + }, + RELAY_SEVEN_MINUTES_AHEAD, + ), + ).pipe(Effect.flip); + + expect(isRelayClockSkewError(error)).toBe(false); + expect(shouldRetryRelayRequest(error)).toBe(false); + expect(error.message).not.toContain("clock"); + }), +); + it.effect("reports an unrecognized access denial without printing its response body", () => Effect.gen(function* () { const error = yield* filterRelayResponse( @@ -108,20 +156,23 @@ it.effect.each([ { status: 403, attempts: 1 }, { status: 429, attempts: 2 }, { status: 503, attempts: 2 }, -])("stops rejected startup links but retries temporary failures: $status", ({ status, attempts }) => - Effect.gen(function* () { - let requests = 0; - const result = yield* Effect.suspend(() => { - requests++; - return filterRelayResponse(response(requests === 1 ? status : 200, "{}")); - }).pipe( - Effect.mapError(relayRequestError), - Effect.retry({ while: shouldRetryRelayRequest, times: 1 }), - Effect.result, - ); - expect(requests).toBe(attempts); - expect(result._tag).toBe(attempts === 1 ? "Failure" : "Success"); - }), + { status: 401, headers: RELAY_SEVEN_MINUTES_AHEAD, attempts: 2 }, +])( + "stops rejected startup links but retries temporary failures: $status", + ({ status, headers, attempts }) => + Effect.gen(function* () { + let requests = 0; + const result = yield* Effect.suspend(() => { + requests++; + return filterRelayResponse(response(requests === 1 ? status : 200, "{}", headers)); + }).pipe( + Effect.mapError(relayRequestError), + Effect.retry({ while: shouldRetryRelayRequest, times: 1 }), + Effect.result, + ); + expect(requests).toBe(attempts); + expect(result._tag).toBe(attempts === 1 ? "Failure" : "Success"); + }), ); it.effect("keeps the relay failure reason and trace when tunnel cleanup fails", () => diff --git a/apps/server/src/cloud/relayResponse.ts b/apps/server/src/cloud/relayResponse.ts index db8a6b88702f..4ec63148b592 100644 --- a/apps/server/src/cloud/relayResponse.ts +++ b/apps/server/src/cloud/relayResponse.ts @@ -1,4 +1,7 @@ import { RelayProtectedError } from "@t3tools/contracts/relay"; +import { clockSkewFromDateHeader, describeClockSkew } from "@t3tools/shared/clockSkew"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; @@ -7,14 +10,17 @@ import { isHttpClientError } from "effect/http/HttpClientError"; /** * A relay request that did not succeed. `unavailable` means it may succeed on - * retry; the relay refused the other kinds. The description is safe to show: - * it carries the relay's own explanation and trace ID, never request details. + * retry; the relay refused the other kinds. A refusal with `clockSkewSeconds` + * happened while this machine's clock was that far from the relay's, so it can + * succeed once the clock is corrected. The description is safe to show: it + * carries the relay's own explanation and trace ID, never request details. */ export class RelayRequestError extends Schema.TaggedError()( "RelayRequestError", { rejection: Schema.Literals(["unauthorized", "forbidden", "rejected", "unavailable"]), description: Schema.String, + clockSkewSeconds: Schema.optional(Schema.Number), }, ) { override get message(): string { @@ -35,7 +41,13 @@ export function relayRequestError(cause: unknown): RelayRequestError { /** Whether a failure may succeed on retry: anything but a relay refusal. */ export const shouldRetryRelayRequest = (error: unknown): boolean => - !isRelayRequestError(error) || error.rejection === "unavailable"; + !isRelayRequestError(error) || + error.rejection === "unavailable" || + error.clockSkewSeconds !== undefined; + +/** A relay refusal caused by this machine's clock, which clears once the clock is fixed. */ +export const isRelayClockSkewError = (error: unknown): error is RelayRequestError => + isRelayRequestError(error) && error.clockSkewSeconds !== undefined; function recoveryHint(error: RelayProtectedError): string { switch (error._tag) { @@ -56,6 +68,10 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func response: HttpClientResponse.HttpClientResponse, ) { if (response.status >= 200 && response.status < 300) return response; + const clockSkewSeconds = clockSkewFromDateHeader( + response.headers.date, + yield* Clock.currentTimeMillis, + ); const decoded = yield* HttpClientResponse.schemaBodyJson(RelayProtectedError)(response).pipe( Effect.option, ); @@ -64,12 +80,33 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func const description = Option.isSome(decoded) ? `T3 Connect: ${decoded.value.message}. ${recoveryHint(decoded.value)} Trace ID: ${decoded.value.traceId}.` : `T3 Connect relay returned HTTP ${response.status} without a recognized error response. Check relay access and any proxy or firewall restrictions, then restart T3 Code.${requestId}`; + // The relay checks the timestamps this machine signs against its own clock + // and answers 400 or 401 when they fall outside its window. With a skewed + // clock that is the likely cause, and nothing but fixing the clock helps. + // Other refusals (a tunnel limit, a missing link) are about the request. + const clockCaused = + clockSkewSeconds !== null && (response.status === 400 || response.status === 401); + const platform = yield* HostProcess.Platform; + const refusal = (rejection: "unauthorized" | "forbidden" | "rejected") => + !clockCaused + ? new RelayRequestError({ rejection, description }) + : new RelayRequestError({ + rejection, + // The relay's own recovery advice does not apply until the clock is + // fixed, so keep only what it said and its trace. + description: `${describeClockSkew(clockSkewSeconds, platform)} Relay response: ${ + Option.isSome(decoded) + ? `${decoded.value.message}. Trace ID: ${decoded.value.traceId}.` + : `HTTP ${response.status}.${requestId}` + }`, + clockSkewSeconds, + }); if (response.status === 401) { - return yield* new RelayRequestError({ rejection: "unauthorized", description }); + return yield* refusal("unauthorized"); } if (response.status === 403) { - return yield* new RelayRequestError({ rejection: "forbidden", description }); + return yield* refusal("forbidden"); } if ( response.status >= 400 && @@ -77,7 +114,7 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func response.status !== 408 && response.status !== 429 ) { - return yield* new RelayRequestError({ rejection: "rejected", description }); + return yield* refusal("rejected"); } return yield* new RelayRequestError({ rejection: "unavailable", description }); }); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 8148479f8318..244d24d46870 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -136,6 +136,7 @@ import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import * as CloudHttp from "./cloud/http.ts"; import * as CloudLink from "./cloud/CloudLink.ts"; import { pendingServiceUpdateExists } from "./cloud/updateHandoff.ts"; +import { isRelayClockSkewError } from "./cloud/relayResponse.ts"; import * as RelayTracing from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; import { @@ -879,20 +880,39 @@ const layerMakeServer = Layer.unwrap( const localOrigin = `http://127.0.0.1:${address.port}`; const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; const recoveryLock = yield* Semaphore.make(1); - let lastRecoveryAtMillis = 0; + // Monotonic, so a wall clock corrected backward (a host that booted + // ahead) cannot push the next recovery hours into the future. + let lastRecoveryAtNanos: bigint | null = null; + // Clock refusals retry until the clock is fixed, which can be hours, + // and a retry only logs once it gives up. Say once why it waits. + let clockSkewReported = false; + const reportClockSkew = (error: unknown) => + isRelayClockSkewError(error) && !clockSkewReported + ? Effect.logWarning(error.message).pipe( + Effect.andThen( + Effect.sync(() => { + clockSkewReported = true; + }), + ), + ) + : Effect.void; const recoverManagedTunnel = (config: RelayManagedEndpointRuntimeConfig) => recoveryLock.withPermits(1)( Effect.gen(function* () { - const elapsed = (yield* Clock.currentTimeMillis) - lastRecoveryAtMillis; - const wait = Duration.toMillis(MANAGED_TUNNEL_RECOVERY_COOLDOWN) - elapsed; - if (wait > 0) yield* Effect.sleep(Duration.millis(wait)); - lastRecoveryAtMillis = yield* Clock.currentTimeMillis; + const now = yield* Clock.monotonicTimeNanos; + if (lastRecoveryAtNanos !== null) { + const elapsed = Number((now - lastRecoveryAtNanos) / 1_000_000n); + const wait = Duration.toMillis(MANAGED_TUNNEL_RECOVERY_COOLDOWN) - elapsed; + if (wait > 0) yield* Effect.sleep(Duration.millis(wait)); + } + lastRecoveryAtNanos = yield* Clock.monotonicTimeNanos; }).pipe( Effect.andThen( cloudLink.recoverManagedTunnel(localOrigin, config, { retryRuntimeFailures: true, }), ), + Effect.tapError(reportClockSkew), Effect.retry({ while: (error) => CloudLink.shouldRetryCloudLink(error) && @@ -976,10 +996,28 @@ const layerMakeServer = Layer.unwrap( ), Effect.asVoid, ); + // A clock refusal used to end registration and start the stored + // tunnel at once. It now retries until the clock is fixed, so start + // the tunnel on the first one rather than after the retry window. + let startedForClockSkew = startedConfirmed; + const startStoredTunnelOnClockSkew = (error: unknown) => + isRelayClockSkewError(error) && !startedForClockSkew + ? Effect.sync(() => { + startedForClockSkew = true; + }).pipe(Effect.andThen(startStoredManagedTunnel)) + : Effect.void; const registerManagedTunnel = retryManagedTunnelRegistration( - cloudLink.registerManagedTunnelRecovery(localOrigin, { - retryRuntimeFailures: true, - }), + cloudLink + .registerManagedTunnelRecovery(localOrigin, { + retryRuntimeFailures: true, + }) + .pipe( + Effect.tapError((error) => + reportClockSkew(error).pipe( + Effect.andThen(startStoredTunnelOnClockSkew(error)), + ), + ), + ), (error) => CloudLink.shouldRetryCloudLink(error) && error._tag !== "CloudLinkEndpointUnavailableError", @@ -1026,6 +1064,14 @@ const layerMakeServer = Layer.unwrap( const reconciledMode = yield* cloudLink .reconcileDesiredLinkIfStillDesired(localOrigin) .pipe( + // A host often boots before its clock syncs. Keep trying for as + // long as the clock is what the relay refuses, so the link comes + // up as soon as the clock is fixed instead of after a restart. + Effect.tapError(reportClockSkew), + Effect.retry({ + while: isRelayClockSkewError, + schedule: Schedule.spaced("30 seconds"), + }), Effect.retry({ while: CloudLink.shouldRetryCloudLink, schedule: Schedule.exponential("1 second").pipe( diff --git a/apps/web/src/components/settings/ConnectionsSettings.tsx b/apps/web/src/components/settings/ConnectionsSettings.tsx index 1ff650a9b77d..f8b651a64a47 100644 --- a/apps/web/src/components/settings/ConnectionsSettings.tsx +++ b/apps/web/src/components/settings/ConnectionsSettings.tsx @@ -1,5 +1,6 @@ import { SessionPermissions } from "./SessionPermissions"; import { AUTH_SCOPE_OPTIONS as PAIRING_SCOPE_OPTIONS } from "@t3tools/shared/authScopeOptions"; +import { describeClockSkew } from "@t3tools/shared/clockSkew"; import { ChevronRightIcon, ChevronsLeftRightEllipsisIcon, @@ -61,7 +62,7 @@ import * as DateTime from "effect/DateTime"; import * as Option from "effect/Option"; import { useCopyToClipboard } from "../../hooks/useCopyToClipboard"; -import { cn } from "../../lib/utils"; +import { cn, isMacPlatform, isWindowsPlatform } from "../../lib/utils"; import { isLocalEnvironmentDisabled } from "../../localEnvironment"; import { formatElapsedDurationLabel, formatExpiresInLabel } from "../../timestampFormat"; import { resolveDesktopPairingUrl, resolveHostedPairingUrl } from "./pairingUrls"; @@ -1845,6 +1846,27 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b ? "Your session does not have permission to manage T3 Connect access." : null; const isBusy = isUpdating || isUpdatingPreference; + // This row only renders in the desktop app, so the browser's platform is the + // host's, which is the clock that needs fixing. + const clockSkewSeconds = primaryCloudLinkState.data?.clockSkewSeconds; + const clockSkewWarning = + clockSkewSeconds === undefined + ? null + : describeClockSkew( + clockSkewSeconds, + isWindowsPlatform(navigator.platform) + ? "win32" + : isMacPlatform(navigator.platform) + ? "darwin" + : "linux", + ); + // The fix happens outside the app, so check again when the user comes back. + const refreshLinkState = primaryCloudLinkState.refresh; + useEffect(() => { + if (clockSkewWarning === null) return; + window.addEventListener("focus", refreshLinkState); + return () => window.removeEventListener("focus", refreshLinkState); + }, [clockSkewWarning, refreshLinkState]); const updateManagedTunnel = async (enabled: boolean) => { if ( @@ -1927,7 +1949,7 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b ? "T3 Connect is out of sync. This environment is linked only to publish agent activity, so other devices cannot connect. Turn T3 Connect on to repair it." : "Make this environment available to your other devices through T3 Connect." } - status={operationError ?? primaryCloudLinkState.error} + status={operationError ?? clockSkewWarning ?? primaryCloudLinkState.error} control={ { + it("reports a clock that is minutes behind the relay", () => { + expect(clockSkewFromDateHeader("Wed, 14 Oct 2026 09:00:00 GMT", relayNow - 7 * 60_000)).toBe( + -420, + ); + }); + + it("reports a clock that is hours ahead of the relay", () => { + expect( + clockSkewFromDateHeader("Wed, 14 Oct 2026 09:00:00 GMT", relayNow + 3 * 60 * 60_000), + ).toBe(10_800); + }); + + it("ignores a gap the relay still accepts", () => { + expect(clockSkewFromDateHeader("Wed, 14 Oct 2026 09:00:00 GMT", relayNow + 59_000)).toBeNull(); + expect(clockSkewFromDateHeader("Wed, 14 Oct 2026 09:00:00 GMT", relayNow - 59_000)).toBeNull(); + }); + + it("cannot judge a response without a usable date", () => { + expect(clockSkewFromDateHeader(undefined, relayNow + 3_600_000)).toBeNull(); + expect(clockSkewFromDateHeader("not a date", relayNow + 3_600_000)).toBeNull(); + }); +}); + +describe("describeClockSkew", () => { + it("says how far off the clock is and how to fix it on Windows", () => { + const message = describeClockSkew(-420, "win32"); + expect(message).toContain("This computer's clock is 7 minutes behind"); + expect(message).toContain('"Sync now"'); + expect(message).toContain("reconnects by itself"); + }); + + it("uses the host's own settings path on macOS and Linux", () => { + expect(describeClockSkew(3_600, "darwin")).toContain("1 hour ahead"); + expect(describeClockSkew(3_600, "darwin")).toContain("System Settings"); + expect(describeClockSkew(2 * 86_400, "linux")).toContain("2 days ahead"); + expect(describeClockSkew(2 * 86_400, "linux")).toContain("timedatectl set-ntp true"); + }); +}); diff --git a/packages/shared/src/clockSkew.ts b/packages/shared/src/clockSkew.ts new file mode 100644 index 000000000000..67ebb265ed10 --- /dev/null +++ b/packages/shared/src/clockSkew.ts @@ -0,0 +1,62 @@ +/** + * T3 Connect's signed requests only work while the host clock is close to the + * relay's. These helpers decide when it is too far off and tell the user how + * to fix it, so the server, CLI and settings all say the same thing. + */ + +// The relay accepts proofs issued up to 60 seconds in its future. Past that, a +// rejection is the clock's fault rather than a coincidence. +export const CLOCK_SKEW_LIMIT_SECONDS = 60; + +/** + * Seconds this machine's clock is ahead of (positive) or behind (negative) the + * relay, judged from the HTTP `Date` header of a relay response received at + * `nowMillis`. `null` when the header is missing, unparseable, or within the + * limit. + */ +export function clockSkewFromDateHeader( + dateHeader: string | undefined, + nowMillis: number, +): number | null { + if (dateHeader === undefined) return null; + const relayMillis = Date.parse(dateHeader); + if (Number.isNaN(relayMillis)) return null; + // `Date` has whole-second precision, so the relay's true time is up to a + // second later than the header says. + const skewSeconds = Math.round((nowMillis - relayMillis - 500) / 1_000); + return Math.abs(skewSeconds) > CLOCK_SKEW_LIMIT_SECONDS ? skewSeconds : null; +} + +function skewAmount(seconds: number): string { + const magnitude = Math.abs(seconds); + const minutes = Math.round(magnitude / 60); + const hours = Math.round(magnitude / (60 * 60)); + const [value, unit] = + magnitude < 90 + ? [magnitude, "second"] + : minutes < 60 + ? [minutes, "minute"] + : hours < 36 + ? [hours, "hour"] + : [Math.round(magnitude / (24 * 60 * 60)), "day"]; + return `${value} ${unit}${value === 1 ? "" : "s"}`; +} + +function clockFixInstructions(platform: string): string { + if (platform === "win32") { + return 'Open Settings > Time & language > Date & time, turn on "Set time automatically", then press "Sync now".'; + } + if (platform === "darwin") { + return 'Open System Settings > General > Date & Time and turn on "Set time and date automatically".'; + } + return "Turn on network time sync, for example with `timedatectl set-ntp true`."; +} + +/** + * What to tell the user when T3 Connect fails because of `seconds` of skew. + * `platform` is the host's `process.platform`, since that is the clock to fix. + */ +export function describeClockSkew(seconds: number, platform: string): string { + const direction = seconds > 0 ? "ahead" : "behind"; + return `This computer's clock is ${skewAmount(seconds)} ${direction}, and T3 Connect needs it to be right. ${clockFixInstructions(platform)} T3 Connect reconnects by itself once the clock is correct.`; +} From 592fd684926acdb219b4b5c60d7d245a0d768113 Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Wed, 14 Oct 2026 01:54:19 -0400 Subject: [PATCH 02/13] fix(server): clock comments say what the check can prove Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cloud/relayResponse.ts | 19 +++++++++++-------- packages/shared/src/clockSkew.ts | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/apps/server/src/cloud/relayResponse.ts b/apps/server/src/cloud/relayResponse.ts index 4ec63148b592..093036137801 100644 --- a/apps/server/src/cloud/relayResponse.ts +++ b/apps/server/src/cloud/relayResponse.ts @@ -11,8 +11,8 @@ import { isHttpClientError } from "effect/http/HttpClientError"; /** * A relay request that did not succeed. `unavailable` means it may succeed on * retry; the relay refused the other kinds. A refusal with `clockSkewSeconds` - * happened while this machine's clock was that far from the relay's, so it can - * succeed once the clock is corrected. The description is safe to show: it + * happened while this machine's clock was that far from the relay's, so it is + * retried in case fixing the clock is all it needs. The description is safe to show: it * carries the relay's own explanation and trace ID, never request details. */ export class RelayRequestError extends Schema.TaggedError()( @@ -39,13 +39,16 @@ export function relayRequestError(cause: unknown): RelayRequestError { }); } -/** Whether a failure may succeed on retry: anything but a relay refusal. */ +/** + * Whether a failure may succeed on retry: anything but a relay refusal, unless + * the clock was off when the relay refused. + */ export const shouldRetryRelayRequest = (error: unknown): boolean => !isRelayRequestError(error) || error.rejection === "unavailable" || error.clockSkewSeconds !== undefined; -/** A relay refusal caused by this machine's clock, which clears once the clock is fixed. */ +/** A relay refusal that came while this machine's clock was too far off to trust. */ export const isRelayClockSkewError = (error: unknown): error is RelayRequestError => isRelayRequestError(error) && error.clockSkewSeconds !== undefined; @@ -81,14 +84,14 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func ? `T3 Connect: ${decoded.value.message}. ${recoveryHint(decoded.value)} Trace ID: ${decoded.value.traceId}.` : `T3 Connect relay returned HTTP ${response.status} without a recognized error response. Check relay access and any proxy or firewall restrictions, then restart T3 Code.${requestId}`; // The relay checks the timestamps this machine signs against its own clock - // and answers 400 or 401 when they fall outside its window. With a skewed - // clock that is the likely cause, and nothing but fixing the clock helps. + // and answers 400 or 401 when they fall outside its window. A skewed clock + // explains that refusal, and the request cannot succeed until it is fixed. // Other refusals (a tunnel limit, a missing link) are about the request. - const clockCaused = + const clockSuspected = clockSkewSeconds !== null && (response.status === 400 || response.status === 401); const platform = yield* HostProcess.Platform; const refusal = (rejection: "unauthorized" | "forbidden" | "rejected") => - !clockCaused + !clockSuspected ? new RelayRequestError({ rejection, description }) : new RelayRequestError({ rejection, diff --git a/packages/shared/src/clockSkew.ts b/packages/shared/src/clockSkew.ts index 67ebb265ed10..e34ebad774a3 100644 --- a/packages/shared/src/clockSkew.ts +++ b/packages/shared/src/clockSkew.ts @@ -5,7 +5,7 @@ */ // The relay accepts proofs issued up to 60 seconds in its future. Past that, a -// rejection is the clock's fault rather than a coincidence. +// skewed clock is enough on its own to explain a rejection. export const CLOCK_SKEW_LIMIT_SECONDS = 60; /** From c3d2389508a9fa1066ad291231fa3de3bcfc3f6c Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Wed, 14 Oct 2026 11:45:46 -0400 Subject: [PATCH 03/13] fix(server): a wrong clock no longer marks a T3 Connect link as removed The relay answers 401 both to a deregistered environment and to a host whose clock is off. A 401 that came with a clock skew is now retried like other clock refusals instead of being marked revoked. Co-Authored-By: Claude Opus 5.5 --- .../src/cloud/CloudLink.lifecycle.test.ts | 46 ++++++++++++++++++- apps/server/src/cloud/CloudLink.ts | 17 ++++--- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.ts index 86239f9e73a9..66a4d760f9eb 100644 --- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts +++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts @@ -716,7 +716,8 @@ describe("releaseManagedTunnelOnShutdown", () => { }); expect((yield* link.linkState()).linkRevoked).toBe(true); // One registration request, no relink, and nothing started or removed. - expect(requests).toHaveLength(1); + // linkState also sends a HEAD to read the relay's clock. + expect(requests.filter((request) => request.method !== "HEAD")).toHaveLength(1); expect(applyConfigCalls).toEqual([]); expect(values.has(RELAY_ENVIRONMENT_CREDENTIAL_SECRET)).toBe(true); expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); @@ -739,6 +740,49 @@ describe("releaseManagedTunnelOnShutdown", () => { ); }); + it.effect( + "does not mark a link revoked when the rejection came while the clock was wrong", + () => { + const { store, values } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"], + ]); + + return Effect.gen(function* () { + const link = yield* CloudLink.CloudLink; + const error = yield* link + .registerManagedTunnelRecovery("http://127.0.0.1:3773") + .pipe(Effect.flip); + expect(error).toMatchObject({ _tag: "RelayRequestError", clockSkewSeconds: -420 }); + expect((yield* link.linkState()).linkRevoked).toBe(false); + expect(values.has(CLOUD_LINK_REVOKED)).toBe(false); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls: [], + requests: [], + respond: () => + Response.json( + { + _tag: "RelayAuthInvalidError", + code: "auth_invalid", + reason: "not_authorized", + traceId: "trace-clock", + }, + // The test clock starts at the epoch, so this relay is 7 minutes ahead. + { status: 401, headers: { date: "Thu, 01 Jan 1970 00:07:00 GMT" } }, + ), + }), + ); + }, + ); + it.effect("does not mark a link that was replaced while its rejection was in flight", () => { const { store, values } = makeMemorySecretStore([ [ diff --git a/apps/server/src/cloud/CloudLink.ts b/apps/server/src/cloud/CloudLink.ts index 516adcfe6f70..be3904452437 100644 --- a/apps/server/src/cloud/CloudLink.ts +++ b/apps/server/src/cloud/CloudLink.ts @@ -1026,10 +1026,10 @@ const make = Effect.gen(function* () { /** * Marks the link revoked after the relay answered 401 to `credential`, usually * because the environment was deregistered. Retrying cannot fix that, and - * relinking is the user's call, so only mark it. Nothing stored is removed: - * the relay also answers 401 for a skewed clock, and the next accepted - * registration clears the mark. Returns false, without marking, when a relink - * replaced the credential while the rejected request was in flight. + * relinking is the user's call, so only mark it. Nothing stored is removed, + * and the next accepted registration clears the mark. Returns false, without + * marking, when a relink replaced the credential while the rejected request + * was in flight. */ const markLinkRevoked = (credential: string, cause: RelayRequestError) => endpointRuntime.withLinkStateLock( @@ -1047,6 +1047,11 @@ const make = Effect.gen(function* () { }), ); + // The relay also answers 401 while this machine's clock is wrong. That + // refusal is retried instead, since fixing the clock is enough to recover. + const isLinkRevocation = (error: RelayRequestError) => + error.rejection === "unauthorized" && error.clockSkewSeconds === undefined; + const registerManagedTunnelRecovery = Effect.fn( "environment.cloud.registerManagedCloudTunnelRecovery", )(function* (localOrigin: string, options?: { readonly retryRuntimeFailures?: boolean }) { @@ -1103,7 +1108,7 @@ const make = Effect.gen(function* () { }).pipe( Effect.catchTags({ RelayRequestError: (error) => - error.rejection === "unauthorized" + isLinkRevocation(error) ? markLinkRevoked(credential, error).pipe( Effect.map((marked) => (marked ? ("revoked" as const) : ("superseded" as const))), ) @@ -1193,7 +1198,7 @@ const make = Effect.gen(function* () { // A running host learns of a Deregister here, when its dead tunnel asks // for a replacement. The rejection still fails the recovery. Effect.tapError((error) => - error.rejection === "unauthorized" ? markLinkRevoked(credential, error) : Effect.void, + isLinkRevocation(error) ? markLinkRevoked(credential, error) : Effect.void, ), ); if (recovered.endpointRuntime.providerKind !== "cloudflare_tunnel") { From 5f740516e797cd21462d6ee7b1538b5845b5773c Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Wed, 14 Oct 2026 12:01:06 -0400 Subject: [PATCH 04/13] chore: keep clock constants private to their modules Co-Authored-By: Claude Opus 5.5 --- packages/client-runtime/src/relay/errorPresentation.ts | 2 +- packages/shared/src/clockSkew.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client-runtime/src/relay/errorPresentation.ts b/packages/client-runtime/src/relay/errorPresentation.ts index c93250120009..c986fdc89dee 100644 --- a/packages/client-runtime/src/relay/errorPresentation.ts +++ b/packages/client-runtime/src/relay/errorPresentation.ts @@ -12,7 +12,7 @@ export const DPOP_UNKNOWN_HINT = export const DPOP_RETRY_HINT = "Hint: Try again. If the problem continues, copy the trace ID."; /** The relay reports a host clock outside its window as an expired or unverifiable link proof. */ -export const LINK_PROOF_CLOCK_HINT = +const LINK_PROOF_CLOCK_HINT = "Hint: Check that the computer running T3 Code sets its date and time automatically, then try again."; function dpopFailureHint(reason: DpopFailureReason | undefined): string { diff --git a/packages/shared/src/clockSkew.ts b/packages/shared/src/clockSkew.ts index e34ebad774a3..47cf1686f1f4 100644 --- a/packages/shared/src/clockSkew.ts +++ b/packages/shared/src/clockSkew.ts @@ -6,7 +6,7 @@ // The relay accepts proofs issued up to 60 seconds in its future. Past that, a // skewed clock is enough on its own to explain a rejection. -export const CLOCK_SKEW_LIMIT_SECONDS = 60; +const CLOCK_SKEW_LIMIT_SECONDS = 60; /** * Seconds this machine's clock is ahead of (positive) or behind (negative) the From 674cf6f23ac0831dc093a2a0208d502475833e56 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 09:35:15 +0000 Subject: [PATCH 05/13] test(server): focus clock skew recovery and revocation coverage --- .../src/cloud/CloudLink.lifecycle.test.ts | 149 +++++++++++++----- apps/server/src/cloud/CloudLink.test.ts | 42 +---- apps/server/src/cloud/relayResponse.test.ts | 126 +++++++-------- .../src/relay/errorPresentation.test.ts | 32 +++- packages/shared/src/clockSkew.test.ts | 45 ------ 5 files changed, 202 insertions(+), 192 deletions(-) delete mode 100644 packages/shared/src/clockSkew.test.ts diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.ts index 66a4d760f9eb..0d835df85fc9 100644 --- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts +++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts @@ -1,6 +1,8 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; +import * as Clock from "effect/Clock"; +import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -715,6 +717,11 @@ describe("releaseManagedTunnelOnShutdown", () => { status: "revoked", }); expect((yield* link.linkState()).linkRevoked).toBe(true); + expect( + yield* link.startManagedTunnelIfOriginConfirmed("http://127.0.0.1:3773", { + requireConfirmedOrigin: false, + }), + ).toBe(false); // One registration request, no relink, and nothing started or removed. // linkState also sends a HEAD to read the relay's clock. expect(requests.filter((request) => request.method !== "HEAD")).toHaveLength(1); @@ -740,48 +747,97 @@ describe("releaseManagedTunnelOnShutdown", () => { ); }); - it.effect( - "does not mark a link revoked when the rejection came while the clock was wrong", - () => { - const { store, values } = makeMemorySecretStore([ - [ - CLOUD_ENDPOINT_RUNTIME_CONFIG, - '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}', - ], - [RELAY_URL_SECRET, "https://relay.example.test"], - [CLOUD_LINKED_USER_ID, "user-123"], - [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], - [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"], - ]); + it.effect("keeps registering a skewed host and reconnects after its clock is corrected", () => { + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "existing-token", + tunnelId: "existing-tunnel", + }; + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + let clockCorrected = false; - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - const error = yield* link + return Effect.gen(function* () { + const link = yield* CloudLink.CloudLink; + const firstFailure = yield* Deferred.make(); + const retryWindowExhausted = yield* Deferred.make(); + const registration = yield* retryManagedTunnelRegistration( + link .registerManagedTunnelRecovery("http://127.0.0.1:3773") - .pipe(Effect.flip); - expect(error).toMatchObject({ _tag: "RelayRequestError", clockSkewSeconds: -420 }); - expect((yield* link.linkState()).linkRevoked).toBe(false); - expect(values.has(CLOUD_LINK_REVOKED)).toBe(false); - }).pipe( - provideReleaseHarness({ - store, - applyConfigCalls: [], - requests: [], - respond: () => - Response.json( - { - _tag: "RelayAuthInvalidError", - code: "auth_invalid", - reason: "not_authorized", - traceId: "trace-clock", + .pipe(Effect.tapError(() => Deferred.succeed(firstFailure, undefined))), + CloudLink.shouldRetryCloudLink, + Deferred.succeed(retryWindowExhausted, undefined), + ).pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.raceFirst(Deferred.await(firstFailure), Fiber.await(registration)); + const state = yield* link.linkState(); + expect(state).toMatchObject({ clockSkewSeconds: -420, linkRevoked: false }); + expect(values.has(CLOUD_LINK_REVOKED)).toBe(false); + + // Clock synchronization can take longer than the normal startup retry window. + yield* TestClock.adjust("15 minutes"); + yield* Deferred.await(retryWindowExhausted); + const attempts = requests.filter((request) => request.method === "POST").length; + expect(attempts).toBeGreaterThan(1); + expect(values.has(CLOUD_LINK_REVOKED)).toBe(false); + expect(applyConfigCalls).toEqual([]); + + clockCorrected = true; + yield* TestClock.setTime((yield* Clock.currentTimeMillis) + 420_000); + yield* TestClock.adjust("30 seconds"); + expect(yield* Fiber.join(registration)).toMatchObject({ status: "ready" }); + expect(requests.filter((request) => request.method === "POST").length).toBeGreaterThan( + attempts, + ); + expect(applyConfigCalls).toEqual([config]); + const recoveredState = yield* link.linkState(); + expect(recoveredState).toMatchObject({ linked: true, linkRevoked: false }); + expect(recoveredState.clockSkewSeconds).toBeUndefined(); + expect(new TextDecoder().decode(values.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET))).toBe( + "environment-credential", + ); + expect( + Option.getOrThrow( + decodeConfirmedOrigin( + new TextDecoder().decode(values.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)), + ), + ), + ).toEqual({ config, origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 } }); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respondEffect: Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + return Response.json( + clockCorrected + ? { status: "ready" } + : { + _tag: "RelayAuthInvalidError", + code: "auth_invalid", + reason: "not_authorized", + traceId: "trace-clock", + }, + { + status: clockCorrected ? 200 : 401, + headers: { + date: DateTime.toDateUtc( + DateTime.makeUnsafe(now + (clockCorrected ? 0 : 420_000)), + ).toUTCString(), }, - // The test clock starts at the epoch, so this relay is 7 minutes ahead. - { status: 401, headers: { date: "Thu, 01 Jan 1970 00:07:00 GMT" } }, - ), + }, + ); }), - ); - }, - ); + }), + ); + }); it.effect("does not mark a link that was replaced while its rejection was in flight", () => { const { store, values } = makeMemorySecretStore([ @@ -1159,10 +1215,18 @@ describe("releaseManagedTunnelOnShutdown", () => { [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], ]); const applyConfigCalls: Array = []; + let clockCorrected = false; return Effect.gen(function* () { const link = yield* CloudLink.CloudLink; + const skewed = yield* Effect.flip(link.recoverManagedTunnel("http://127.0.0.1:3773")); + expect(CloudLink.shouldRetryCloudLink(skewed)).toBe(true); + expect((yield* link.linkState()).linkRevoked).toBe(false); + expect(values.has(CLOUD_LINK_REVOKED)).toBe(false); + + clockCorrected = true; const error = yield* Effect.flip(link.recoverManagedTunnel("http://127.0.0.1:3773")); + expect(CloudLink.shouldRetryCloudLink(error)).toBe(false); expect(error).toMatchObject({ _tag: "RelayRequestError", rejection: "unauthorized" }); expect((yield* link.linkState()).linkRevoked).toBe(true); expect(applyConfigCalls).toEqual([]); @@ -1180,7 +1244,14 @@ describe("releaseManagedTunnelOnShutdown", () => { reason: "not_authorized", traceId: "trace-recovery-revoked", }, - { status: 401 }, + { + status: 401, + headers: { + date: clockCorrected + ? "Thu, 01 Jan 1970 00:00:00 GMT" + : "Thu, 01 Jan 1970 00:07:00 GMT", + }, + }, ), }), ); diff --git a/apps/server/src/cloud/CloudLink.test.ts b/apps/server/src/cloud/CloudLink.test.ts index 8948976234a3..fd196a1b33ae 100644 --- a/apps/server/src/cloud/CloudLink.test.ts +++ b/apps/server/src/cloud/CloudLink.test.ts @@ -5,7 +5,7 @@ import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as FetchHttpClient from "effect/http/FetchHttpClient"; -import { HttpClient, HttpClientResponse, HttpServer } from "effect/http"; +import { HttpClient, HttpServer } from "effect/http"; import * as NetAddress from "effect/net/NetAddress"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -34,8 +34,6 @@ const withService = ( readonly relayFails?: boolean; readonly failActivityRead?: boolean; readonly failHoldRead?: boolean; - /** The `Date` the relay answers a clock check with. */ - readonly relayDate?: string; /** The first relay call reports itself, then waits for this before answering. */ readonly holdFirstRelayCall?: { readonly started: () => void; @@ -44,7 +42,6 @@ const withService = ( }, body: (input: { readonly preferences: { readonly update: CloudLink.CloudLink["Service"]["updatePreferences"] }; - readonly linkState: CloudLink.CloudLink["Service"]["linkState"]; readonly stored: Map; readonly relayCalls: Array; }) => Effect.Effect, @@ -108,27 +105,13 @@ const withService = ( Layer.mock(ServerConfig.ServerConfig)({} as ServerConfig.ServerConfig["Service"]), Layer.succeed( HttpClient.HttpClient, - HttpClient.make((request) => - options.relayDate === undefined - ? Effect.die("unused") - : Effect.succeed( - HttpClientResponse.fromWeb( - request, - new Response(null, { status: 404, headers: { date: options.relayDate } }), - ), - ), - ), + HttpClient.make(() => Effect.die("unused")), ), NodeServices.layer, ); return yield* Effect.gen(function* () { const link = yield* CloudLink.CloudLink; - return yield* body({ - preferences: { update: link.updatePreferences }, - linkState: link.linkState, - stored, - relayCalls, - }); + return yield* body({ preferences: { update: link.updatePreferences }, stored, relayCalls }); }).pipe( Effect.provide(CloudLink.layer.pipe(Layer.provide(layerDependencies))), Effect.provideService(FetchHttpClient.Fetch, fetch), @@ -289,22 +272,3 @@ it.effect("an update the client walks away from still finishes", () => { }), ); }); - -// The test clock starts at the epoch. -it.effect("reports how far off this machine's clock is when it breaks T3 Connect", () => - withService({ relayDate: "Thu, 01 Jan 1970 00:07:00 GMT" }, ({ linkState }) => - Effect.gen(function* () { - const state = yield* linkState(); - assert.equal(state.clockSkewSeconds, -420); - }), - ), -); - -it.effect("leaves the clock out of the link state when it is close enough", () => - withService({ relayDate: "Thu, 01 Jan 1970 00:00:30 GMT" }, ({ linkState }) => - Effect.gen(function* () { - const state = yield* linkState(); - assert.isUndefined(state.clockSkewSeconds); - }), - ), -); diff --git a/apps/server/src/cloud/relayResponse.test.ts b/apps/server/src/cloud/relayResponse.test.ts index 99094d2109d8..a851c267c418 100644 --- a/apps/server/src/cloud/relayResponse.test.ts +++ b/apps/server/src/cloud/relayResponse.test.ts @@ -1,18 +1,16 @@ import { expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as TestClock from "effect/testing/TestClock"; import { HttpClientRequest, HttpClientResponse } from "effect/http"; import * as HttpClientError from "effect/http/HttpClientError"; import { filterRelayResponse, - isRelayClockSkewError, relayRequestError, shouldRetryRelayRequest, } from "./relayResponse.ts"; -// The test clock starts at the epoch, so this relay is seven minutes ahead of it. -const RELAY_SEVEN_MINUTES_AHEAD = { date: "Thu, 01 Jan 1970 00:07:00 GMT" }; - const response = ( status: number, body: string | Record, @@ -47,18 +45,24 @@ it("reports a transport failure category without exposing request or cause detai it.effect("reports the tunnel limit and relay trace instead of a generic 403", () => Effect.gen(function* () { const error = yield* filterRelayResponse( - response(403, { - _tag: "RelayEnvironmentLinkLimitExceededError", - code: "environment_link_limit_exceeded", - maxTunnels: 3, - traceId: "trace-limit", - }), + response( + 403, + { + _tag: "RelayEnvironmentLinkLimitExceededError", + code: "environment_link_limit_exceeded", + maxTunnels: 3, + traceId: "trace-limit", + }, + { date: "Thu, 01 Jan 1970 00:07:00 GMT" }, + ), ).pipe(Effect.mapError(relayRequestError), Effect.flip); expect(error).toMatchObject({ _tag: "RelayRequestError", rejection: "forbidden" }); expect(error.message).toContain("at most 3 tunnels"); expect(error.message).toContain("Unlink an unused environment"); expect(error.message).toContain("Trace ID: trace-limit"); + expect(shouldRetryRelayRequest(error)).toBe(false); + expect(error.clockSkewSeconds).toBeUndefined(); }), ); @@ -80,48 +84,37 @@ it.effect("makes revoked authorization actionable and non-retryable", () => }), ); -it.effect("blames a refused link proof on this machine's clock when it is far off", () => +it.effect("keeps a rejected proof retryable until the host clock is within 60 seconds", () => Effect.gen(function* () { - const error = yield* filterRelayResponse( - response( - 400, - { - _tag: "RelayEnvironmentLinkProofInvalidError", - code: "environment_link_proof_invalid", - reason: "invalid_signature_or_scope", - traceId: "trace-clock", - }, - RELAY_SEVEN_MINUTES_AHEAD, - ), - ).pipe(Effect.flip); - - expect(error).toMatchObject({ rejection: "rejected", clockSkewSeconds: -420 }); - expect(isRelayClockSkewError(error)).toBe(true); - expect(shouldRetryRelayRequest(error)).toBe(true); - expect(error.message).toContain("clock is 7 minutes behind"); - expect(error.message).toContain("Trace ID: trace-clock"); - }), -); - -it.effect("keeps a refusal about the request itself permanent despite a skewed clock", () => - Effect.gen(function* () { - const error = yield* filterRelayResponse( - response( - 403, - { - _tag: "RelayEnvironmentLinkLimitExceededError", - code: "environment_link_limit_exceeded", - maxTunnels: 3, - traceId: "trace-limit", - }, - RELAY_SEVEN_MINUTES_AHEAD, - ), - ).pipe(Effect.flip); - - expect(isRelayClockSkewError(error)).toBe(false); - expect(shouldRetryRelayRequest(error)).toBe(false); - expect(error.message).not.toContain("clock"); - }), + const rejectedProof = () => + filterRelayResponse( + response( + 400, + { + _tag: "RelayEnvironmentLinkProofInvalidError", + code: "environment_link_proof_invalid", + reason: "invalid_signature_or_scope", + traceId: "trace-clock", + }, + { date: "Thu, 01 Jan 1970 00:07:00 GMT" }, + ), + ).pipe(Effect.flip); + + const skewed = yield* rejectedProof(); + expect(skewed.clockSkewSeconds).toBe(-420); + expect(shouldRetryRelayRequest(skewed)).toBe(true); + expect(skewed.message).toContain("clock is 7 minutes behind"); + expect(skewed.message).toContain('"Sync now"'); + expect(skewed.message).toContain("reconnects by itself"); + expect(skewed.message).toContain("Trace ID: trace-clock"); + + yield* TestClock.setTime(359_000); + expect(shouldRetryRelayRequest(yield* rejectedProof())).toBe(true); + yield* TestClock.setTime(360_000); + const withinTolerance = yield* rejectedProof(); + expect(withinTolerance.clockSkewSeconds).toBeUndefined(); + expect(shouldRetryRelayRequest(withinTolerance)).toBe(false); + }).pipe(Effect.provideService(HostProcess.Platform, "win32")), ); it.effect("reports an unrecognized access denial without printing its response body", () => @@ -156,23 +149,20 @@ it.effect.each([ { status: 403, attempts: 1 }, { status: 429, attempts: 2 }, { status: 503, attempts: 2 }, - { status: 401, headers: RELAY_SEVEN_MINUTES_AHEAD, attempts: 2 }, -])( - "stops rejected startup links but retries temporary failures: $status", - ({ status, headers, attempts }) => - Effect.gen(function* () { - let requests = 0; - const result = yield* Effect.suspend(() => { - requests++; - return filterRelayResponse(response(requests === 1 ? status : 200, "{}", headers)); - }).pipe( - Effect.mapError(relayRequestError), - Effect.retry({ while: shouldRetryRelayRequest, times: 1 }), - Effect.result, - ); - expect(requests).toBe(attempts); - expect(result._tag).toBe(attempts === 1 ? "Failure" : "Success"); - }), +])("stops rejected startup links but retries temporary failures: $status", ({ status, attempts }) => + Effect.gen(function* () { + let requests = 0; + const result = yield* Effect.suspend(() => { + requests++; + return filterRelayResponse(response(requests === 1 ? status : 200, "{}")); + }).pipe( + Effect.mapError(relayRequestError), + Effect.retry({ while: shouldRetryRelayRequest, times: 1 }), + Effect.result, + ); + expect(requests).toBe(attempts); + expect(result._tag).toBe(attempts === 1 ? "Failure" : "Success"); + }), ); it.effect("keeps the relay failure reason and trace when tunnel cleanup fails", () => diff --git a/packages/client-runtime/src/relay/errorPresentation.test.ts b/packages/client-runtime/src/relay/errorPresentation.test.ts index a27810c4366e..6b1e12887ded 100644 --- a/packages/client-runtime/src/relay/errorPresentation.test.ts +++ b/packages/client-runtime/src/relay/errorPresentation.test.ts @@ -1,4 +1,8 @@ -import { RelayAuthInvalidError } from "@t3tools/contracts/relay"; +import { + RelayAuthInvalidError, + RelayEnvironmentLinkProofExpiredError, + RelayEnvironmentLinkProofInvalidError, +} from "@t3tools/contracts/relay"; import { describe, expect, it } from "@effect/vitest"; import { @@ -54,4 +58,30 @@ describe("relayProtectedErrorMessage", () => { expect(relayProtectedErrorMessage(error)).toBe("Relay rejected the cloud session token."); }); + it("points clients to the host clock only when a link proof could have a time failure", () => { + const expired = new RelayEnvironmentLinkProofExpiredError({ + code: "environment_link_proof_expired", + traceId: "trace-expired", + }); + const invalid = new RelayEnvironmentLinkProofInvalidError({ + code: "environment_link_proof_invalid", + reason: "invalid_signature_or_scope", + traceId: "trace-invalid", + }); + const mismatch = new RelayEnvironmentLinkProofInvalidError({ + code: "environment_link_proof_invalid", + reason: "descriptor_mismatch", + traceId: "trace-mismatch", + }); + + expect(relayProtectedErrorMessage(expired)).toContain( + "computer running T3 Code sets its date and time automatically", + ); + expect(relayProtectedErrorMessage(invalid)).toContain( + "computer running T3 Code sets its date and time automatically", + ); + expect(relayProtectedErrorMessage(mismatch)).toBe( + "Relay rejected the environment link proof (descriptor_mismatch).", + ); + }); }); diff --git a/packages/shared/src/clockSkew.test.ts b/packages/shared/src/clockSkew.test.ts deleted file mode 100644 index c7b5462872d1..000000000000 --- a/packages/shared/src/clockSkew.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { describe, expect, it } from "vite-plus/test"; - -import { clockSkewFromDateHeader, describeClockSkew } from "./clockSkew.ts"; - -const relayNow = Date.parse("Wed, 14 Oct 2026 09:00:00 GMT"); - -describe("clockSkewFromDateHeader", () => { - it("reports a clock that is minutes behind the relay", () => { - expect(clockSkewFromDateHeader("Wed, 14 Oct 2026 09:00:00 GMT", relayNow - 7 * 60_000)).toBe( - -420, - ); - }); - - it("reports a clock that is hours ahead of the relay", () => { - expect( - clockSkewFromDateHeader("Wed, 14 Oct 2026 09:00:00 GMT", relayNow + 3 * 60 * 60_000), - ).toBe(10_800); - }); - - it("ignores a gap the relay still accepts", () => { - expect(clockSkewFromDateHeader("Wed, 14 Oct 2026 09:00:00 GMT", relayNow + 59_000)).toBeNull(); - expect(clockSkewFromDateHeader("Wed, 14 Oct 2026 09:00:00 GMT", relayNow - 59_000)).toBeNull(); - }); - - it("cannot judge a response without a usable date", () => { - expect(clockSkewFromDateHeader(undefined, relayNow + 3_600_000)).toBeNull(); - expect(clockSkewFromDateHeader("not a date", relayNow + 3_600_000)).toBeNull(); - }); -}); - -describe("describeClockSkew", () => { - it("says how far off the clock is and how to fix it on Windows", () => { - const message = describeClockSkew(-420, "win32"); - expect(message).toContain("This computer's clock is 7 minutes behind"); - expect(message).toContain('"Sync now"'); - expect(message).toContain("reconnects by itself"); - }); - - it("uses the host's own settings path on macOS and Linux", () => { - expect(describeClockSkew(3_600, "darwin")).toContain("1 hour ahead"); - expect(describeClockSkew(3_600, "darwin")).toContain("System Settings"); - expect(describeClockSkew(2 * 86_400, "linux")).toContain("2 days ahead"); - expect(describeClockSkew(2 * 86_400, "linux")).toContain("timedatectl set-ntp true"); - }); -}); From 9837a08e4f29ea2c1c185f49db5718f7af77417b Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 09:47:42 +0000 Subject: [PATCH 06/13] fix(server): recover T3 Connect after host clock correction --- .../src/cloud/CloudLink.lifecycle.test.ts | 28 +++- apps/server/src/cloud/CloudLink.ts | 37 +++--- .../src/cloud/managedTunnelStartup.test.ts | 113 ++++++++++++++++ apps/server/src/cloud/managedTunnelStartup.ts | 55 +++++++- apps/server/src/cloud/relayClock.ts | 18 +-- apps/server/src/cloud/relayResponse.ts | 28 +--- apps/server/src/server.ts | 125 +++++------------- .../settings/ConnectionsSettings.tsx | 13 +- docs/user/remote-access.md | 4 +- packages/contracts/src/environmentHttp.ts | 1 + packages/shared/src/clockSkew.ts | 19 +-- 11 files changed, 269 insertions(+), 172 deletions(-) diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.ts index b99913be2767..4bd5576a892b 100644 --- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts +++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts @@ -17,6 +17,8 @@ import * as Stream from "effect/Stream"; import { HttpClient, HttpClientResponse, HttpServer, type HttpClientRequest } from "effect/http"; import * as NetAddress from "effect/net/NetAddress"; +import * as HostProcess from "@t3tools/shared/HostProcess"; + import { DESKTOP_UPDATE_RESTART_MARKER_FILE, EnvironmentId } from "@t3tools/contracts"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; @@ -686,6 +688,11 @@ describe("releaseManagedTunnelOnShutdown", () => { return Effect.gen(function* () { const link = yield* CloudLink.CloudLink; + expect( + yield* link.startManagedTunnelIfOriginConfirmed("http://127.0.0.1:3773", { + requireConfirmedOrigin: false, + }), + ).toBe(true); expect(yield* link.registerManagedTunnelRecovery("http://127.0.0.1:3773")).toEqual({ status: "revoked", }); @@ -695,10 +702,15 @@ describe("releaseManagedTunnelOnShutdown", () => { requireConfirmedOrigin: false, }), ).toBe(false); - // One registration request, no relink, and nothing started or removed. - // linkState also sends a HEAD to read the relay's clock. expect(requests.filter((request) => request.method !== "HEAD")).toHaveLength(1); - expect(applyConfigCalls).toEqual([]); + expect(applyConfigCalls).toEqual([ + { + providerKind: "cloudflare_tunnel", + connectorToken: "existing-token", + tunnelId: "existing-tunnel", + }, + null, + ]); expect(values.has(RELAY_ENVIRONMENT_CREDENTIAL_SECRET)).toBe(true); expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); }).pipe( @@ -750,7 +762,11 @@ describe("releaseManagedTunnelOnShutdown", () => { ).pipe(Effect.forkChild({ startImmediately: true })); yield* Effect.raceFirst(Deferred.await(firstFailure), Fiber.await(registration)); const state = yield* link.linkState(); - expect(state).toMatchObject({ clockSkewSeconds: -420, linkRevoked: false }); + expect(state).toMatchObject({ + clockSkewSeconds: -420, + linkRevoked: false, + hostPlatform: yield* HostProcess.Platform, + }); expect(values.has(CLOUD_LINK_REVOKED)).toBe(false); // Clock synchronization can take longer than the normal startup retry window. @@ -1202,7 +1218,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(CloudLink.shouldRetryCloudLink(error)).toBe(false); expect(error).toMatchObject({ _tag: "RelayRequestError", rejection: "unauthorized" }); expect((yield* link.linkState()).linkRevoked).toBe(true); - expect(applyConfigCalls).toEqual([]); + expect(applyConfigCalls).toEqual([null]); expect(values.has(RELAY_ENVIRONMENT_CREDENTIAL_SECRET)).toBe(true); }).pipe( provideReleaseHarness({ @@ -1358,7 +1374,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(error).toMatchObject({ _tag: "RelayRequestError", rejection }); expect(CloudLink.shouldRetryCloudLink(error)).toBe(false); expect(requests).toHaveLength(1); - expect(applyConfigCalls).toEqual([]); + expect(applyConfigCalls).toEqual(status === 401 ? [null] : []); }).pipe( provideReleaseHarness({ store, diff --git a/apps/server/src/cloud/CloudLink.ts b/apps/server/src/cloud/CloudLink.ts index a8e1c48078d2..f77743511d1c 100644 --- a/apps/server/src/cloud/CloudLink.ts +++ b/apps/server/src/cloud/CloudLink.ts @@ -43,6 +43,7 @@ import { import { isSecureRelayUrl } from "@t3tools/shared/relayUrl"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -647,8 +648,12 @@ const make = Effect.gen(function* () { }) { return yield* endpointRuntime.withLinkStateLock( Effect.gen(function* () { - const currentConfig = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); + const [currentConfig, revoked] = yield* Effect.all([ + secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + secrets.get(CLOUD_LINK_REVOKED), + ]); if ( + Option.isSome(revoked) || Option.isNone(currentConfig) || bytesToString(currentConfig.value) !== input.configJson ) { @@ -710,11 +715,12 @@ const make = Effect.gen(function* () { }); return yield* endpointRuntime.withLinkStateLock( Effect.gen(function* () { - const [runtimeBytes, markerBytes] = yield* Effect.all([ + const [runtimeBytes, markerBytes, revoked] = yield* Effect.all([ secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), secrets.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN), + secrets.get(CLOUD_LINK_REVOKED), ]); - if (Option.isNone(runtimeBytes)) return false; + if (Option.isNone(runtimeBytes) || Option.isSome(revoked)) return false; const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeBytes.value))); if (config === null || config.providerKind !== "cloudflare_tunnel") return false; // With the marker required, only a config the relay already confirmed on @@ -969,14 +975,8 @@ const make = Effect.gen(function* () { ); }); - /** - * Marks the link revoked after the relay answered 401 to `credential`, usually - * because the environment was deregistered. Retrying cannot fix that, and - * relinking is the user's call, so only mark it. Nothing stored is removed, - * and the next accepted registration clears the mark. Returns false, without - * marking, when a relink replaced the credential while the rejected request - * was in flight. - */ + // Check the credential under the same lock as connector startup so an old + // rejection cannot stop a replacement link. const markLinkRevoked = (credential: string, cause: RelayRequestError) => endpointRuntime.withLinkStateLock( Effect.gen(function* () { @@ -989,6 +989,7 @@ const make = Effect.gen(function* () { { cause }, ); yield* secrets.set(CLOUD_LINK_REVOKED, stringToBytes("true")); + yield* endpointRuntime.applyConfig(null); return true; }), ); @@ -1338,12 +1339,18 @@ const make = Effect.gen(function* () { const linkState = Effect.fn("environment.cloud.linkState")( function* () { const state = yield* readCloudLinkState(); - // Only a configured link has a relay whose clock matters. - if (state.relayUrl === null) return state; - const clockSkewSeconds = yield* checkRelayClock(state.relayUrl).pipe( + const relayUrl = + state.relayUrl ?? + ((yield* readCliDesiredCloudLink.pipe(withSecrets)) + ? Option.getOrNull(yield* relayUrlConfig.pipe(Effect.option)) + : null); + if (relayUrl === null) return state; + const clockSkewSeconds = yield* checkRelayClock(relayUrl).pipe( Effect.provideService(HttpClient.HttpClient, httpClient), ); - return clockSkewSeconds === null ? state : { ...state, clockSkewSeconds }; + return clockSkewSeconds === null + ? state + : { ...state, clockSkewSeconds, hostPlatform: yield* HostProcess.Platform }; }, Effect.catchIf(ServerSecretStore.isSecretStoreError, internalError("read-relay-config")), ); diff --git a/apps/server/src/cloud/managedTunnelStartup.test.ts b/apps/server/src/cloud/managedTunnelStartup.test.ts index e87f3e8e363e..c69becb5cded 100644 --- a/apps/server/src/cloud/managedTunnelStartup.test.ts +++ b/apps/server/src/cloud/managedTunnelStartup.test.ts @@ -1,4 +1,6 @@ import { describe, expect, it } from "@effect/vitest"; +import * as Clock from "effect/Clock"; +import { RelayRequestError, shouldRetryRelayRequest } from "./relayResponse.ts"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -6,6 +8,8 @@ import * as TestClock from "effect/testing/TestClock"; import { managedTunnelStartupAction, + makeManagedTunnelRecovery, + retryDesiredManagedTunnelLink, retryManagedTunnelRegistration, } from "./managedTunnelStartup.ts"; @@ -129,3 +133,112 @@ describe("retryManagedTunnelRegistration", () => { }), ); }); + +it.effect( + "retries the first desired link beyond ten minutes until clock correction, but stops on a permanent refusal", + () => + Effect.gen(function* () { + let corrected = false; + let attempts = 0; + const firstAttempt = yield* Deferred.make(); + const reconcile = Effect.suspend(() => { + attempts += 1; + return Deferred.succeed(firstAttempt, undefined).pipe( + Effect.andThen( + corrected + ? Effect.succeed("managed" as const) + : Effect.fail( + new RelayRequestError({ + rejection: "unauthorized", + description: "clock skew", + clockSkewSeconds: -420, + }), + ), + ), + ); + }); + const fiber = yield* Effect.forkChild( + retryDesiredManagedTunnelLink(reconcile, shouldRetryRelayRequest), + { startImmediately: true }, + ); + yield* Deferred.await(firstAttempt); + yield* TestClock.adjust("15 minutes"); + expect(attempts).toBeGreaterThan(20); + corrected = true; + yield* TestClock.adjust("30 seconds"); + expect(yield* Fiber.join(fiber)).toBe("managed"); + let rejectedAttempts = 0; + const error = yield* Effect.flip( + retryDesiredManagedTunnelLink( + Effect.suspend(() => { + rejectedAttempts += 1; + return Effect.fail( + new RelayRequestError({ rejection: "unauthorized", description: "revoked" }), + ); + }), + shouldRetryRelayRequest, + ), + ); + expect(error.rejection).toBe("unauthorized"); + expect(rejectedAttempts).toBe(1); + }), +); + +it.effect("keeps recovery cooldowns through forward and backward wall-clock corrections", () => + Effect.gen(function* () { + const clock = yield* Clock.Clock; + let wallOffset = 3_600_000; + const wallMillis = () => clock.currentTimeMillisUnsafe() + wallOffset; + const hostClock = { + ...clock, + currentTimeMillisUnsafe: wallMillis, + currentTimeMillis: Effect.sync(wallMillis), + currentTimeNanosUnsafe: () => BigInt(wallMillis()) * 1_000_000n, + currentTimeNanos: Effect.sync(() => BigInt(wallMillis()) * 1_000_000n), + }; + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "token", + tunnelId: "tunnel", + }; + let attempts = 0; + const firstAttempt = yield* Deferred.make(); + const recover = yield* makeManagedTunnelRecovery( + () => + Effect.gen(function* () { + attempts += 1; + if (attempts === 1) { + yield* Deferred.succeed(firstAttempt, undefined); + return yield* new RelayRequestError({ + rejection: "unauthorized", + description: "clock skew", + clockSkewSeconds: 3600, + }); + } + return true; + }), + shouldRetryRelayRequest, + ); + const fiber = yield* Effect.forkChild( + recover(config).pipe(Effect.provideService(Clock.Clock, hostClock)), + { startImmediately: true }, + ); + yield* Deferred.await(firstAttempt); + wallOffset = 0; + yield* TestClock.adjust("119 seconds"); + expect(attempts).toBe(1); + yield* TestClock.adjust("1 second"); + expect(attempts).toBe(2); + expect(yield* Fiber.join(fiber)).toBe(true); + wallOffset = 3_600_000; + const next = yield* Effect.forkChild( + recover(config).pipe(Effect.provideService(Clock.Clock, hostClock)), + { startImmediately: true }, + ); + yield* TestClock.adjust("119 seconds"); + expect(attempts).toBe(2); + yield* TestClock.adjust("1 second"); + expect(attempts).toBe(3); + expect(yield* Fiber.join(next)).toBe(true); + }), +); diff --git a/apps/server/src/cloud/managedTunnelStartup.ts b/apps/server/src/cloud/managedTunnelStartup.ts index bf442e848ad7..baede590c4bf 100644 --- a/apps/server/src/cloud/managedTunnelStartup.ts +++ b/apps/server/src/cloud/managedTunnelStartup.ts @@ -1,8 +1,12 @@ import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; +import * as Clock from "effect/Clock"; +import * as Semaphore from "effect/Semaphore"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Schedule from "effect/Schedule"; +import { isRelayClockSkewError } from "./relayResponse.ts"; + // "revoked" leads to no action: relinking a deregistered environment is the // user's call, so startup neither retries nor relinks it. export type ManagedTunnelRegistrationResult = @@ -69,9 +73,58 @@ export const retryManagedTunnelRegistration = ( // Every managed host shares one relay, so a host stuck in a bad loop must not // turn into a fleet-wide request storm. -export const MANAGED_TUNNEL_RECOVERY_COOLDOWN = Duration.minutes(2); +const MANAGED_TUNNEL_RECOVERY_COOLDOWN = Duration.minutes(2); // Existing hosts register on their first boot after an upgrade, and desktop // auto-update delivers that boot to many hosts at once. Spread the first // registration so the relay and Cloudflare see a ramp instead of a spike. export const MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER = Duration.seconds(30); + +export const retryDesiredManagedTunnelLink = ( + reconcile: Effect.Effect, + isRetryable: (error: E) => boolean, +) => + reconcile.pipe( + Effect.retry({ while: isRelayClockSkewError, schedule: Schedule.spaced("30 seconds") }), + Effect.retry({ + while: isRetryable, + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), + ), + }), + ); + +export const makeManagedTunnelRecovery = Effect.fnUntraced(function* ( + recover: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect, + isRetryable: (error: E) => boolean, +) { + const lock = yield* Semaphore.make(1); + let lastAttemptNanos: bigint | null = null; + return (config: RelayManagedEndpointRuntimeConfig) => + lock.withPermits(1)( + Effect.gen(function* () { + // Clock corrections must neither bypass nor extend the relay cooldown. + const now = yield* Clock.monotonicTimeNanos; + if (lastAttemptNanos !== null) { + const elapsedMillis = Number(now - lastAttemptNanos) / 1_000_000; + const wait = Duration.toMillis(MANAGED_TUNNEL_RECOVERY_COOLDOWN) - elapsedMillis; + if (wait > 0) yield* Effect.sleep(Duration.millis(wait)); + } + lastAttemptNanos = yield* Clock.monotonicTimeNanos; + return yield* recover(config); + }).pipe( + Effect.retry({ + while: isRetryable, + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.jittered, + ), + }), + ), + ); +}); diff --git a/apps/server/src/cloud/relayClock.ts b/apps/server/src/cloud/relayClock.ts index 2a4f0ae4b5ba..97125432a61e 100644 --- a/apps/server/src/cloud/relayClock.ts +++ b/apps/server/src/cloud/relayClock.ts @@ -3,17 +3,13 @@ import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; import { HttpClient, HttpClientRequest } from "effect/http"; -/** - * Seconds this machine's clock is off from the relay's when that is enough to - * break T3 Connect, `null` when it is close enough or the relay cannot be - * reached. Any relay response carries its time, so the request needs no auth. - */ -export const checkRelayClock = (relayUrl: string) => - Effect.gen(function* () { +// Status also works before authorization succeeds, so this probe needs no credentials. +export const checkRelayClock = Effect.fn("cloud.checkRelayClock")( + function* (relayUrl: string) { const httpClient = yield* HttpClient.HttpClient; const response = yield* httpClient.execute(HttpClientRequest.head(`${relayUrl}/`)); return clockSkewFromDateHeader(response.headers.date, yield* Clock.currentTimeMillis); - }).pipe( - Effect.timeout("2 seconds"), - Effect.orElseSucceed(() => null), - ); + }, + Effect.timeout("2 seconds"), + Effect.orElseSucceed(() => null), +); diff --git a/apps/server/src/cloud/relayResponse.ts b/apps/server/src/cloud/relayResponse.ts index cb96da64e80a..3b287fce9fbc 100644 --- a/apps/server/src/cloud/relayResponse.ts +++ b/apps/server/src/cloud/relayResponse.ts @@ -8,13 +8,7 @@ import * as Schema from "effect/Schema"; import * as HttpClientResponse from "effect/http/HttpClientResponse"; import { isHttpClientError } from "effect/http/HttpClientError"; -/** - * A relay request that did not succeed. `unavailable` means it may succeed on - * retry; the relay refused the other kinds. A refusal with `clockSkewSeconds` - * happened while this machine's clock was that far from the relay's, so it is - * retried in case fixing the clock is all it needs. The description is safe to show: it - * carries the relay's own explanation and trace ID, never request details. - */ +// Descriptions carry relay diagnostics and trace IDs, never request credentials. export class RelayRequestError extends Schema.TaggedError()( "RelayRequestError", { @@ -39,10 +33,6 @@ export function relayRequestError(cause: unknown): RelayRequestError { }); } -/** - * Whether a failure may succeed on retry: anything but a relay refusal, unless - * the clock was off when the relay refused. - */ export const shouldRetryRelayRequest = (error: unknown): boolean => !isRelayRequestError(error) || error.rejection === "unavailable" || @@ -60,7 +50,7 @@ function recoveryHint(error: RelayProtectedError): string { return "Run `t3 connect login` to check this machine's authorization. If the stored credential was revoked, sign out with `t3 connect logout`, then run `t3 connect` again. Restart T3 Code after signing in."; case "RelayEnvironmentLinkProofExpiredError": case "RelayEnvironmentLinkProofInvalidError": - return "Check this machine's date and time, update T3 Code, then restart it."; + return "Check this machine's date and time. T3 Connect retries expired proofs automatically; update T3 Code if this continues."; default: return "Retry when the relay is available. If this continues, include the trace ID when reporting it."; } @@ -82,14 +72,11 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func const description = Option.isSome(decoded) ? `T3 Connect: ${decoded.value.message}. ${recoveryHint(decoded.value)} Trace ID: ${decoded.value.traceId}.` : `T3 Connect relay returned HTTP ${response.status} without a recognized error response. Check relay access and any proxy or firewall restrictions, then restart T3 Code.${requestId}`; - // The relay checks the timestamps this machine signs against its own clock - // and answers 400 or 401 when they fall outside its window. A skewed clock - // explains that refusal, and the request cannot succeed until it is fixed. - // Other refusals (a tunnel limit, a missing link) are about the request. + // Timestamp rejections can clear after clock sync; unrelated HTTP statuses stay permanent. const clockSuspected = clockSkewSeconds !== null && (response.status === 400 || response.status === 401); const platform = yield* HostProcess.Platform; - const refusal = (rejection: "unauthorized" | "forbidden" | "rejected") => + const refusal = (rejection: RelayRequestError["rejection"]) => !clockSuspected ? new RelayRequestError({ rejection, description }) : new RelayRequestError({ @@ -104,16 +91,13 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func clockSkewSeconds, }); - // The only 401 that clears itself: a proof signed with a skewed clock (a host - // that booted before NTP) expires on arrival, and the next attempt signs a new - // one. It is recognized only from a schema-decoded relay body. Every other - // 401, including a revoked bearer or an undecodable body, stays permanent. + // A decoded expired proof remains retryable even without a usable Date header. if ( response.status === 401 && Option.isSome(decoded) && decoded.value._tag === "RelayEnvironmentLinkProofExpiredError" ) { - return yield* new RelayRequestError({ rejection: "unavailable", description }); + return yield* refusal("unavailable"); } if (response.status === 401) { return yield* refusal("unauthorized"); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index e9afb5694c78..c16893fac411 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,8 +1,5 @@ import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; -import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; -import * as Clock from "effect/Clock"; import * as Random from "effect/Random"; -import * as Semaphore from "effect/Semaphore"; import * as DatabaseStorage from "./persistence/DatabaseStorage.ts"; import * as StorageCleanup from "./storageCleanup.ts"; import * as PullRequestSyncReactor from "./orchestration-v2/PullRequestSyncReactor.ts"; @@ -21,7 +18,6 @@ import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; -import * as Schedule from "effect/Schedule"; import { FetchHttpClient, HttpRouter, HttpServer } from "effect/http"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; @@ -145,7 +141,8 @@ import * as RelayTracing from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; import { MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER, - MANAGED_TUNNEL_RECOVERY_COOLDOWN, + makeManagedTunnelRecovery, + retryDesiredManagedTunnelLink, managedTunnelStartupAction, retryManagedTunnelRegistration, } from "./cloud/managedTunnelStartup.ts"; @@ -850,12 +847,6 @@ const layerMakeServer = Layer.unwrap( if (typeof address === "string" || !("port" in address)) return; const localOrigin = `http://127.0.0.1:${address.port}`; const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; - const recoveryLock = yield* Semaphore.make(1); - // Monotonic, so a wall clock corrected backward (a host that booted - // ahead) cannot push the next recovery hours into the future. - let lastRecoveryAtNanos: bigint | null = null; - // Clock refusals retry until the clock is fixed, which can be hours, - // and a retry only logs once it gives up. Say once why it waits. let clockSkewReported = false; const reportClockSkew = (error: unknown) => isRelayClockSkewError(error) && !clockSkewReported @@ -867,34 +858,20 @@ const layerMakeServer = Layer.unwrap( ), ) : Effect.void; - const recoverManagedTunnel = (config: RelayManagedEndpointRuntimeConfig) => - recoveryLock.withPermits(1)( - Effect.gen(function* () { - const now = yield* Clock.monotonicTimeNanos; - if (lastRecoveryAtNanos !== null) { - const elapsed = Number((now - lastRecoveryAtNanos) / 1_000_000n); - const wait = Duration.toMillis(MANAGED_TUNNEL_RECOVERY_COOLDOWN) - elapsed; - if (wait > 0) yield* Effect.sleep(Duration.millis(wait)); - } - lastRecoveryAtNanos = yield* Clock.monotonicTimeNanos; - }).pipe( - Effect.andThen( - cloudLink.recoverManagedTunnel(localOrigin, config, { - retryRuntimeFailures: true, - }), - ), - Effect.tapError(reportClockSkew), - Effect.retry({ - while: (error) => - CloudLink.shouldRetryCloudLink(error) && - error._tag !== "CloudLinkEndpointUnavailableError", - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), - ), - Schedule.jittered, - ), - }), + const recoverManagedTunnel = yield* makeManagedTunnelRecovery( + (config) => + cloudLink + .recoverManagedTunnel(localOrigin, config, { + retryRuntimeFailures: true, + }) + .pipe(Effect.tapError(reportClockSkew)), + (error) => + CloudLink.shouldRetryCloudLink(error) && + error._tag !== "CloudLinkEndpointUnavailableError", + ); + yield* endpointRuntime.recoveryRequests.pipe( + Stream.runForEach((config) => + recoverManagedTunnel(config).pipe( Effect.tap((recovered) => recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, ), @@ -906,9 +883,7 @@ const layerMakeServer = Layer.unwrap( }), ), ), - ); - yield* endpointRuntime.recoveryRequests.pipe( - Stream.runForEach(recoverManagedTunnel), + ), Effect.forkScoped, ); const wantsCliLink = hasCloudPublicConfig @@ -960,28 +935,12 @@ const layerMakeServer = Layer.unwrap( ), Effect.asVoid, ); - // A clock refusal used to end registration and start the stored - // tunnel at once. It now retries until the clock is fixed, so start - // the tunnel on the first one rather than after the retry window. - let startedForClockSkew = startedConfirmed; - const startStoredTunnelOnClockSkew = (error: unknown) => - isRelayClockSkewError(error) && !startedForClockSkew - ? Effect.sync(() => { - startedForClockSkew = true; - }).pipe(Effect.andThen(startStoredManagedTunnel)) - : Effect.void; const registerManagedTunnel = retryManagedTunnelRegistration( cloudLink .registerManagedTunnelRecovery(localOrigin, { retryRuntimeFailures: true, }) - .pipe( - Effect.tapError((error) => - reportClockSkew(error).pipe( - Effect.andThen(startStoredTunnelOnClockSkew(error)), - ), - ), - ), + .pipe(Effect.tapError(reportClockSkew)), (error) => CloudLink.shouldRetryCloudLink(error) && error._tag !== "CloudLinkEndpointUnavailableError", @@ -1021,37 +980,23 @@ const layerMakeServer = Layer.unwrap( yield* endpointRuntime.requestRecovery(startupAction.config); } if (startupAction.action === "reconcile_link") { - const reconciledMode = yield* cloudLink - .reconcileDesiredLinkIfStillDesired(localOrigin) - .pipe( - // A host often boots before its clock syncs. Keep trying for as - // long as the clock is what the relay refuses, so the link comes - // up as soon as the clock is fixed instead of after a restart. - Effect.tapError(reportClockSkew), - Effect.retry({ - while: isRelayClockSkewError, - schedule: Schedule.spaced("30 seconds"), - }), - Effect.retry({ - while: CloudLink.shouldRetryCloudLink, - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), - ), - Schedule.upTo({ duration: "10 minutes" }), - ), - }), - Effect.tap((mode) => - mode === null - ? Effect.void - : Effect.logInfo("T3 Connect desired link reconciled on startup"), - ), - Effect.catch((cause) => - Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { - cause, - }).pipe(Effect.as(null)), - ), - ); + const reconciledMode = yield* retryDesiredManagedTunnelLink( + cloudLink + .reconcileDesiredLinkIfStillDesired(localOrigin) + .pipe(Effect.tapError(reportClockSkew)), + CloudLink.shouldRetryCloudLink, + ).pipe( + Effect.tap((mode) => + mode === null + ? Effect.void + : Effect.logInfo("T3 Connect desired link reconciled on startup"), + ), + Effect.catch((cause) => + Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { + cause, + }).pipe(Effect.as(null)), + ), + ); if (reconciledMode === "managed") { const afterReconcile = yield* registerManagedTunnel; if (afterReconcile.status === "recovery_required") { diff --git a/apps/web/src/components/settings/ConnectionsSettings.tsx b/apps/web/src/components/settings/ConnectionsSettings.tsx index 43d0427fdf12..0d90f7d10418 100644 --- a/apps/web/src/components/settings/ConnectionsSettings.tsx +++ b/apps/web/src/components/settings/ConnectionsSettings.tsx @@ -61,7 +61,7 @@ import * as DateTime from "effect/DateTime"; import * as Option from "effect/Option"; import { useCopyToClipboard } from "../../hooks/useCopyToClipboard"; -import { cn, isMacPlatform, isWindowsPlatform } from "../../lib/utils"; +import { cn } from "../../lib/utils"; import { isLocalEnvironmentDisabled } from "../../localEnvironment"; import { formatElapsedDurationLabel, formatExpiresInLabel } from "../../timestampFormat"; import { resolveDesktopPairingUrl, resolveHostedPairingUrl } from "./pairingUrls"; @@ -1807,20 +1807,11 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b ? "Your session does not have permission to manage T3 Connect access." : null; const isBusy = isUpdating || isUpdatingPreference; - // This row only renders in the desktop app, so the browser's platform is the - // host's, which is the clock that needs fixing. const clockSkewSeconds = primaryCloudLinkState.data?.clockSkewSeconds; const clockSkewWarning = clockSkewSeconds === undefined ? null - : describeClockSkew( - clockSkewSeconds, - isWindowsPlatform(navigator.platform) - ? "win32" - : isMacPlatform(navigator.platform) - ? "darwin" - : "linux", - ); + : describeClockSkew(clockSkewSeconds, primaryCloudLinkState.data?.hostPlatform ?? ""); // The fix happens outside the app, so check again when the user comes back. const refreshLinkState = primaryCloudLinkState.refresh; useEffect(() => { diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index ab4ddd71bc81..d2bc7d2b9f46 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -339,8 +339,8 @@ when SSH closes, see [background-service troubleshooting](./background-service.m | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | | `environment_link_limit_exceeded` or managed tunnel limit | Deregister an unused environment, then restart T3 Code on the host. | | `auth_invalid` or `invalid_bearer` | Run `t3 connect login`. If credentials were revoked, run `t3 connect logout`, then `t3 connect` again. Restart the server after signing in. | -| "This computer's clock is … behind" or "… ahead" | Turn on automatic date and time on the host and sync it. T3 Connect reconnects by itself, without a restart. | -| Expired or invalid link proof | Check the host's date and time, update T3 Code, then restart it. | +| "The host computer's clock is … behind" or "… ahead" | Turn on automatic date and time on the host and sync it. T3 Connect reconnects by itself, without a restart. | +| Expired or invalid link proof | Check the host's automatic date and time. T3 Connect retries expired proofs automatically. Update T3 Code if it continues. | | HTTP 403 without a recognized error | Check relay access, proxies, and firewall rules. Keep any Cloudflare Ray ID for a bug report. | | HTTP 408, 429, or 5xx | Check network and relay availability. Startup retries temporary failures for up to ten minutes. | diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 7334d82771ea..fa278f34b5b8 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -420,6 +420,7 @@ export const EnvironmentCloudLinkStateResult = Schema.Struct({ // Seconds this environment's clock is ahead of (positive) or behind the // relay's, present only when the gap is large enough to break T3 Connect. clockSkewSeconds: Schema.optional(Schema.Number), + hostPlatform: Schema.optional(Schema.String), }); export type EnvironmentCloudLinkStateResult = typeof EnvironmentCloudLinkStateResult.Type; diff --git a/packages/shared/src/clockSkew.ts b/packages/shared/src/clockSkew.ts index 47cf1686f1f4..971343b1b73e 100644 --- a/packages/shared/src/clockSkew.ts +++ b/packages/shared/src/clockSkew.ts @@ -1,11 +1,4 @@ -/** - * T3 Connect's signed requests only work while the host clock is close to the - * relay's. These helpers decide when it is too far off and tell the user how - * to fix it, so the server, CLI and settings all say the same thing. - */ - -// The relay accepts proofs issued up to 60 seconds in its future. Past that, a -// skewed clock is enough on its own to explain a rejection. +// The relay permits proofs issued up to 60 seconds in its future. const CLOCK_SKEW_LIMIT_SECONDS = 60; /** @@ -49,14 +42,12 @@ function clockFixInstructions(platform: string): string { if (platform === "darwin") { return 'Open System Settings > General > Date & Time and turn on "Set time and date automatically".'; } - return "Turn on network time sync, for example with `timedatectl set-ntp true`."; + return platform === "linux" + ? "Turn on network time sync, for example with `timedatectl set-ntp true`." + : "Turn on automatic date and time on the computer running T3 Code."; } -/** - * What to tell the user when T3 Connect fails because of `seconds` of skew. - * `platform` is the host's `process.platform`, since that is the clock to fix. - */ export function describeClockSkew(seconds: number, platform: string): string { const direction = seconds > 0 ? "ahead" : "behind"; - return `This computer's clock is ${skewAmount(seconds)} ${direction}, and T3 Connect needs it to be right. ${clockFixInstructions(platform)} T3 Connect reconnects by itself once the clock is correct.`; + return `The host computer's clock is ${skewAmount(seconds)} ${direction}, and T3 Connect needs it to be right. ${clockFixInstructions(platform)} T3 Connect reconnects by itself once the clock is correct.`; } From aad0f0e00a582920e5871218a3ccd392eda9c323 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 10:09:48 +0000 Subject: [PATCH 07/13] fix(cloud): show accurate host clock diagnostics --- apps/server/src/cloud/relayResponse.ts | 2 +- apps/web/src/components/settings/ConnectionsSettings.tsx | 9 +++++++-- packages/shared/src/clockSkew.ts | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/server/src/cloud/relayResponse.ts b/apps/server/src/cloud/relayResponse.ts index 3b287fce9fbc..fb0f9bc4b2d4 100644 --- a/apps/server/src/cloud/relayResponse.ts +++ b/apps/server/src/cloud/relayResponse.ts @@ -83,7 +83,7 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func rejection, // The relay's own recovery advice does not apply until the clock is // fixed, so keep only what it said and its trace. - description: `${describeClockSkew(clockSkewSeconds, platform)} Relay response: ${ + description: `${describeClockSkew(clockSkewSeconds, platform)} T3 Connect reconnects by itself once the clock is correct. Relay response: ${ Option.isSome(decoded) ? `${decoded.value.message}. Trace ID: ${decoded.value.traceId}.` : `HTTP ${response.status}.${requestId}` diff --git a/apps/web/src/components/settings/ConnectionsSettings.tsx b/apps/web/src/components/settings/ConnectionsSettings.tsx index 0d90f7d10418..ce21ee146c79 100644 --- a/apps/web/src/components/settings/ConnectionsSettings.tsx +++ b/apps/web/src/components/settings/ConnectionsSettings.tsx @@ -1915,10 +1915,15 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b /> } /> - ) : linkRevoked ? ( + ) : linkRevoked || clockSkewWarning !== null ? ( ) : null} 0 ? "ahead" : "behind"; - return `The host computer's clock is ${skewAmount(seconds)} ${direction}, and T3 Connect needs it to be right. ${clockFixInstructions(platform)} T3 Connect reconnects by itself once the clock is correct.`; + return `The host computer's clock is ${skewAmount(seconds)} ${direction}, and T3 Connect needs it to be right. ${clockFixInstructions(platform)}`; } From 4a3c345bbc2e73f451d17260c37df97409817985 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 10:22:23 +0000 Subject: [PATCH 08/13] test(cloud): cover stale successes after link revocation --- .../src/cloud/CloudLink.lifecycle.test.ts | 128 +++++++++++++++++- 1 file changed, 124 insertions(+), 4 deletions(-) diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.ts index 4bd5576a892b..206e44012c48 100644 --- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts +++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts @@ -12,6 +12,7 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import * as TestClock from "effect/testing/TestClock"; import * as Stream from "effect/Stream"; import { HttpClient, HttpClientResponse, HttpServer, type HttpClientRequest } from "effect/http"; @@ -273,9 +274,9 @@ describe("releaseManagedTunnelOnShutdown", () => { return Option.getOrThrow(mtime).getTime(); }); - const provideReleaseHarness = - (harness: ReleaseHarness) => - (effect: Effect.Effect) => + const provideReleaseHarness = (harness: ReleaseHarness) => { + const linkStateLock = Semaphore.makeUnsafe(1); + return (effect: Effect.Effect) => effect.pipe( Effect.provide(CloudLink.layer), Effect.provideService(HttpServer.HttpServer, idleHttpServer), @@ -307,7 +308,7 @@ describe("releaseManagedTunnelOnShutdown", () => { recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, tunnelConnected: Stream.empty, - withLinkStateLock: (effect) => effect, + withLinkStateLock: linkStateLock.withPermits(1), }), ), Effect.provideService( @@ -347,6 +348,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ), Effect.scoped, ); + }; const managedLinkSecrets = [ [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"], @@ -732,6 +734,124 @@ describe("releaseManagedTunnelOnShutdown", () => { ); }); + it.effect("does not let a delayed relay success undo a newer revocation", () => + Effect.gen(function* () { + const outcomes = yield* Effect.forEach( + ["registration", "recovery"] as const, + (acceptedRequest) => + Effect.gen(function* () { + const config = { + providerKind: "cloudflare_tunnel", + connectorToken: "existing-token", + tunnelId: "existing-tunnel", + } as const; + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + const requestStarted = yield* Deferred.make(); + const response = yield* Deferred.make(); + + return yield* Effect.gen(function* () { + const link = yield* CloudLink.CloudLink; + const localOrigin = "http://127.0.0.1:3773"; + expect( + yield* link.startManagedTunnelIfOriginConfirmed(localOrigin, { + requireConfirmedOrigin: false, + }), + ).toBe(true); + const pending = yield* ( + acceptedRequest === "registration" + ? link.registerManagedTunnelRecovery(localOrigin).pipe(Effect.asVoid) + : link.recoverManagedTunnel(localOrigin, config).pipe(Effect.asVoid) + ).pipe(Effect.forkChild()); + yield* Deferred.await(requestStarted); + + if (acceptedRequest === "registration") { + const error = yield* Effect.flip(link.recoverManagedTunnel(localOrigin, config)); + expect(error).toMatchObject({ + _tag: "RelayRequestError", + rejection: "unauthorized", + }); + expect(CloudLink.shouldRetryCloudLink(error)).toBe(false); + } else { + expect(yield* link.registerManagedTunnelRecovery(localOrigin)).toEqual({ + status: "revoked", + }); + } + expect(values.has(CLOUD_LINK_REVOKED)).toBe(true); + expect(applyConfigCalls).toEqual([config, null]); + expect(requests.map((request) => request.headers.authorization)).toEqual([ + "Bearer environment-credential", + "Bearer environment-credential", + ]); + const stoppedAt = applyConfigCalls.length; + + yield* Deferred.succeed( + response, + Response.json( + acceptedRequest === "registration" + ? { status: "ready" } + : { + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: { + ...config, + connectorToken: "new-token", + tunnelId: "new-tunnel", + }, + }, + ), + ); + yield* Fiber.join(pending); + + return { + acceptedRequest, + revoked: values.has(CLOUD_LINK_REVOKED), + configsAfterStop: applyConfigCalls + .slice(stoppedAt) + .filter((config) => config !== null), + }; + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + onRequest: () => Deferred.succeed(requestStarted, undefined), + respondEffect: Effect.suspend(() => + requests.length === 1 + ? Deferred.await(response) + : Effect.succeed( + Response.json( + { + _tag: "RelayAuthInvalidError", + code: "auth_invalid", + reason: "not_authorized", + traceId: "trace-revoked", + }, + { status: 401, headers: { date: "Thu, 01 Jan 1970 00:00:00 GMT" } }, + ), + ), + ), + }), + ); + }), + ); + + expect(outcomes).toEqual([ + { acceptedRequest: "registration", revoked: true, configsAfterStop: [] }, + { acceptedRequest: "recovery", revoked: true, configsAfterStop: [] }, + ]); + }), + ); + it.effect("keeps registering a skewed host and reconnects after its clock is corrected", () => { const config = { providerKind: "cloudflare_tunnel" as const, From 6a39f70bca4305db86153d8c02615ab8cf97c6d4 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 10:25:48 +0000 Subject: [PATCH 09/13] fix(cloud): preserve newer link revocations --- apps/server/src/cloud/CloudLink.ts | 70 ++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/apps/server/src/cloud/CloudLink.ts b/apps/server/src/cloud/CloudLink.ts index f77743511d1c..bdb7acb21cae 100644 --- a/apps/server/src/cloud/CloudLink.ts +++ b/apps/server/src/cloud/CloudLink.ts @@ -533,6 +533,8 @@ const make = Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; + // Accepted responses may heal old markers, but cannot undo a newer refusal. + let revocationRevision = 0; const withSecrets = Effect.provideService(ServerSecretStore.ServerSecretStore, secrets); const withRuntimeFiles = ( @@ -640,23 +642,36 @@ const make = Effect.gen(function* () { Effect.catchTags({ PlatformError: internalError("generate-link-proof") }), ); + const isCurrentManagedTunnel = Effect.fnUntraced(function* (input: { + readonly configJson: string; + readonly credential: string; + readonly revocationRevision: number; + }) { + const [currentConfig, currentCredential] = yield* Effect.all([ + secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + ]); + return ( + revocationRevision === input.revocationRevision && + Option.isSome(currentConfig) && + bytesToString(currentConfig.value) === input.configJson && + Option.isSome(currentCredential) && + bytesToString(currentCredential.value) === input.credential + ); + }); + const activateManagedTunnel = Effect.fn("environment.cloud.activateManagedTunnel")( function* (input: { readonly config: RelayManagedEndpointRuntimeConfig; readonly configJson: string; readonly origin: RelayManagedEndpointOrigin; + readonly credential: string; + readonly revocationRevision: number; }) { return yield* endpointRuntime.withLinkStateLock( Effect.gen(function* () { - const [currentConfig, revoked] = yield* Effect.all([ - secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), - secrets.get(CLOUD_LINK_REVOKED), - ]); - if ( - Option.isSome(revoked) || - Option.isNone(currentConfig) || - bytesToString(currentConfig.value) !== input.configJson - ) { + const revoked = yield* secrets.get(CLOUD_LINK_REVOKED); + if (Option.isSome(revoked) || !(yield* isCurrentManagedTunnel(input))) { return null; } const status = yield* endpointRuntime.applyConfig(input.config); @@ -682,6 +697,8 @@ const make = Effect.gen(function* () { readonly config: RelayManagedEndpointRuntimeConfig; readonly configJson: string; readonly origin: RelayManagedEndpointOrigin; + readonly credential: string; + readonly revocationRevision: number; }, retryRuntimeFailures: boolean, ) => { @@ -984,6 +1001,7 @@ const make = Effect.gen(function* () { if (Option.isNone(current) || bytesToString(current.value) !== credential) { return false; } + revocationRevision++; yield* Effect.logWarning( "T3 Connect no longer accepts this environment's link. Link it again to restore remote access.", { cause }, @@ -1002,6 +1020,7 @@ const make = Effect.gen(function* () { const registerManagedTunnelRecovery = Effect.fn( "environment.cloud.registerManagedCloudTunnelRecovery", )(function* (localOrigin: string, options?: { readonly retryRuntimeFailures?: boolean }) { + const requestRevocationRevision = revocationRevision; const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), secrets.get(RELAY_URL_SECRET), @@ -1065,16 +1084,26 @@ const make = Effect.gen(function* () { if (registered === "revoked" || registered === "superseded") { return { status: registered }; } - yield* secrets.remove(CLOUD_LINK_REVOKED); + const activation = { + config, + configJson: bytesToString(runtimeConfig.value), + origin, + credential, + revocationRevision: requestRevocationRevision, + }; + const accepted = yield* endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + if (!(yield* isCurrentManagedTunnel(activation))) return false; + yield* secrets.remove(CLOUD_LINK_REVOKED); + return true; + }), + ); + if (!accepted) return { status: "superseded" as const }; if (registered.status === "recovery_required") { return { status: registered.status, config }; } const endpointRuntimeStatus = yield* activateManagedTunnelWithRetry( - { - config, - configJson: bytesToString(runtimeConfig.value), - origin, - }, + activation, options?.retryRuntimeFailures === true, ); return endpointRuntimeStatus === null @@ -1087,6 +1116,7 @@ const make = Effect.gen(function* () { expectedConfig?: RelayManagedEndpointRuntimeConfig, options?: { readonly retryRuntimeFailures?: boolean }, ) { + const requestRevocationRevision = revocationRevision; const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), secrets.get(RELAY_URL_SECRET), @@ -1157,10 +1187,12 @@ const make = Effect.gen(function* () { ); const stored = yield* endpointRuntime.withLinkStateLock( Effect.gen(function* () { - const currentConfig = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); if ( - Option.isNone(currentConfig) || - bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value) + !(yield* isCurrentManagedTunnel({ + configJson: bytesToString(runtimeConfig.value), + credential, + revocationRevision: requestRevocationRevision, + })) ) { return false; } @@ -1176,6 +1208,8 @@ const make = Effect.gen(function* () { config: recovered.endpointRuntime, configJson: encoded, origin, + credential, + revocationRevision: requestRevocationRevision, }, options?.retryRuntimeFailures === true, ); From 8cfc1a4a70918701768582448ba826d1eee89051 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 10:54:40 +0000 Subject: [PATCH 10/13] test(cloud): cover delayed recovery authorization refusals --- .../src/cloud/CloudLink.lifecycle.test.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.ts index 206e44012c48..5b951887ef00 100644 --- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts +++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts @@ -1366,6 +1366,70 @@ describe("releaseManagedTunnelOnShutdown", () => { ); }); + it.effect( + "revokes a genuine recovery refusal delayed in transit with an accurate host clock", + () => + Effect.gen(function* () { + const { store, values } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + const requestStarted = yield* Deferred.make(); + const response = yield* Deferred.make(); + + yield* Effect.gen(function* () { + const link = yield* CloudLink.CloudLink; + const recovery = yield* Effect.flip( + link.recoverManagedTunnel("http://127.0.0.1:3773"), + ).pipe(Effect.forkChild({ startImmediately: true })); + const sentAt = yield* Deferred.await(requestStarted); + const refusal = Response.json( + { + _tag: "RelayAuthInvalidError", + code: "auth_invalid", + reason: "not_authorized", + traceId: "trace-delayed-recovery-refusal", + }, + { + status: 401, + headers: { date: DateTime.toDateUtc(DateTime.makeUnsafe(sentAt)).toUTCString() }, + }, + ); + + // The Date header predates transport delay, not a change to the host's clock. + yield* TestClock.adjust("75 seconds"); + yield* Deferred.succeed(response, refusal); + const error = yield* Fiber.join(recovery); + + expect(error).toMatchObject({ _tag: "RelayRequestError", rejection: "unauthorized" }); + expect(requests).toHaveLength(1); + expect({ + retryable: CloudLink.shouldRetryCloudLink(error), + revoked: values.has(CLOUD_LINK_REVOKED), + applyConfigCalls, + }).toEqual({ retryable: false, revoked: true, applyConfigCalls: [null] }); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + onRequest: () => + Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => Deferred.succeed(requestStarted, now)), + ), + respondEffect: Deferred.await(response), + }), + ); + }), + ); + it.effect("allows managed tunnel provisioning to take longer than ten seconds", () => Effect.gen(function* () { const oldConfig = From dd453f4c5a9643cd59cfa8d21224590826e2652c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 11:00:06 +0000 Subject: [PATCH 11/13] fix(cloud): account for relay response latency --- apps/server/src/cli/connect.ts | 5 ++--- apps/server/src/cloud/CloudLink.ts | 9 ++++----- apps/server/src/cloud/relayClock.ts | 11 +++++++++-- apps/server/src/cloud/relayResponse.ts | 14 ++++++++++++++ packages/shared/src/clockSkew.ts | 16 +++++++++++++--- 5 files changed, 42 insertions(+), 13 deletions(-) diff --git a/apps/server/src/cli/connect.ts b/apps/server/src/cli/connect.ts index 1195afb2908d..f5c41e4272bf 100644 --- a/apps/server/src/cli/connect.ts +++ b/apps/server/src/cli/connect.ts @@ -31,7 +31,7 @@ import * as BootService from "../cloud/bootService.ts"; import * as CliState from "../cloud/CliState.ts"; import * as CliTokenManager from "../cloud/CliTokenManager.ts"; import { checkRelayClock } from "../cloud/relayClock.ts"; -import { filterRelayResponse } from "../cloud/relayResponse.ts"; +import { filterRelayClient } from "../cloud/relayResponse.ts"; import { CLOUD_LINK_REVOKED, CLOUD_LINKED_USER_ID, @@ -343,13 +343,12 @@ const unlinkRelayEnvironment = Effect.fn("cloud.cli.unlink_relay_environment")(f const environment = yield* ServerEnvironment.ServerEnvironmentIdentity; const environmentId = yield* environment.getEnvironmentId; const relayUrl = yield* relayUrlConfig; - const httpClient = yield* HttpClient.HttpClient; + const httpClient = filterRelayClient(yield* HttpClient.HttpClient); const response = yield* HttpClientRequest.delete( `${relayUrl}/v1/client/environment-links/${encodeURIComponent(environmentId)}`, ).pipe( HttpClientRequest.bearerToken(token.value.accessToken), httpClient.execute, - Effect.flatMap(filterRelayResponse), Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)), withRelayClientTracing, ); diff --git a/apps/server/src/cloud/CloudLink.ts b/apps/server/src/cloud/CloudLink.ts index bdb7acb21cae..f04e5a854091 100644 --- a/apps/server/src/cloud/CloudLink.ts +++ b/apps/server/src/cloud/CloudLink.ts @@ -99,7 +99,7 @@ import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { relayUrlConfig } from "./publicConfig.ts"; import { checkRelayClock } from "./relayClock.ts"; import { - filterRelayResponse, + filterRelayClient, relayRequestError, type RelayRequestError, shouldRetryRelayRequest, @@ -527,6 +527,7 @@ const make = Effect.gen(function* () { const environmentAuth = yield* EnvironmentAuth.EnvironmentAuth; const cliTokenManager = yield* CliTokenManager.CloudCliTokenManager; const httpClient = yield* HttpClient.HttpClient; + const relayHttpClient = filterRelayClient(httpClient); const httpServer = yield* HttpServer.HttpServer; const awarenessRelay = yield* AgentAwarenessRelay.AgentAwarenessRelay; const crypto = yield* Crypto.Crypto; @@ -853,8 +854,7 @@ const make = Effect.gen(function* () { HttpClientRequest.post(input.url).pipe( HttpClientRequest.bearerToken(input.token), HttpClientRequest.bodyJson(input.payload), - Effect.flatMap(httpClient.execute), - Effect.flatMap(filterRelayResponse), + Effect.flatMap(relayHttpClient.execute), Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)), Effect.timeout(input.timeout ?? "10 seconds"), Effect.mapError(relayRequestError), @@ -1307,8 +1307,7 @@ const make = Effect.gen(function* () { `${bytesToString(relayUrl.value)}/v1/client/environment-links/${encodeURIComponent(environmentId)}/tunnel`, ).pipe( HttpClientRequest.bearerToken(token.value.accessToken), - httpClient.execute, - Effect.flatMap(filterRelayResponse), + relayHttpClient.execute, Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)), withRelayClientTracing, ); diff --git a/apps/server/src/cloud/relayClock.ts b/apps/server/src/cloud/relayClock.ts index 97125432a61e..170acc73a87f 100644 --- a/apps/server/src/cloud/relayClock.ts +++ b/apps/server/src/cloud/relayClock.ts @@ -1,5 +1,6 @@ import { clockSkewFromDateHeader } from "@t3tools/shared/clockSkew"; import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import { HttpClient, HttpClientRequest } from "effect/http"; @@ -7,8 +8,14 @@ import { HttpClient, HttpClientRequest } from "effect/http"; export const checkRelayClock = Effect.fn("cloud.checkRelayClock")( function* (relayUrl: string) { const httpClient = yield* HttpClient.HttpClient; - const response = yield* httpClient.execute(HttpClientRequest.head(`${relayUrl}/`)); - return clockSkewFromDateHeader(response.headers.date, yield* Clock.currentTimeMillis); + const [duration, response] = yield* httpClient + .execute(HttpClientRequest.head(`${relayUrl}/`)) + .pipe(Effect.timed); + return clockSkewFromDateHeader( + response.headers.date, + yield* Clock.currentTimeMillis, + Duration.toMillis(duration), + ); }, Effect.timeout("2 seconds"), Effect.orElseSucceed(() => null), diff --git a/apps/server/src/cloud/relayResponse.ts b/apps/server/src/cloud/relayResponse.ts index fb0f9bc4b2d4..3d128b4c1d85 100644 --- a/apps/server/src/cloud/relayResponse.ts +++ b/apps/server/src/cloud/relayResponse.ts @@ -2,9 +2,11 @@ import { RelayProtectedError } from "@t3tools/contracts/relay"; import { clockSkewFromDateHeader, describeClockSkew } from "@t3tools/shared/clockSkew"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; +import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientResponse from "effect/http/HttpClientResponse"; import { isHttpClientError } from "effect/http/HttpClientError"; @@ -58,11 +60,13 @@ function recoveryHint(error: RelayProtectedError): string { export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(function* ( response: HttpClientResponse.HttpClientResponse, + requestDurationMillis = 0, ) { if (response.status >= 200 && response.status < 300) return response; const clockSkewSeconds = clockSkewFromDateHeader( response.headers.date, yield* Clock.currentTimeMillis, + requestDurationMillis, ); const decoded = yield* HttpClientResponse.schemaBodyJson(RelayProtectedError)(response).pipe( Effect.option, @@ -115,3 +119,13 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func } return yield* new RelayRequestError({ rejection: "unavailable", description }); }); + +export const filterRelayClient = (client: HttpClient.HttpClient) => + HttpClient.transformResponse(client, (response) => + response.pipe( + Effect.timed, + Effect.flatMap(([duration, result]) => + filterRelayResponse(result, Duration.toMillis(duration)), + ), + ), + ); diff --git a/packages/shared/src/clockSkew.ts b/packages/shared/src/clockSkew.ts index 18461e41c3cb..b036074f1b7f 100644 --- a/packages/shared/src/clockSkew.ts +++ b/packages/shared/src/clockSkew.ts @@ -4,12 +4,13 @@ const CLOCK_SKEW_LIMIT_SECONDS = 60; /** * Seconds this machine's clock is ahead of (positive) or behind (negative) the * relay, judged from the HTTP `Date` header of a relay response received at - * `nowMillis`. `null` when the header is missing, unparseable, or within the - * limit. + * `nowMillis`, allowing for the request's monotonic elapsed time. `null` when + * the header is missing, unparseable, or does not establish skew beyond the limit. */ export function clockSkewFromDateHeader( dateHeader: string | undefined, nowMillis: number, + requestDurationMillis = 0, ): number | null { if (dateHeader === undefined) return null; const relayMillis = Date.parse(dateHeader); @@ -17,7 +18,16 @@ export function clockSkewFromDateHeader( // `Date` has whole-second precision, so the relay's true time is up to a // second later than the header says. const skewSeconds = Math.round((nowMillis - relayMillis - 500) / 1_000); - return Math.abs(skewSeconds) > CLOCK_SKEW_LIMIT_SECONDS ? skewSeconds : null; + // Date may originate anywhere between request start and receipt. Transport + // delay alone cannot establish that the host clock is ahead. + const minimumSkewSeconds = Math.round( + (nowMillis - relayMillis - 500 - requestDurationMillis) / 1_000, + ); + return minimumSkewSeconds > CLOCK_SKEW_LIMIT_SECONDS + ? minimumSkewSeconds + : skewSeconds < -CLOCK_SKEW_LIMIT_SECONDS + ? skewSeconds + : null; } function skewAmount(seconds: number): string { From 4cc1674c72e4f68b6f32712180218368d760a7d7 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 11:11:21 +0000 Subject: [PATCH 12/13] test(cloud): cover clock correction during recovery --- .../src/cloud/CloudLink.lifecycle.test.ts | 112 +++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-) diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.ts index 5b951887ef00..214abc0acd3d 100644 --- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts +++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts @@ -19,6 +19,7 @@ import { HttpClient, HttpClientResponse, HttpServer, type HttpClientRequest } fr import * as NetAddress from "effect/net/NetAddress"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { decodeRelayJwt } from "@t3tools/shared/relayJwt"; import { DESKTOP_UPDATE_RESTART_MARKER_FILE, EnvironmentId } from "@t3tools/contracts"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; @@ -35,7 +36,10 @@ import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; -import { RelayManagedEndpointRecoveryRegistrationRequest } from "@t3tools/contracts/relay"; +import { + RelayManagedEndpointRecoveryRegistrationRequest, + RelayManagedEndpointRecoveryRequest, +} from "@t3tools/contracts/relay"; import { CLOUD_ENDPOINT_CONFIRMED_ORIGIN, CLOUD_ENDPOINT_RUNTIME_CONFIG, @@ -84,6 +88,9 @@ const idleHttpServer = HttpServer.HttpServer.of({ const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect( Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest), ); +const decodeManagedTunnelRecovery = Schema.decodeSync( + Schema.fromJsonString(RelayManagedEndpointRecoveryRequest), +); function makeSecretStore( create: ServerSecretStore.ServerSecretStore["Service"]["create"], @@ -1430,6 +1437,109 @@ describe("releaseManagedTunnelOnShutdown", () => { }), ); + it.effect( + "retries recovery when the host clock corrects while its rejected proof is in flight", + () => + Effect.gen(function* () { + const { store, values } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const nextConfig = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "new-token", + tunnelId: "new-tunnel", + }; + const applyConfigCalls: Array = []; + const requests: Array = []; + const requestStarted = yield* Deferred.make(); + const response = yield* Deferred.make(); + const clock = yield* Clock.Clock; + let wallOffset = -420_000; + const wallMillis = () => clock.currentTimeMillisUnsafe() + wallOffset; + const hostClock = { + ...clock, + currentTimeMillisUnsafe: wallMillis, + currentTimeMillis: Effect.sync(wallMillis), + currentTimeNanosUnsafe: () => BigInt(wallMillis()) * 1_000_000n, + currentTimeNanos: Effect.sync(() => BigInt(wallMillis()) * 1_000_000n), + }; + yield* TestClock.setTime(420_000); + + yield* Effect.gen(function* () { + const link = yield* CloudLink.CloudLink; + const recovery = yield* Effect.flip( + link.recoverManagedTunnel("http://127.0.0.1:3773"), + ).pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(requestStarted); + const refusal = Response.json( + { + _tag: "RelayAuthInvalidError", + code: "auth_invalid", + reason: "not_authorized", + traceId: "trace-inflight-clock-correction", + }, + { status: 401, headers: { date: "Thu, 01 Jan 1970 00:07:00 GMT" } }, + ); + + // Time sync changes the wall clock, not the elapsed request duration. + yield* TestClock.adjust("2 seconds"); + wallOffset = 0; + yield* Deferred.succeed(response, refusal); + const error = yield* Fiber.join(recovery); + expect(error).toMatchObject({ _tag: "RelayRequestError" }); + expect({ + retryable: CloudLink.shouldRetryCloudLink(error), + revoked: values.has(CLOUD_LINK_REVOKED), + applyConfigCalls, + }).toEqual({ retryable: true, revoked: false, applyConfigCalls: [] }); + + expect(yield* link.recoverManagedTunnel("http://127.0.0.1:3773")).toBe(true); + expect(applyConfigCalls).toEqual([nextConfig]); + expect(values.has(CLOUD_LINK_REVOKED)).toBe(false); + expect(requests).toHaveLength(2); + expect( + requests.map(({ body }) => { + if (body._tag !== "Uint8Array") throw new Error("Expected a JSON recovery request"); + const { proof } = decodeManagedTunnelRecovery(new TextDecoder().decode(body.body)); + const { iat, exp } = decodeRelayJwt(proof); + return { iat, exp }; + }), + ).toEqual([ + { iat: 0, exp: 60 }, + { iat: 422, exp: 482 }, + ]); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + onRequest: () => Deferred.succeed(requestStarted, undefined), + respondEffect: Effect.suspend(() => + requests.length === 1 + ? Deferred.await(response) + : Effect.succeed( + Response.json({ + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: nextConfig, + }), + ), + ), + }), + Effect.provideService(Clock.Clock, hostClock), + ); + }), + ); + it.effect("allows managed tunnel provisioning to take longer than ten seconds", () => Effect.gen(function* () { const oldConfig = From 219a8946d2b5400665f99fd0b6d47bf727cedcf6 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 11:15:17 +0000 Subject: [PATCH 13/13] fix(cloud): retry proofs rejected during clock correction --- apps/server/src/cloud/relayResponse.ts | 36 +++++++++++++++++++++----- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/apps/server/src/cloud/relayResponse.ts b/apps/server/src/cloud/relayResponse.ts index 3d128b4c1d85..e9a40b4bc63b 100644 --- a/apps/server/src/cloud/relayResponse.ts +++ b/apps/server/src/cloud/relayResponse.ts @@ -61,11 +61,13 @@ function recoveryHint(error: RelayProtectedError): string { export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(function* ( response: HttpClientResponse.HttpClientResponse, requestDurationMillis = 0, + requestStartedAtMillis?: number, ) { if (response.status >= 200 && response.status < 300) return response; + const nowMillis = yield* Clock.currentTimeMillis; const clockSkewSeconds = clockSkewFromDateHeader( response.headers.date, - yield* Clock.currentTimeMillis, + nowMillis, requestDurationMillis, ); const decoded = yield* HttpClientResponse.schemaBodyJson(RelayProtectedError)(response).pipe( @@ -76,6 +78,27 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func const description = Option.isSome(decoded) ? `T3 Connect: ${decoded.value.message}. ${recoveryHint(decoded.value)} Trace ID: ${decoded.value.traceId}.` : `T3 Connect relay returned HTTP ${response.status} without a recognized error response. Check relay access and any proxy or firewall restrictions, then restart T3 Code.${requestId}`; + // The proof predates any in-flight clock correction. Project the starting + // wall clock forward with monotonic time to check the clock that signed it. + if ( + (response.status === 400 || response.status === 401) && + clockSkewSeconds === null && + requestStartedAtMillis !== undefined && + clockSkewFromDateHeader( + response.headers.date, + requestStartedAtMillis + requestDurationMillis, + requestDurationMillis, + ) !== null + ) { + return yield* new RelayRequestError({ + rejection: "unavailable", + description: `The host computer's clock changed during the T3 Connect relay request. T3 Connect will retry with a fresh proof. Relay response: ${ + Option.isSome(decoded) + ? `${decoded.value.message}. Trace ID: ${decoded.value.traceId}.` + : `HTTP ${response.status}.${requestId}` + }`, + }); + } // Timestamp rejections can clear after clock sync; unrelated HTTP statuses stay permanent. const clockSuspected = clockSkewSeconds !== null && (response.status === 400 || response.status === 401); @@ -122,10 +145,9 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func export const filterRelayClient = (client: HttpClient.HttpClient) => HttpClient.transformResponse(client, (response) => - response.pipe( - Effect.timed, - Effect.flatMap(([duration, result]) => - filterRelayResponse(result, Duration.toMillis(duration)), - ), - ), + Effect.gen(function* () { + const startedAtMillis = yield* Clock.currentTimeMillis; + const [duration, result] = yield* response.pipe(Effect.timed); + return yield* filterRelayResponse(result, Duration.toMillis(duration), startedAtMillis); + }), );