From bef303e98f0b2ddbfad707abe7efa9f7c39618ab Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 19 Oct 2026 23:28:41 -0700 Subject: [PATCH 1/8] feat(connect): t3 connect checks that other devices can reach this machine Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cli/connect.ts | 116 +++++++- .../src/cloud/CloudLink.lifecycle.test.ts | 19 ++ apps/server/src/cloud/CloudLink.test.ts | 4 + apps/server/src/cloud/CloudLink.ts | 7 +- .../src/cloud/ConnectReachability.test.ts | 83 ++++++ apps/server/src/cloud/ConnectReachability.ts | 280 ++++++++++++++++++ .../src/cloud/ManagedEndpointRuntime.ts | 75 ++++- apps/server/src/cloud/http.test.ts | 6 + apps/server/src/cloud/http.ts | 5 + apps/server/src/server.ts | 3 +- .../settings/ConnectionsSettings.tsx | 18 +- packages/contracts/src/environmentHttp.ts | 53 ++++ packages/tailscale/src/tailscale.test.ts | 21 ++ packages/tailscale/src/tailscale.ts | 50 ++++ 14 files changed, 727 insertions(+), 13 deletions(-) create mode 100644 apps/server/src/cloud/ConnectReachability.test.ts create mode 100644 apps/server/src/cloud/ConnectReachability.ts diff --git a/apps/server/src/cli/connect.ts b/apps/server/src/cli/connect.ts index 9cc59335115b..5dc3329c5fe6 100644 --- a/apps/server/src/cli/connect.ts +++ b/apps/server/src/cli/connect.ts @@ -1,6 +1,10 @@ import { + AuthRelayReadScope, AuthRelayWriteScope, + type AuthEnvironmentScope, + type EnvironmentConnectReachability, EnvironmentHttpApi, + formatConnectReachable, type RelayClientInstallProgressEvent, type RelayClientInstallProgressStage, } from "@t3tools/contracts"; @@ -9,6 +13,7 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as RelayClient from "@t3tools/shared/relayClient"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Config from "effect/Config"; import * as Console from "effect/Console"; import * as Crypto from "effect/Crypto"; @@ -29,6 +34,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as BootService from "../cloud/bootService.ts"; import * as CliState from "../cloud/CliState.ts"; import * as CliTokenManager from "../cloud/CliTokenManager.ts"; +import { formatReachabilityFailure } from "../cloud/ConnectReachability.ts"; import { filterRelayResponse } from "../cloud/relayResponse.ts"; import { CLOUD_LINK_REVOKED, @@ -142,6 +148,8 @@ interface CloudCliStatus { readonly relayUrl: string | null; readonly publishAgentActivity: boolean; readonly relayClient: RelayClient.RelayClientStatus; + /** `null` when no server answered the live check, or none ran because T3 Connect is off. */ + readonly reachability: EnvironmentConnectReachability | null; } function formatRelayClientStatus(executable: RelayClient.RelayClientStatus): ReadonlyArray { @@ -199,8 +207,19 @@ function formatCloudStatus(status: CloudCliStatus, options?: { readonly json?: b ` Relay: ${status.relayUrl ?? "not provisioned"}`, ` Publish agent activity: ${status.publishAgentActivity ? "enabled" : "disabled"}`, ...formatRelayClientStatus(status.relayClient), - "", - "This is saved setup, not a live connection check. Check the background service with `t3 service status`.", + ...(!status.desired + ? [] + : status.reachability === null + ? [ + "", + "No running server answered, so reachability was not checked. Check the background service with `t3 service status`.", + ] + : status.reachability.status === "reachable" + ? [ + "", + `✓ Other devices can reach this machine · ${formatConnectReachable(status.reachability)}`, + ] + : ["", formatReachabilityFailure(status.reachability)]), ...(nextStep ? ["", `Next: ${nextStep}`] : []), ].join("\n"); } @@ -261,11 +280,12 @@ export const acquireRelayClientForLink = Effect.fn("cloud.cli.acquire_relay_clie const withCloudCliSessionToken = ( environmentAuth: EnvironmentAuth.EnvironmentAuth["Service"], + scope: AuthEnvironmentScope, run: (token: string) => Effect.Effect, ) => Effect.acquireUseRelease( environmentAuth.issueSession({ - scopes: [AuthRelayWriteScope], + scopes: [scope], subject: "cloud-cli", label: "t3 connect cli", }), @@ -287,7 +307,7 @@ const runLiveCloudUnlink = Effect.fn("cloud.cli.run_live_unlink")(function* () { const environmentAuth = yield* EnvironmentAuth.EnvironmentAuth; const result = yield* Effect.exit( - withCloudCliSessionToken(environmentAuth, (token) => + withCloudCliSessionToken(environmentAuth, AuthRelayWriteScope, (token) => HttpApiClient.make(EnvironmentHttpApi, { baseUrl: runtimeState.value.origin, }).pipe( @@ -303,6 +323,73 @@ const runLiveCloudUnlink = Effect.fn("cloud.cli.run_live_unlink")(function* () { : ({ status: "failed", cause: result.cause } satisfies LiveCloudActionResult); }); +// One check probes the endpoint, the tunnel edge, and Tailscale, each with its own timeout. +const CLOUD_CLI_REACHABILITY_TIMEOUT = Duration.seconds(30); +const CLOUD_CLI_REACHABILITY_WAIT = Duration.seconds(60); +const CLOUD_CLI_REACHABILITY_RETRY = Duration.seconds(5); + +/** Asks the running server to check T3 Connect now. `null` when no server answers. */ +const checkLiveReachability = Effect.fn("cloud.cli.check_live_reachability")(function* () { + const config = yield* ServerConfig.ServerConfig; + const runtimeState = yield* readPersistedServerRuntimeState(config.serverRuntimeStatePath); + if (Option.isNone(runtimeState)) return null; + const environmentAuth = yield* EnvironmentAuth.EnvironmentAuth; + return yield* withCloudCliSessionToken(environmentAuth, AuthRelayReadScope, (token) => + HttpApiClient.make(EnvironmentHttpApi, { baseUrl: runtimeState.value.origin }).pipe( + Effect.flatMap((client) => + client.connect.checkReachability({ headers: { authorization: `Bearer ${token}` } }), + ), + Effect.timeout(CLOUD_CLI_REACHABILITY_TIMEOUT), + ), + ).pipe(Effect.orElseSucceed(() => null)); +}); + +function formatAlertBox(text: string): string { + const lines = text.split("\n"); + const width = Math.max(...lines.map((line) => line.length)); + const color = process.stdout.isTTY === true && !process.env.NO_COLOR; + const red = (line: string) => (color ? `\x1b[1;31m${line}\x1b[0m` : line); + return [ + `┌${"─".repeat(width + 2)}┐`, + ...lines.map((line) => `│ ${line.padEnd(width)} │`), + `└${"─".repeat(width + 2)}┘`, + ] + .map(red) + .join("\n"); +} + +/** + * Waits for the server that `t3 connect` just started to prove other devices + * can reach it, so a broken tunnel fails setup instead of looking green. + */ +const waitForReachability = Effect.fn("cloud.cli.wait_for_reachability")(function* () { + yield* Console.log("\nChecking that other devices can reach this machine..."); + const deadline = + (yield* Clock.currentTimeMillis) + Duration.toMillis(CLOUD_CLI_REACHABILITY_WAIT); + let last: EnvironmentConnectReachability | null = null; + while (true) { + last = (yield* checkLiveReachability()) ?? last; + if (last?.status === "reachable") { + yield* Console.log( + `✓ Other devices can reach this machine · ${formatConnectReachable(last)}`, + ); + return; + } + if ((yield* Clock.currentTimeMillis) >= deadline) break; + yield* Effect.sleep(CLOUD_CLI_REACHABILITY_RETRY); + } + yield* Console.error( + formatAlertBox( + last === null + ? "✕ Could not check T3 Connect\n The T3 server on this machine did not answer.\n Fix: run `t3 service status`, then `t3 connect status`." + : formatReachabilityFailure(last), + ), + ); + yield* Effect.sync(() => { + process.exitCode = 1; + }); +}); + type RelayUnlinkResult = | { readonly status: "not-authenticated" } | { readonly status: "revoked" } @@ -551,7 +638,9 @@ const connectStatusCommand = Command.make("status", { ...projectLocationFlags, json: jsonFlag, }).pipe( - Command.withDescription("Show persisted T3 Connect and relay client state."), + Command.withDescription( + "Show T3 Connect setup and check that other devices can reach this machine.", + ), Command.withHandler((flags) => runCloudCommand( flags, @@ -567,6 +656,7 @@ const connectStatusCommand = Command.make("status", { publishAgentActivity, revoked, executable, + reachability, ] = yield* Effect.all( [ CliState.readCliDesiredCloudLink, @@ -576,6 +666,11 @@ const connectStatusCommand = Command.make("status", { secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET), secrets.get(CLOUD_LINK_REVOKED), relayClient.resolve, + CliState.readCliDesiredCloudLink.pipe( + Effect.flatMap((desired) => + desired ? checkLiveReachability() : Effect.succeed(null), + ), + ), ], { concurrency: "unbounded" }, ); @@ -590,6 +685,7 @@ const connectStatusCommand = Command.make("status", { Option.isSome(publishAgentActivity) ? bytesToString(publishAgentActivity.value) : null, ), relayClient: executable, + reachability, }; yield* Console.log(formatCloudStatus(status, { json: flags.json })); }), @@ -703,7 +799,7 @@ export const reportServiceOnboarding = Effect.fn("cli.connect.report_service_onb } const serveCommand = yield* resolveCliCommand("serve"); yield* Console.log( - `\nNext\n Start the server with \`${serveCommand}\` to make this machine reachable.`, + `\nNext\n Start the server with \`${serveCommand}\` to make this machine reachable.\n It checks that other devices can reach this machine when it starts.`, ); }, ); @@ -724,9 +820,13 @@ export const connectCommand = Command.make("connect", { } yield* Console.log(`✓ Authorized${connectedAs(linked.identity)}`); - yield* reportServiceOnboarding( - yield* CliService.recoverServiceOnboardingOffer(CliService.offerServiceDuringOnboarding), + const outcome = yield* CliService.recoverServiceOnboardingOffer( + CliService.offerServiceDuringOnboarding, ); + yield* reportServiceOnboarding(outcome); + if (outcome !== "skipped") { + yield* waitForReachability(); + } }), ), ), diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.ts index fea003fe3d58..653a3ce07710 100644 --- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts +++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts @@ -43,6 +43,7 @@ import { RELAY_URL_SECRET, } from "./config.ts"; import * as CloudLink from "./CloudLink.ts"; +import * as ConnectReachability from "./ConnectReachability.ts"; import { consumeCloudReplayGuards } from "./CloudLink.ts"; import { pendingServiceUpdateExists } from "./updateHandoff.ts"; import { @@ -154,6 +155,13 @@ describe("reconcileDesiredCloudLink", () => { expect(error._tag).toBe("CloudLinkAuthorizationMissingError"); }).pipe( Effect.provide(CloudLink.layer), + Effect.provideService( + ConnectReachability.ConnectReachability, + ConnectReachability.ConnectReachability.of({ + check: Effect.die("unused"), + latest: Effect.succeed(Option.none()), + }), + ), Effect.provideService(HttpServer.HttpServer, idleHttpServer), Effect.provideService( ServerConfigModule.ServerConfig, @@ -177,6 +185,8 @@ describe("reconcileDesiredCloudLink", () => { recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, tunnelConnected: Stream.empty, + inspectConnector: Effect.succeed(null), + readinessChanged: Stream.empty, withLinkStateLock: (effect) => effect, } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]), ), @@ -276,6 +286,13 @@ describe("releaseManagedTunnelOnShutdown", () => { (effect: Effect.Effect) => effect.pipe( Effect.provide(CloudLink.layer), + Effect.provideService( + ConnectReachability.ConnectReachability, + ConnectReachability.ConnectReachability.of({ + check: Effect.die("unused"), + latest: Effect.succeed(Option.none()), + }), + ), Effect.provideService(HttpServer.HttpServer, { ...idleHttpServer, address: harness.address ?? idleHttpServer.address, @@ -308,6 +325,8 @@ describe("releaseManagedTunnelOnShutdown", () => { recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, tunnelConnected: Stream.empty, + inspectConnector: Effect.succeed(null), + readinessChanged: Stream.empty, withLinkStateLock: (effect) => effect, }), ), diff --git a/apps/server/src/cloud/CloudLink.test.ts b/apps/server/src/cloud/CloudLink.test.ts index 0f2de771777c..db945509a56f 100644 --- a/apps/server/src/cloud/CloudLink.test.ts +++ b/apps/server/src/cloud/CloudLink.test.ts @@ -16,6 +16,7 @@ import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; import * as CloudLink from "./CloudLink.ts"; +import * as ConnectReachability from "./ConnectReachability.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET, @@ -94,6 +95,9 @@ const withService = ( }), Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ requestCatchUp: () => Effect.void }), Layer.mock(ManagedEndpointRuntime.CloudManagedEndpointRuntime)({}), + Layer.mock(ConnectReachability.ConnectReachability)({ + latest: Effect.succeed(Option.none()), + }), Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), Layer.mock(CliTokenManager.CloudCliTokenManager)({}), Layer.mock(HttpServer.HttpServer)({ diff --git a/apps/server/src/cloud/CloudLink.ts b/apps/server/src/cloud/CloudLink.ts index 12d6aee49310..9cab063ef672 100644 --- a/apps/server/src/cloud/CloudLink.ts +++ b/apps/server/src/cloud/CloudLink.ts @@ -96,6 +96,7 @@ import { RELAY_URL_SECRET, } from "./config.ts"; import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts"; +import * as ConnectReachability from "./ConnectReachability.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { relayUrlConfig } from "./publicConfig.ts"; import { @@ -529,6 +530,7 @@ const make = Effect.gen(function* () { const secrets = yield* ServerSecretStore.ServerSecretStore; const environment = yield* ServerEnvironment.ServerEnvironment; const endpointRuntime = yield* ManagedEndpointRuntime.CloudManagedEndpointRuntime; + const reachability = yield* ConnectReachability.ConnectReachability; const environmentAuth = yield* EnvironmentAuth.EnvironmentAuth; const cliTokenManager = yield* CliTokenManager.CloudCliTokenManager; const httpClient = yield* HttpClient.HttpClient; @@ -1314,6 +1316,7 @@ const make = Effect.gen(function* () { publishAgentActivity, holdWebhooks, revoked, + latestReachability, ] = yield* Effect.all( [ secrets.get(CLOUD_LINKED_USER_ID), @@ -1323,8 +1326,9 @@ const make = Effect.gen(function* () { secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET), secrets.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET), secrets.get(CLOUD_LINK_REVOKED), + reachability.latest, ], - { concurrency: 7 }, + { concurrency: 8 }, ); return { linked: Option.isSome(cloudUserId), @@ -1338,6 +1342,7 @@ const make = Effect.gen(function* () { : false, holdWebhooksWhileOffline: Option.isSome(holdWebhooks) && bytesToString(holdWebhooks.value) === "true", + ...(Option.isSome(latestReachability) ? { reachability: latestReachability.value } : {}), } satisfies EnvironmentCloudLinkStateResult; }); diff --git a/apps/server/src/cloud/ConnectReachability.test.ts b/apps/server/src/cloud/ConnectReachability.test.ts new file mode 100644 index 000000000000..9e100b2c0160 --- /dev/null +++ b/apps/server/src/cloud/ConnectReachability.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { + classifyReachability, + formatReachabilityFailure, + type ReachabilityFacts, +} from "./ConnectReachability.ts"; + +const checkedAt = "2026-10-19T00:00:00.000Z"; +const connected = { readyConnections: 4, hostname: "env.example.test", protocol: "quic" }; +const notConnected = { readyConnections: 0, hostname: null, protocol: null }; +const facts = (overrides: Partial): ReachabilityFacts => ({ + managedTunnelConfigured: true, + connector: connected, + endpoint: null, + edgeReachable: null, + exitNode: null, + ...overrides, +}); + +describe("classifyReachability", () => { + it("blames the exit node when the tunnel edge times out through it", () => { + const result = classifyReachability( + facts({ + connector: notConnected, + edgeReachable: false, + exitNode: { name: "home-pi", online: true }, + }), + checkedAt, + ); + + expect(result.status === "unreachable" && formatReachabilityFailure(result)).toBe( + [ + "✕ T3 Connect is NOT reachable from other devices", + " Cloudflare's tunnel port (7844) times out from this machine.", + ' Tailscale exit node "home-pi" is on. Traffic to Cloudflare goes through it.', + " Fix: tailscale set --exit-node= (or allow outbound 7844 on that network)", + ].join("\n"), + ); + }); + + it("names a generic blocker when no exit node is on", () => { + const result = classifyReachability( + facts({ connector: notConnected, edgeReachable: false }), + checkedAt, + ); + + expect(result).toMatchObject({ + cause: "edge_unreachable", + hint: "A VPN, proxy, or firewall may block it.", + }); + }); + + it.each([ + [{ kind: "status", status: 502 }, "origin_unreachable"], + [{ kind: "status", status: 530 }, "no_connector"], + [{ kind: "status", status: 404 }, "unexpected_response"], + [{ kind: "dns_failed" }, "dns_failed"], + [{ kind: "network_error" }, "timeout"], + ] as const)("maps endpoint result %o to %s", (endpoint, cause) => { + expect(classifyReachability(facts({ endpoint, edgeReachable: true }), checkedAt)).toMatchObject( + { status: "unreachable", cause }, + ); + }); + + it("waits on a connector that has not connected while the edge answers", () => { + expect( + classifyReachability(facts({ connector: notConnected, edgeReachable: true }), checkedAt), + ).toMatchObject({ cause: "connector_not_ready" }); + }); + + it("reports connections and protocol when the endpoint answers", () => { + expect( + classifyReachability(facts({ endpoint: { kind: "status", status: 200 } }), checkedAt), + ).toEqual({ status: "reachable", checkedAt, readyConnections: 4, protocol: "quic" }); + }); + + it("does not probe a tunnel that was never set up", () => { + expect( + classifyReachability(facts({ managedTunnelConfigured: false, connector: null }), checkedAt), + ).toMatchObject({ cause: "not_linked" }); + }); +}); diff --git a/apps/server/src/cloud/ConnectReachability.ts b/apps/server/src/cloud/ConnectReachability.ts new file mode 100644 index 000000000000..31349736e01c --- /dev/null +++ b/apps/server/src/cloud/ConnectReachability.ts @@ -0,0 +1,280 @@ +import * as NodeNet from "node:net"; + +import { type EnvironmentConnectReachability, formatConnectReachable } from "@t3tools/contracts"; +import { readTailscaleExitNode, type TailscaleExitNode } from "@t3tools/tailscale"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; +import { HttpClient } from "effect/http"; +import { ChildProcessSpawner } from "effect/process"; + +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import { CLOUD_ENDPOINT_RUNTIME_CONFIG } from "./config.ts"; +import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; + +// cloudflared finds the edge through the SRV record `_v2-origintunneld._tcp.argotunnel.com`, +// which names these two hosts on port 7844. QUIC and HTTP/2 both use that port. +const TUNNEL_EDGE_HOSTS = ["region1.v2.argotunnel.com", "region2.v2.argotunnel.com"]; +const TUNNEL_EDGE_PORT = 7844; +const EDGE_CONNECT_TIMEOUT = Duration.seconds(5); +const ENDPOINT_REQUEST_TIMEOUT = Duration.seconds(10); +// cloudflared registers several connections within a few seconds; they are one change. +const READINESS_CHANGE_DEBOUNCE = Duration.seconds(3); + +export type EndpointProbe = + | { readonly kind: "status"; readonly status: number } + | { readonly kind: "dns_failed" | "timeout" | "network_error" }; + +/** Everything one check learned. `null` means that step did not run. */ +export interface ReachabilityFacts { + readonly managedTunnelConfigured: boolean; + readonly connector: ManagedEndpointRuntime.ConnectorFacts | null; + readonly endpoint: EndpointProbe | null; + readonly edgeReachable: boolean | null; + readonly exitNode: TailscaleExitNode | null; +} + +type Unreachable = Extract; + +function exitNodeHint(exitNode: TailscaleExitNode | null): string | null { + if (!exitNode) return null; + const name = exitNode.name ? ` "${exitNode.name}"` : ""; + return `Tailscale exit node${name} is on. Traffic to Cloudflare goes through it.`; +} + +/** Turns what one check learned into the answer every surface shows. */ +export function classifyReachability( + facts: ReachabilityFacts, + checkedAt: string, +): EnvironmentConnectReachability { + const { connector, endpoint, exitNode } = facts; + const fail = ( + cause: Unreachable["cause"], + reason: string, + fix: string, + hint: string | null = null, + ): Unreachable => ({ status: "unreachable", checkedAt, cause, reason, hint, fix }); + const restart = "Restart T3 Code. If it stays, run `t3 connect` again."; + + if (!facts.managedTunnelConfigured) { + return fail( + "not_linked", + "The server has not set up a T3 Connect tunnel.", + "Run `t3 connect`, then restart the server (`t3 service restart`) so it applies the link.", + ); + } + if (connector === null) { + return fail( + "connector_not_running", + "The tunnel client (cloudflared) is not running.", + "Restart T3 Code. If it stays down, check the server log for relay client errors.", + ); + } + const status = endpoint?.kind === "status" ? endpoint.status : null; + if (status === 200) { + return { + status: "reachable", + checkedAt, + readyConnections: connector.readyConnections ?? 0, + protocol: connector.protocol, + }; + } + if (facts.edgeReachable === false) { + return fail( + "edge_unreachable", + `Cloudflare's tunnel port (${TUNNEL_EDGE_PORT}) times out from this machine.`, + exitNode + ? `tailscale set --exit-node= (or allow outbound ${TUNNEL_EDGE_PORT} on that network)` + : `Allow outbound TCP and UDP port ${TUNNEL_EDGE_PORT}, or turn off the VPN or firewall that blocks it.`, + exitNodeHint(exitNode) ?? "A VPN, proxy, or firewall may block it.", + ); + } + if (status === 502) { + return fail( + "origin_unreachable", + "The tunnel is up, but it cannot reach the T3 server on this machine.", + "Restart T3 Code so the tunnel uses the port the server listens on.", + ); + } + if (endpoint === null || connector.readyConnections === 0) { + return fail( + "connector_not_ready", + "cloudflared has not connected to Cloudflare yet.", + "Wait a minute. If it stays, check the network and the server log.", + ); + } + // Cloudflare answers 530 (error 1033) when the tunnel has no live connector. + if (status === 530) { + return fail( + "no_connector", + "Cloudflare has no live connection from this machine for the tunnel.", + restart, + ); + } + if (status !== null) { + return fail( + "unexpected_response", + `The T3 Connect address answered with HTTP ${status}.`, + restart, + ); + } + return endpoint.kind === "dns_failed" + ? fail( + "dns_failed", + "The T3 Connect address does not resolve from this machine.", + "Check the DNS settings on this machine or its VPN.", + exitNodeHint(exitNode), + ) + : fail( + "timeout", + "The T3 Connect address did not answer from this machine.", + "Check the network, VPN, or proxy on this machine.", + exitNodeHint(exitNode), + ); +} + +/** The loud diagnosis the CLI prints and the server logs. */ +export function formatReachabilityFailure(result: Unreachable): string { + return [ + "✕ T3 Connect is NOT reachable from other devices", + ` ${result.reason}`, + ...(result.hint ? [` ${result.hint}`] : []), + ` Fix: ${result.fix}`, + ].join("\n"); +} + +function hasErrorCode(error: unknown, codes: ReadonlySet): boolean { + let current = error; + for (let depth = 0; depth < 6 && typeof current === "object" && current !== null; depth++) { + if ("code" in current && typeof current.code === "string" && codes.has(current.code)) { + return true; + } + current = "cause" in current ? current.cause : undefined; + } + return false; +} + +const DNS_ERROR_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "EAI_NONAME"]); + +const canOpenTcp = (host: string, port: number) => + Effect.callback((resume) => { + const socket = NodeNet.connect({ host, port }); + const finish = (connected: boolean) => { + socket.destroy(); + resume(Effect.succeed(connected)); + }; + socket.setTimeout(Duration.toMillis(EDGE_CONNECT_TIMEOUT), () => finish(false)); + socket.once("connect", () => finish(true)); + socket.once("error", () => finish(false)); + return Effect.sync(() => socket.destroy()); + }); + +export class ConnectReachability extends Context.Service< + ConnectReachability, + { + /** Runs a check now. The server also runs one when the tunnel's readiness changes. */ + readonly check: Effect.Effect; + readonly latest: Effect.Effect>; + } +>()("t3/cloud/ConnectReachability") {} + +const make = Effect.gen(function* () { + const runtime = yield* ManagedEndpointRuntime.CloudManagedEndpointRuntime; + const secrets = yield* ServerSecretStore.ServerSecretStore; + const httpClient = yield* HttpClient.HttpClient; + const latestRef = yield* Ref.make(Option.none()); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const readExitNode = readTailscaleExitNode.pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.orElseSucceed(() => null), + ); + + const probeEndpoint = (hostname: string) => + httpClient.get(`https://${hostname}/.well-known/t3/environment`).pipe( + Effect.map((response): EndpointProbe => ({ kind: "status", status: response.status })), + Effect.scoped, + Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT), + Effect.map(Option.getOrElse((): EndpointProbe => ({ kind: "timeout" }))), + Effect.catch((error) => + Effect.succeed({ + kind: hasErrorCode(error, DNS_ERROR_CODES) ? "dns_failed" : "network_error", + }), + ), + ); + + const gatherFacts = Effect.gen(function* () { + const managedTunnelConfigured = Option.isSome( + yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG).pipe(Effect.orElseSucceed(Option.none)), + ); + const connector = managedTunnelConfigured ? yield* runtime.inspectConnector : null; + const endpoint = + connector?.hostname && connector.readyConnections !== 0 + ? yield* probeEndpoint(connector.hostname) + : null; + const facts: ReachabilityFacts = { + managedTunnelConfigured, + connector, + endpoint, + edgeReachable: null, + exitNode: null, + }; + if (connector === null || (endpoint?.kind === "status" && endpoint.status === 200)) { + return facts; + } + const [edgeResults, exitNode] = yield* Effect.all( + [ + Effect.forEach(TUNNEL_EDGE_HOSTS, (host) => canOpenTcp(host, TUNNEL_EDGE_PORT), { + concurrency: "unbounded", + }), + readExitNode, + ], + { concurrency: 2 }, + ); + return { ...facts, edgeReachable: edgeResults.some(Boolean), exitNode }; + }); + + const check = Effect.gen(function* () { + const facts = yield* gatherFacts; + const result = classifyReachability(facts, DateTime.formatIso(yield* DateTime.now)); + const previous = yield* Ref.getAndSet(latestRef, Option.some(result)); + const changed = Option.match(previous, { + onNone: () => true, + onSome: (last) => + last.status !== result.status || + (last.status === "unreachable" && + result.status === "unreachable" && + last.cause !== result.cause), + }); + if (!changed || (result.status === "unreachable" && result.cause === "not_linked")) { + return result; + } + if (result.status === "unreachable") { + // A connector that has not connected yet is usually still starting; the + // runtime restarts it, and an unreachable edge is reported as its own cause. + const log = result.cause === "connector_not_ready" ? Effect.logWarning : Effect.logError; + yield* log(`\n${formatReachabilityFailure(result)}`).pipe( + Effect.annotateLogs({ cause: result.cause }), + ); + } else { + yield* Effect.logInfo( + `T3 Connect is reachable from other devices. ${formatConnectReachable(result)}`, + ); + } + return result; + }).pipe(Effect.withSpan("ConnectReachability.check")); + + yield* runtime.readinessChanged.pipe( + Stream.debounce(READINESS_CHANGE_DEBOUNCE), + Stream.runForEach(() => check), + Effect.forkScoped, + ); + + return ConnectReachability.of({ check, latest: Ref.get(latestRef) }); +}); + +export const layer = Layer.effect(ConnectReachability, make); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 1ec54b5e839c..2bc742a17063 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -11,10 +11,11 @@ import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; +import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; -import { HttpClient } from "effect/http"; +import { HttpClient, HttpClientResponse } from "effect/http"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; @@ -42,12 +43,24 @@ export type CloudManagedEndpointRuntimeStatus = readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"]; }; +/** What the running cloudflared reports about itself. `null` fields are not known yet. */ +export interface ConnectorFacts { + readonly readyConnections: number | null; + /** The public tunnel hostname from the remote config cloudflared receives once it connects. */ + readonly hostname: string | null; + readonly protocol: string | null; +} + export class CloudManagedEndpointRuntime extends Context.Service< CloudManagedEndpointRuntime, { readonly applyConfig: ( config: RelayManagedEndpointRuntimeConfig | null, ) => Effect.Effect; + /** `null` when no connector is running. */ + readonly inspectConnector: Effect.Effect; + /** Emits when a connector registers a connection or its readiness changes. */ + readonly readinessChanged: Stream.Stream; readonly recoveryRequests: Stream.Stream; readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect; readonly tunnelConnected: Stream.Stream; @@ -64,6 +77,7 @@ interface ActiveConnector { readonly startedAtMillis: number; readonly registered: Deferred.Deferred; readonly metricsAddress: Deferred.Deferred; + readonly protocol: Ref.Ref; } // Without the backoff below, a relay client that fails instantly (a stale @@ -86,6 +100,17 @@ const CONNECTOR_READINESS_REQUEST_TIMEOUT = Duration.seconds(5); const CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART = 6; const CONNECTOR_NOT_READY_CHECKS_MAX = 60; const RELAY_CLIENT_METRICS_LINE = /\bStarting metrics server on (127\.0\.0\.1:\d+)\/metrics\b/u; +const RELAY_CLIENT_PROTOCOL = /\bprotocol=(\w+)/u; +// cloudflared's metrics server: `/ready` answers 200 or 503 with this body, and +// `/config` holds the ingress rules the edge pushed after the first connection. +const ConnectorReadyBody = Schema.Struct({ readyConnections: Schema.Number }); +const ConnectorConfigBody = Schema.Struct({ + config: Schema.Struct({ + ingress: Schema.optional( + Schema.NullOr(Schema.Array(Schema.Struct({ hostname: Schema.optional(Schema.String) }))), + ), + }), +}); // Registration resets cloudflared's own fallback, even when QUIC stalls again // immediately. Keep that evidence across registrations and connector restarts. const QUIC_STALLS_BEFORE_HTTP2 = 30; @@ -173,6 +198,7 @@ export const make = Effect.gen(function* () { const desiredConfigRef = yield* Ref.make(null); const recoveryRequests = yield* Queue.sliding(1); const tunnelConnections = yield* Queue.sliding(1); + const readinessChanges = yield* Queue.sliding(1); const reconcileSemaphore = yield* Semaphore.make(1); const restartDelayRef = yield* Ref.make(0); const linkStateSemaphore = yield* Semaphore.make(1); @@ -403,6 +429,7 @@ export const make = Effect.gen(function* () { const watchConnectorReadiness = Effect.fnUntraced(function* (connector: ActiveConnector) { const readyUrl = `http://${yield* Deferred.await(connector.metricsAddress)}/ready`; let notReadyChecks = 0; + let lastReady: boolean | null = null; while (true) { yield* Effect.sleep(CONNECTOR_READINESS_INTERVAL); const status = yield* httpClient.get(readyUrl).pipe( @@ -413,6 +440,11 @@ export const make = Effect.gen(function* () { Effect.orElseSucceed(() => Option.none()), ); if (Option.isNone(status)) continue; + if (status.value === 200 || status.value === 503) { + const ready = status.value === 200; + if (ready !== lastReady) yield* Queue.offer(readinessChanges, undefined); + lastReady = ready; + } if (status.value === 200) { notReadyChecks = 0; yield* Ref.set(notReadyChecksBeforeRestartRef, CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART); @@ -489,10 +521,14 @@ export const make = Effect.gen(function* () { case "connected": rejectedRegistrations = 0; return Deferred.succeed(connector.registered, undefined).pipe( + Effect.andThen( + Ref.set(connector.protocol, RELAY_CLIENT_PROTOCOL.exec(line)?.[1] ?? null), + ), Effect.andThen( Effect.logInfo("Relay client tunnel connection registered", attributes), ), Effect.andThen(Queue.offer(tunnelConnections, undefined)), + Effect.andThen(Queue.offer(readinessChanges, undefined)), Effect.andThen( RelayClient.isPinnedManagedRelayClient(connector.executable) ? Ref.getAndSet(prunedRef, true).pipe( @@ -666,6 +702,7 @@ export const make = Effect.gen(function* () { startedAtMillis: yield* Clock.currentTimeMillis, registered: yield* Deferred.make(), metricsAddress: yield* Deferred.make(), + protocol: yield* Ref.make(null), } satisfies ActiveConnector; yield* Ref.set(activeRef, connector); yield* Effect.forkIn(observeConnectorOutput(connector), connectorScope); @@ -712,8 +749,44 @@ export const make = Effect.gen(function* () { ), ); + const readMetrics = >(url: string, schema: S) => + httpClient.get(url).pipe( + Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)), + Effect.scoped, + Effect.withTracerEnabled(false), + Effect.timeoutOption(CONNECTOR_READINESS_REQUEST_TIMEOUT), + Effect.map(Option.getOrNull), + Effect.orElseSucceed(() => null), + ); + + const inspectConnector = Effect.gen(function* () { + const active = yield* Ref.get(activeRef); + if (!active || !(yield* active.child.isRunning.pipe(Effect.orElseSucceed(() => false)))) { + return null; + } + const protocol = yield* Ref.get(active.protocol); + if (!(yield* Deferred.isDone(active.metricsAddress))) { + return { readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts; + } + const metricsUrl = `http://${yield* Deferred.await(active.metricsAddress)}`; + const [ready, config] = yield* Effect.all( + [ + readMetrics(`${metricsUrl}/ready`, ConnectorReadyBody), + readMetrics(`${metricsUrl}/config`, ConnectorConfigBody), + ], + { concurrency: 2 }, + ); + return { + readyConnections: ready?.readyConnections ?? null, + hostname: config?.config.ingress?.find((rule) => rule.hostname)?.hostname ?? null, + protocol, + } satisfies ConnectorFacts; + }); + const runtime = CloudManagedEndpointRuntime.of({ applyConfig, + inspectConnector, + readinessChanged: Stream.fromQueue(readinessChanges), recoveryRequests: Stream.fromQueue(recoveryRequests), requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid), tunnelConnected: Stream.fromQueue(tunnelConnections), diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 3e3aae1423c5..0d0f3006a07a 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -13,6 +13,7 @@ import { } from "@t3tools/contracts/relay"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import { HttpClient, HttpServer } from "effect/http"; import * as Etag from "effect/http/Etag"; @@ -34,6 +35,7 @@ import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; import * as CloudLink from "./CloudLink.ts"; import * as ConnectHttp from "./http.ts"; +import * as ConnectReachability from "./ConnectReachability.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; const DESKTOP_BOOTSTRAP_TOKEN = "test-desktop-bootstrap-token"; @@ -75,6 +77,9 @@ const layerLinkProofRoutes = HttpApiBuilder.layer(LinkProofTestApi).pipe( Layer.provide([AuthHttp.layer, ConnectHttp.layer]), Layer.provide(AuthHttp.layerAuthenticatedAuth), Layer.provide(CloudLink.layer), + Layer.provide( + Layer.mock(ConnectReachability.ConnectReachability)({ latest: Effect.succeed(Option.none()) }), + ), Layer.provide(EnvironmentAuth.layer), Layer.provide([ SqlitePersistence.layerMemory, @@ -285,6 +290,7 @@ const answerHealthWith = async (failure: HealthFailure) => { answerHealthRequest: () => Effect.fail(failure), }), ), + Layer.provide(Layer.mock(ConnectReachability.ConnectReachability)({})), Layer.provide(AuthHttp.layerAuthenticatedAuth), Layer.provide(Layer.mock(EnvironmentAuth.EnvironmentAuth)({})), Layer.provideMerge( diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index e9357c2ccc46..568bf9379383 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -17,6 +17,7 @@ import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import { requireEnvironmentScope } from "../auth/http.ts"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as CloudLink from "./CloudLink.ts"; +import * as ConnectReachability from "./ConnectReachability.ts"; import type { RelayRequestError } from "./relayResponse.ts"; import { traceRelayRequest } from "./traceRelayRequest.ts"; @@ -107,6 +108,7 @@ export const layer = HttpApiBuilder.group( "connect", Effect.fnUntraced(function* (handlers) { const cloudLink = yield* CloudLink.CloudLink; + const reachability = yield* ConnectReachability.ConnectReachability; return handlers .handle("linkProof", ({ payload }) => Effect.gen(function* () { @@ -127,6 +129,9 @@ export const layer = HttpApiBuilder.group( Effect.andThen(toHttpError(cloudLink.linkState())), ), ) + .handle("checkReachability", () => + requireEnvironmentScope(AuthRelayReadScope).pipe(Effect.andThen(reachability.check)), + ) .handle("unlink", () => requireEnvironmentScope(AuthRelayWriteScope).pipe( Effect.andThen(toHttpError(cloudLink.unlink())), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 0b0f9c2ac3e4..48d701103c07 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -140,6 +140,7 @@ import { import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import * as CloudHttp from "./cloud/http.ts"; import * as CloudLink from "./cloud/CloudLink.ts"; +import * as ConnectReachability from "./cloud/ConnectReachability.ts"; import { pendingServiceUpdateExists } from "./cloud/updateHandoff.ts"; import * as RelayTracing from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; @@ -1047,7 +1048,7 @@ const layerMakeServer = Layer.unwrap( ); return layerServerApplication.pipe( - Layer.provide(CloudLink.layer), + Layer.provide(CloudLink.layer.pipe(Layer.provideMerge(ConnectReachability.layer))), Layer.provideMerge(layerRuntimeServices), Layer.provideMerge( McpSessionRegistry.layer.pipe( diff --git a/apps/web/src/components/settings/ConnectionsSettings.tsx b/apps/web/src/components/settings/ConnectionsSettings.tsx index a3759c0dbeb4..adfb0ea78188 100644 --- a/apps/web/src/components/settings/ConnectionsSettings.tsx +++ b/apps/web/src/components/settings/ConnectionsSettings.tsx @@ -41,6 +41,7 @@ import { type DesktopServerExposureState, type DesktopWslState, type EnvironmentId, + formatConnectReachable, resolveEnvironmentMachineKind, } from "@t3tools/contracts"; import { @@ -1810,6 +1811,17 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b ? "Your session does not have permission to manage T3 Connect access." : null; const isBusy = isUpdating || isUpdatingPreference; + const reachability = managedTunnelActive ? primaryCloudLinkState.data?.reachability : undefined; + const reachabilityStatus = + reachability?.status === "reachable" ? ( + formatConnectReachable(reachability) + ) : reachability?.status === "unreachable" ? ( + + {[reachability.reason, reachability.hint, `Fix: ${reachability.fix}`] + .filter(Boolean) + .join(" ")} + + ) : null; const updateManagedTunnel = async (enabled: boolean) => { if ( @@ -1885,12 +1897,14 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b linkRevoked ? "This environment was removed from T3 Connect. Turn it on to link it again." : managedTunnelActive - ? "This environment is available to your other devices through T3 Connect." + ? reachability?.status === "unreachable" + ? "T3 Connect is on, but other devices cannot reach this environment." + : "This environment is available to your other devices through T3 Connect." : managedTunnelOutOfSync ? "T3 Connect is out of sync. This environment is linked only to publish agent activity, so other devices cannot connect. Turn T3 Connect on to repair it." : "Make this environment available to your other devices through T3 Connect." } - status={operationError ?? primaryCloudLinkState.error} + status={operationError ?? primaryCloudLinkState.error ?? reachabilityStatus} control={ , +): string { + const connections = `${result.readyConnections} connection${result.readyConnections === 1 ? "" : "s"}`; + return [ + "Tunnel connected", + connections, + ...(result.protocol ? [result.protocol.toUpperCase()] : []), + ].join(" · "); +} + export const EnvironmentCloudLinkStateResult = Schema.Struct({ linked: Schema.Boolean, cloudUserId: Schema.NullOr(Schema.String), @@ -417,6 +461,8 @@ export const EnvironmentCloudLinkStateResult = Schema.Struct({ publishAgentActivity: Schema.Boolean, holdWebhooksWhileOffline: Schema.optional(Schema.Boolean), linkRevoked: Schema.optional(Schema.Boolean), + /** The latest check the server ran on its own, absent until one finishes. */ + reachability: Schema.optional(EnvironmentConnectReachability), }); export type EnvironmentCloudLinkStateResult = typeof EnvironmentCloudLinkStateResult.Type; @@ -689,6 +735,13 @@ class EnvironmentConnectHttpApi extends HttpApiGroup.make("connect") error: EnvironmentHttpCloudErrors, }).middleware(EnvironmentAuthenticatedAuth), ) + .add( + HttpApiEndpoint.post("checkReachability", "/api/connect/reachability", { + headers: OptionalBearerHeaders, + success: EnvironmentConnectReachability, + error: EnvironmentHttpCloudErrors, + }).middleware(EnvironmentAuthenticatedAuth), + ) .add( HttpApiEndpoint.post("unlink", "/api/connect/unlink", { headers: OptionalBearerHeaders, diff --git a/packages/tailscale/src/tailscale.test.ts b/packages/tailscale/src/tailscale.test.ts index da20dcd60427..ba9fbe2adfad 100644 --- a/packages/tailscale/src/tailscale.test.ts +++ b/packages/tailscale/src/tailscale.test.ts @@ -15,6 +15,7 @@ import { disableTailscaleServe, ensureTailscaleServe, isTailscaleIpv4Address, + parseTailscaleExitNode, parseTailscaleMagicDnsName, parseTailscaleSshEnabled, parseTailscaleStatus, @@ -147,6 +148,26 @@ describe("tailscale", () => { }), ); + it.effect("names the exit node in use", () => + Effect.gen(function* () { + const inUse = yield* parseTailscaleExitNode( + JSON.stringify({ + ExitNodeStatus: { ID: "nX", Online: true, TailscaleIPs: ["100.101.102.103/32"] }, + Peer: { + "nodekey:a": { HostName: "laptop", DNSName: "laptop.tail.ts.net.", ExitNode: false }, + "nodekey:b": { HostName: "home-pi", DNSName: "home-pi.tail.ts.net.", ExitNode: true }, + }, + }), + ); + assert.deepEqual(inUse, { name: "home-pi", online: true }); + + const unnamed = yield* parseTailscaleExitNode(`{"ExitNodeStatus":{"Online":false}}`); + assert.deepEqual(unnamed, { name: null, online: false }); + + assert.equal(yield* parseTailscaleExitNode(tailscaleStatusJson), null); + }), + ); + it.effect("reads the HTTPS serve capability from the status capability map", () => Effect.gen(function* () { const enabled = yield* parseTailscaleStatus( diff --git a/packages/tailscale/src/tailscale.ts b/packages/tailscale/src/tailscale.ts index 28664e23fa9b..6d5225d60ce8 100644 --- a/packages/tailscale/src/tailscale.ts +++ b/packages/tailscale/src/tailscale.ts @@ -286,6 +286,56 @@ export const readTailscaleStatus = readTailscaleOutput("status", ["status", "--j Effect.flatMap(parseTailscaleStatus), ); +const TailscaleExitNodeJson = Schema.Struct({ + ExitNodeStatus: Schema.optional(Schema.Struct({ Online: Schema.optional(Schema.Unknown) })), + Peer: Schema.optional( + Schema.Record( + Schema.String, + Schema.Struct({ + ExitNode: Schema.optional(Schema.Unknown), + HostName: Schema.optional(Schema.Unknown), + DNSName: Schema.optional(Schema.Unknown), + }), + ), + ), +}); + +const decodeTailscaleExitNodeJson = Schema.decodeEffect( + Schema.fromJsonString(TailscaleExitNodeJson), +); + +export interface TailscaleExitNode { + readonly name: string | null; + readonly online: boolean; +} + +/** + * `ExitNodeStatus` is present only while an exit node is in use, but it has no + * name. The name comes from the peer marked `ExitNode: true`. + */ +export const parseTailscaleExitNode = ( + rawStatusJson: string, +): Effect.Effect => + decodeTailscaleExitNodeJson(rawStatusJson).pipe( + Effect.mapError((cause) => new TailscaleStatusParseError({ cause })), + Effect.map((parsed) => { + if (parsed.ExitNodeStatus === undefined) return null; + const peer = Object.values(parsed.Peer ?? {}).find( + (candidate) => candidate.ExitNode === true, + ); + const hostName = typeof peer?.HostName === "string" ? peer.HostName.trim() : ""; + const dnsLabel = typeof peer?.DNSName === "string" ? (peer.DNSName.split(".")[0] ?? "") : ""; + return { + name: hostName || dnsLabel || null, + online: parsed.ExitNodeStatus.Online === true, + }; + }), + ); + +export const readTailscaleExitNode = readTailscaleOutput("status", ["status", "--json"]).pipe( + Effect.flatMap(parseTailscaleExitNode), +); + const TailscalePrefsJson = Schema.Struct({ RunSSH: Schema.optional(Schema.Unknown), WantRunning: Schema.optional(Schema.Unknown), From 8aa548cdc6a02d6d9aab7ee0f10813cf8d32f799 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 19 Oct 2026 23:31:08 -0700 Subject: [PATCH 2/8] fix(web): host T3 Connect row refreshes reachability every 30s Co-Authored-By: Claude Opus 5.5 --- apps/web/src/components/settings/ConnectionsSettings.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/web/src/components/settings/ConnectionsSettings.tsx b/apps/web/src/components/settings/ConnectionsSettings.tsx index adfb0ea78188..3dccec651c7a 100644 --- a/apps/web/src/components/settings/ConnectionsSettings.tsx +++ b/apps/web/src/components/settings/ConnectionsSettings.tsx @@ -15,6 +15,7 @@ import { type ReactNode, memo, useEffect, + useEffectEvent, useId, useRef, useState, @@ -1811,6 +1812,13 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b ? "Your session does not have permission to manage T3 Connect access." : null; const isBusy = isUpdating || isUpdatingPreference; + const refreshLinkState = useEffectEvent(primaryCloudLinkState.refresh); + // The server rechecks reachability when the tunnel's 30 s readiness poll changes. + useEffect(() => { + if (!managedTunnelActive) return; + const interval = window.setInterval(() => refreshLinkState(), 30_000); + return () => window.clearInterval(interval); + }, [managedTunnelActive]); const reachability = managedTunnelActive ? primaryCloudLinkState.data?.reachability : undefined; const reachabilityStatus = reachability?.status === "reachable" ? ( From 15758da9daf498ca081fce027df66b917d30d680 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 19 Oct 2026 23:32:04 -0700 Subject: [PATCH 3/8] fix(web): keep polling T3 Connect reachability after a failed read Co-Authored-By: Claude Opus 5.5 --- apps/web/src/components/settings/ConnectionsSettings.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/web/src/components/settings/ConnectionsSettings.tsx b/apps/web/src/components/settings/ConnectionsSettings.tsx index 3dccec651c7a..b59d3607372f 100644 --- a/apps/web/src/components/settings/ConnectionsSettings.tsx +++ b/apps/web/src/components/settings/ConnectionsSettings.tsx @@ -1814,11 +1814,11 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b const isBusy = isUpdating || isUpdatingPreference; const refreshLinkState = useEffectEvent(primaryCloudLinkState.refresh); // The server rechecks reachability when the tunnel's 30 s readiness poll changes. + // Poll while mounted, so a failed read does not stop later ones. useEffect(() => { - if (!managedTunnelActive) return; const interval = window.setInterval(() => refreshLinkState(), 30_000); return () => window.clearInterval(interval); - }, [managedTunnelActive]); + }, []); const reachability = managedTunnelActive ? primaryCloudLinkState.data?.reachability : undefined; const reachabilityStatus = reachability?.status === "reachable" ? ( From 15c86d2afcb318119e95d822edf4d02af58cc206 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 19 Oct 2026 23:45:18 -0700 Subject: [PATCH 4/8] fix(connect): address review findings on reachability causes and status Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cli/connect.ts | 17 ++++++++------- .../src/cloud/ConnectReachability.test.ts | 21 +++++++++++++++++++ apps/server/src/cloud/ConnectReachability.ts | 8 ++++++- .../settings/ConnectionsSettings.tsx | 4 +++- packages/tailscale/src/tailscale.test.ts | 6 +++++- packages/tailscale/src/tailscale.ts | 5 +++-- 6 files changed, 48 insertions(+), 13 deletions(-) diff --git a/apps/server/src/cli/connect.ts b/apps/server/src/cli/connect.ts index 5dc3329c5fe6..94f338cef087 100644 --- a/apps/server/src/cli/connect.ts +++ b/apps/server/src/cli/connect.ts @@ -148,7 +148,8 @@ interface CloudCliStatus { readonly relayUrl: string | null; readonly publishAgentActivity: boolean; readonly relayClient: RelayClient.RelayClientStatus; - /** `null` when no server answered the live check, or none ran because T3 Connect is off. */ + readonly linkMode: CliState.CliDesiredLinkMode; + /** `null` when no server answered the live check, or none ran because no tunnel is wanted. */ readonly reachability: EnvironmentConnectReachability | null; } @@ -207,7 +208,7 @@ function formatCloudStatus(status: CloudCliStatus, options?: { readonly json?: b ` Relay: ${status.relayUrl ?? "not provisioned"}`, ` Publish agent activity: ${status.publishAgentActivity ? "enabled" : "disabled"}`, ...formatRelayClientStatus(status.relayClient), - ...(!status.desired + ...(!status.desired || status.linkMode !== "managed" ? [] : status.reachability === null ? [ @@ -656,7 +657,7 @@ const connectStatusCommand = Command.make("status", { publishAgentActivity, revoked, executable, - reachability, + linkMode, ] = yield* Effect.all( [ CliState.readCliDesiredCloudLink, @@ -666,14 +667,13 @@ const connectStatusCommand = Command.make("status", { secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET), secrets.get(CLOUD_LINK_REVOKED), relayClient.resolve, - CliState.readCliDesiredCloudLink.pipe( - Effect.flatMap((desired) => - desired ? checkLiveReachability() : Effect.succeed(null), - ), - ), + CliState.readCliDesiredLinkMode, ], { concurrency: "unbounded" }, ); + // A publish-only link has no tunnel for other devices to reach. + const reachability = + desired && linkMode === "managed" ? yield* checkLiveReachability() : null; const status: CloudCliStatus = { desired, authenticated, @@ -685,6 +685,7 @@ const connectStatusCommand = Command.make("status", { Option.isSome(publishAgentActivity) ? bytesToString(publishAgentActivity.value) : null, ), relayClient: executable, + linkMode, reachability, }; yield* Console.log(formatCloudStatus(status, { json: flags.json })); diff --git a/apps/server/src/cloud/ConnectReachability.test.ts b/apps/server/src/cloud/ConnectReachability.test.ts index 9e100b2c0160..fa5fcc647abf 100644 --- a/apps/server/src/cloud/ConnectReachability.test.ts +++ b/apps/server/src/cloud/ConnectReachability.test.ts @@ -63,6 +63,27 @@ describe("classifyReachability", () => { ); }); + it("keeps the endpoint cause when the TCP edge test cannot explain it", () => { + // QUIC (UDP) can connect while TCP 7844 is blocked. + expect( + classifyReachability( + facts({ endpoint: { kind: "status", status: 502 }, edgeReachable: false }), + checkedAt, + ), + ).toMatchObject({ cause: "origin_unreachable" }); + // Failed DNS also fails the edge lookups. + expect( + classifyReachability( + facts({ + connector: { readyConnections: null, hostname: "env.example.test", protocol: null }, + endpoint: { kind: "dns_failed" }, + edgeReachable: false, + }), + checkedAt, + ), + ).toMatchObject({ cause: "dns_failed" }); + }); + it("waits on a connector that has not connected while the edge answers", () => { expect( classifyReachability(facts({ connector: notConnected, edgeReachable: true }), checkedAt), diff --git a/apps/server/src/cloud/ConnectReachability.ts b/apps/server/src/cloud/ConnectReachability.ts index 31349736e01c..4c20ca2ff00c 100644 --- a/apps/server/src/cloud/ConnectReachability.ts +++ b/apps/server/src/cloud/ConnectReachability.ts @@ -84,7 +84,13 @@ export function classifyReachability( protocol: connector.protocol, }; } - if (facts.edgeReachable === false) { + // The edge test is TCP only. With QUIC connections up, or with DNS already failing, a + // failed TCP test would hide the real cause. + if ( + facts.edgeReachable === false && + !(connector.readyConnections && connector.readyConnections > 0) && + endpoint?.kind !== "dns_failed" + ) { return fail( "edge_unreachable", `Cloudflare's tunnel port (${TUNNEL_EDGE_PORT}) times out from this machine.`, diff --git a/apps/web/src/components/settings/ConnectionsSettings.tsx b/apps/web/src/components/settings/ConnectionsSettings.tsx index b59d3607372f..62e5c49d13fe 100644 --- a/apps/web/src/components/settings/ConnectionsSettings.tsx +++ b/apps/web/src/components/settings/ConnectionsSettings.tsx @@ -1907,7 +1907,9 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b : managedTunnelActive ? reachability?.status === "unreachable" ? "T3 Connect is on, but other devices cannot reach this environment." - : "This environment is available to your other devices through T3 Connect." + : reachability?.status === "reachable" + ? "This environment is available to your other devices through T3 Connect." + : "T3 Connect is on. Reachability has not been checked yet." : managedTunnelOutOfSync ? "T3 Connect is out of sync. This environment is linked only to publish agent activity, so other devices cannot connect. Turn T3 Connect on to repair it." : "Make this environment available to your other devices through T3 Connect." diff --git a/packages/tailscale/src/tailscale.test.ts b/packages/tailscale/src/tailscale.test.ts index ba9fbe2adfad..4ae89521b612 100644 --- a/packages/tailscale/src/tailscale.test.ts +++ b/packages/tailscale/src/tailscale.test.ts @@ -155,7 +155,11 @@ describe("tailscale", () => { ExitNodeStatus: { ID: "nX", Online: true, TailscaleIPs: ["100.101.102.103/32"] }, Peer: { "nodekey:a": { HostName: "laptop", DNSName: "laptop.tail.ts.net.", ExitNode: false }, - "nodekey:b": { HostName: "home-pi", DNSName: "home-pi.tail.ts.net.", ExitNode: true }, + "nodekey:b": { + HostName: "raspberrypi", + DNSName: "home-pi.tail.ts.net.", + ExitNode: true, + }, }, }), ); diff --git a/packages/tailscale/src/tailscale.ts b/packages/tailscale/src/tailscale.ts index 6d5225d60ce8..97c5d38a1cce 100644 --- a/packages/tailscale/src/tailscale.ts +++ b/packages/tailscale/src/tailscale.ts @@ -323,10 +323,11 @@ export const parseTailscaleExitNode = ( const peer = Object.values(parsed.Peer ?? {}).find( (candidate) => candidate.ExitNode === true, ); - const hostName = typeof peer?.HostName === "string" ? peer.HostName.trim() : ""; + // The DNS label is the tailnet machine name; HostName is the OS name and ignores renames. const dnsLabel = typeof peer?.DNSName === "string" ? (peer.DNSName.split(".")[0] ?? "") : ""; + const hostName = typeof peer?.HostName === "string" ? peer.HostName.trim() : ""; return { - name: hostName || dnsLabel || null, + name: dnsLabel || hostName || null, online: parsed.ExitNodeStatus.Online === true, }; }), From 62539aa2e0dfa492119a8e0e7722279e1b8b6964 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 20 Oct 2026 00:03:04 -0700 Subject: [PATCH 5/8] fix(connect): recheck reachability every 5 minutes and keep 502 over edge failures Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cloud/ConnectReachability.test.ts | 10 ++++++++++ apps/server/src/cloud/ConnectReachability.ts | 12 ++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/apps/server/src/cloud/ConnectReachability.test.ts b/apps/server/src/cloud/ConnectReachability.test.ts index fa5fcc647abf..fbf21bdd0022 100644 --- a/apps/server/src/cloud/ConnectReachability.test.ts +++ b/apps/server/src/cloud/ConnectReachability.test.ts @@ -71,6 +71,16 @@ describe("classifyReachability", () => { checkedAt, ), ).toMatchObject({ cause: "origin_unreachable" }); + expect( + classifyReachability( + facts({ + connector: { readyConnections: null, hostname: "env.example.test", protocol: null }, + endpoint: { kind: "status", status: 502 }, + edgeReachable: false, + }), + checkedAt, + ), + ).toMatchObject({ cause: "origin_unreachable" }); // Failed DNS also fails the edge lookups. expect( classifyReachability( diff --git a/apps/server/src/cloud/ConnectReachability.ts b/apps/server/src/cloud/ConnectReachability.ts index 4c20ca2ff00c..7783d18436e7 100644 --- a/apps/server/src/cloud/ConnectReachability.ts +++ b/apps/server/src/cloud/ConnectReachability.ts @@ -25,6 +25,8 @@ const EDGE_CONNECT_TIMEOUT = Duration.seconds(5); const ENDPOINT_REQUEST_TIMEOUT = Duration.seconds(10); // cloudflared registers several connections within a few seconds; they are one change. const READINESS_CHANGE_DEBOUNCE = Duration.seconds(3); +// Readiness changes miss failures and recoveries past cloudflared (DNS, origin, edge errors). +const RECHECK_INTERVAL = Duration.minutes(5); export type EndpointProbe = | { readonly kind: "status"; readonly status: number } @@ -84,10 +86,11 @@ export function classifyReachability( protocol: connector.protocol, }; } - // The edge test is TCP only. With QUIC connections up, or with DNS already failing, a - // failed TCP test would hide the real cause. + // The edge test is TCP only. With QUIC connections up, a 502 from a working tunnel, or + // DNS already failing, a failed TCP test would hide the real cause. if ( facts.edgeReachable === false && + status !== 502 && !(connector.readyConnections && connector.readyConnections > 0) && endpoint?.kind !== "dns_failed" ) { @@ -279,6 +282,11 @@ const make = Effect.gen(function* () { Stream.runForEach(() => check), Effect.forkScoped, ); + yield* Effect.sleep(RECHECK_INTERVAL).pipe( + Effect.andThen(check), + Effect.forever, + Effect.forkScoped, + ); return ConnectReachability.of({ check, latest: Ref.get(latestRef) }); }); From 075e79d4c1a1f1ff392a73ea30511fe376178609 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 20 Oct 2026 00:24:04 -0700 Subject: [PATCH 6/8] fix(connect): a slow older reachability check no longer replaces a newer result Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cloud/ConnectReachability.ts | 21 ++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/apps/server/src/cloud/ConnectReachability.ts b/apps/server/src/cloud/ConnectReachability.ts index 7783d18436e7..1fd0ae83422f 100644 --- a/apps/server/src/cloud/ConnectReachability.ts +++ b/apps/server/src/cloud/ConnectReachability.ts @@ -196,7 +196,11 @@ const make = Effect.gen(function* () { const runtime = yield* ManagedEndpointRuntime.CloudManagedEndpointRuntime; const secrets = yield* ServerSecretStore.ServerSecretStore; const httpClient = yield* HttpClient.HttpClient; - const latestRef = yield* Ref.make(Option.none()); + // `order` ranks checks by start, so a slow older one cannot replace a newer result. + const latestRef = yield* Ref.make( + Option.none<{ readonly order: number; readonly result: EnvironmentConnectReachability }>(), + ); + const checkCounter = yield* Ref.make(0); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const readExitNode = readTailscaleExitNode.pipe( Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), @@ -248,12 +252,18 @@ const make = Effect.gen(function* () { }); const check = Effect.gen(function* () { + const order = yield* Ref.updateAndGet(checkCounter, (count) => count + 1); const facts = yield* gatherFacts; const result = classifyReachability(facts, DateTime.formatIso(yield* DateTime.now)); - const previous = yield* Ref.getAndSet(latestRef, Option.some(result)); + const previous = yield* Ref.modify(latestRef, (current) => + Option.isSome(current) && current.value.order > order + ? [null, current] + : [current, Option.some({ order, result })], + ); + if (previous === null) return result; const changed = Option.match(previous, { onNone: () => true, - onSome: (last) => + onSome: ({ result: last }) => last.status !== result.status || (last.status === "unreachable" && result.status === "unreachable" && @@ -288,7 +298,10 @@ const make = Effect.gen(function* () { Effect.forkScoped, ); - return ConnectReachability.of({ check, latest: Ref.get(latestRef) }); + return ConnectReachability.of({ + check, + latest: Ref.get(latestRef).pipe(Effect.map(Option.map(({ result }) => result))), + }); }); export const layer = Layer.effect(ConnectReachability, make); From 7f606900d370e978beb919627e03eeb5b794f2dd Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 20 Oct 2026 00:58:23 -0700 Subject: [PATCH 7/8] docs(connect): T3 Connect troubleshooting guide, linked from every error Adds docs/user/t3-connect-troubleshooting.md with one section per failure: what we think is wrong and how to fix it. The CLI box, server log, host T3 Connect row, and the web and mobile connection Details link to the matching section as "More info". A test fails when a heading is renamed without updating the linked anchors. Co-Authored-By: Claude Opus 5.5 --- .../connection/ConnectionDiagnosisDetails.tsx | 11 +- .../src/cloud/ConnectReachability.test.ts | 26 ++++ apps/server/src/cloud/ConnectReachability.ts | 7 +- .../settings/ConnectionDiagnosisDetails.tsx | 10 ++ .../settings/ConnectionsSettings.tsx | 11 +- docs/README.md | 1 + docs/user/background-service.md | 2 +- docs/user/remote-access.md | 23 +-- docs/user/t3-connect-troubleshooting.md | 144 ++++++++++++++++++ .../src/connection/diagnosis.test.ts | 8 + .../src/connection/diagnosis.ts | 58 +++++++ packages/contracts/src/environmentHttp.ts | 38 +++++ 12 files changed, 315 insertions(+), 24 deletions(-) create mode 100644 docs/user/t3-connect-troubleshooting.md diff --git a/apps/mobile/src/features/connection/ConnectionDiagnosisDetails.tsx b/apps/mobile/src/features/connection/ConnectionDiagnosisDetails.tsx index 0fbfd68b480e..df0e8c165929 100644 --- a/apps/mobile/src/features/connection/ConnectionDiagnosisDetails.tsx +++ b/apps/mobile/src/features/connection/ConnectionDiagnosisDetails.tsx @@ -12,7 +12,7 @@ import type { EnvironmentId } from "@t3tools/contracts"; import Constants from "expo-constants"; import * as Option from "effect/Option"; import { AsyncResult } from "effect/reactivity"; -import { View } from "react-native"; +import { Linking, View } from "react-native"; import { AppText as Text } from "../../components/ui/AppText"; import { Button } from "../../components/ui/Button"; @@ -52,6 +52,7 @@ export function ConnectionDiagnosisDetails(props: { readonly diagnosis: ConnectionDiagnosis; }) { const { diagnosis } = props; + const { moreInfoUrl } = diagnosis; return ( @@ -87,6 +88,14 @@ export function ConnectionDiagnosisDetails(props: { }); }} /> + {moreInfoUrl ? ( +