feat(web): update feed and usage endpoints for the desktop app

surfsense_web/app/api/updates/[file]/route.ts

@@ -0,0 +1,12 @@
+import { recordAfterResponse } from "@/lib/desktop-telemetry/record-after-response";
+import { serveUpdateManifest } from "@/lib/desktop-telemetry/update-feed/serve-update-manifest";
+
+// Each check must reach the handler to be counted. The GitHub fetch inside
+// keeps its own 15-minute cache.
+export const dynamic = "force-dynamic";
+
+/** The desktop update feed at updates.surfsense.com (docs/contracts/05-desktop-telemetry.md). */
+export async function GET(request: Request, { params }: { params: Promise<{ file: string }> }) {
+	const { file } = await params;
+	return serveUpdateManifest(request, file, { fetchManifest: fetch, record: recordAfterResponse });
+}

surfsense_web/app/api/usage/route.ts

@@ -0,0 +1,9 @@
+import { recordAfterResponse } from "@/lib/desktop-telemetry/record-after-response";
+import { receiveUsageReport } from "@/lib/desktop-telemetry/usage-report/receive-usage-report";
+
+export const dynamic = "force-dynamic";
+
+/** Opt-in usage reports at usage.surfsense.com (docs/contracts/05-desktop-telemetry.md). */
+export async function POST(request: Request) {
+	return receiveUsageReport(request, { record: recordAfterResponse });
+}

surfsense_web/lib/desktop-telemetry/desktop-dimensions.ts

@@ -0,0 +1,51 @@
+/**
+ * App version, platform and OS as the desktop app reports them
+ * (docs/contracts/05-desktop-telemetry.md). Both routes accept only these
+ * spellings, so nothing free-form reaches an event.
+ */
+
+export const DESKTOP_PLATFORMS = [
+	"win32-x64",
+	"win32-arm64",
+	"darwin-arm64",
+	"darwin-x64",
+	"linux-x64",
+	"linux-arm64",
+] as const;
+
+const FIXED_OS_NAMES = new Set(["windows-10", "windows-11", "linux"]);
+const MACOS_MAJOR = /^macos-[1-9][0-9]$/;
+
+// Plain x.y.z only: a pre-release tag is free text and could carry a name, so
+// a pre-release build's checks count as "invalid" and its usage reports get 400.
+export const APP_VERSION = /^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$/;
+
+export function isDesktopPlatform(value: unknown): value is (typeof DESKTOP_PLATFORMS)[number] {
+	return typeof value === "string" && (DESKTOP_PLATFORMS as readonly string[]).includes(value);
+}
+
+export function isDesktopOs(value: unknown): value is string {
+	return typeof value === "string" && (FIXED_OS_NAMES.has(value) || MACOS_MAJOR.test(value));
+}
+
+export function isAppVersion(value: unknown): value is string {
+	return typeof value === "string" && APP_VERSION.test(value);
+}
+
+export type UpdateCheckDimensions = {
+	app_version: string;
+	platform: string;
+	os: string;
+};
+
+/** Reads the three X-SurfSense-* headers, folding anything unexpected into one bucket. */
+export function readUpdateCheckDimensions(headers: Headers): UpdateCheckDimensions {
+	const version = headers.get("x-surfsense-version");
+	const platform = headers.get("x-surfsense-platform");
+	const os = headers.get("x-surfsense-os");
+	return {
+		app_version: isAppVersion(version) ? version : "invalid",
+		platform: isDesktopPlatform(platform) ? platform : "other",
+		os: isDesktopOs(os) ? os : "other",
+	};
+}

surfsense_web/lib/desktop-telemetry/personless-capture.ts

@@ -0,0 +1,61 @@
+import { randomUUID } from "node:crypto";
+import { PostHog, type PostHogOptions } from "posthog-node";
+
+/** An event with no person behind it; every value is an enum, flag or fixed format. */
+export type PersonlessEvent = {
+	event: string;
+	properties: Record<string, string | number | boolean>;
+};
+
+export type CapturePersonless = (event: PersonlessEvent) => Promise<void>;
+
+type CaptureConfig = {
+	apiKey: string | undefined;
+	host: string | undefined;
+	fetch?: PostHogOptions["fetch"];
+};
+
+/**
+ * Sends one event straight away, with GeoIP off and no person profile, so
+ * PostHog keeps neither a location from our server's address nor a person
+ * (docs/contracts/05-desktop-telemetry.md). Never throws: posthog-node logs
+ * and drops a failed send.
+ */
+export function createPersonlessCapture({ apiKey, host, fetch }: CaptureConfig): CapturePersonless {
+	if (!apiKey) return async () => {};
+
+	const client = new PostHog(apiKey, {
+		host: host || undefined,
+		disableGeoip: true,
+		flushAt: 1,
+		flushInterval: 0,
+		requestTimeout: 5000,
+		fetchRetryCount: 1,
+		fetch,
+	});
+
+	return async ({ event, properties }) => {
+		await client.captureImmediate({
+			// A fresh id per event: there is no one to count twice and nothing to join on.
+			distinctId: randomUUID(),
+			event,
+			properties: { ...properties, $process_person_profile: false },
+			disableGeoip: true,
+		});
+	};
+}
+
+let captureFromEnv: CapturePersonless | undefined;
+
+/**
+ * The routes' capture. Does nothing until DESKTOP_TELEMETRY_POSTHOG_KEY is set.
+ * The key's project must live on the instance NEXT_PUBLIC_POSTHOG_HOST names
+ * (fixed at build time), or PostHog drops every event without an error.
+ */
+export function capturePersonlessEvent(event: PersonlessEvent): Promise<void> {
+	captureFromEnv ??= createPersonlessCapture({
+		apiKey: process.env.DESKTOP_TELEMETRY_POSTHOG_KEY,
+		host: process.env.NEXT_PUBLIC_POSTHOG_HOST,
+	});
+	return captureFromEnv(event);
+}

surfsense_web/lib/desktop-telemetry/record-after-response.ts

@@ -0,0 +1,11 @@
+import { after } from "next/server";
+import { capturePersonlessEvent, type PersonlessEvent } from "./personless-capture";
+
+/**
+ * Captures once the response has gone, so PostHog being slow or down never
+ * delays an update check or a usage report. On Vercel, after() keeps the
+ * function alive until the send settles.
+ */
+export function recordAfterResponse(event: PersonlessEvent): void {
+	after(() => capturePersonlessEvent(event));
+}

surfsense_web/lib/desktop-telemetry/request-country.ts

@@ -0,0 +1,10 @@
+const COUNTRY_CODE = /^[A-Z]{2}$/;
+
+/**
+ * Country from the header Vercel derives at its edge, so the address itself
+ * is never read here (docs/contracts/05-desktop-telemetry.md).
+ */
+export function requestCountry(headers: Headers): string {
+	const country = headers.get("x-vercel-ip-country");
+	return country && COUNTRY_CODE.test(country) ? country : "unknown";
+}

surfsense_web/lib/desktop-telemetry/update-feed/channel-files.ts

@@ -0,0 +1,7 @@
+// The four 2.x channel files. latest*.yml belongs to the legacy 0.0.x bridge
+// on GitHub and must never be served from here.
+const CHANNEL_FILE = /^stable(-mac|-linux(-arm64)?)?\.yml$/;
+
+export function isChannelFile(file: string): boolean {
+	return CHANNEL_FILE.test(file);
+}

surfsense_web/lib/desktop-telemetry/update-feed/latest-github-manifest.ts

@@ -0,0 +1,30 @@
+const LATEST_DOWNLOAD = "https://github.com/MODSetter/SurfSense/releases/latest/download";
+// Real manifests are about 500 bytes.
+const MAX_MANIFEST_CHARS = 64 * 1024;
+
+export type FetchManifest = (
+	url: string,
+	init: RequestInit & { next?: { revalidate?: number } }
+) => Promise<Response>;
+
+/**
+ * The channel file of the latest published release. "latest" only moves on
+ * publish, so a draft merged to main early is never served. Each revalidation
+ * is still a download, so 15 minutes of caching adds about 96 a day per file
+ * (more if Vercel caches per region) to GitHub's stable*.yml counts.
+ */
+export async function fetchLatestGithubManifest(
+	file: string,
+	fetchManifest: FetchManifest
+): Promise<string> {
+	const response = await fetchManifest(`${LATEST_DOWNLOAD}/${file}`, {
+		redirect: "follow",
+		next: { revalidate: 900 },
+		// Well inside the app's 10 s, so it still has time to fall back to GitHub.
+		signal: AbortSignal.timeout(5000),
+	});
+	if (response.status !== 200) throw new Error(`GitHub answered ${response.status}`);
+	const text = await response.text();
+	if (text.length > MAX_MANIFEST_CHARS) throw new Error("manifest is too large");
+	return text;
+}

surfsense_web/lib/desktop-telemetry/update-feed/pin-manifest-to-release.ts

@@ -0,0 +1,114 @@
+import { APP_VERSION } from "../desktop-dimensions";
+
+const RELEASE_DOWNLOAD = "https://github.com/MODSetter/SurfSense/releases/download";
+const RELEASE_FILE = /^SurfSense[-A-Za-z0-9._]*\.(exe|zip|dmg|AppImage|deb)$/;
+
+const SHA512 = /^[A-Za-z0-9+/]{86}==$/;
+const BYTE_COUNT = /^[0-9]{1,13}$/;
+const ISO_DATE = /^(['"]?)[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{1,3})?Z\1$/;
+const PERCENTAGE = /^(100|[1-9]?[0-9])(\.[0-9]+)?$/;
+
+// Every line electron-builder writes for our releases, value shapes included.
+// With no YAML parser here, a line outside this list could hide a url (a flow
+// mapping, a quoted key, a folded scalar), so it fails the whole manifest and
+// the app falls back to GitHub.
+const TOP_LEVEL_LINE = /^([A-Za-z][A-Za-z0-9]*):(?: (.*))?$/;
+const FILE_START_LINE = /^ {2}- url: (.*)$/;
+const FILE_FIELD_LINE = /^ {4}([A-Za-z][A-Za-z0-9]*): (.*)$/;
+
+const TOP_LEVEL_VALUES = new Map<string, RegExp>([
+	["version", APP_VERSION],
+	["files", /^$/],
+	["path", /^\S+$/],
+	["sha512", SHA512],
+	["releaseDate", ISO_DATE],
+	["stagingPercentage", PERCENTAGE],
+]);
+
+const FILE_FIELD_VALUES = new Map<string, RegExp>([
+	["sha512", SHA512],
+	["size", BYTE_COUNT],
+	["blockMapSize", BYTE_COUNT],
+	["isAdminRightsRequired", /^(true|false)$/],
+]);
+
+type Line =
+	| { kind: "top"; key: string; value: string }
+	| { kind: "file-start"; url: string }
+	| { kind: "file-field"; key: string; value: string };
+
+function parseLine(line: string, number: number): Line {
+	const fileStart = FILE_START_LINE.exec(line);
+	if (fileStart) return { kind: "file-start", url: fileStart[1] };
+
+	const fileField = FILE_FIELD_LINE.exec(line);
+	if (fileField && FILE_FIELD_VALUES.get(fileField[1])?.test(fileField[2])) {
+		return { kind: "file-field", key: fileField[1], value: fileField[2] };
+	}
+
+	const topLevel = TOP_LEVEL_LINE.exec(line);
+	const value = topLevel?.[2] ?? "";
+	if (topLevel && TOP_LEVEL_VALUES.get(topLevel[1])?.test(value)) {
+		return { kind: "top", key: topLevel[1], value };
+	}
+
+	throw new Error(`line ${number} is not part of an update manifest`);
+}
+
+/** Checks the nesting and that no key repeats, which the app's YAML parser would reject. */
+function checkStructure(lines: Line[]): string {
+	const topKeys = new Set<string>();
+	let inFiles = false;
+	let fileKeys: Set<string> | null = null;
+	let fileCount = 0;
+
+	for (const line of lines) {
+		if (line.kind === "top") {
+			if (topKeys.has(line.key)) throw new Error(`${line.key} appears twice`);
+			topKeys.add(line.key);
+			inFiles = line.key === "files";
+			fileKeys = null;
+		} else if (line.kind === "file-start") {
+			if (!inFiles) throw new Error("a file entry sits outside files:");
+			fileKeys = new Set(["url"]);
+			fileCount += 1;
+		} else {
+			if (!fileKeys) throw new Error(`${line.key} sits outside a file entry`);
+			if (fileKeys.has(line.key)) throw new Error(`${line.key} appears twice in one file entry`);
+			fileKeys.add(line.key);
+		}
+	}
+
+	const version = lines.find((line) => line.kind === "top" && line.key === "version");
+	if (!version || version.kind !== "top") throw new Error("no version");
+	if (fileCount === 0) throw new Error("no files");
+	return version.value;
+}
+
+/** Accepts a bare release file name, or the exact versioned URL for it. */
+function releaseFileUrl(value: string, version: string): string {
+	const prefix = `${RELEASE_DOWNLOAD}/v${version}/`;
+	const name = value.startsWith(prefix) ? value.slice(prefix.length) : value;
+	if (!RELEASE_FILE.test(name)) throw new Error("a file is not a SurfSense release file");
+	return `${prefix}${name}`;
+}
+
+/**
+ * Points every url: and path: at the GitHub release for the manifest's own
+ * version, leaving hashes, sizes and dates as they are. On Linux the sha512
+ * here is the only check on an update, so anything unexpected throws and the
+ * route answers 502 rather than serve it (docs/contracts/05-desktop-telemetry.md).
+ */
+export function pinManifestToRelease(manifest: string): string {
+	const body = manifest.endsWith("\n") ? manifest.slice(0, -1) : manifest;
+	const lines = body.split("\n").map((line, index) => parseLine(line, index + 1));
+	const version = checkStructure(lines);
+
+	const pinned = lines.map((line) => {
+		if (line.kind === "file-start") return `  - url: ${releaseFileUrl(line.url, version)}`;
+		if (line.kind === "file-field") return `    ${line.key}: ${line.value}`;
+		if (line.key === "path") return `path: ${releaseFileUrl(line.value, version)}`;
+		return line.value === "" ? `${line.key}:` : `${line.key}: ${line.value}`;
+	});
+	return `${pinned.join("\n")}\n`;
+}

surfsense_web/lib/desktop-telemetry/update-feed/serve-update-manifest.ts

@@ -0,0 +1,55 @@
+import { readUpdateCheckDimensions } from "../desktop-dimensions";
+import type { PersonlessEvent } from "../personless-capture";
+import { requestCountry } from "../request-country";
+import { isChannelFile } from "./channel-files";
+import { type FetchManifest, fetchLatestGithubManifest } from "./latest-github-manifest";
+import { pinManifestToRelease } from "./pin-manifest-to-release";
+
+export type UpdateFeedDeps = {
+	fetchManifest: FetchManifest;
+	record: (event: PersonlessEvent) => void;
+};
+
+const NO_STORE = { "Cache-Control": "no-store" };
+
+/**
+ * GET /api/updates/<file> (docs/contracts/05-desktop-telemetry.md, part A).
+ * Every failure is a 502, which sends the app to GitHub directly, so this
+ * route can be down or wrong without blocking an update. Reads only the
+ * X-SurfSense-* headers and the country header: never the address or the
+ * User-Agent.
+ */
+export async function serveUpdateManifest(
+	request: Request,
+	file: string,
+	deps: UpdateFeedDeps
+): Promise<Response> {
+	if (!isChannelFile(file)) return new Response("Not found\n", { status: 404, headers: NO_STORE });
+
+	// Counted whatever GitHub answers: the app checked either way. Next answers
+	// HEAD with this handler too, but the app only GETs, so a HEAD is a monitor
+	// or a scanner and is not an update check.
+	if (request.method !== "HEAD") {
+		deps.record({
+			event: "desktop_update_check",
+			properties: {
+				file,
+				...readUpdateCheckDimensions(request.headers),
+				country: requestCountry(request.headers),
+			},
+		});
+	}
+
+	try {
+		const manifest = pinManifestToRelease(
+			await fetchLatestGithubManifest(file, deps.fetchManifest)
+		);
+		return new Response(manifest, {
+			status: 200,
+			headers: { "Content-Type": "text/yaml", ...NO_STORE },
+		});
+	} catch (error) {
+		console.warn(`[updates] ${file} not served: ${error instanceof Error ? error.message : error}`);
+		return new Response("Bad gateway\n", { status: 502, headers: NO_STORE });
+	}
+}

surfsense_web/lib/desktop-telemetry/usage-report/read-text-within.ts

@@ -0,0 +1,25 @@
+/**
+ * The body as UTF-8 text, or null as soon as it passes `limitBytes`. Stops
+ * reading there, so an oversized or endless body costs no more than the limit.
+ * Throws on invalid UTF-8.
+ */
+export async function readTextWithin(request: Request, limitBytes: number): Promise<string | null> {
+	const declared = Number(request.headers.get("content-length"));
+	if (Number.isFinite(declared) && declared > limitBytes) return null;
+	if (!request.body) return "";
+
+	const reader = request.body.getReader();
+	const chunks: Uint8Array[] = [];
+	let size = 0;
+	for (;;) {
+		const { done, value } = await reader.read();
+		if (done) break;
+		size += value.byteLength;
+		if (size > limitBytes) {
+			await reader.cancel();
+			return null;
+		}
+		chunks.push(value);
+	}
+	return new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks));
+}