From a7e95e484003b087a2329e4ff4cc49aa2bb7b781 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:58:12 +0530 Subject: [PATCH 01/13] fix(server): bound editor discovery when loading config --- apps/server/src/server.test.ts | 18 +++++++++++++++--- apps/server/src/ws.ts | 8 +++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 871b79eca90e..6a3c6f825594 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -4218,6 +4218,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { it.effect("routes websocket rpc subscribeServerConfig streams snapshot then update", () => Effect.gen(function* () { + const editorDiscoveryStarted = yield* Deferred.make(); const providers = [ { instanceId: ProviderInstanceId.make("codex"), @@ -4254,15 +4255,25 @@ it.layer(NodeServices.layer)("server router seam", (it) => { providerRegistry: { getProviders: Effect.succeed(providers), }, + externalLauncher: { + resolveAvailableEditors: () => + Deferred.succeed(editorDiscoveryStarted, undefined).pipe( + Effect.andThen(Effect.never), + ), + }, }, }); const wsUrl = yield* getWsServerUrl("/ws"); - const events = yield* Effect.scoped( + const eventsFiber = yield* Effect.scoped( withWsRpcClient(wsUrl, (client) => client[WS_METHODS.subscribeServerConfig]({}).pipe(Stream.take(2), Stream.runCollect), ), - ); + ).pipe(Effect.forkChild); + yield* Deferred.await(editorDiscoveryStarted); + yield* Effect.yieldNow; + yield* TestClock.adjust(Duration.seconds(2)); + const events = yield* Fiber.join(eventsFiber); const [first, second] = Array.from(events); assert.equal(first?.type, "snapshot"); @@ -4271,7 +4282,8 @@ it.layer(NodeServices.layer)("server router seam", (it) => { assert.deepEqual(first.config.keybindings, []); assert.deepEqual(first.config.issues, []); assert.deepEqual(first.config.providers, providers); - assert.equal(first.config.observability.logsDirectoryPath.endsWith("/logs"), true); + assert.deepEqual(first.config.availableEditors, []); + assert.equal(first.config.observability.logsDirectoryPath.split(/[\\/]/).at(-1), "logs"); assert.equal(first.config.observability.localTracingEnabled, true); assert.equal(first.config.observability.otlpTracesUrl, "http://localhost:4318/v1/traces"); assert.equal(first.config.observability.otlpTracesEnabled, true); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index f6f46d1e76ed..99429c5b222e 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -1074,6 +1074,12 @@ const makeWsRpcLayer = ( ); const environment = yield* serverEnvironment.getDescriptor; const auth = yield* serverAuth.getDescriptor(); + const availableEditorsResult = yield* externalLauncher + .resolveAvailableEditors() + .pipe(Effect.timeoutOption(Duration.seconds(2))); + if (Option.isNone(availableEditorsResult)) { + yield* Effect.logWarning("editor discovery timed out while loading server config"); + } return { environment, @@ -1083,7 +1089,7 @@ const makeWsRpcLayer = ( keybindings: keybindingsConfig.keybindings, issues: keybindingsConfig.issues, providers, - availableEditors: yield* externalLauncher.resolveAvailableEditors(), + availableEditors: Option.getOrElse(availableEditorsResult, () => []), observability: { logsDirectoryPath: config.logsDir, localTracingEnabled: true, From a421c409895b3cc6715d6f3bdc3c2281e850b2d8 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:58:32 +0530 Subject: [PATCH 02/13] feat(contracts): define token usage query contracts --- packages/contracts/src/index.ts | 1 + packages/contracts/src/rpc.ts | 9 +++ packages/contracts/src/tokenUsage.test.ts | 71 ++++++++++++++++++++ packages/contracts/src/tokenUsage.ts | 80 +++++++++++++++++++++++ 4 files changed, 161 insertions(+) create mode 100644 packages/contracts/src/tokenUsage.test.ts create mode 100644 packages/contracts/src/tokenUsage.ts diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 5936cb0fc17c..a62a6e4eec0a 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -10,6 +10,7 @@ export * from "./terminal.ts"; export * from "./provider.ts"; export * from "./providerInstance.ts"; export * from "./providerRuntime.ts"; +export * from "./tokenUsage.ts"; export * from "./model.ts"; export * from "./keybindings.ts"; export * from "./server.ts"; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index fa2d23b8ef29..cfec710ea997 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -135,6 +135,7 @@ import { ServerUpsertKeybindingResult, } from "./server.ts"; import { ServerSettings, ServerSettingsError, ServerSettingsPatch } from "./settings.ts"; +import { TokenUsageQueryError, TokenUsageQueryInput, TokenUsageQueryResult } from "./tokenUsage.ts"; import { SourceControlCloneRepositoryInput, SourceControlCloneRepositoryResult, @@ -206,6 +207,7 @@ export const WS_METHODS = { // Server meta serverProbe: "server.probe", serverGetConfig: "server.getConfig", + serverGetTokenUsage: "server.getTokenUsage", serverRefreshProviders: "server.refreshProviders", serverUpdateProvider: "server.updateProvider", serverUpdateServer: "server.updateServer", @@ -263,6 +265,12 @@ export const WsServerGetConfigRpc = Rpc.make(WS_METHODS.serverGetConfig, { error: Schema.Union([KeybindingsConfigError, ServerSettingsError, EnvironmentAuthorizationError]), }); +export const WsServerGetTokenUsageRpc = Rpc.make(WS_METHODS.serverGetTokenUsage, { + payload: TokenUsageQueryInput, + success: TokenUsageQueryResult, + error: Schema.Union([TokenUsageQueryError, EnvironmentAuthorizationError]), +}); + export const WsServerRefreshProvidersRpc = Rpc.make(WS_METHODS.serverRefreshProviders, { payload: Schema.Struct({ /** @@ -701,6 +709,7 @@ export const WsSubscribeAuthAccessRpc = Rpc.make(WS_METHODS.subscribeAuthAccess, export const WsRpcGroup = RpcGroup.make( WsServerProbeRpc, WsServerGetConfigRpc, + WsServerGetTokenUsageRpc, WsServerRefreshProvidersRpc, WsServerUpdateProviderRpc, WsServerUpdateServerRpc, diff --git a/packages/contracts/src/tokenUsage.test.ts b/packages/contracts/src/tokenUsage.test.ts new file mode 100644 index 000000000000..d3ff909ac5c3 --- /dev/null +++ b/packages/contracts/src/tokenUsage.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vite-plus/test"; +import * as Schema from "effect/Schema"; + +import { + ProviderTokenUsageObservation, + TokenUsageQueryInput, + TokenUsageQueryResult, +} from "./tokenUsage.ts"; + +const decodeObservation = Schema.decodeUnknownSync(ProviderTokenUsageObservation); +const decodeQuery = Schema.decodeUnknownSync(TokenUsageQueryInput); +const decodeResult = Schema.decodeUnknownSync(TokenUsageQueryResult); + +describe("token usage contracts", () => { + it("preserves nullable metrics and provenance", () => { + const observation = decodeObservation({ + sourceObservationId: "codex:turn:turn-1", + sourceKind: "codex.thread-token-usage.last", + model: "gpt-5.4", + reasoningLevel: "high", + metrics: { + inputTokens: 120, + cachedInputTokens: null, + outputTokens: 40, + reasoningOutputTokens: null, + totalTokens: 160, + }, + metricsProvenance: "exact", + modelProvenance: "inferred", + reasoningProvenance: "inferred", + }); + + expect(observation.metrics.cachedInputTokens).toBeNull(); + expect(observation.metricsProvenance).toBe("exact"); + }); + + it("rejects malformed dates and negative counts", () => { + expect(() => + decodeQuery({ fromDate: "2026-1-01", toDate: "2026-01-02", timeZone: "UTC" }), + ).toThrow(); + expect(() => + decodeObservation({ + sourceObservationId: "x", + sourceKind: "test", + model: null, + reasoningLevel: null, + metrics: { + inputTokens: -1, + cachedInputTokens: null, + outputTokens: null, + reasoningOutputTokens: null, + totalTokens: null, + }, + metricsProvenance: "unknown", + modelProvenance: "unknown", + reasoningProvenance: "unknown", + }), + ).toThrow(); + }); + + it("decodes sparse daily results", () => { + const result = decodeResult({ + fromDate: "2026-01-01", + toDate: "2026-01-03", + timeZone: "America/New_York", + trackingStartedAt: null, + days: [], + }); + expect(result.days).toEqual([]); + }); +}); diff --git a/packages/contracts/src/tokenUsage.ts b/packages/contracts/src/tokenUsage.ts new file mode 100644 index 000000000000..4c27aab4c545 --- /dev/null +++ b/packages/contracts/src/tokenUsage.ts @@ -0,0 +1,80 @@ +import * as Schema from "effect/Schema"; + +import { NonNegativeInt } from "./baseSchemas.ts"; + +const TrimmedNonEmptyString = Schema.Trimmed.check(Schema.isNonEmpty()); + +export const TokenUsageProvenance = Schema.Literals(["exact", "inferred", "unknown"]); +export type TokenUsageProvenance = typeof TokenUsageProvenance.Type; + +export const TokenUsageMetrics = Schema.Struct({ + inputTokens: Schema.NullOr(NonNegativeInt), + cachedInputTokens: Schema.NullOr(NonNegativeInt), + outputTokens: Schema.NullOr(NonNegativeInt), + reasoningOutputTokens: Schema.NullOr(NonNegativeInt), + totalTokens: Schema.NullOr(NonNegativeInt), +}); +export type TokenUsageMetrics = typeof TokenUsageMetrics.Type; + +/** + * An immutable provider accounting observation. This is deliberately separate + * from context-window telemetry: observations may be summed only after the + * server has deduplicated them by sourceObservationId. + */ +export const ProviderTokenUsageObservation = Schema.Struct({ + sourceObservationId: TrimmedNonEmptyString, + sourceKind: TrimmedNonEmptyString, + model: Schema.NullOr(TrimmedNonEmptyString), + reasoningLevel: Schema.NullOr(TrimmedNonEmptyString), + metrics: TokenUsageMetrics, + metricsProvenance: TokenUsageProvenance, + modelProvenance: TokenUsageProvenance, + reasoningProvenance: TokenUsageProvenance, +}); +export type ProviderTokenUsageObservation = typeof ProviderTokenUsageObservation.Type; + +export const TokenUsageCalendarDate = Schema.String.check(Schema.isPattern(/^\d{4}-\d{2}-\d{2}$/)); +export type TokenUsageCalendarDate = typeof TokenUsageCalendarDate.Type; + +export const TokenUsageQueryInput = Schema.Struct({ + fromDate: TokenUsageCalendarDate, + toDate: TokenUsageCalendarDate, + timeZone: TrimmedNonEmptyString, +}); +export type TokenUsageQueryInput = typeof TokenUsageQueryInput.Type; + +export const TokenUsageBreakdown = Schema.Struct({ + provider: TrimmedNonEmptyString, + model: Schema.NullOr(TrimmedNonEmptyString), + reasoningLevel: Schema.NullOr(TrimmedNonEmptyString), + metrics: TokenUsageMetrics, + metricsProvenance: TokenUsageProvenance, + modelProvenance: TokenUsageProvenance, + reasoningProvenance: TokenUsageProvenance, +}); +export type TokenUsageBreakdown = typeof TokenUsageBreakdown.Type; + +export const TokenUsageDay = Schema.Struct({ + date: TokenUsageCalendarDate, + metrics: TokenUsageMetrics, + provenance: TokenUsageProvenance, + breakdown: Schema.Array(TokenUsageBreakdown), +}); +export type TokenUsageDay = typeof TokenUsageDay.Type; + +export const TokenUsageQueryResult = Schema.Struct({ + fromDate: TokenUsageCalendarDate, + toDate: TokenUsageCalendarDate, + timeZone: TrimmedNonEmptyString, + trackingStartedAt: Schema.NullOr(Schema.String), + days: Schema.Array(TokenUsageDay), +}); +export type TokenUsageQueryResult = typeof TokenUsageQueryResult.Type; + +export class TokenUsageQueryError extends Schema.TaggedErrorClass()( + "TokenUsageQueryError", + { + reason: Schema.Literals(["invalid-range", "invalid-time-zone", "persistence"]), + message: Schema.String, + }, +) {} From 64f415a5badfd332cbdbca626e188ce156fcdaeb Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:58:33 +0530 Subject: [PATCH 03/13] feat(contracts): add provider usage observations --- .../contracts/src/providerRuntime.test.ts | 38 +++++++++++++++++++ packages/contracts/src/providerRuntime.ts | 3 ++ 2 files changed, 41 insertions(+) diff --git a/packages/contracts/src/providerRuntime.test.ts b/packages/contracts/src/providerRuntime.test.ts index 0e6ced58fd52..60498f388523 100644 --- a/packages/contracts/src/providerRuntime.test.ts +++ b/packages/contracts/src/providerRuntime.test.ts @@ -181,4 +181,42 @@ describe("ProviderRuntimeEvent", () => { expect(parsed.payload.usage.maxTokens).toBe(200000); expect(parsed.payload.usage.usedTokens).toBe(31251); }); + + it("decodes accounting observations separately from context snapshots", () => { + const parsed = decodeRuntimeEvent({ + type: "thread.token-usage.updated", + eventId: "event-token-accounting-1", + provider: "codex", + createdAt: "2026-02-28T00:00:04.000Z", + threadId: "thread-1", + turnId: "turn-1", + payload: { + usage: { usedTokens: 160 }, + accounting: [ + { + sourceObservationId: "codex:turn:turn-1", + sourceKind: "codex.thread-token-usage.last", + model: null, + reasoningLevel: null, + metrics: { + inputTokens: 120, + cachedInputTokens: null, + outputTokens: 40, + reasoningOutputTokens: null, + totalTokens: 160, + }, + metricsProvenance: "exact", + modelProvenance: "unknown", + reasoningProvenance: "unknown", + }, + ], + }, + }); + + expect(parsed.type).toBe("thread.token-usage.updated"); + if (parsed.type !== "thread.token-usage.updated") { + throw new Error("expected thread.token-usage.updated"); + } + expect(parsed.payload.accounting?.[0]?.metrics.totalTokens).toBe(160); + }); }); diff --git a/packages/contracts/src/providerRuntime.ts b/packages/contracts/src/providerRuntime.ts index eb2563eff004..168f3755a492 100644 --- a/packages/contracts/src/providerRuntime.ts +++ b/packages/contracts/src/providerRuntime.ts @@ -14,6 +14,7 @@ import { TurnId, } from "./baseSchemas.ts"; import { ProviderInstanceId, ProviderDriverKind } from "./providerInstance.ts"; +import { ProviderTokenUsageObservation } from "./tokenUsage.ts"; const TrimmedNonEmptyStringSchema = TrimmedNonEmptyString; const UnknownRecordSchema = Schema.Record(Schema.String, Schema.Unknown); @@ -325,6 +326,7 @@ export type ThreadTokenUsageSnapshot = typeof ThreadTokenUsageSnapshot.Type; const ThreadTokenUsageUpdatedPayload = Schema.Struct({ usage: ThreadTokenUsageSnapshot, + accounting: Schema.optional(Schema.Array(ProviderTokenUsageObservation)), }); export type ThreadTokenUsageUpdatedPayload = typeof ThreadTokenUsageUpdatedPayload.Type; @@ -364,6 +366,7 @@ const TurnCompletedPayload = Schema.Struct({ stopReason: Schema.optional(Schema.NullOr(TrimmedNonEmptyStringSchema)), usage: Schema.optional(Schema.Unknown), modelUsage: Schema.optional(UnknownRecordSchema), + accounting: Schema.optional(Schema.Array(ProviderTokenUsageObservation)), totalCostUsd: Schema.optional(Schema.Number), errorMessage: Schema.optional(TrimmedNonEmptyStringSchema), }); From f7217d2629b1066d06ccb4abe0f2fcf35387a014 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:58:40 +0530 Subject: [PATCH 04/13] feat(server): add durable token usage ledger --- .../src/persistence/Layers/TokenUsage.test.ts | 175 ++++++++ .../src/persistence/Layers/TokenUsage.ts | 421 ++++++++++++++++++ apps/server/src/persistence/Migrations.ts | 2 + .../Migrations/035_ProviderTokenUsageFacts.ts | 55 +++ .../src/persistence/Services/TokenUsage.ts | 40 ++ 5 files changed, 693 insertions(+) create mode 100644 apps/server/src/persistence/Layers/TokenUsage.test.ts create mode 100644 apps/server/src/persistence/Layers/TokenUsage.ts create mode 100644 apps/server/src/persistence/Migrations/035_ProviderTokenUsageFacts.ts create mode 100644 apps/server/src/persistence/Services/TokenUsage.ts diff --git a/apps/server/src/persistence/Layers/TokenUsage.test.ts b/apps/server/src/persistence/Layers/TokenUsage.test.ts new file mode 100644 index 000000000000..d5875d716099 --- /dev/null +++ b/apps/server/src/persistence/Layers/TokenUsage.test.ts @@ -0,0 +1,175 @@ +import { assert, it } from "@effect/vitest"; +import { + EventId, + ProviderDriverKind, + ProviderInstanceId, + ThreadId, + TurnId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { SqlitePersistenceMemory } from "./Sqlite.ts"; +import { TokenUsageRepositoryLive } from "./TokenUsage.ts"; +import { TokenUsageRepository } from "../Services/TokenUsage.ts"; + +const layer = it.layer(TokenUsageRepositoryLive.pipe(Layer.provideMerge(SqlitePersistenceMemory))); + +layer("TokenUsageRepository", (it) => { + it.effect("deduplicates provider observations and buckets them in the requested timezone", () => + Effect.gen(function* () { + const repository = yield* TokenUsageRepository; + const fact = { + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeEventId: EventId.make("event-1"), + threadId: ThreadId.make("thread-1"), + turnId: TurnId.make("turn-1"), + occurredAt: "2026-01-02T02:00:00.000Z", + observation: { + sourceObservationId: "codex:turn:turn-1", + sourceKind: "codex.thread-token-usage.last", + model: "gpt-5.4", + reasoningLevel: "high", + metrics: { + inputTokens: 100, + cachedInputTokens: 20, + outputTokens: 50, + reasoningOutputTokens: 10, + totalTokens: 150, + }, + metricsProvenance: "exact" as const, + modelProvenance: "inferred" as const, + reasoningProvenance: "inferred" as const, + }, + }; + + yield* repository.record(fact); + yield* repository.record(fact); + const olderMetrics = { + inputTokens: 90, + cachedInputTokens: 10, + outputTokens: 60, + reasoningOutputTokens: 5, + totalTokens: 150, + }; + yield* repository.record({ + ...fact, + runtimeEventId: EventId.make("event-replayed-older"), + occurredAt: "2026-01-02T01:00:00.000Z", + observation: { + ...fact.observation, + model: "stale-model", + metrics: olderMetrics, + }, + }); + yield* repository.record({ + ...fact, + runtimeEventId: EventId.make("event-richer-attribution"), + occurredAt: "2026-01-02T01:00:00.000Z", + observation: { + ...fact.observation, + model: "gpt-5.4-reported", + modelProvenance: "exact", + metrics: olderMetrics, + }, + }); + + const result = yield* repository.query({ + fromDate: "2026-01-01", + toDate: "2026-01-02", + timeZone: "America/New_York", + }); + + assert.strictEqual(result.days.length, 1); + assert.strictEqual(result.days[0]?.date, "2026-01-01"); + assert.strictEqual(result.days[0]?.metrics.totalTokens, 150); + assert.strictEqual(result.days[0]?.metrics.inputTokens, 100); + assert.strictEqual(result.days[0]?.metrics.outputTokens, 50); + assert.strictEqual(result.days[0]?.breakdown[0]?.model, "gpt-5.4-reported"); + assert.strictEqual(result.days[0]?.breakdown[0]?.modelProvenance, "exact"); + assert.strictEqual(result.days[0]?.breakdown[0]?.reasoningLevel, "high"); + }), + ); + + it.effect("isolates identical source observation ids by provider instance", () => + Effect.gen(function* () { + const repository = yield* TokenUsageRepository; + const observation = { + sourceObservationId: "codex:turn:shared-turn", + sourceKind: "codex.thread-token-usage.last", + model: "gpt-5.4", + reasoningLevel: null, + metrics: { + inputTokens: 10, + cachedInputTokens: 0, + outputTokens: 5, + reasoningOutputTokens: 0, + totalTokens: 15, + }, + metricsProvenance: "exact" as const, + modelProvenance: "exact" as const, + reasoningProvenance: "unknown" as const, + }; + for (const [instanceId, eventId] of [ + ["codex_work", "event-work"], + ["codex_personal", "event-personal"], + ] as const) { + yield* repository.record({ + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make(instanceId), + runtimeEventId: EventId.make(eventId), + threadId: ThreadId.make("thread-shared"), + turnId: TurnId.make("shared-turn"), + occurredAt: "2026-01-03T12:00:00.000Z", + observation, + }); + } + + const result = yield* repository.query({ + fromDate: "2026-01-03", + toDate: "2026-01-03", + timeZone: "UTC", + }); + assert.strictEqual(result.days[0]?.metrics.totalTokens, 30); + }), + ); + + it.effect("keeps unavailable component metrics unknown", () => + Effect.gen(function* () { + const repository = yield* TokenUsageRepository; + yield* repository.record({ + provider: ProviderDriverKind.make("claudeAgent"), + providerInstanceId: null, + runtimeEventId: EventId.make("event-unknown-components"), + threadId: ThreadId.make("thread-2"), + turnId: TurnId.make("turn-2"), + occurredAt: "2026-02-01T12:00:00.000Z", + observation: { + sourceObservationId: "claude:turn:turn-2", + sourceKind: "claude.result.usage", + model: "claude-opus-4-6", + reasoningLevel: null, + metrics: { + inputTokens: null, + cachedInputTokens: null, + outputTokens: null, + reasoningOutputTokens: null, + totalTokens: 500, + }, + metricsProvenance: "exact", + modelProvenance: "exact", + reasoningProvenance: "unknown", + }, + }); + + const result = yield* repository.query({ + fromDate: "2026-02-01", + toDate: "2026-02-01", + timeZone: "UTC", + }); + assert.strictEqual(result.days[0]?.metrics.totalTokens, 500); + assert.strictEqual(result.days[0]?.metrics.inputTokens, null); + }), + ); +}); diff --git a/apps/server/src/persistence/Layers/TokenUsage.ts b/apps/server/src/persistence/Layers/TokenUsage.ts new file mode 100644 index 000000000000..0a88a20e0102 --- /dev/null +++ b/apps/server/src/persistence/Layers/TokenUsage.ts @@ -0,0 +1,421 @@ +import { + type TokenUsageBreakdown, + type TokenUsageDay, + type TokenUsageMetrics, + type TokenUsageProvenance, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; + +import { toPersistenceDecodeError, toPersistenceSqlError } from "../Errors.ts"; +import { + RecordTokenUsageFactInput, + TokenUsageRepository, + type TokenUsageRepositoryShape, +} from "../Services/TokenUsage.ts"; + +const QueryBounds = Schema.Struct({ + fromUtc: Schema.String, + toUtc: Schema.String, +}); + +const UsageFactRow = Schema.Struct({ + provider: Schema.String, + occurredAt: Schema.String, + model: Schema.NullOr(Schema.String), + reasoningLevel: Schema.NullOr(Schema.String), + inputTokens: Schema.NullOr(Schema.Number), + cachedInputTokens: Schema.NullOr(Schema.Number), + outputTokens: Schema.NullOr(Schema.Number), + reasoningOutputTokens: Schema.NullOr(Schema.Number), + totalTokens: Schema.NullOr(Schema.Number), + metricsProvenance: Schema.Literals(["exact", "inferred", "unknown"]), + modelProvenance: Schema.Literals(["exact", "inferred", "unknown"]), + reasoningProvenance: Schema.Literals(["exact", "inferred", "unknown"]), +}); +type UsageFactRow = typeof UsageFactRow.Type; + +const TrackingStartRow = Schema.Struct({ trackingStartedAt: Schema.NullOr(Schema.String) }); + +function toPersistenceSqlOrDecodeError(sqlOperation: string, decodeOperation: string) { + return (cause: unknown) => + Schema.isSchemaError(cause) + ? toPersistenceDecodeError(decodeOperation)(cause) + : toPersistenceSqlError(sqlOperation)(cause); +} + +function addUtcDays(calendarDate: string, days: number): string { + const date = new Date(`${calendarDate}T00:00:00.000Z`); + date.setUTCDate(date.getUTCDate() + days); + return date.toISOString(); +} + +function localCalendarDate(occurredAt: string, timeZone: string): string { + const parts = new Intl.DateTimeFormat("en-US", { + timeZone, + year: "numeric", + month: "2-digit", + day: "2-digit", + }).formatToParts(new Date(occurredAt)); + const value = (type: Intl.DateTimeFormatPartTypes) => + parts.find((part) => part.type === type)?.value ?? ""; + return `${value("year")}-${value("month")}-${value("day")}`; +} + +function combineProvenance(values: ReadonlyArray): TokenUsageProvenance { + if (values.some((value) => value === "unknown")) return "unknown"; + if (values.some((value) => value === "inferred")) return "inferred"; + return "exact"; +} + +function sumKnown( + rows: ReadonlyArray, + key: keyof Pick< + UsageFactRow, + "inputTokens" | "cachedInputTokens" | "outputTokens" | "reasoningOutputTokens" | "totalTokens" + >, +): number | null { + let total = 0; + for (const row of rows) { + const value = row[key]; + if (value === null) return null; + total += value; + } + return total; +} + +function aggregateMetrics(rows: ReadonlyArray): TokenUsageMetrics { + return { + inputTokens: sumKnown(rows, "inputTokens"), + cachedInputTokens: sumKnown(rows, "cachedInputTokens"), + outputTokens: sumKnown(rows, "outputTokens"), + reasoningOutputTokens: sumKnown(rows, "reasoningOutputTokens"), + totalTokens: sumKnown(rows, "totalTokens"), + }; +} + +function aggregateRows(rows: ReadonlyArray, date: string): TokenUsageDay { + const byDimension = new Map(); + for (const row of rows) { + const key = JSON.stringify([row.provider, row.model, row.reasoningLevel]); + const current = byDimension.get(key) ?? []; + current.push(row); + byDimension.set(key, current); + } + + const breakdown: TokenUsageBreakdown[] = [...byDimension.values()] + .map((group) => ({ + provider: group[0]!.provider, + model: group[0]!.model, + reasoningLevel: group[0]!.reasoningLevel, + metrics: aggregateMetrics(group), + metricsProvenance: combineProvenance(group.map((row) => row.metricsProvenance)), + modelProvenance: combineProvenance(group.map((row) => row.modelProvenance)), + reasoningProvenance: combineProvenance(group.map((row) => row.reasoningProvenance)), + })) + .toSorted((left, right) => + `${left.provider}:${left.model ?? ""}:${left.reasoningLevel ?? ""}`.localeCompare( + `${right.provider}:${right.model ?? ""}:${right.reasoningLevel ?? ""}`, + ), + ); + + return { + date, + metrics: aggregateMetrics(rows), + provenance: combineProvenance(rows.map((row) => row.metricsProvenance)), + breakdown, + }; +} + +const makeTokenUsageRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + const recordFact = SqlSchema.void({ + Request: RecordTokenUsageFactInput, + execute: (input) => { + const providerIdentity = input.providerInstanceId ?? `driver:${input.provider}`; + const { metrics } = input.observation; + return sql` + INSERT INTO provider_token_usage_facts ( + provider_identity, + provider_driver, + provider_instance_id, + source_observation_id, + source_kind, + runtime_event_id, + thread_id, + turn_id, + occurred_at, + model, + reasoning_level, + input_tokens, + cached_input_tokens, + output_tokens, + reasoning_output_tokens, + total_tokens, + metrics_provenance, + model_provenance, + reasoning_provenance + ) VALUES ( + ${providerIdentity}, + ${input.provider}, + ${input.providerInstanceId}, + ${input.observation.sourceObservationId}, + ${input.observation.sourceKind}, + ${input.runtimeEventId}, + ${input.threadId}, + ${input.turnId}, + ${input.occurredAt}, + ${input.observation.model}, + ${input.observation.reasoningLevel}, + ${metrics.inputTokens}, + ${metrics.cachedInputTokens}, + ${metrics.outputTokens}, + ${metrics.reasoningOutputTokens}, + ${metrics.totalTokens}, + ${input.observation.metricsProvenance}, + ${input.observation.modelProvenance}, + ${input.observation.reasoningProvenance} + ) + ON CONFLICT (provider_identity, source_observation_id) + DO UPDATE SET + runtime_event_id = excluded.runtime_event_id, + thread_id = excluded.thread_id, + turn_id = COALESCE(excluded.turn_id, provider_token_usage_facts.turn_id), + occurred_at = MAX(excluded.occurred_at, provider_token_usage_facts.occurred_at), + model = CASE + WHEN excluded.model IS NULL THEN provider_token_usage_facts.model + WHEN provider_token_usage_facts.model IS NULL THEN excluded.model + WHEN CASE excluded.model_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + > CASE provider_token_usage_facts.model_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + THEN excluded.model + ELSE provider_token_usage_facts.model + END, + reasoning_level = CASE + WHEN excluded.reasoning_level IS NULL THEN provider_token_usage_facts.reasoning_level + WHEN provider_token_usage_facts.reasoning_level IS NULL THEN excluded.reasoning_level + WHEN CASE excluded.reasoning_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + > CASE provider_token_usage_facts.reasoning_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + THEN excluded.reasoning_level + ELSE provider_token_usage_facts.reasoning_level + END, + input_tokens = CASE + WHEN ( + CASE excluded.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + > CASE provider_token_usage_facts.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + ) OR ( + excluded.metrics_provenance = provider_token_usage_facts.metrics_provenance + AND excluded.total_tokens IS NOT NULL + AND ( + provider_token_usage_facts.total_tokens IS NULL + OR excluded.total_tokens > provider_token_usage_facts.total_tokens + OR ( + excluded.total_tokens = provider_token_usage_facts.total_tokens + AND excluded.occurred_at > provider_token_usage_facts.occurred_at + ) + ) + ) THEN excluded.input_tokens + ELSE provider_token_usage_facts.input_tokens + END, + cached_input_tokens = CASE + WHEN ( + CASE excluded.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + > CASE provider_token_usage_facts.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + ) OR ( + excluded.metrics_provenance = provider_token_usage_facts.metrics_provenance + AND excluded.total_tokens IS NOT NULL + AND ( + provider_token_usage_facts.total_tokens IS NULL + OR excluded.total_tokens > provider_token_usage_facts.total_tokens + OR ( + excluded.total_tokens = provider_token_usage_facts.total_tokens + AND excluded.occurred_at > provider_token_usage_facts.occurred_at + ) + ) + ) THEN excluded.cached_input_tokens + ELSE provider_token_usage_facts.cached_input_tokens + END, + output_tokens = CASE + WHEN ( + CASE excluded.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + > CASE provider_token_usage_facts.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + ) OR ( + excluded.metrics_provenance = provider_token_usage_facts.metrics_provenance + AND excluded.total_tokens IS NOT NULL + AND ( + provider_token_usage_facts.total_tokens IS NULL + OR excluded.total_tokens > provider_token_usage_facts.total_tokens + OR ( + excluded.total_tokens = provider_token_usage_facts.total_tokens + AND excluded.occurred_at > provider_token_usage_facts.occurred_at + ) + ) + ) THEN excluded.output_tokens + ELSE provider_token_usage_facts.output_tokens + END, + reasoning_output_tokens = CASE + WHEN ( + CASE excluded.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + > CASE provider_token_usage_facts.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + ) OR ( + excluded.metrics_provenance = provider_token_usage_facts.metrics_provenance + AND excluded.total_tokens IS NOT NULL + AND ( + provider_token_usage_facts.total_tokens IS NULL + OR excluded.total_tokens > provider_token_usage_facts.total_tokens + OR ( + excluded.total_tokens = provider_token_usage_facts.total_tokens + AND excluded.occurred_at > provider_token_usage_facts.occurred_at + ) + ) + ) THEN excluded.reasoning_output_tokens + ELSE provider_token_usage_facts.reasoning_output_tokens + END, + total_tokens = CASE + WHEN ( + CASE excluded.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + > CASE provider_token_usage_facts.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + ) OR ( + excluded.metrics_provenance = provider_token_usage_facts.metrics_provenance + AND excluded.total_tokens IS NOT NULL + AND ( + provider_token_usage_facts.total_tokens IS NULL + OR excluded.total_tokens > provider_token_usage_facts.total_tokens + OR ( + excluded.total_tokens = provider_token_usage_facts.total_tokens + AND excluded.occurred_at > provider_token_usage_facts.occurred_at + ) + ) + ) THEN excluded.total_tokens + ELSE provider_token_usage_facts.total_tokens + END, + metrics_provenance = CASE + WHEN CASE excluded.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + > CASE provider_token_usage_facts.metrics_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + THEN excluded.metrics_provenance + ELSE provider_token_usage_facts.metrics_provenance + END, + model_provenance = CASE + WHEN CASE excluded.model_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + > CASE provider_token_usage_facts.model_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + THEN excluded.model_provenance + ELSE provider_token_usage_facts.model_provenance + END, + reasoning_provenance = CASE + WHEN CASE excluded.reasoning_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + > CASE provider_token_usage_facts.reasoning_provenance + WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END + THEN excluded.reasoning_provenance + ELSE provider_token_usage_facts.reasoning_provenance + END + `; + }, + }); + + const listFacts = SqlSchema.findAll({ + Request: QueryBounds, + Result: UsageFactRow, + execute: ({ fromUtc, toUtc }) => sql` + SELECT + provider_driver AS "provider", + occurred_at AS "occurredAt", + model, + reasoning_level AS "reasoningLevel", + input_tokens AS "inputTokens", + cached_input_tokens AS "cachedInputTokens", + output_tokens AS "outputTokens", + reasoning_output_tokens AS "reasoningOutputTokens", + total_tokens AS "totalTokens", + metrics_provenance AS "metricsProvenance", + model_provenance AS "modelProvenance", + reasoning_provenance AS "reasoningProvenance" + FROM provider_token_usage_facts + WHERE occurred_at >= ${fromUtc} + AND occurred_at < ${toUtc} + AND total_tokens IS NOT NULL + ORDER BY occurred_at ASC, id ASC + `, + }); + + const readTrackingStart = SqlSchema.findOne({ + Request: Schema.Struct({}), + Result: TrackingStartRow, + execute: () => sql` + SELECT MIN(occurred_at) AS "trackingStartedAt" + FROM provider_token_usage_facts + WHERE total_tokens IS NOT NULL + `, + }); + + const record: TokenUsageRepositoryShape["record"] = (input) => + recordFact(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "TokenUsageRepository.record:query", + "TokenUsageRepository.record:encodeRequest", + ), + ), + ); + + const query: TokenUsageRepositoryShape["query"] = (input) => + Effect.gen(function* () { + const rows = yield* listFacts({ + fromUtc: addUtcDays(input.fromDate, -2), + toUtc: addUtcDays(input.toDate, 3), + }); + const trackingStart = yield* readTrackingStart({}); + const byDate = new Map(); + for (const row of rows) { + const date = localCalendarDate(row.occurredAt, input.timeZone); + if (date < input.fromDate || date > input.toDate) continue; + const current = byDate.get(date) ?? []; + current.push(row); + byDate.set(date, current); + } + return { + fromDate: input.fromDate, + toDate: input.toDate, + timeZone: input.timeZone, + trackingStartedAt: trackingStart.trackingStartedAt, + days: [...byDate.entries()] + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([date, dayRows]) => aggregateRows(dayRows, date)), + }; + }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "TokenUsageRepository.query:query", + "TokenUsageRepository.query:decodeRows", + ), + ), + ); + + return TokenUsageRepository.of({ record, query }); +}); + +export const TokenUsageRepositoryLive = Layer.effect( + TokenUsageRepository, + makeTokenUsageRepository, +); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index d25895671a91..01788dc1da9a 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -47,6 +47,7 @@ import Migration0031 from "./Migrations/031_AuthAuthorizationScopes.ts"; import Migration0032 from "./Migrations/032_AuthPairingProofKeyThumbprint.ts"; import Migration0033 from "./Migrations/033_ProjectionThreadsSettled.ts"; import Migration0034 from "./Migrations/034_ProjectionThreadsSnoozed.ts"; +import Migration0035 from "./Migrations/035_ProviderTokenUsageFacts.ts"; /** * Migration loader with all migrations defined inline. @@ -93,6 +94,7 @@ export const migrationEntries = [ [32, "AuthPairingProofKeyThumbprint", Migration0032], [33, "ProjectionThreadsSettled", Migration0033], [34, "ProjectionThreadsSnoozed", Migration0034], + [35, "ProviderTokenUsageFacts", Migration0035], ] as const; export const makeMigrationLoader = (throughId?: number) => diff --git a/apps/server/src/persistence/Migrations/035_ProviderTokenUsageFacts.ts b/apps/server/src/persistence/Migrations/035_ProviderTokenUsageFacts.ts new file mode 100644 index 000000000000..811e2d5c2e2a --- /dev/null +++ b/apps/server/src/persistence/Migrations/035_ProviderTokenUsageFacts.ts @@ -0,0 +1,55 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + yield* sql` + CREATE TABLE IF NOT EXISTS provider_token_usage_facts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider_identity TEXT NOT NULL, + provider_driver TEXT NOT NULL, + provider_instance_id TEXT, + source_observation_id TEXT NOT NULL, + source_kind TEXT NOT NULL, + runtime_event_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + turn_id TEXT, + occurred_at TEXT NOT NULL, + model TEXT, + reasoning_level TEXT, + input_tokens INTEGER CHECK (input_tokens IS NULL OR input_tokens >= 0), + cached_input_tokens INTEGER CHECK (cached_input_tokens IS NULL OR cached_input_tokens >= 0), + output_tokens INTEGER CHECK (output_tokens IS NULL OR output_tokens >= 0), + reasoning_output_tokens INTEGER CHECK ( + reasoning_output_tokens IS NULL OR reasoning_output_tokens >= 0 + ), + total_tokens INTEGER CHECK (total_tokens IS NULL OR total_tokens >= 0), + metrics_provenance TEXT NOT NULL CHECK ( + metrics_provenance IN ('exact', 'inferred', 'unknown') + ), + model_provenance TEXT NOT NULL CHECK ( + model_provenance IN ('exact', 'inferred', 'unknown') + ), + reasoning_provenance TEXT NOT NULL CHECK ( + reasoning_provenance IN ('exact', 'inferred', 'unknown') + ), + UNIQUE (provider_identity, source_observation_id) + ) + `; + + yield* sql` + CREATE INDEX IF NOT EXISTS idx_provider_token_usage_facts_occurred_at + ON provider_token_usage_facts(occurred_at) + `; + + yield* sql` + CREATE INDEX IF NOT EXISTS idx_provider_token_usage_facts_time_model_reasoning + ON provider_token_usage_facts(occurred_at, model, reasoning_level) + `; + + yield* sql` + CREATE INDEX IF NOT EXISTS idx_provider_token_usage_facts_thread_turn + ON provider_token_usage_facts(thread_id, turn_id) + `; +}); diff --git a/apps/server/src/persistence/Services/TokenUsage.ts b/apps/server/src/persistence/Services/TokenUsage.ts new file mode 100644 index 000000000000..0b9702faf329 --- /dev/null +++ b/apps/server/src/persistence/Services/TokenUsage.ts @@ -0,0 +1,40 @@ +import { + EventId, + ProviderDriverKind, + ProviderInstanceId, + ProviderTokenUsageObservation, + ThreadId, + TokenUsageQueryInput, + TokenUsageQueryResult, + TurnId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import type * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import type { ProjectionRepositoryError } from "../Errors.ts"; + +export const RecordTokenUsageFactInput = Schema.Struct({ + provider: ProviderDriverKind, + providerInstanceId: Schema.NullOr(ProviderInstanceId), + runtimeEventId: EventId, + threadId: ThreadId, + turnId: Schema.NullOr(TurnId), + occurredAt: Schema.String, + observation: ProviderTokenUsageObservation, +}); +export type RecordTokenUsageFactInput = typeof RecordTokenUsageFactInput.Type; + +export interface TokenUsageRepositoryShape { + readonly record: ( + input: RecordTokenUsageFactInput, + ) => Effect.Effect; + readonly query: ( + input: TokenUsageQueryInput, + ) => Effect.Effect; +} + +export class TokenUsageRepository extends Context.Service< + TokenUsageRepository, + TokenUsageRepositoryShape +>()("t3/persistence/Services/TokenUsage/TokenUsageRepository") {} From 2777d32602d95ae5e4c25f317fa11fd1330bc8b1 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:58:52 +0530 Subject: [PATCH 05/13] feat(server): capture provider token usage --- .../OrchestrationEngineHarness.integration.ts | 2 + .../Layers/ProviderRuntimeIngestion.test.ts | 85 +++++++++++++ .../Layers/ProviderRuntimeIngestion.ts | 52 ++++++++ .../src/provider/Layers/ClaudeAdapter.test.ts | 65 ++++++++++ .../src/provider/Layers/ClaudeAdapter.ts | 116 +++++++++++++++++- .../src/provider/Layers/CodexAdapter.test.ts | 20 +++ .../src/provider/Layers/CodexAdapter.ts | 25 +++- apps/server/src/server.ts | 7 +- 8 files changed, 368 insertions(+), 4 deletions(-) diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.ts index ebc4f984b86d..86ba471648f4 100644 --- a/apps/server/integration/OrchestrationEngineHarness.integration.ts +++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts @@ -30,6 +30,7 @@ import { ProjectionCheckpointRepositoryLive } from "../src/persistence/Layers/Pr import { ProjectionPendingApprovalRepositoryLive } from "../src/persistence/Layers/ProjectionPendingApprovals.ts"; import { ProviderSessionRuntimeRepositoryLive } from "../src/persistence/Layers/ProviderSessionRuntime.ts"; import { makeSqlitePersistenceLive } from "../src/persistence/Layers/Sqlite.ts"; +import { TokenUsageRepositoryLive } from "../src/persistence/Layers/TokenUsage.ts"; import { ProjectionCheckpointRepository } from "../src/persistence/Services/ProjectionCheckpoints.ts"; import { ProjectionPendingApprovalRepository } from "../src/persistence/Services/ProjectionPendingApprovals.ts"; import { makeAdapterRegistryMock } from "../src/provider/testUtils/providerAdapterRegistryMock.ts"; @@ -310,6 +311,7 @@ export const makeOrchestrationIntegrationHarness = ( const runtimeIngestionLayer = ProviderRuntimeIngestionLive.pipe( Layer.provideMerge(runtimeServicesLayer), Layer.provideMerge(serverSettingsLayer), + Layer.provide(TokenUsageRepositoryLive), ); const gitWorkflowLayer = Layer.mock(GitWorkflowService)({ renameBranch: (input: { diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts index 74ece50cd318..410aa9fa528e 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts @@ -36,6 +36,10 @@ import { afterEach, describe, expect, it } from "vite-plus/test"; import { OrchestrationEventStoreLive } from "../../persistence/Layers/OrchestrationEventStore.ts"; import { OrchestrationCommandReceiptRepositoryLive } from "../../persistence/Layers/OrchestrationCommandReceipts.ts"; import { SqlitePersistenceMemory } from "../../persistence/Layers/Sqlite.ts"; +import { + type RecordTokenUsageFactInput, + TokenUsageRepository, +} from "../../persistence/Services/TokenUsage.ts"; import { ProviderService, type ProviderServiceShape, @@ -222,6 +226,14 @@ describe("ProviderRuntimeIngestion", () => { const workspaceRoot = makeTempDir("t3-provider-project-"); NodeFS.mkdirSync(NodePath.join(workspaceRoot, ".git")); const provider = createProviderServiceHarness(); + const recordedTokenUsageFacts: Array = []; + const tokenUsageRepositoryLayer = Layer.succeed(TokenUsageRepository, { + record: (input) => + Effect.sync(() => { + recordedTokenUsageFacts.push(input); + }), + query: () => Effect.die("TokenUsageRepository.query is not used in this test harness"), + }); const orchestrationLayer = OrchestrationEngineLive.pipe( Layer.provide(OrchestrationProjectionSnapshotQueryLive), Layer.provide(OrchestrationProjectionPipelineLive), @@ -237,6 +249,7 @@ describe("ProviderRuntimeIngestion", () => { const layer = ProviderRuntimeIngestionLive.pipe( Layer.provideMerge(orchestrationLayer), Layer.provideMerge(projectionSnapshotLayer), + Layer.provideMerge(tokenUsageRepositoryLayer), Layer.provideMerge(SqlitePersistenceMemory), Layer.provideMerge(Layer.succeed(ProviderService, provider.service)), Layer.provideMerge(makeTestServerSettingsLayer(options?.serverSettings)), @@ -315,6 +328,7 @@ describe("ProviderRuntimeIngestion", () => { readModel: () => Effect.runPromise(snapshotQuery.getSnapshot()), emit: provider.emit, setProviderSession: provider.setSession, + recordedTokenUsageFacts, drain, }; } @@ -2944,6 +2958,77 @@ describe("ProviderRuntimeIngestion", () => { }); }); + it("persists explicit accounting observations without summing context snapshots", async () => { + const harness = await createHarness(); + + const accountingObservation = { + sourceObservationId: "codex:turn:turn-accounting", + sourceKind: "codex.thread-token-usage.last", + model: null, + reasoningLevel: null, + metrics: { + inputTokens: 120, + cachedInputTokens: 0, + outputTokens: 6, + reasoningOutputTokens: 0, + totalTokens: 126, + }, + metricsProvenance: "exact" as const, + modelProvenance: "unknown" as const, + reasoningProvenance: "unknown" as const, + }; + harness.emit({ + type: "thread.token-usage.updated", + eventId: asEventId("evt-thread-token-accounting-work"), + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex_work"), + createdAt: "2026-01-01T12:00:00.000Z", + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-accounting"), + payload: { + usage: { usedTokens: 126 }, + accounting: [accountingObservation], + }, + }); + harness.emit({ + type: "thread.token-usage.updated", + eventId: asEventId("evt-thread-token-accounting-personal"), + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex_personal"), + createdAt: "2026-01-01T12:01:00.000Z", + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-accounting"), + payload: { + usage: { usedTokens: 50 }, + accounting: [ + { + ...accountingObservation, + metrics: { + ...accountingObservation.metrics, + inputTokens: 45, + outputTokens: 5, + totalTokens: 50, + }, + }, + ], + }, + }); + await harness.drain(); + + expect(harness.recordedTokenUsageFacts).toHaveLength(2); + expect( + harness.recordedTokenUsageFacts.map((fact) => fact.providerInstanceId).toSorted(), + ).toEqual(["codex_personal", "codex_work"]); + expect( + harness.recordedTokenUsageFacts.reduce( + (total, fact) => total + (fact.observation.metrics.totalTokens ?? 0), + 0, + ), + ).toBe(176); + expect(harness.recordedTokenUsageFacts[0]?.observation.model).toBe("gpt-5-codex"); + expect(harness.recordedTokenUsageFacts[0]?.observation.modelProvenance).toBe("inferred"); + }); + it("projects Codex camelCase token usage payloads into normalized thread activities", async () => { const harness = await createHarness(); const now = "2026-01-01T00:00:00.000Z"; diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index a8a51b302608..531402d2e8d9 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -26,10 +26,12 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; +import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; import { ProjectionTurnRepository } from "../../persistence/Services/ProjectionTurns.ts"; import { ProjectionTurnRepositoryLive } from "../../persistence/Layers/ProjectionTurns.ts"; +import { TokenUsageRepository } from "../../persistence/Services/TokenUsage.ts"; import { isGitRepository } from "../../git/Utils.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; @@ -692,6 +694,7 @@ const make = Effect.gen(function* () { const projectionSnapshotQuery = yield* ProjectionSnapshotQuery; const providerService = yield* ProviderService; const projectionTurnRepository = yield* ProjectionTurnRepository; + const tokenUsageRepository = yield* TokenUsageRepository; const serverSettingsService = yield* ServerSettingsService; const providerCommandId = (event: ProviderRuntimeEvent, tag: string) => crypto.randomUUIDv4.pipe( @@ -1307,6 +1310,55 @@ const make = Effect.gen(function* () { const now = event.createdAt; const eventTurnId = toTurnId(event.turnId); + const accountingObservations = + event.type === "thread.token-usage.updated" || event.type === "turn.completed" + ? (event.payload.accounting ?? []) + : []; + if (accountingObservations.length > 0) { + const fallbackModel = thread.modelSelection.model; + const fallbackReasoningLevel = + getModelSelectionStringOptionValue(thread.modelSelection, "reasoningEffort") ?? + getModelSelectionStringOptionValue(thread.modelSelection, "effort") ?? + getModelSelectionStringOptionValue(thread.modelSelection, "variant") ?? + null; + for (const observation of accountingObservations) { + const model = observation.model ?? fallbackModel; + const reasoningLevel = observation.reasoningLevel ?? fallbackReasoningLevel; + yield* tokenUsageRepository + .record({ + provider: event.provider, + providerInstanceId: event.providerInstanceId ?? null, + runtimeEventId: event.eventId, + threadId: thread.id, + turnId: eventTurnId ?? null, + occurredAt: event.createdAt, + observation: { + ...observation, + model, + reasoningLevel, + modelProvenance: + observation.model === null && model !== null + ? "inferred" + : observation.modelProvenance, + reasoningProvenance: + observation.reasoningLevel === null && reasoningLevel !== null + ? "inferred" + : observation.reasoningProvenance, + }, + }) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("provider token usage accounting failed", { + provider: event.provider, + eventId: event.eventId, + threadId: thread.id, + turnId: eventTurnId ?? null, + cause: Cause.pretty(cause), + }), + ), + ); + } + } const activeTurnId = thread.session?.activeTurnId ?? null; const pendingTurnStart = yield* projectionTurnRepository.getPendingTurnStartByThreadId({ threadId: thread.id, diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts index 760f0e7fbab2..059abc9b24ee 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts @@ -1974,6 +1974,22 @@ describe("ClaudeAdapterLive", () => { }, modelUsage: { "claude-opus-4-6": { + inputTokens: 4, + outputTokens: 300, + cacheReadInputTokens: 9000, + cacheCreationInputTokens: 1000, + webSearchRequests: 0, + costUSD: 0.1, + contextWindow: 200000, + maxOutputTokens: 64000, + }, + "claude-haiku-4-5": { + inputTokens: 0, + outputTokens: 379, + cacheReadInputTokens: 12144, + cacheCreationInputTokens: 1715, + webSearchRequests: 0, + costUSD: 0.02, contextWindow: 200000, maxOutputTokens: 64000, }, @@ -1995,6 +2011,44 @@ describe("ClaudeAdapterLive", () => { }, }); } + const completedEvent = runtimeEvents.find((event) => event.type === "turn.completed"); + assert.equal(completedEvent?.type, "turn.completed"); + if (completedEvent?.type === "turn.completed") { + assert.deepEqual(completedEvent.payload.accounting, [ + { + sourceObservationId: `claude:turn:${completedEvent.turnId}:model:claude-opus-4-6`, + sourceKind: "claude.result.model-usage", + model: "claude-opus-4-6", + reasoningLevel: null, + metrics: { + inputTokens: 4, + cachedInputTokens: 10000, + outputTokens: 300, + reasoningOutputTokens: null, + totalTokens: 10304, + }, + metricsProvenance: "exact", + modelProvenance: "exact", + reasoningProvenance: "unknown", + }, + { + sourceObservationId: `claude:turn:${completedEvent.turnId}:model:claude-haiku-4-5`, + sourceKind: "claude.result.model-usage", + model: "claude-haiku-4-5", + reasoningLevel: null, + metrics: { + inputTokens: 0, + cachedInputTokens: 13859, + outputTokens: 379, + reasoningOutputTokens: null, + totalTokens: 14238, + }, + metricsProvenance: "exact", + modelProvenance: "exact", + reasoningProvenance: "unknown", + }, + ]); + } }).pipe( Effect.provideService(Random.Random, makeDeterministicRandomService()), Effect.provide(harness.layer), @@ -2058,6 +2112,12 @@ describe("ClaudeAdapterLive", () => { }, }); } + const completedEvent = runtimeEvents.find((event) => event.type === "turn.completed"); + assert.equal(completedEvent?.type, "turn.completed"); + if (completedEvent?.type === "turn.completed") { + assert.equal(completedEvent.payload.accounting?.[0]?.metrics.totalTokens, 535000); + assert.equal(completedEvent.payload.accounting?.[0]?.metricsProvenance, "inferred"); + } }).pipe( Effect.provideService(Random.Random, makeDeterministicRandomService()), Effect.provide(harness.layer), @@ -3232,6 +3292,11 @@ describe("ClaudeAdapterLive", () => { provider: ProviderDriverKind.make("claudeAgent"), runtimeMode: "full-access", }); + const startedEvent = yield* Stream.runHead(adapter.streamEvents); + assert.equal(startedEvent._tag, "Some"); + if (startedEvent._tag === "Some") { + assert.equal(startedEvent.value.providerInstanceId, customInstanceId); + } yield* adapter.sendTurn({ threadId: session.threadId, input: "hello", diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 285d9dac6088..9e5fd7645923 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -380,6 +380,100 @@ function claudeUsageOutputTokens(usage: Record): number { return finiteNonNegativeInteger(usage.output_tokens) ?? 0; } +function claudeAccountingObservations( + result: SDKResultMessage | undefined, + turnId: TurnId, + fallbackModel: string | undefined, +) { + if (!result?.usage || typeof result.usage !== "object" || Array.isArray(result.usage)) { + return []; + } + + const perModel = Object.entries(result.modelUsage ?? {}).flatMap(([model, modelUsage]) => { + const inputTokens = finiteNonNegativeInteger(modelUsage.inputTokens); + const outputTokens = finiteNonNegativeInteger(modelUsage.outputTokens); + const cacheReadTokens = finiteNonNegativeInteger(modelUsage.cacheReadInputTokens); + const cacheCreationTokens = finiteNonNegativeInteger(modelUsage.cacheCreationInputTokens); + if ( + inputTokens === undefined || + outputTokens === undefined || + cacheReadTokens === undefined || + cacheCreationTokens === undefined + ) { + return []; + } + const cachedInputTokens = cacheReadTokens + cacheCreationTokens; + return [ + { + sourceObservationId: `claude:turn:${turnId}:model:${model}`, + sourceKind: "claude.result.model-usage", + model, + reasoningLevel: null, + metrics: { + inputTokens, + cachedInputTokens, + outputTokens, + reasoningOutputTokens: null, + totalTokens: inputTokens + cachedInputTokens + outputTokens, + }, + metricsProvenance: "exact" as const, + modelProvenance: "exact" as const, + reasoningProvenance: "unknown" as const, + }, + ]; + }); + if (perModel.length > 0) { + return perModel; + } + + const usage = result.usage as Record; + const inputTokens = finiteNonNegativeInteger(usage.input_tokens); + const cacheCreationTokens = finiteNonNegativeInteger(usage.cache_creation_input_tokens); + const cacheReadTokens = finiteNonNegativeInteger(usage.cache_read_input_tokens); + const outputTokens = finiteNonNegativeInteger(usage.output_tokens); + const explicitTotal = finiteNonNegativeInteger(usage.total_tokens); + const hasCompleteComponents = + inputTokens !== undefined && + cacheCreationTokens !== undefined && + cacheReadTokens !== undefined && + outputTokens !== undefined; + const cachedInputTokens = hasCompleteComponents + ? cacheCreationTokens + cacheReadTokens + : undefined; + const totalTokens = + explicitTotal ?? + (hasCompleteComponents ? inputTokens + cachedInputTokens! + outputTokens : undefined); + if (totalTokens === undefined) { + return []; + } + + const reportedModels = Object.keys(result.modelUsage ?? {}); + const model = reportedModels.length === 1 ? reportedModels[0] : fallbackModel; + return [ + { + sourceObservationId: `claude:turn:${turnId}`, + sourceKind: "claude.result.usage", + model: model ?? null, + reasoningLevel: null, + metrics: { + inputTokens: inputTokens ?? null, + cachedInputTokens: cachedInputTokens ?? null, + outputTokens: outputTokens ?? null, + reasoningOutputTokens: null, + totalTokens, + }, + metricsProvenance: hasCompleteComponents ? ("exact" as const) : ("inferred" as const), + modelProvenance: + reportedModels.length === 1 + ? ("exact" as const) + : model + ? ("inferred" as const) + : ("unknown" as const), + reasoningProvenance: "unknown" as const, + }, + ]; +} + function lastClaudeUsageIteration( value: Record, ): Record | undefined { @@ -1385,7 +1479,10 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( const makeEventStamp = () => Effect.all({ eventId: nextEventId, createdAt: nowIso }); const offerRuntimeEvent = (event: ProviderRuntimeEvent): Effect.Effect => - Queue.offer(runtimeEventQueue, event).pipe(Effect.asVoid); + Queue.offer(runtimeEventQueue, { + ...event, + providerInstanceId: boundInstanceId, + }).pipe(Effect.asVoid); const logNativeSdkMessage = Effect.fn("logNativeSdkMessage")(function* ( context: ClaudeSessionContext, @@ -1953,6 +2050,13 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( : undefined); const turnState = context.turnState; + const accountingObservations = turnState + ? claudeAccountingObservations( + result, + turnState.turnId, + context.currentApiModelId ?? context.session.model, + ) + : []; if (!turnState) { yield* emitThreadTokenUsage(context, usageSnapshot, { rawMethod: "claude/result", @@ -2046,6 +2150,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( ...(result?.stop_reason !== undefined ? { stopReason: result.stop_reason } : {}), ...(result?.usage ? { usage: result.usage } : {}), ...(result?.modelUsage ? { modelUsage: result.modelUsage } : {}), + ...(accountingObservations.length > 0 ? { accounting: accountingObservations } : {}), ...(typeof result?.total_cost_usd === "number" ? { totalCostUsd: result.total_cost_usd } : {}), @@ -3795,7 +3900,14 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( createdAt: turnStartedStamp.createdAt, threadId: context.session.threadId, turnId, - payload: modelSelection?.model ? { model: modelSelection.model } : {}, + payload: { + ...(modelSelection?.model ? { model: modelSelection.model } : {}), + ...(modelSelection + ? { + effort: getModelSelectionStringOptionValue(modelSelection, "effort") ?? undefined, + } + : {}), + }, providerRefs: {}, }); } diff --git a/apps/server/src/provider/Layers/CodexAdapter.test.ts b/apps/server/src/provider/Layers/CodexAdapter.test.ts index 4ae654a51873..8c3ffe0a97c7 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.test.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.test.ts @@ -521,6 +521,7 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { id: asEventId("evt-msg-complete"), kind: "notification", provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex_work"), createdAt: "2026-01-01T00:00:00.000Z", method: "item/completed", threadId: asThreadId("thread-1"), @@ -549,6 +550,7 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { if (firstEvent.value.type !== "item.completed") { return; } + NodeAssert.equal(firstEvent.value.providerInstanceId, "codex_work"); NodeAssert.equal(firstEvent.value.itemId, "msg_1"); NodeAssert.equal(firstEvent.value.turnId, "turn-1"); NodeAssert.equal(firstEvent.value.payload.itemType, "assistant_message"); @@ -1148,6 +1150,24 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { lastReasoningOutputTokens: 0, compactsAutomatically: true, }); + NodeAssert.deepEqual(firstEvent.value.payload.accounting, [ + { + sourceObservationId: "codex:turn:turn-1", + sourceKind: "codex.thread-token-usage.last", + model: null, + reasoningLevel: null, + metrics: { + inputTokens: 120, + cachedInputTokens: 0, + outputTokens: 6, + reasoningOutputTokens: 0, + totalTokens: 126, + }, + metricsProvenance: "exact", + modelProvenance: "unknown", + reasoningProvenance: "unknown", + }, + ]); }), ); }); diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 38a5887cdc3e..8a451dfc15bf 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -159,7 +159,7 @@ function normalizeCodexTokenUsage( ): ThreadTokenUsageSnapshot | undefined { const totalProcessedTokens = usage.total.totalTokens; const usedTokens = usage.last.totalTokens; - if (usedTokens === undefined || usedTokens <= 0) { + if (usedTokens === undefined || usedTokens < 0) { return undefined; } @@ -442,6 +442,7 @@ function runtimeEventBase( return { eventId: event.id, provider: event.provider, + ...(event.providerInstanceId ? { providerInstanceId: event.providerInstanceId } : {}), threadId: canonicalThreadId, createdAt: event.createdAt, ...(event.turnId ? { turnId: event.turnId } : {}), @@ -747,6 +748,28 @@ function mapToRuntimeEvents( ...runtimeEventBase(event, canonicalThreadId), payload: { usage: normalizedUsage, + ...(event.turnId + ? { + accounting: [ + { + sourceObservationId: `codex:turn:${event.turnId}`, + sourceKind: "codex.thread-token-usage.last", + model: null, + reasoningLevel: null, + metrics: { + inputTokens: normalizedUsage.lastInputTokens ?? null, + cachedInputTokens: normalizedUsage.lastCachedInputTokens ?? null, + outputTokens: normalizedUsage.lastOutputTokens ?? null, + reasoningOutputTokens: normalizedUsage.lastReasoningOutputTokens ?? null, + totalTokens: normalizedUsage.lastUsedTokens ?? normalizedUsage.usedTokens, + }, + metricsProvenance: "exact" as const, + modelProvenance: "unknown" as const, + reasoningProvenance: "unknown" as const, + }, + ], + } + : {}), }, }, ]; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 66b9823afb3e..e9908b1035a0 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -16,6 +16,7 @@ import { fixPath } from "./os-jank.ts"; import { websocketRpcRouteLayer } from "./ws.ts"; import * as ExternalLauncher from "./process/externalLauncher.ts"; import { layerConfig as SqlitePersistenceLayerLive } from "./persistence/Layers/Sqlite.ts"; +import { TokenUsageRepositoryLive } from "./persistence/Layers/TokenUsage.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; import { ProviderSessionDirectoryLive } from "./provider/Layers/ProviderSessionDirectory.ts"; @@ -287,7 +288,7 @@ const ProviderRuntimeLayerLive = ProviderSessionReaperLive.pipe( Layer.provideMerge(OrchestrationLayerLive), ); -const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( +const RuntimeCoreServicesLive = ReactorLayerLive.pipe( // Core Services Layer.provideMerge(CheckpointingLayerLive), Layer.provideMerge(SourceControlProviderRegistryLayerLive), @@ -295,9 +296,13 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( Layer.provideMerge(VcsLayerLive), Layer.provideMerge(ProviderRuntimeLayerLive), Layer.provideMerge(Layer.mergeAll(TerminalLayerLive, PreviewLayerLive)), + Layer.provideMerge(TokenUsageRepositoryLive), Layer.provideMerge(PersistenceLayerLive), Layer.provideMerge(Keybindings.layer), Layer.provideMerge(ProviderRegistryLive), +); + +const RuntimeCoreDependenciesLive = RuntimeCoreServicesLive.pipe( // The instance registry is the new routing keystone — text generation, // adapter lookup, and runtime ingestion all resolve `ProviderInstanceId` // through this layer. Built-in drivers come from `BUILT_IN_DRIVERS`; From f6af1974abadf4d2502010f082f4c3e3e8089967 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:59:01 +0530 Subject: [PATCH 06/13] feat(server): expose token usage query --- apps/server/src/server.test.ts | 142 +++++++++++++++++++- apps/server/src/ws.ts | 50 +++++++ packages/client-runtime/src/state/server.ts | 4 + 3 files changed, 195 insertions(+), 1 deletion(-) diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 6a3c6f825594..67095fb12c3c 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -82,6 +82,7 @@ import { OrchestrationListenerCallbackError } from "./orchestration/Errors.ts"; import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts"; import { SqlitePersistenceMemory } from "./persistence/Layers/Sqlite.ts"; import { PersistenceSqlError } from "./persistence/Errors.ts"; +import * as TokenUsageRepository from "./persistence/Services/TokenUsage.ts"; import * as ProviderRegistry from "./provider/Services/ProviderRegistry.ts"; import { makeManualOnlyProviderMaintenanceCapabilities } from "./provider/providerMaintenance.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; @@ -340,6 +341,7 @@ const buildAppUnderTest = (options?: { terminalManager?: Partial; orchestrationEngine?: Partial; projectionSnapshotQuery?: Partial; + tokenUsageRepository?: Partial; checkpointDiffQuery?: Partial; browserTraceCollector?: Partial; serverLifecycleEvents?: Partial; @@ -529,7 +531,7 @@ const buildAppUnderTest = (options?: { }) : VcsStatusBroadcaster.layer.pipe(Layer.provide(gitWorkflowLayer)); - const servedRoutesLayer = HttpRouter.serve(makeRoutesLayer, { + const servedRoutesLayerBase = HttpRouter.serve(makeRoutesLayer, { disableListenLog: true, disableLogger: true, }).pipe( @@ -741,6 +743,16 @@ const buildAppUnderTest = (options?: { ), ); + const servedRoutesLayer = options?.layers?.tokenUsageRepository + ? servedRoutesLayerBase.pipe( + Layer.provide( + Layer.mock(TokenUsageRepository.TokenUsageRepository)({ + ...options.layers.tokenUsageRepository, + }), + ), + ) + : servedRoutesLayerBase; + const appLayer = servedRoutesLayer.pipe( Layer.provide( Layer.mock(BrowserTraceCollector.BrowserTraceCollector)({ @@ -3149,6 +3161,42 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("rejects server.getTokenUsage for management-only access tokens", () => + Effect.gen(function* () { + yield* buildAppUnderTest(); + + const { response: exchangeResponse, body: tokenBody } = yield* exchangeAccessToken( + defaultDesktopBootstrapToken, + { scope: "access:write" }, + ); + assert.equal(exchangeResponse.status, 200); + const wsTicketResponse = yield* HttpClient.post("/api/auth/websocket-ticket", { + headers: { + authorization: `Bearer ${tokenBody.access_token ?? ""}`, + }, + }); + const wsTicketBody = (yield* wsTicketResponse.json) as { readonly ticket: string }; + assert.equal(wsTicketResponse.status, 200); + + const wsUrl = `${yield* getWsServerUrl("/ws", { authenticated: false })}?wsTicket=${encodeURIComponent(wsTicketBody.ticket)}`; + const error = yield* Effect.flip( + Effect.scoped( + withWsRpcClient(wsUrl, (client) => + client[WS_METHODS.serverGetTokenUsage]({ + fromDate: "2026-01-01", + toDate: "2026-01-02", + timeZone: "UTC", + }), + ), + ), + ); + assert.equal(error._tag, "EnvironmentAuthorizationError"); + if (error._tag === "EnvironmentAuthorizationError") { + assert.equal(error.requiredScope, "orchestration:read"); + } + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("includes CORS headers on remote auth success responses", () => Effect.gen(function* () { yield* buildAppUnderTest(); @@ -4216,6 +4264,98 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("routes websocket rpc server.getTokenUsage validates and returns aggregates", () => + Effect.gen(function* () { + const query = vi.fn((input) => + Effect.succeed({ + ...input, + trackingStartedAt: null, + days: [], + }), + ); + yield* buildAppUnderTest({ + layers: { + tokenUsageRepository: { query }, + }, + }); + + const wsUrl = yield* getWsServerUrl("/ws"); + const call = (input: { fromDate: string; toDate: string; timeZone: string }) => + Effect.scoped( + withWsRpcClient(wsUrl, (client) => client[WS_METHODS.serverGetTokenUsage](input)), + ); + const result = yield* call({ + fromDate: "2026-01-01", + toDate: "2026-01-02", + timeZone: "America/New_York", + }); + assert.deepEqual(result, { + fromDate: "2026-01-01", + toDate: "2026-01-02", + timeZone: "America/New_York", + trackingStartedAt: null, + days: [], + }); + + for (const input of [ + { fromDate: "2026-02-30", toDate: "2026-03-01", timeZone: "UTC" }, + { fromDate: "2026-03-02", toDate: "2026-03-01", timeZone: "UTC" }, + { fromDate: "2025-01-01", toDate: "2026-01-01", timeZone: "UTC" }, + ]) { + const error = yield* Effect.flip(call(input)); + assert.equal(error._tag, "TokenUsageQueryError"); + if (error._tag === "TokenUsageQueryError") { + assert.equal(error.reason, "invalid-range"); + } + } + + const timeZoneError = yield* Effect.flip( + call({ fromDate: "2026-01-01", toDate: "2026-01-02", timeZone: "Mars/Olympus" }), + ); + assert.equal(timeZoneError._tag, "TokenUsageQueryError"); + if (timeZoneError._tag === "TokenUsageQueryError") { + assert.equal(timeZoneError.reason, "invalid-time-zone"); + } + assert.equal(query.mock.calls.length, 1); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + + it.effect("routes websocket rpc server.getTokenUsage maps persistence failures", () => + Effect.gen(function* () { + yield* buildAppUnderTest({ + layers: { + tokenUsageRepository: { + query: () => + Effect.fail( + new PersistenceSqlError({ + operation: "TokenUsageRepository.query:test", + detail: "forced token usage query failure", + }), + ), + }, + }, + }); + + const wsUrl = yield* getWsServerUrl("/ws"); + const error = yield* Effect.flip( + Effect.scoped( + withWsRpcClient(wsUrl, (client) => + client[WS_METHODS.serverGetTokenUsage]({ + fromDate: "2026-01-01", + toDate: "2026-01-02", + timeZone: "UTC", + }), + ), + ), + ); + assert.equal(error._tag, "TokenUsageQueryError"); + if (error._tag === "TokenUsageQueryError") { + assert.equal(error.reason, "persistence"); + assert.equal(error.message, "Failed to load token usage."); + } + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("routes websocket rpc subscribeServerConfig streams snapshot then update", () => Effect.gen(function* () { const editorDiscoveryStarted = yield* Deferred.make(); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 99429c5b222e..26ac3c9f6c64 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -53,6 +53,7 @@ import { AssetWorkspaceContextResolutionError, EnvironmentAuthorizationError, ThreadId, + TokenUsageQueryError, type TerminalAttachStreamEvent, type TerminalError, type TerminalEvent, @@ -71,6 +72,7 @@ import * as ExternalLauncher from "./process/externalLauncher.ts"; import { normalizeDispatchCommand } from "./orchestration/Normalizer.ts"; import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts"; import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts"; +import { TokenUsageRepository } from "./persistence/Services/TokenUsage.ts"; import { observeRpcEffect as instrumentRpcEffect, observeRpcStream as instrumentRpcStream, @@ -295,6 +297,7 @@ const RPC_REQUIRED_SCOPE = new Map([ [ORCHESTRATION_WS_METHODS.subscribeThread, AuthOrchestrationReadScope], [WS_METHODS.serverProbe, AuthOrchestrationReadScope], [WS_METHODS.serverGetConfig, AuthOrchestrationReadScope], + [WS_METHODS.serverGetTokenUsage, AuthOrchestrationReadScope], [WS_METHODS.serverRefreshProviders, AuthOrchestrationOperateScope], [WS_METHODS.serverUpdateProvider, AuthOrchestrationOperateScope], [WS_METHODS.serverUpdateServer, AuthOrchestrationOperateScope], @@ -407,6 +410,7 @@ const makeWsRpcLayer = ( const currentSessionId = currentSession.sessionId; const crypto = yield* Crypto.Crypto; const projectionSnapshotQuery = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery; + const tokenUsageRepository = yield* Effect.serviceOption(TokenUsageRepository); const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService; const checkpointDiffQuery = yield* CheckpointDiffQuery.CheckpointDiffQuery; const keybindings = yield* Keybindings.Keybindings; @@ -1468,6 +1472,52 @@ const makeWsRpcLayer = ( observeRpcEffect(WS_METHODS.serverGetConfig, loadServerConfig, { "rpc.aggregate": "server", }), + [WS_METHODS.serverGetTokenUsage]: (input) => + observeRpcEffect( + WS_METHODS.serverGetTokenUsage, + Effect.gen(function* () { + const from = new Date(`${input.fromDate}T00:00:00.000Z`); + const to = new Date(`${input.toDate}T00:00:00.000Z`); + const validDates = + Number.isFinite(from.getTime()) && + Number.isFinite(to.getTime()) && + from.toISOString().slice(0, 10) === input.fromDate && + to.toISOString().slice(0, 10) === input.toDate; + const inclusiveDays = validDates + ? Math.floor((to.getTime() - from.getTime()) / 86_400_000) + 1 + : 0; + if (!validDates || inclusiveDays < 1 || inclusiveDays > 365) { + return yield* new TokenUsageQueryError({ + reason: "invalid-range", + message: "Token usage range must contain between 1 and 365 valid calendar days.", + }); + } + try { + new Intl.DateTimeFormat("en-US", { timeZone: input.timeZone }).format(from); + } catch { + return yield* new TokenUsageQueryError({ + reason: "invalid-time-zone", + message: `Unknown IANA time zone: ${input.timeZone}`, + }); + } + if (Option.isNone(tokenUsageRepository)) { + return yield* new TokenUsageQueryError({ + reason: "persistence", + message: "Token usage storage is unavailable.", + }); + } + return yield* tokenUsageRepository.value.query(input).pipe( + Effect.mapError( + () => + new TokenUsageQueryError({ + reason: "persistence", + message: "Failed to load token usage.", + }), + ), + ); + }), + { "rpc.aggregate": "server" }, + ), [WS_METHODS.serverRefreshProviders]: (input) => observeRpcEffect( WS_METHODS.serverRefreshProviders, diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index ea7f5fb6d755..0d275cce5fb7 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -298,6 +298,10 @@ export function createServerEnvironmentAtoms( label: "environment-data:server:process-resource-history", tag: WS_METHODS.serverGetProcessResourceHistory, }), + tokenUsage: createEnvironmentRpcQueryAtomFamily(runtime, { + label: "environment-data:server:token-usage", + tag: WS_METHODS.serverGetTokenUsage, + }), configProjection, welcome: createEnvironmentRpcSubscriptionAtomFamily(runtime, { label: "environment-data:server:welcome", From d8495e4bb2c02570a503bc95895f7319e73b5a0c Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:59:09 +0530 Subject: [PATCH 07/13] feat(web): add token usage settings page --- .../settings/SettingsSidebarNav.tsx | 3 + .../settings/UsageSettingsPanel.test.tsx | 66 ++++ .../settings/UsageSettingsPanel.tsx | 318 ++++++++++++++++++ apps/web/src/hooks/useTokenUsage.ts | 63 ++++ apps/web/src/lib/tokenUsage.test.ts | 56 +++ apps/web/src/lib/tokenUsage.ts | 105 ++++++ apps/web/src/routeTree.gen.ts | 21 ++ apps/web/src/routes/settings.usage.tsx | 11 + 8 files changed, 643 insertions(+) create mode 100644 apps/web/src/components/settings/UsageSettingsPanel.test.tsx create mode 100644 apps/web/src/components/settings/UsageSettingsPanel.tsx create mode 100644 apps/web/src/hooks/useTokenUsage.ts create mode 100644 apps/web/src/lib/tokenUsage.test.ts create mode 100644 apps/web/src/lib/tokenUsage.ts create mode 100644 apps/web/src/routes/settings.usage.tsx diff --git a/apps/web/src/components/settings/SettingsSidebarNav.tsx b/apps/web/src/components/settings/SettingsSidebarNav.tsx index ebbd1b1e4988..e88d7d633bc6 100644 --- a/apps/web/src/components/settings/SettingsSidebarNav.tsx +++ b/apps/web/src/components/settings/SettingsSidebarNav.tsx @@ -2,6 +2,7 @@ import { useCallback, type ComponentType } from "react"; import { ArchiveIcon, ArrowLeftIcon, + BarChart3Icon, BotIcon, FlaskConicalIcon, GitBranchIcon, @@ -26,6 +27,7 @@ export type SettingsSectionPath = | "/settings/general" | "/settings/keybindings" | "/settings/providers" + | "/settings/usage" | "/settings/source-control" | "/settings/connections" | "/settings/beta" @@ -39,6 +41,7 @@ export const SETTINGS_NAV_ITEMS: ReadonlyArray<{ { label: "General", to: "/settings/general", icon: Settings2Icon }, { label: "Keybindings", to: "/settings/keybindings", icon: KeyboardIcon }, { label: "Providers", to: "/settings/providers", icon: BotIcon }, + { label: "Usage", to: "/settings/usage", icon: BarChart3Icon }, { label: "Source Control", to: "/settings/source-control", icon: GitBranchIcon }, { label: "Connections", to: "/settings/connections", icon: Link2Icon }, { label: "Beta", to: "/settings/beta", icon: FlaskConicalIcon }, diff --git a/apps/web/src/components/settings/UsageSettingsPanel.test.tsx b/apps/web/src/components/settings/UsageSettingsPanel.test.tsx new file mode 100644 index 000000000000..37a6a1e2fc43 --- /dev/null +++ b/apps/web/src/components/settings/UsageSettingsPanel.test.tsx @@ -0,0 +1,66 @@ +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { UsageSettingsContent } from "./UsageSettingsPanel"; + +describe("UsageSettingsContent", () => { + it("renders the contribution grid and model reasoning breakdown", () => { + const html = renderToStaticMarkup( + , + ); + + expect(html).toContain("Daily token usage"); + expect(html.match(/role="row"/g)).toHaveLength(7); + expect(html.match(/role="gridcell"/g)).toHaveLength(3); + expect(html).toContain("150 tokens, exact"); + expect(html).toContain("gpt-5.4"); + expect(html).toContain("high"); + expect(html).toContain("UTC"); + }); +}); diff --git a/apps/web/src/components/settings/UsageSettingsPanel.tsx b/apps/web/src/components/settings/UsageSettingsPanel.tsx new file mode 100644 index 000000000000..3566ed656600 --- /dev/null +++ b/apps/web/src/components/settings/UsageSettingsPanel.tsx @@ -0,0 +1,318 @@ +import type { TokenUsageDay, TokenUsageQueryResult } from "@t3tools/contracts"; +import { BarChart3Icon, InfoIcon, RefreshCwIcon } from "lucide-react"; +import { type KeyboardEvent, useEffect, useMemo, useRef, useState } from "react"; + +import { useTokenUsage } from "../../hooks/useTokenUsage"; +import { + buildTokenUsageGrid, + formatTokenCount, + tokenUsageDateAfterKey, + type TokenUsageDateRange, + type TokenUsageGridCell, +} from "../../lib/tokenUsage"; +import { cn } from "../../lib/utils"; +import { Button } from "../ui/button"; +import { SettingsPageContainer, SettingsSection } from "./settingsLayout"; + +const WEEKDAY_LABELS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; + +function dateLabel(date: string): string { + return new Intl.DateTimeFormat(undefined, { dateStyle: "long" }).format( + new Date(`${date}T12:00:00`), + ); +} + +function timestampLabel(timestamp: string, timeZone: string): string { + return new Intl.DateTimeFormat(undefined, { dateStyle: "long", timeZone }).format( + new Date(timestamp), + ); +} + +function cellLabel(cell: TokenUsageGridCell): string { + if (cell.day === null) return `${dateLabel(cell.date)}: no recorded usage`; + const total = cell.day.metrics.totalTokens; + return `${dateLabel(cell.date)}: ${ + total === null ? "token total unknown" : `${total.toLocaleString()} tokens` + }, ${cell.day.provenance}`; +} + +function cellClass(cell: TokenUsageGridCell, selected: boolean): string { + const total = cell.day?.metrics.totalTokens ?? null; + return cn( + "size-3.5 rounded-[3px] border transition-transform hover:scale-125 focus-visible:z-10 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring", + cell.day === null && "border-dashed border-border/70 bg-muted/20", + cell.day !== null && total === null && "border-amber-500/50 bg-amber-500/25", + cell.day !== null && total === 0 && "border-border bg-muted/60", + total !== null && + total > 0 && + cell.intensity === 1 && + "border-emerald-500/30 bg-emerald-500/30", + total !== null && + total > 0 && + cell.intensity === 2 && + "border-emerald-500/40 bg-emerald-500/50", + total !== null && + total > 0 && + cell.intensity === 3 && + "border-emerald-500/50 bg-emerald-500/70", + total !== null && total > 0 && cell.intensity === 4 && "border-emerald-500/60 bg-emerald-500", + selected && "ring-2 ring-foreground ring-offset-2 ring-offset-background", + ); +} + +function Metric({ label, value }: { label: string; value: number | null }) { + return ( +
+
+ {label} +
+
{formatTokenCount(value)}
+
+ ); +} + +function SelectedDayDetails({ day, date }: { day: TokenUsageDay | null; date: string }) { + return ( +
+
+

{dateLabel(date)}

+

+ {day === null + ? "No provider accounting observation was recorded on this day." + : `${day.provenance} provider accounting coverage`} +

+
+
+ + + + + +
+ {day && day.breakdown.length > 0 ? ( +
+ + + + + + + + + + + + + + {day.breakdown.map((row) => ( + + + + + + + + + + ))} + +
ProviderModelReasoningInputOutputReasoning tokensTotal
{row.provider} + {row.model ?? "Unknown"} + {row.modelProvenance !== "exact" ? ( + + ({row.modelProvenance}) + + ) : null} + + {row.reasoningLevel ?? "Unknown"} + {row.reasoningProvenance !== "exact" ? ( + + ({row.reasoningProvenance}) + + ) : null} + + {formatTokenCount(row.metrics.inputTokens)} + + {formatTokenCount(row.metrics.outputTokens)} + + {formatTokenCount(row.metrics.reasoningOutputTokens)} + + {formatTokenCount(row.metrics.totalTokens)} +
+
+ ) : null} +
+ ); +} + +export function UsageSettingsContent({ + data, + error, + isPending, + refresh, + range, + timeZone, +}: { + readonly data: TokenUsageQueryResult | null; + readonly error: string | null; + readonly isPending: boolean; + readonly refresh: () => void; + readonly range: TokenUsageDateRange; + readonly timeZone: string; +}) { + const [selectedDate, setSelectedDate] = useState(range.toDate); + const initializedSelection = useRef(false); + const days = data?.days ?? []; + const daysByDate = useMemo(() => new Map(days.map((day) => [day.date, day])), [days]); + const weeks = useMemo(() => buildTokenUsageGrid(range.dates, days), [days, range.dates]); + + useEffect(() => { + if (initializedSelection.current || data === null) return; + initializedSelection.current = true; + if (days.length > 0 && !daysByDate.has(range.toDate)) { + setSelectedDate(days.at(-1)?.date ?? range.toDate); + } + }, [data, days, daysByDate, range.toDate]); + + const selectedDay = daysByDate.get(selectedDate) ?? null; + const selectAndFocusDate = (date: string) => { + initializedSelection.current = true; + setSelectedDate(date); + requestAnimationFrame(() => { + document.querySelector(`[data-usage-date="${date}"]`)?.focus(); + }); + }; + const handleGridKeyDown = (event: KeyboardEvent, date: string) => { + const nextDate = tokenUsageDateAfterKey(range.dates, date, event.key); + if (!nextDate) return; + event.preventDefault(); + selectAndFocusDate(nextDate); + }; + + return ( + + } + headerAction={ + + } + > +
+
+ +

+ Usage is recorded on this T3 environment from this version forward. Provider coverage + varies, and unavailable values remain unknown. Days use {timeZone}. +

+
+ + {error ? ( +
+

Could not load token usage

+

{error}

+ +
+ ) : ( + <> +
+
+
+

Last 365 days

+

+ {data?.trackingStartedAt + ? `Tracking since ${timestampLabel(data.trackingStartedAt, timeZone)}` + : isPending + ? "Loading recorded usage…" + : "No usage has been recorded yet."} +

+
+
+ Less + {[0, 1, 2, 3, 4].map((level) => ( + + ))} + More +
+
+ +
+
+
+ {WEEKDAY_LABELS.map((label, index) => ( + {index % 2 === 1 ? label : ""} + ))} +
+
+ {WEEKDAY_LABELS.map((label, dayIndex) => ( +
+ {weeks.map((week, weekIndex) => { + const cell = week[dayIndex]; + return cell ? ( +
+ ))} +
+
+
+
+ + + + )} +
+ + + ); +} + +export function UsageSettingsPanel() { + const state = useTokenUsage(); + return ; +} diff --git a/apps/web/src/hooks/useTokenUsage.ts b/apps/web/src/hooks/useTokenUsage.ts new file mode 100644 index 000000000000..6e7cd153022b --- /dev/null +++ b/apps/web/src/hooks/useTokenUsage.ts @@ -0,0 +1,63 @@ +import { useCallback, useEffect, useMemo, useState } from "react"; + +import { serverEnvironment } from "../state/server"; +import { usePrimaryEnvironment } from "../state/environments"; +import { useEnvironmentQuery } from "../state/query"; +import { trailingTokenUsageRange } from "../lib/tokenUsage"; + +export function useTokenUsage() { + const environmentId = usePrimaryEnvironment()?.environmentId ?? null; + const [calendarRevision, setCalendarRevision] = useState(0); + const timeZone = useMemo( + () => Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC", + [calendarRevision], + ); + const range = useMemo(() => trailingTokenUsageRange(), [calendarRevision]); + const queryAtom = useMemo( + () => + environmentId === null + ? null + : serverEnvironment.tokenUsage({ + environmentId, + input: { + fromDate: range.fromDate, + toDate: range.toDate, + timeZone, + }, + }), + [environmentId, range.fromDate, range.toDate, timeZone], + ); + const query = useEnvironmentQuery(queryAtom); + const refresh = useCallback(() => { + setCalendarRevision((revision) => revision + 1); + query.refresh(); + }, [query.refresh]); + + useEffect(() => { + const now = new Date(); + const nextDay = new Date(now); + nextDay.setHours(24, 0, 0, 0); + const timeout = window.setTimeout( + () => setCalendarRevision((revision) => revision + 1), + Math.max(1, nextDay.getTime() - now.getTime() + 100), + ); + const refreshVisibleCalendar = () => { + if (document.visibilityState === "visible") { + setCalendarRevision((revision) => revision + 1); + } + }; + document.addEventListener("visibilitychange", refreshVisibleCalendar); + return () => { + window.clearTimeout(timeout); + document.removeEventListener("visibilitychange", refreshVisibleCalendar); + }; + }, [calendarRevision]); + + return { + ...query, + refresh, + environmentId, + range, + timeZone, + }; +} diff --git a/apps/web/src/lib/tokenUsage.test.ts b/apps/web/src/lib/tokenUsage.test.ts new file mode 100644 index 000000000000..96c67e30c138 --- /dev/null +++ b/apps/web/src/lib/tokenUsage.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + buildTokenUsageGrid, + tokenUsageDateAfterKey, + tokenUsageIntensity, + trailingTokenUsageRange, +} from "./tokenUsage"; + +describe("token usage presentation helpers", () => { + it("creates exactly 365 local calendar dates", () => { + const range = trailingTokenUsageRange(new Date(2026, 0, 1, 12), 365); + expect(range.dates).toHaveLength(365); + expect(range.toDate).toBe("2026-01-01"); + expect(range.fromDate).toBe("2025-01-02"); + }); + + it("lays dates out by weekday and retains sparse usage", () => { + const dates = ["2026-01-01", "2026-01-02", "2026-01-03"]; + const weeks = buildTokenUsageGrid(dates, [ + { + date: "2026-01-02", + metrics: { + inputTokens: 10, + cachedInputTokens: 0, + outputTokens: 5, + reasoningOutputTokens: 0, + totalTokens: 15, + }, + provenance: "exact", + breakdown: [], + }, + ]); + + expect(weeks).toHaveLength(1); + expect(weeks[0]?.[4]?.date).toBe("2026-01-01"); + expect(weeks[0]?.[4]?.day).toBeNull(); + expect(weeks[0]?.[5]?.day?.metrics.totalTokens).toBe(15); + }); + + it("keeps unknown and zero totals at the empty intensity", () => { + expect(tokenUsageIntensity(null, 100)).toBe(0); + expect(tokenUsageIntensity(0, 100)).toBe(0); + expect(tokenUsageIntensity(100, 100)).toBe(4); + }); + + it("moves through recorded and unrecorded grid dates with arrow keys", () => { + const dates = Array.from( + { length: 15 }, + (_, index) => `2026-01-${String(index + 1).padStart(2, "0")}`, + ); + expect(tokenUsageDateAfterKey(dates, "2026-01-08", "ArrowLeft")).toBe("2026-01-01"); + expect(tokenUsageDateAfterKey(dates, "2026-01-08", "ArrowDown")).toBe("2026-01-09"); + expect(tokenUsageDateAfterKey(dates, "2026-01-08", "End")).toBe("2026-01-15"); + }); +}); diff --git a/apps/web/src/lib/tokenUsage.ts b/apps/web/src/lib/tokenUsage.ts new file mode 100644 index 000000000000..ae2547a6d44a --- /dev/null +++ b/apps/web/src/lib/tokenUsage.ts @@ -0,0 +1,105 @@ +import type { TokenUsageCalendarDate, TokenUsageDay } from "@t3tools/contracts"; + +export interface TokenUsageDateRange { + readonly fromDate: TokenUsageCalendarDate; + readonly toDate: TokenUsageCalendarDate; + readonly dates: ReadonlyArray; +} + +export interface TokenUsageGridCell { + readonly date: TokenUsageCalendarDate; + readonly day: TokenUsageDay | null; + readonly intensity: 0 | 1 | 2 | 3 | 4; +} + +export function formatLocalCalendarDate(date: Date): TokenUsageCalendarDate { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, "0"); + const day = String(date.getDate()).padStart(2, "0"); + return `${year}-${month}-${day}`; +} + +export function trailingTokenUsageRange(today = new Date(), count = 365): TokenUsageDateRange { + const end = new Date(today.getFullYear(), today.getMonth(), today.getDate()); + const dates: TokenUsageCalendarDate[] = []; + for (let offset = count - 1; offset >= 0; offset -= 1) { + const date = new Date(end); + date.setDate(end.getDate() - offset); + dates.push(formatLocalCalendarDate(date)); + } + return { + fromDate: dates[0]!, + toDate: dates.at(-1)!, + dates, + }; +} + +function totalFor(day: TokenUsageDay | null): number | null { + return day?.metrics.totalTokens ?? null; +} + +export function tokenUsageIntensity(total: number | null, maximum: number): 0 | 1 | 2 | 3 | 4 { + if (total === null || total <= 0 || maximum <= 0) return 0; + const ratio = Math.log1p(total) / Math.log1p(maximum); + if (ratio <= 0.25) return 1; + if (ratio <= 0.5) return 2; + if (ratio <= 0.75) return 3; + return 4; +} + +export function buildTokenUsageGrid( + dates: ReadonlyArray, + days: ReadonlyArray, +): ReadonlyArray> { + if (dates.length === 0) return []; + const byDate = new Map(days.map((day) => [day.date, day])); + const maximum = Math.max(0, ...days.map((day) => day.metrics.totalTokens ?? 0)); + const cells: Array = []; + const firstDate = new Date(`${dates[0]}T12:00:00`); + for (let padding = 0; padding < firstDate.getDay(); padding += 1) { + cells.push(null); + } + for (const date of dates) { + const day = byDate.get(date) ?? null; + const total = totalFor(day); + cells.push({ date, day, intensity: tokenUsageIntensity(total, maximum) }); + } + while (cells.length % 7 !== 0) cells.push(null); + + const weeks: Array> = []; + for (let index = 0; index < cells.length; index += 7) { + weeks.push(cells.slice(index, index + 7)); + } + return weeks; +} + +export function tokenUsageDateAfterKey( + dates: ReadonlyArray, + currentDate: string, + key: string, +): TokenUsageCalendarDate | null { + const index = dates.indexOf(currentDate); + if (index < 0) return null; + const offset = + key === "ArrowUp" + ? -1 + : key === "ArrowDown" + ? 1 + : key === "ArrowLeft" + ? -7 + : key === "ArrowRight" + ? 7 + : key === "Home" + ? -index + : key === "End" + ? dates.length - index - 1 + : null; + return offset === null ? null : (dates[index + offset] ?? null); +} + +export function formatTokenCount(value: number | null): string { + if (value === null) return "Unknown"; + return new Intl.NumberFormat(undefined, { notation: "compact", maximumFractionDigits: 1 }).format( + value, + ); +} diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index 563d1b43755e..f0c99fef20ee 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -14,6 +14,7 @@ import { Route as PairRouteImport } from './routes/pair' import { Route as ConnectRouteImport } from './routes/connect' import { Route as ChatRouteImport } from './routes/_chat' import { Route as ChatIndexRouteImport } from './routes/_chat.index' +import { Route as SettingsUsageRouteImport } from './routes/settings.usage' import { Route as SettingsSourceControlRouteImport } from './routes/settings.source-control' import { Route as SettingsProvidersRouteImport } from './routes/settings.providers' import { Route as SettingsKeybindingsRouteImport } from './routes/settings.keybindings' @@ -50,6 +51,11 @@ const ChatIndexRoute = ChatIndexRouteImport.update({ path: '/', getParentRoute: () => ChatRoute, } as any) +const SettingsUsageRoute = SettingsUsageRouteImport.update({ + id: '/usage', + path: '/usage', + getParentRoute: () => SettingsRoute, +} as any) const SettingsSourceControlRoute = SettingsSourceControlRouteImport.update({ id: '/source-control', path: '/source-control', @@ -121,6 +127,7 @@ export interface FileRoutesByFullPath { '/settings/keybindings': typeof SettingsKeybindingsRoute '/settings/providers': typeof SettingsProvidersRoute '/settings/source-control': typeof SettingsSourceControlRoute + '/settings/usage': typeof SettingsUsageRoute '/$environmentId/$threadId': typeof ChatEnvironmentIdThreadIdRoute '/draft/$draftId': typeof ChatDraftDraftIdRoute } @@ -137,6 +144,7 @@ export interface FileRoutesByTo { '/settings/keybindings': typeof SettingsKeybindingsRoute '/settings/providers': typeof SettingsProvidersRoute '/settings/source-control': typeof SettingsSourceControlRoute + '/settings/usage': typeof SettingsUsageRoute '/': typeof ChatIndexRoute '/$environmentId/$threadId': typeof ChatEnvironmentIdThreadIdRoute '/draft/$draftId': typeof ChatDraftDraftIdRoute @@ -156,6 +164,7 @@ export interface FileRoutesById { '/settings/keybindings': typeof SettingsKeybindingsRoute '/settings/providers': typeof SettingsProvidersRoute '/settings/source-control': typeof SettingsSourceControlRoute + '/settings/usage': typeof SettingsUsageRoute '/_chat/': typeof ChatIndexRoute '/_chat/$environmentId/$threadId': typeof ChatEnvironmentIdThreadIdRoute '/_chat/draft/$draftId': typeof ChatDraftDraftIdRoute @@ -176,6 +185,7 @@ export interface FileRouteTypes { | '/settings/keybindings' | '/settings/providers' | '/settings/source-control' + | '/settings/usage' | '/$environmentId/$threadId' | '/draft/$draftId' fileRoutesByTo: FileRoutesByTo @@ -192,6 +202,7 @@ export interface FileRouteTypes { | '/settings/keybindings' | '/settings/providers' | '/settings/source-control' + | '/settings/usage' | '/' | '/$environmentId/$threadId' | '/draft/$draftId' @@ -210,6 +221,7 @@ export interface FileRouteTypes { | '/settings/keybindings' | '/settings/providers' | '/settings/source-control' + | '/settings/usage' | '/_chat/' | '/_chat/$environmentId/$threadId' | '/_chat/draft/$draftId' @@ -260,6 +272,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ChatIndexRouteImport parentRoute: typeof ChatRoute } + '/settings/usage': { + id: '/settings/usage' + path: '/usage' + fullPath: '/settings/usage' + preLoaderRoute: typeof SettingsUsageRouteImport + parentRoute: typeof SettingsRoute + } '/settings/source-control': { id: '/settings/source-control' path: '/source-control' @@ -363,6 +382,7 @@ interface SettingsRouteChildren { SettingsKeybindingsRoute: typeof SettingsKeybindingsRoute SettingsProvidersRoute: typeof SettingsProvidersRoute SettingsSourceControlRoute: typeof SettingsSourceControlRoute + SettingsUsageRoute: typeof SettingsUsageRoute } const SettingsRouteChildren: SettingsRouteChildren = { @@ -374,6 +394,7 @@ const SettingsRouteChildren: SettingsRouteChildren = { SettingsKeybindingsRoute: SettingsKeybindingsRoute, SettingsProvidersRoute: SettingsProvidersRoute, SettingsSourceControlRoute: SettingsSourceControlRoute, + SettingsUsageRoute: SettingsUsageRoute, } const SettingsRouteWithChildren = SettingsRoute._addFileChildren( diff --git a/apps/web/src/routes/settings.usage.tsx b/apps/web/src/routes/settings.usage.tsx new file mode 100644 index 000000000000..86c63a5f3865 --- /dev/null +++ b/apps/web/src/routes/settings.usage.tsx @@ -0,0 +1,11 @@ +import { createFileRoute } from "@tanstack/react-router"; + +import { UsageSettingsPanel } from "../components/settings/UsageSettingsPanel"; + +function SettingsUsageRoute() { + return ; +} + +export const Route = createFileRoute("/settings/usage")({ + component: SettingsUsageRoute, +}); From 57d2853d50c7a80f684ee6ff015ce83396698ab5 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:59:16 +0530 Subject: [PATCH 08/13] docs: record token usage analytics implementation --- task.md | 138 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 task.md diff --git a/task.md b/task.md new file mode 100644 index 000000000000..b350c7231862 --- /dev/null +++ b/task.md @@ -0,0 +1,138 @@ +# Token Usage Analytics — Implementation Plan + +## Product scope and decisions + +- [x] Track usage for the current local T3 environment; do not merge other environments. +- [x] Ship a web-only **Settings → Usage** page in this change. +- [x] Show the browser-local trailing 365 calendar days. +- [x] Store immutable UTC timestamps and bucket them with the requested IANA timezone. +- [x] Record accurate usage going forward; do not backfill context-window snapshots. +- [x] Keep missing values unknown—never convert unavailable usage, model, or reasoning data to zero. +- [x] Keep context-window telemetry separate from token accounting. + +## 1. Define usage accounting contracts + +- [x] Add schema-only token usage contracts. + - [x] Define accounting provenance (`exact`, `inferred`, `unknown`). + - [x] Define nullable input, cached-input, output, reasoning-output, and total token metrics. + - [x] Define requested and reported model/reasoning attribution fields. + - [x] Define an idempotent provider accounting observation. + - [x] Define validated calendar-date aggregate-query fields. + - [x] Define sparse daily totals and model/reasoning breakdown response schemas. +- [x] Register the typed `server.getTokenUsage` RPC. +- [x] Add focused contract tests for dates, unknown metrics, and provenance. + +## 2. Add the durable usage ledger + +- [x] Add and register migration 35. + - [x] Create the usage fact table with non-negative constraints. + - [x] Add stable source-observation uniqueness for replay safety. + - [x] Add timestamp, model/reasoning, and diagnostic indexes. + - [x] Avoid historical snapshot backfill. +- [x] Add the usage persistence service and live SQL layer. + - [x] Record exact/inferred observations idempotently. + - [x] Replace duplicate metrics as one coherent observation, never per-column maxima. + - [x] Preserve stronger model/reasoning attribution during replay. + - [x] Aggregate a bounded UTC range into browser-local calendar days. + - [x] Group selected-day data by provider, model, and reasoning level. + - [x] Exclude non-numeric records from totals while retaining known zero usage. +- [x] Add focused migration/repository coverage. + - [x] Duplicate observations do not inflate totals. + - [x] Older/replayed observations do not create mixed metrics. + - [x] Stronger exact attribution wins over inferred attribution. + - [x] UTC-boundary timestamps bucket correctly. + - [x] Unknown metrics stay unknown. + +## 3. Capture provider usage + +- [x] Codex accounting. + - [x] Keep existing context-window snapshots unchanged. + - [x] Emit a separate exact observation from provider-reported last-turn usage. + - [x] Deduplicate cumulative last-turn updates by provider turn. + - [x] Capture requested model and reasoning effort when provider payloads omit them. +- [x] Claude accounting. + - [x] Normalize final SDK result usage only; do not count progress/context snapshots. + - [x] Preserve exact per-model usage from `modelUsage`. + - [x] Mark requested-model fallback and partial aggregate metrics as inferred. + - [x] Preserve failed/cancelled turn usage when reported. +- [x] Other providers. + - [x] Do not fabricate usage when stable terminal accounting data is unavailable. +- [x] Feed explicit accounting observations through provider runtime ingestion. + - [x] Persist model/reasoning dimensions at observation time. + - [x] Isolate accounting failures from the existing chat projection flow. +- [x] Add focused adapter, ingestion, and replay tests. + +## 4. Expose an authenticated aggregate query + +- [x] Add `server.getTokenUsage` to the WebSocket authorization scope map. +- [x] Implement the unary server handler and query-service dependency. +- [x] Validate IANA timezone and inclusive ranges of at most 365 days. +- [x] Return typed failures for invalid input and persistence errors. +- [x] Verify authenticated query behavior in the isolated integrated browser environment. + +## 5. Build Settings → Usage + +- [x] Add the Usage settings navigation item and file route. +- [x] Add primary-environment request state for the typed usage RPC. +- [x] Add date/grid presentation helpers. + - [x] Calculate exactly 365 browser-local dates. + - [x] Lay out weeks and weekdays across month/year boundaries. + - [x] Derive intensity from known daily totals only. +- [x] Build the usage panel. + - [x] Render an accessible GitHub-style daily grid. + - [x] Distinguish absent, unknown, exact, inferred, and known-zero states. + - [x] Support roving focus and arrow-key selection across every day. + - [x] Show the selected day’s provider/model/reasoning breakdown. + - [x] Show loading, retryable error, empty, and sparse-coverage states. + - [x] Explain timezone, provider coverage, and tracking-start behavior. +- [x] Add focused helper and component tests. +- [x] Regenerate the TanStack route tree through the web build workflow. + +## 6. Verify and review + +- [x] Run focused contract tests. +- [x] Run focused repository, provider, ingestion, and RPC-path tests. +- [x] Run focused web helper and component tests. +- [x] Run affected-package formatting, lint, type checks, and web build. +- [x] Run one integrated `test-t3-app` browser verification. + - [x] Authenticate through an isolated environment pairing URL. + - [x] Verify Settings → Usage navigation and route. + - [x] Verify the 365-day grid and local-timezone label. + - [x] Verify empty state and seeded populated usage. + - [x] Verify day selection and model/reasoning detail rendering. + - [x] Verify 365 grid cells, roving selected state, and no page errors. + - [x] Stop and remove the isolated verification environment. +- [x] Obtain independent diff reviews and resolve all blocker/high findings. +- [x] Resolve post-implementation QA findings. + - [x] Reject stale equal-total metric replays unless their observation timestamp is newer. + - [x] Preserve custom Codex and Claude provider instance IDs through accounting persistence. + - [x] Render seven valid ARIA rows containing all 365 gridcells. + - [x] Recompute the local date range/timezone at midnight, visibility return, and manual retry. + - [x] Add direct authenticated, authorization, validation, and persistence-failure RPC tests. + - [x] Bound Windows editor discovery so server configuration cannot remain blocked indefinitely. +- [x] Confirm no files are staged. + +## Verification evidence + +- `vp test run` across the nine focused contract, provider runtime, usage repository, adapter, ingestion, web, and server integration files: **268/268 passed**. +- Affected package type checks for contracts, client-runtime, server, and web: **passed**; the server layer composition reports zero TypeScript errors. +- Targeted lint for the changed production files and focused new tests: **passed**. +- Web build and generated route tree: **passed**. +- `apps/server/src/server.test.ts`: **115/115 passed**, including direct token-usage RPC and bounded editor-discovery coverage on Windows. +- Isolated browser verification: + - authenticated and opened `/settings/usage`; + - observed exactly 365 accessible grid cells owned by seven ARIA rows; + - verified one roving tab stop and keyboard selection across weeks; + - verified empty state; + - seeded disposable Codex and Claude facts after stopping the server; + - verified daily total, per-model rows, reasoning levels, inferred labels, and prior-day selection; + - observed no page errors; + - stopped the environment and removed disposable state. + +## Residual risks + +- Accurate accounting is available going forward only; existing context-window history is intentionally not backfilled. +- Codex and Claude are the currently supported accounting sources. Other providers remain sparse until they expose stable terminal usage identities and metrics. +- Codex accounting treats `tokenUsage.last` as the cumulative last-turn snapshot and keeps the strongest/latest coherent observation for that provider turn. +- Windows editor discovery is now bounded to two seconds while loading server configuration; a timeout returns an empty editor list and logs a warning instead of blocking environment recovery. +- `ProviderRuntimeIngestion.test.ts` has a pre-existing targeted-lint violation (`no-manual-effect-runtime-in-tests`) outside the added accounting test; all focused tests and package type checks pass. From c47b1999bf43775700aaa1b26c9afaf33f34a131 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Sun, 9 Aug 2026 02:03:45 +0530 Subject: [PATCH 09/13] Revert "feat(web): add token usage settings page" This reverts commit d8495e4bb2c02570a503bc95895f7319e73b5a0c. --- .../settings/UsageSettingsPanel.test.tsx | 66 ---- .../settings/UsageSettingsPanel.tsx | 318 ------------------ apps/web/src/hooks/useTokenUsage.ts | 63 ---- apps/web/src/lib/tokenUsage.test.ts | 56 --- apps/web/src/lib/tokenUsage.ts | 105 ------ apps/web/src/routeTree.gen.ts | 21 -- apps/web/src/routes/settings.usage.tsx | 11 - task.md | 138 -------- 8 files changed, 778 deletions(-) delete mode 100644 apps/web/src/components/settings/UsageSettingsPanel.test.tsx delete mode 100644 apps/web/src/components/settings/UsageSettingsPanel.tsx delete mode 100644 apps/web/src/hooks/useTokenUsage.ts delete mode 100644 apps/web/src/lib/tokenUsage.test.ts delete mode 100644 apps/web/src/lib/tokenUsage.ts delete mode 100644 apps/web/src/routes/settings.usage.tsx delete mode 100644 task.md diff --git a/apps/web/src/components/settings/UsageSettingsPanel.test.tsx b/apps/web/src/components/settings/UsageSettingsPanel.test.tsx deleted file mode 100644 index 37a6a1e2fc43..000000000000 --- a/apps/web/src/components/settings/UsageSettingsPanel.test.tsx +++ /dev/null @@ -1,66 +0,0 @@ -import { renderToStaticMarkup } from "react-dom/server"; -import { describe, expect, it, vi } from "vite-plus/test"; - -import { UsageSettingsContent } from "./UsageSettingsPanel"; - -describe("UsageSettingsContent", () => { - it("renders the contribution grid and model reasoning breakdown", () => { - const html = renderToStaticMarkup( - , - ); - - expect(html).toContain("Daily token usage"); - expect(html.match(/role="row"/g)).toHaveLength(7); - expect(html.match(/role="gridcell"/g)).toHaveLength(3); - expect(html).toContain("150 tokens, exact"); - expect(html).toContain("gpt-5.4"); - expect(html).toContain("high"); - expect(html).toContain("UTC"); - }); -}); diff --git a/apps/web/src/components/settings/UsageSettingsPanel.tsx b/apps/web/src/components/settings/UsageSettingsPanel.tsx deleted file mode 100644 index 3566ed656600..000000000000 --- a/apps/web/src/components/settings/UsageSettingsPanel.tsx +++ /dev/null @@ -1,318 +0,0 @@ -import type { TokenUsageDay, TokenUsageQueryResult } from "@t3tools/contracts"; -import { BarChart3Icon, InfoIcon, RefreshCwIcon } from "lucide-react"; -import { type KeyboardEvent, useEffect, useMemo, useRef, useState } from "react"; - -import { useTokenUsage } from "../../hooks/useTokenUsage"; -import { - buildTokenUsageGrid, - formatTokenCount, - tokenUsageDateAfterKey, - type TokenUsageDateRange, - type TokenUsageGridCell, -} from "../../lib/tokenUsage"; -import { cn } from "../../lib/utils"; -import { Button } from "../ui/button"; -import { SettingsPageContainer, SettingsSection } from "./settingsLayout"; - -const WEEKDAY_LABELS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; - -function dateLabel(date: string): string { - return new Intl.DateTimeFormat(undefined, { dateStyle: "long" }).format( - new Date(`${date}T12:00:00`), - ); -} - -function timestampLabel(timestamp: string, timeZone: string): string { - return new Intl.DateTimeFormat(undefined, { dateStyle: "long", timeZone }).format( - new Date(timestamp), - ); -} - -function cellLabel(cell: TokenUsageGridCell): string { - if (cell.day === null) return `${dateLabel(cell.date)}: no recorded usage`; - const total = cell.day.metrics.totalTokens; - return `${dateLabel(cell.date)}: ${ - total === null ? "token total unknown" : `${total.toLocaleString()} tokens` - }, ${cell.day.provenance}`; -} - -function cellClass(cell: TokenUsageGridCell, selected: boolean): string { - const total = cell.day?.metrics.totalTokens ?? null; - return cn( - "size-3.5 rounded-[3px] border transition-transform hover:scale-125 focus-visible:z-10 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring", - cell.day === null && "border-dashed border-border/70 bg-muted/20", - cell.day !== null && total === null && "border-amber-500/50 bg-amber-500/25", - cell.day !== null && total === 0 && "border-border bg-muted/60", - total !== null && - total > 0 && - cell.intensity === 1 && - "border-emerald-500/30 bg-emerald-500/30", - total !== null && - total > 0 && - cell.intensity === 2 && - "border-emerald-500/40 bg-emerald-500/50", - total !== null && - total > 0 && - cell.intensity === 3 && - "border-emerald-500/50 bg-emerald-500/70", - total !== null && total > 0 && cell.intensity === 4 && "border-emerald-500/60 bg-emerald-500", - selected && "ring-2 ring-foreground ring-offset-2 ring-offset-background", - ); -} - -function Metric({ label, value }: { label: string; value: number | null }) { - return ( -
-
- {label} -
-
{formatTokenCount(value)}
-
- ); -} - -function SelectedDayDetails({ day, date }: { day: TokenUsageDay | null; date: string }) { - return ( -
-
-

{dateLabel(date)}

-

- {day === null - ? "No provider accounting observation was recorded on this day." - : `${day.provenance} provider accounting coverage`} -

-
-
- - - - - -
- {day && day.breakdown.length > 0 ? ( -
- - - - - - - - - - - - - - {day.breakdown.map((row) => ( - - - - - - - - - - ))} - -
ProviderModelReasoningInputOutputReasoning tokensTotal
{row.provider} - {row.model ?? "Unknown"} - {row.modelProvenance !== "exact" ? ( - - ({row.modelProvenance}) - - ) : null} - - {row.reasoningLevel ?? "Unknown"} - {row.reasoningProvenance !== "exact" ? ( - - ({row.reasoningProvenance}) - - ) : null} - - {formatTokenCount(row.metrics.inputTokens)} - - {formatTokenCount(row.metrics.outputTokens)} - - {formatTokenCount(row.metrics.reasoningOutputTokens)} - - {formatTokenCount(row.metrics.totalTokens)} -
-
- ) : null} -
- ); -} - -export function UsageSettingsContent({ - data, - error, - isPending, - refresh, - range, - timeZone, -}: { - readonly data: TokenUsageQueryResult | null; - readonly error: string | null; - readonly isPending: boolean; - readonly refresh: () => void; - readonly range: TokenUsageDateRange; - readonly timeZone: string; -}) { - const [selectedDate, setSelectedDate] = useState(range.toDate); - const initializedSelection = useRef(false); - const days = data?.days ?? []; - const daysByDate = useMemo(() => new Map(days.map((day) => [day.date, day])), [days]); - const weeks = useMemo(() => buildTokenUsageGrid(range.dates, days), [days, range.dates]); - - useEffect(() => { - if (initializedSelection.current || data === null) return; - initializedSelection.current = true; - if (days.length > 0 && !daysByDate.has(range.toDate)) { - setSelectedDate(days.at(-1)?.date ?? range.toDate); - } - }, [data, days, daysByDate, range.toDate]); - - const selectedDay = daysByDate.get(selectedDate) ?? null; - const selectAndFocusDate = (date: string) => { - initializedSelection.current = true; - setSelectedDate(date); - requestAnimationFrame(() => { - document.querySelector(`[data-usage-date="${date}"]`)?.focus(); - }); - }; - const handleGridKeyDown = (event: KeyboardEvent, date: string) => { - const nextDate = tokenUsageDateAfterKey(range.dates, date, event.key); - if (!nextDate) return; - event.preventDefault(); - selectAndFocusDate(nextDate); - }; - - return ( - - } - headerAction={ - - } - > -
-
- -

- Usage is recorded on this T3 environment from this version forward. Provider coverage - varies, and unavailable values remain unknown. Days use {timeZone}. -

-
- - {error ? ( -
-

Could not load token usage

-

{error}

- -
- ) : ( - <> -
-
-
-

Last 365 days

-

- {data?.trackingStartedAt - ? `Tracking since ${timestampLabel(data.trackingStartedAt, timeZone)}` - : isPending - ? "Loading recorded usage…" - : "No usage has been recorded yet."} -

-
-
- Less - {[0, 1, 2, 3, 4].map((level) => ( - - ))} - More -
-
- -
-
-
- {WEEKDAY_LABELS.map((label, index) => ( - {index % 2 === 1 ? label : ""} - ))} -
-
- {WEEKDAY_LABELS.map((label, dayIndex) => ( -
- {weeks.map((week, weekIndex) => { - const cell = week[dayIndex]; - return cell ? ( -
- ))} -
-
-
-
- - - - )} -
- - - ); -} - -export function UsageSettingsPanel() { - const state = useTokenUsage(); - return ; -} diff --git a/apps/web/src/hooks/useTokenUsage.ts b/apps/web/src/hooks/useTokenUsage.ts deleted file mode 100644 index 6e7cd153022b..000000000000 --- a/apps/web/src/hooks/useTokenUsage.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { useCallback, useEffect, useMemo, useState } from "react"; - -import { serverEnvironment } from "../state/server"; -import { usePrimaryEnvironment } from "../state/environments"; -import { useEnvironmentQuery } from "../state/query"; -import { trailingTokenUsageRange } from "../lib/tokenUsage"; - -export function useTokenUsage() { - const environmentId = usePrimaryEnvironment()?.environmentId ?? null; - const [calendarRevision, setCalendarRevision] = useState(0); - const timeZone = useMemo( - () => Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC", - [calendarRevision], - ); - const range = useMemo(() => trailingTokenUsageRange(), [calendarRevision]); - const queryAtom = useMemo( - () => - environmentId === null - ? null - : serverEnvironment.tokenUsage({ - environmentId, - input: { - fromDate: range.fromDate, - toDate: range.toDate, - timeZone, - }, - }), - [environmentId, range.fromDate, range.toDate, timeZone], - ); - const query = useEnvironmentQuery(queryAtom); - const refresh = useCallback(() => { - setCalendarRevision((revision) => revision + 1); - query.refresh(); - }, [query.refresh]); - - useEffect(() => { - const now = new Date(); - const nextDay = new Date(now); - nextDay.setHours(24, 0, 0, 0); - const timeout = window.setTimeout( - () => setCalendarRevision((revision) => revision + 1), - Math.max(1, nextDay.getTime() - now.getTime() + 100), - ); - const refreshVisibleCalendar = () => { - if (document.visibilityState === "visible") { - setCalendarRevision((revision) => revision + 1); - } - }; - document.addEventListener("visibilitychange", refreshVisibleCalendar); - return () => { - window.clearTimeout(timeout); - document.removeEventListener("visibilitychange", refreshVisibleCalendar); - }; - }, [calendarRevision]); - - return { - ...query, - refresh, - environmentId, - range, - timeZone, - }; -} diff --git a/apps/web/src/lib/tokenUsage.test.ts b/apps/web/src/lib/tokenUsage.test.ts deleted file mode 100644 index 96c67e30c138..000000000000 --- a/apps/web/src/lib/tokenUsage.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { describe, expect, it } from "vite-plus/test"; - -import { - buildTokenUsageGrid, - tokenUsageDateAfterKey, - tokenUsageIntensity, - trailingTokenUsageRange, -} from "./tokenUsage"; - -describe("token usage presentation helpers", () => { - it("creates exactly 365 local calendar dates", () => { - const range = trailingTokenUsageRange(new Date(2026, 0, 1, 12), 365); - expect(range.dates).toHaveLength(365); - expect(range.toDate).toBe("2026-01-01"); - expect(range.fromDate).toBe("2025-01-02"); - }); - - it("lays dates out by weekday and retains sparse usage", () => { - const dates = ["2026-01-01", "2026-01-02", "2026-01-03"]; - const weeks = buildTokenUsageGrid(dates, [ - { - date: "2026-01-02", - metrics: { - inputTokens: 10, - cachedInputTokens: 0, - outputTokens: 5, - reasoningOutputTokens: 0, - totalTokens: 15, - }, - provenance: "exact", - breakdown: [], - }, - ]); - - expect(weeks).toHaveLength(1); - expect(weeks[0]?.[4]?.date).toBe("2026-01-01"); - expect(weeks[0]?.[4]?.day).toBeNull(); - expect(weeks[0]?.[5]?.day?.metrics.totalTokens).toBe(15); - }); - - it("keeps unknown and zero totals at the empty intensity", () => { - expect(tokenUsageIntensity(null, 100)).toBe(0); - expect(tokenUsageIntensity(0, 100)).toBe(0); - expect(tokenUsageIntensity(100, 100)).toBe(4); - }); - - it("moves through recorded and unrecorded grid dates with arrow keys", () => { - const dates = Array.from( - { length: 15 }, - (_, index) => `2026-01-${String(index + 1).padStart(2, "0")}`, - ); - expect(tokenUsageDateAfterKey(dates, "2026-01-08", "ArrowLeft")).toBe("2026-01-01"); - expect(tokenUsageDateAfterKey(dates, "2026-01-08", "ArrowDown")).toBe("2026-01-09"); - expect(tokenUsageDateAfterKey(dates, "2026-01-08", "End")).toBe("2026-01-15"); - }); -}); diff --git a/apps/web/src/lib/tokenUsage.ts b/apps/web/src/lib/tokenUsage.ts deleted file mode 100644 index ae2547a6d44a..000000000000 --- a/apps/web/src/lib/tokenUsage.ts +++ /dev/null @@ -1,105 +0,0 @@ -import type { TokenUsageCalendarDate, TokenUsageDay } from "@t3tools/contracts"; - -export interface TokenUsageDateRange { - readonly fromDate: TokenUsageCalendarDate; - readonly toDate: TokenUsageCalendarDate; - readonly dates: ReadonlyArray; -} - -export interface TokenUsageGridCell { - readonly date: TokenUsageCalendarDate; - readonly day: TokenUsageDay | null; - readonly intensity: 0 | 1 | 2 | 3 | 4; -} - -export function formatLocalCalendarDate(date: Date): TokenUsageCalendarDate { - const year = date.getFullYear(); - const month = String(date.getMonth() + 1).padStart(2, "0"); - const day = String(date.getDate()).padStart(2, "0"); - return `${year}-${month}-${day}`; -} - -export function trailingTokenUsageRange(today = new Date(), count = 365): TokenUsageDateRange { - const end = new Date(today.getFullYear(), today.getMonth(), today.getDate()); - const dates: TokenUsageCalendarDate[] = []; - for (let offset = count - 1; offset >= 0; offset -= 1) { - const date = new Date(end); - date.setDate(end.getDate() - offset); - dates.push(formatLocalCalendarDate(date)); - } - return { - fromDate: dates[0]!, - toDate: dates.at(-1)!, - dates, - }; -} - -function totalFor(day: TokenUsageDay | null): number | null { - return day?.metrics.totalTokens ?? null; -} - -export function tokenUsageIntensity(total: number | null, maximum: number): 0 | 1 | 2 | 3 | 4 { - if (total === null || total <= 0 || maximum <= 0) return 0; - const ratio = Math.log1p(total) / Math.log1p(maximum); - if (ratio <= 0.25) return 1; - if (ratio <= 0.5) return 2; - if (ratio <= 0.75) return 3; - return 4; -} - -export function buildTokenUsageGrid( - dates: ReadonlyArray, - days: ReadonlyArray, -): ReadonlyArray> { - if (dates.length === 0) return []; - const byDate = new Map(days.map((day) => [day.date, day])); - const maximum = Math.max(0, ...days.map((day) => day.metrics.totalTokens ?? 0)); - const cells: Array = []; - const firstDate = new Date(`${dates[0]}T12:00:00`); - for (let padding = 0; padding < firstDate.getDay(); padding += 1) { - cells.push(null); - } - for (const date of dates) { - const day = byDate.get(date) ?? null; - const total = totalFor(day); - cells.push({ date, day, intensity: tokenUsageIntensity(total, maximum) }); - } - while (cells.length % 7 !== 0) cells.push(null); - - const weeks: Array> = []; - for (let index = 0; index < cells.length; index += 7) { - weeks.push(cells.slice(index, index + 7)); - } - return weeks; -} - -export function tokenUsageDateAfterKey( - dates: ReadonlyArray, - currentDate: string, - key: string, -): TokenUsageCalendarDate | null { - const index = dates.indexOf(currentDate); - if (index < 0) return null; - const offset = - key === "ArrowUp" - ? -1 - : key === "ArrowDown" - ? 1 - : key === "ArrowLeft" - ? -7 - : key === "ArrowRight" - ? 7 - : key === "Home" - ? -index - : key === "End" - ? dates.length - index - 1 - : null; - return offset === null ? null : (dates[index + offset] ?? null); -} - -export function formatTokenCount(value: number | null): string { - if (value === null) return "Unknown"; - return new Intl.NumberFormat(undefined, { notation: "compact", maximumFractionDigits: 1 }).format( - value, - ); -} diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index 9e22ae642d9e..eb31a8de91cd 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -15,7 +15,6 @@ import { Route as PairRouteImport } from './routes/pair' import { Route as ConnectRouteImport } from './routes/connect' import { Route as ChatRouteImport } from './routes/_chat' import { Route as ChatIndexRouteImport } from './routes/_chat.index' -import { Route as SettingsUsageRouteImport } from './routes/settings.usage' import { Route as SettingsSourceControlRouteImport } from './routes/settings.source-control' import { Route as SettingsProvidersRouteImport } from './routes/settings.providers' import { Route as SettingsKeybindingsRouteImport } from './routes/settings.keybindings' @@ -57,11 +56,6 @@ const ChatIndexRoute = ChatIndexRouteImport.update({ path: '/', getParentRoute: () => ChatRoute, } as any) -const SettingsUsageRoute = SettingsUsageRouteImport.update({ - id: '/usage', - path: '/usage', - getParentRoute: () => SettingsRoute, -} as any) const SettingsSourceControlRoute = SettingsSourceControlRouteImport.update({ id: '/source-control', path: '/source-control', @@ -134,7 +128,6 @@ export interface FileRoutesByFullPath { '/settings/keybindings': typeof SettingsKeybindingsRoute '/settings/providers': typeof SettingsProvidersRoute '/settings/source-control': typeof SettingsSourceControlRoute - '/settings/usage': typeof SettingsUsageRoute '/$environmentId/$threadId': typeof ChatEnvironmentIdThreadIdRoute '/draft/$draftId': typeof ChatDraftDraftIdRoute } @@ -152,7 +145,6 @@ export interface FileRoutesByTo { '/settings/keybindings': typeof SettingsKeybindingsRoute '/settings/providers': typeof SettingsProvidersRoute '/settings/source-control': typeof SettingsSourceControlRoute - '/settings/usage': typeof SettingsUsageRoute '/': typeof ChatIndexRoute '/$environmentId/$threadId': typeof ChatEnvironmentIdThreadIdRoute '/draft/$draftId': typeof ChatDraftDraftIdRoute @@ -173,7 +165,6 @@ export interface FileRoutesById { '/settings/keybindings': typeof SettingsKeybindingsRoute '/settings/providers': typeof SettingsProvidersRoute '/settings/source-control': typeof SettingsSourceControlRoute - '/settings/usage': typeof SettingsUsageRoute '/_chat/': typeof ChatIndexRoute '/_chat/$environmentId/$threadId': typeof ChatEnvironmentIdThreadIdRoute '/_chat/draft/$draftId': typeof ChatDraftDraftIdRoute @@ -195,7 +186,6 @@ export interface FileRouteTypes { | '/settings/keybindings' | '/settings/providers' | '/settings/source-control' - | '/settings/usage' | '/$environmentId/$threadId' | '/draft/$draftId' fileRoutesByTo: FileRoutesByTo @@ -213,7 +203,6 @@ export interface FileRouteTypes { | '/settings/keybindings' | '/settings/providers' | '/settings/source-control' - | '/settings/usage' | '/' | '/$environmentId/$threadId' | '/draft/$draftId' @@ -233,7 +222,6 @@ export interface FileRouteTypes { | '/settings/keybindings' | '/settings/providers' | '/settings/source-control' - | '/settings/usage' | '/_chat/' | '/_chat/$environmentId/$threadId' | '/_chat/draft/$draftId' @@ -292,13 +280,6 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ChatIndexRouteImport parentRoute: typeof ChatRoute } - '/settings/usage': { - id: '/settings/usage' - path: '/usage' - fullPath: '/settings/usage' - preLoaderRoute: typeof SettingsUsageRouteImport - parentRoute: typeof SettingsRoute - } '/settings/source-control': { id: '/settings/source-control' path: '/source-control' @@ -402,7 +383,6 @@ interface SettingsRouteChildren { SettingsKeybindingsRoute: typeof SettingsKeybindingsRoute SettingsProvidersRoute: typeof SettingsProvidersRoute SettingsSourceControlRoute: typeof SettingsSourceControlRoute - SettingsUsageRoute: typeof SettingsUsageRoute } const SettingsRouteChildren: SettingsRouteChildren = { @@ -414,7 +394,6 @@ const SettingsRouteChildren: SettingsRouteChildren = { SettingsKeybindingsRoute: SettingsKeybindingsRoute, SettingsProvidersRoute: SettingsProvidersRoute, SettingsSourceControlRoute: SettingsSourceControlRoute, - SettingsUsageRoute: SettingsUsageRoute, } const SettingsRouteWithChildren = SettingsRoute._addFileChildren( diff --git a/apps/web/src/routes/settings.usage.tsx b/apps/web/src/routes/settings.usage.tsx deleted file mode 100644 index 86c63a5f3865..000000000000 --- a/apps/web/src/routes/settings.usage.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import { createFileRoute } from "@tanstack/react-router"; - -import { UsageSettingsPanel } from "../components/settings/UsageSettingsPanel"; - -function SettingsUsageRoute() { - return ; -} - -export const Route = createFileRoute("/settings/usage")({ - component: SettingsUsageRoute, -}); diff --git a/task.md b/task.md deleted file mode 100644 index b350c7231862..000000000000 --- a/task.md +++ /dev/null @@ -1,138 +0,0 @@ -# Token Usage Analytics — Implementation Plan - -## Product scope and decisions - -- [x] Track usage for the current local T3 environment; do not merge other environments. -- [x] Ship a web-only **Settings → Usage** page in this change. -- [x] Show the browser-local trailing 365 calendar days. -- [x] Store immutable UTC timestamps and bucket them with the requested IANA timezone. -- [x] Record accurate usage going forward; do not backfill context-window snapshots. -- [x] Keep missing values unknown—never convert unavailable usage, model, or reasoning data to zero. -- [x] Keep context-window telemetry separate from token accounting. - -## 1. Define usage accounting contracts - -- [x] Add schema-only token usage contracts. - - [x] Define accounting provenance (`exact`, `inferred`, `unknown`). - - [x] Define nullable input, cached-input, output, reasoning-output, and total token metrics. - - [x] Define requested and reported model/reasoning attribution fields. - - [x] Define an idempotent provider accounting observation. - - [x] Define validated calendar-date aggregate-query fields. - - [x] Define sparse daily totals and model/reasoning breakdown response schemas. -- [x] Register the typed `server.getTokenUsage` RPC. -- [x] Add focused contract tests for dates, unknown metrics, and provenance. - -## 2. Add the durable usage ledger - -- [x] Add and register migration 35. - - [x] Create the usage fact table with non-negative constraints. - - [x] Add stable source-observation uniqueness for replay safety. - - [x] Add timestamp, model/reasoning, and diagnostic indexes. - - [x] Avoid historical snapshot backfill. -- [x] Add the usage persistence service and live SQL layer. - - [x] Record exact/inferred observations idempotently. - - [x] Replace duplicate metrics as one coherent observation, never per-column maxima. - - [x] Preserve stronger model/reasoning attribution during replay. - - [x] Aggregate a bounded UTC range into browser-local calendar days. - - [x] Group selected-day data by provider, model, and reasoning level. - - [x] Exclude non-numeric records from totals while retaining known zero usage. -- [x] Add focused migration/repository coverage. - - [x] Duplicate observations do not inflate totals. - - [x] Older/replayed observations do not create mixed metrics. - - [x] Stronger exact attribution wins over inferred attribution. - - [x] UTC-boundary timestamps bucket correctly. - - [x] Unknown metrics stay unknown. - -## 3. Capture provider usage - -- [x] Codex accounting. - - [x] Keep existing context-window snapshots unchanged. - - [x] Emit a separate exact observation from provider-reported last-turn usage. - - [x] Deduplicate cumulative last-turn updates by provider turn. - - [x] Capture requested model and reasoning effort when provider payloads omit them. -- [x] Claude accounting. - - [x] Normalize final SDK result usage only; do not count progress/context snapshots. - - [x] Preserve exact per-model usage from `modelUsage`. - - [x] Mark requested-model fallback and partial aggregate metrics as inferred. - - [x] Preserve failed/cancelled turn usage when reported. -- [x] Other providers. - - [x] Do not fabricate usage when stable terminal accounting data is unavailable. -- [x] Feed explicit accounting observations through provider runtime ingestion. - - [x] Persist model/reasoning dimensions at observation time. - - [x] Isolate accounting failures from the existing chat projection flow. -- [x] Add focused adapter, ingestion, and replay tests. - -## 4. Expose an authenticated aggregate query - -- [x] Add `server.getTokenUsage` to the WebSocket authorization scope map. -- [x] Implement the unary server handler and query-service dependency. -- [x] Validate IANA timezone and inclusive ranges of at most 365 days. -- [x] Return typed failures for invalid input and persistence errors. -- [x] Verify authenticated query behavior in the isolated integrated browser environment. - -## 5. Build Settings → Usage - -- [x] Add the Usage settings navigation item and file route. -- [x] Add primary-environment request state for the typed usage RPC. -- [x] Add date/grid presentation helpers. - - [x] Calculate exactly 365 browser-local dates. - - [x] Lay out weeks and weekdays across month/year boundaries. - - [x] Derive intensity from known daily totals only. -- [x] Build the usage panel. - - [x] Render an accessible GitHub-style daily grid. - - [x] Distinguish absent, unknown, exact, inferred, and known-zero states. - - [x] Support roving focus and arrow-key selection across every day. - - [x] Show the selected day’s provider/model/reasoning breakdown. - - [x] Show loading, retryable error, empty, and sparse-coverage states. - - [x] Explain timezone, provider coverage, and tracking-start behavior. -- [x] Add focused helper and component tests. -- [x] Regenerate the TanStack route tree through the web build workflow. - -## 6. Verify and review - -- [x] Run focused contract tests. -- [x] Run focused repository, provider, ingestion, and RPC-path tests. -- [x] Run focused web helper and component tests. -- [x] Run affected-package formatting, lint, type checks, and web build. -- [x] Run one integrated `test-t3-app` browser verification. - - [x] Authenticate through an isolated environment pairing URL. - - [x] Verify Settings → Usage navigation and route. - - [x] Verify the 365-day grid and local-timezone label. - - [x] Verify empty state and seeded populated usage. - - [x] Verify day selection and model/reasoning detail rendering. - - [x] Verify 365 grid cells, roving selected state, and no page errors. - - [x] Stop and remove the isolated verification environment. -- [x] Obtain independent diff reviews and resolve all blocker/high findings. -- [x] Resolve post-implementation QA findings. - - [x] Reject stale equal-total metric replays unless their observation timestamp is newer. - - [x] Preserve custom Codex and Claude provider instance IDs through accounting persistence. - - [x] Render seven valid ARIA rows containing all 365 gridcells. - - [x] Recompute the local date range/timezone at midnight, visibility return, and manual retry. - - [x] Add direct authenticated, authorization, validation, and persistence-failure RPC tests. - - [x] Bound Windows editor discovery so server configuration cannot remain blocked indefinitely. -- [x] Confirm no files are staged. - -## Verification evidence - -- `vp test run` across the nine focused contract, provider runtime, usage repository, adapter, ingestion, web, and server integration files: **268/268 passed**. -- Affected package type checks for contracts, client-runtime, server, and web: **passed**; the server layer composition reports zero TypeScript errors. -- Targeted lint for the changed production files and focused new tests: **passed**. -- Web build and generated route tree: **passed**. -- `apps/server/src/server.test.ts`: **115/115 passed**, including direct token-usage RPC and bounded editor-discovery coverage on Windows. -- Isolated browser verification: - - authenticated and opened `/settings/usage`; - - observed exactly 365 accessible grid cells owned by seven ARIA rows; - - verified one roving tab stop and keyboard selection across weeks; - - verified empty state; - - seeded disposable Codex and Claude facts after stopping the server; - - verified daily total, per-model rows, reasoning levels, inferred labels, and prior-day selection; - - observed no page errors; - - stopped the environment and removed disposable state. - -## Residual risks - -- Accurate accounting is available going forward only; existing context-window history is intentionally not backfilled. -- Codex and Claude are the currently supported accounting sources. Other providers remain sparse until they expose stable terminal usage identities and metrics. -- Codex accounting treats `tokenUsage.last` as the cumulative last-turn snapshot and keeps the strongest/latest coherent observation for that provider turn. -- Windows editor discovery is now bounded to two seconds while loading server configuration; a timeout returns an empty editor list and logs a warning instead of blocking environment recovery. -- `ProviderRuntimeIngestion.test.ts` has a pre-existing targeted-lint violation (`no-manual-effect-runtime-in-tests`) outside the added accounting test; all focused tests and package type checks pass. From 99c283a0b53ad0f84a081870cac2e9e399a894f4 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Sun, 9 Aug 2026 02:04:05 +0530 Subject: [PATCH 10/13] Revert "feat(server): capture provider token usage" This reverts commit 2777d32602d95ae5e4c25f317fa11fd1330bc8b1. --- .../OrchestrationEngineHarness.integration.ts | 2 - .../Layers/ProviderRuntimeIngestion.ts | 52 -------- .../src/provider/Layers/ClaudeAdapter.test.ts | 65 ---------- .../src/provider/Layers/ClaudeAdapter.ts | 116 +----------------- .../src/provider/Layers/CodexAdapter.test.ts | 20 --- .../src/provider/Layers/CodexAdapter.ts | 25 +--- apps/server/src/server.ts | 7 +- 7 files changed, 4 insertions(+), 283 deletions(-) diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.ts index d5f8fb616a82..71ef59a0910c 100644 --- a/apps/server/integration/OrchestrationEngineHarness.integration.ts +++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts @@ -30,7 +30,6 @@ import { ProjectionCheckpointRepositoryLive } from "../src/persistence/Layers/Pr import { ProjectionPendingApprovalRepositoryLive } from "../src/persistence/Layers/ProjectionPendingApprovals.ts"; import { ProviderSessionRuntimeRepositoryLive } from "../src/persistence/Layers/ProviderSessionRuntime.ts"; import { makeSqlitePersistenceLive } from "../src/persistence/Layers/Sqlite.ts"; -import { TokenUsageRepositoryLive } from "../src/persistence/Layers/TokenUsage.ts"; import { ProjectionCheckpointRepository } from "../src/persistence/Services/ProjectionCheckpoints.ts"; import { ProjectionPendingApprovalRepository } from "../src/persistence/Services/ProjectionPendingApprovals.ts"; import { makeAdapterRegistryMock } from "../src/provider/testUtils/providerAdapterRegistryMock.ts"; @@ -320,7 +319,6 @@ export const makeOrchestrationIntegrationHarness = ( const runtimeIngestionLayer = ProviderRuntimeIngestionLive.pipe( Layer.provideMerge(runtimeServicesLayer), Layer.provideMerge(serverSettingsLayer), - Layer.provide(TokenUsageRepositoryLive), ); const gitWorkflowLayer = Layer.mock(GitWorkflowService)({ renameBranch: (input: { diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index 76c3a6f4748e..40307cd9f25e 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -28,12 +28,10 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; -import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; import { ProjectionTurnRepository } from "../../persistence/Services/ProjectionTurns.ts"; import { ProjectionTurnRepositoryLive } from "../../persistence/Layers/ProjectionTurns.ts"; -import { TokenUsageRepository } from "../../persistence/Services/TokenUsage.ts"; import { isGitRepository } from "../../git/Utils.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; import { ThreadBackgroundLivenessService } from "../ThreadBackgroundLiveness.ts"; @@ -876,7 +874,6 @@ const make = Effect.gen(function* () { const projectionSnapshotQuery = yield* ProjectionSnapshotQuery; const providerService = yield* ProviderService; const projectionTurnRepository = yield* ProjectionTurnRepository; - const tokenUsageRepository = yield* TokenUsageRepository; const serverSettingsService = yield* ServerSettingsService; const providerCommandId = (event: ProviderRuntimeEvent, tag: string) => crypto.randomUUIDv4.pipe( @@ -1492,55 +1489,6 @@ const make = Effect.gen(function* () { const now = event.createdAt; const eventTurnId = toTurnId(event.turnId); - const accountingObservations = - event.type === "thread.token-usage.updated" || event.type === "turn.completed" - ? (event.payload.accounting ?? []) - : []; - if (accountingObservations.length > 0) { - const fallbackModel = thread.modelSelection.model; - const fallbackReasoningLevel = - getModelSelectionStringOptionValue(thread.modelSelection, "reasoningEffort") ?? - getModelSelectionStringOptionValue(thread.modelSelection, "effort") ?? - getModelSelectionStringOptionValue(thread.modelSelection, "variant") ?? - null; - for (const observation of accountingObservations) { - const model = observation.model ?? fallbackModel; - const reasoningLevel = observation.reasoningLevel ?? fallbackReasoningLevel; - yield* tokenUsageRepository - .record({ - provider: event.provider, - providerInstanceId: event.providerInstanceId ?? null, - runtimeEventId: event.eventId, - threadId: thread.id, - turnId: eventTurnId ?? null, - occurredAt: event.createdAt, - observation: { - ...observation, - model, - reasoningLevel, - modelProvenance: - observation.model === null && model !== null - ? "inferred" - : observation.modelProvenance, - reasoningProvenance: - observation.reasoningLevel === null && reasoningLevel !== null - ? "inferred" - : observation.reasoningProvenance, - }, - }) - .pipe( - Effect.catchCause((cause) => - Effect.logWarning("provider token usage accounting failed", { - provider: event.provider, - eventId: event.eventId, - threadId: thread.id, - turnId: eventTurnId ?? null, - cause: Cause.pretty(cause), - }), - ), - ); - } - } const activeTurnId = thread.session?.activeTurnId ?? null; const pendingTurnStart = yield* projectionTurnRepository.getPendingTurnStartByThreadId({ threadId: thread.id, diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts index 8d4f97d7dc12..d3d768b53844 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts @@ -2312,22 +2312,6 @@ describe("ClaudeAdapterLive", () => { }, modelUsage: { "claude-opus-4-6": { - inputTokens: 4, - outputTokens: 300, - cacheReadInputTokens: 9000, - cacheCreationInputTokens: 1000, - webSearchRequests: 0, - costUSD: 0.1, - contextWindow: 200000, - maxOutputTokens: 64000, - }, - "claude-haiku-4-5": { - inputTokens: 0, - outputTokens: 379, - cacheReadInputTokens: 12144, - cacheCreationInputTokens: 1715, - webSearchRequests: 0, - costUSD: 0.02, contextWindow: 200000, maxOutputTokens: 64000, }, @@ -2349,44 +2333,6 @@ describe("ClaudeAdapterLive", () => { }, }); } - const completedEvent = runtimeEvents.find((event) => event.type === "turn.completed"); - assert.equal(completedEvent?.type, "turn.completed"); - if (completedEvent?.type === "turn.completed") { - assert.deepEqual(completedEvent.payload.accounting, [ - { - sourceObservationId: `claude:turn:${completedEvent.turnId}:model:claude-opus-4-6`, - sourceKind: "claude.result.model-usage", - model: "claude-opus-4-6", - reasoningLevel: null, - metrics: { - inputTokens: 4, - cachedInputTokens: 10000, - outputTokens: 300, - reasoningOutputTokens: null, - totalTokens: 10304, - }, - metricsProvenance: "exact", - modelProvenance: "exact", - reasoningProvenance: "unknown", - }, - { - sourceObservationId: `claude:turn:${completedEvent.turnId}:model:claude-haiku-4-5`, - sourceKind: "claude.result.model-usage", - model: "claude-haiku-4-5", - reasoningLevel: null, - metrics: { - inputTokens: 0, - cachedInputTokens: 13859, - outputTokens: 379, - reasoningOutputTokens: null, - totalTokens: 14238, - }, - metricsProvenance: "exact", - modelProvenance: "exact", - reasoningProvenance: "unknown", - }, - ]); - } }).pipe( Effect.provideService(Random.Random, makeDeterministicRandomService()), Effect.provide(harness.layer), @@ -2450,12 +2396,6 @@ describe("ClaudeAdapterLive", () => { }, }); } - const completedEvent = runtimeEvents.find((event) => event.type === "turn.completed"); - assert.equal(completedEvent?.type, "turn.completed"); - if (completedEvent?.type === "turn.completed") { - assert.equal(completedEvent.payload.accounting?.[0]?.metrics.totalTokens, 535000); - assert.equal(completedEvent.payload.accounting?.[0]?.metricsProvenance, "inferred"); - } }).pipe( Effect.provideService(Random.Random, makeDeterministicRandomService()), Effect.provide(harness.layer), @@ -3630,11 +3570,6 @@ describe("ClaudeAdapterLive", () => { provider: ProviderDriverKind.make("claudeAgent"), runtimeMode: "full-access", }); - const startedEvent = yield* Stream.runHead(adapter.streamEvents); - assert.equal(startedEvent._tag, "Some"); - if (startedEvent._tag === "Some") { - assert.equal(startedEvent.value.providerInstanceId, customInstanceId); - } yield* adapter.sendTurn({ threadId: session.threadId, input: "hello", diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 0e9f9eb16f51..92445522cc49 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -448,100 +448,6 @@ function claudeUsageOutputTokens(usage: Record): number { return finiteNonNegativeInteger(usage.output_tokens) ?? 0; } -function claudeAccountingObservations( - result: SDKResultMessage | undefined, - turnId: TurnId, - fallbackModel: string | undefined, -) { - if (!result?.usage || typeof result.usage !== "object" || Array.isArray(result.usage)) { - return []; - } - - const perModel = Object.entries(result.modelUsage ?? {}).flatMap(([model, modelUsage]) => { - const inputTokens = finiteNonNegativeInteger(modelUsage.inputTokens); - const outputTokens = finiteNonNegativeInteger(modelUsage.outputTokens); - const cacheReadTokens = finiteNonNegativeInteger(modelUsage.cacheReadInputTokens); - const cacheCreationTokens = finiteNonNegativeInteger(modelUsage.cacheCreationInputTokens); - if ( - inputTokens === undefined || - outputTokens === undefined || - cacheReadTokens === undefined || - cacheCreationTokens === undefined - ) { - return []; - } - const cachedInputTokens = cacheReadTokens + cacheCreationTokens; - return [ - { - sourceObservationId: `claude:turn:${turnId}:model:${model}`, - sourceKind: "claude.result.model-usage", - model, - reasoningLevel: null, - metrics: { - inputTokens, - cachedInputTokens, - outputTokens, - reasoningOutputTokens: null, - totalTokens: inputTokens + cachedInputTokens + outputTokens, - }, - metricsProvenance: "exact" as const, - modelProvenance: "exact" as const, - reasoningProvenance: "unknown" as const, - }, - ]; - }); - if (perModel.length > 0) { - return perModel; - } - - const usage = result.usage as Record; - const inputTokens = finiteNonNegativeInteger(usage.input_tokens); - const cacheCreationTokens = finiteNonNegativeInteger(usage.cache_creation_input_tokens); - const cacheReadTokens = finiteNonNegativeInteger(usage.cache_read_input_tokens); - const outputTokens = finiteNonNegativeInteger(usage.output_tokens); - const explicitTotal = finiteNonNegativeInteger(usage.total_tokens); - const hasCompleteComponents = - inputTokens !== undefined && - cacheCreationTokens !== undefined && - cacheReadTokens !== undefined && - outputTokens !== undefined; - const cachedInputTokens = hasCompleteComponents - ? cacheCreationTokens + cacheReadTokens - : undefined; - const totalTokens = - explicitTotal ?? - (hasCompleteComponents ? inputTokens + cachedInputTokens! + outputTokens : undefined); - if (totalTokens === undefined) { - return []; - } - - const reportedModels = Object.keys(result.modelUsage ?? {}); - const model = reportedModels.length === 1 ? reportedModels[0] : fallbackModel; - return [ - { - sourceObservationId: `claude:turn:${turnId}`, - sourceKind: "claude.result.usage", - model: model ?? null, - reasoningLevel: null, - metrics: { - inputTokens: inputTokens ?? null, - cachedInputTokens: cachedInputTokens ?? null, - outputTokens: outputTokens ?? null, - reasoningOutputTokens: null, - totalTokens, - }, - metricsProvenance: hasCompleteComponents ? ("exact" as const) : ("inferred" as const), - modelProvenance: - reportedModels.length === 1 - ? ("exact" as const) - : model - ? ("inferred" as const) - : ("unknown" as const), - reasoningProvenance: "unknown" as const, - }, - ]; -} - function lastClaudeUsageIteration( value: Record, ): Record | undefined { @@ -1768,10 +1674,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( const makeEventStamp = () => Effect.all({ eventId: nextEventId, createdAt: nowIso }); const offerRuntimeEvent = (event: ProviderRuntimeEvent): Effect.Effect => - Queue.offer(runtimeEventQueue, { - ...event, - providerInstanceId: boundInstanceId, - }).pipe(Effect.asVoid); + Queue.offer(runtimeEventQueue, event).pipe(Effect.asVoid); const logNativeSdkMessage = Effect.fnUntraced(function* ( context: ClaudeSessionContext, @@ -2339,13 +2242,6 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( : undefined); const turnState = context.turnState; - const accountingObservations = turnState - ? claudeAccountingObservations( - result, - turnState.turnId, - context.currentApiModelId ?? context.session.model, - ) - : []; if (!turnState) { yield* emitThreadTokenUsage(context, usageSnapshot, { rawMethod: "claude/result", @@ -2439,7 +2335,6 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( ...(result?.stop_reason !== undefined ? { stopReason: result.stop_reason } : {}), ...(result?.usage ? { usage: result.usage } : {}), ...(result?.modelUsage ? { modelUsage: result.modelUsage } : {}), - ...(accountingObservations.length > 0 ? { accounting: accountingObservations } : {}), ...(typeof result?.total_cost_usd === "number" ? { totalCostUsd: result.total_cost_usd } : {}), @@ -4482,14 +4377,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( createdAt: turnStartedStamp.createdAt, threadId: context.session.threadId, turnId, - payload: { - ...(modelSelection?.model ? { model: modelSelection.model } : {}), - ...(modelSelection - ? { - effort: getModelSelectionStringOptionValue(modelSelection, "effort") ?? undefined, - } - : {}), - }, + payload: modelSelection?.model ? { model: modelSelection.model } : {}, providerRefs: {}, }); } diff --git a/apps/server/src/provider/Layers/CodexAdapter.test.ts b/apps/server/src/provider/Layers/CodexAdapter.test.ts index 40003df05ff4..7b8fbec56660 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.test.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.test.ts @@ -521,7 +521,6 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { id: asEventId("evt-msg-complete"), kind: "notification", provider: ProviderDriverKind.make("codex"), - providerInstanceId: ProviderInstanceId.make("codex_work"), createdAt: "2026-01-01T00:00:00.000Z", method: "item/completed", threadId: asThreadId("thread-1"), @@ -550,7 +549,6 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { if (firstEvent.value.type !== "item.completed") { return; } - NodeAssert.equal(firstEvent.value.providerInstanceId, "codex_work"); NodeAssert.equal(firstEvent.value.itemId, "msg_1"); NodeAssert.equal(firstEvent.value.turnId, "turn-1"); NodeAssert.equal(firstEvent.value.payload.itemType, "assistant_message"); @@ -1150,24 +1148,6 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { lastReasoningOutputTokens: 0, compactsAutomatically: true, }); - NodeAssert.deepEqual(firstEvent.value.payload.accounting, [ - { - sourceObservationId: "codex:turn:turn-1", - sourceKind: "codex.thread-token-usage.last", - model: null, - reasoningLevel: null, - metrics: { - inputTokens: 120, - cachedInputTokens: 0, - outputTokens: 6, - reasoningOutputTokens: 0, - totalTokens: 126, - }, - metricsProvenance: "exact", - modelProvenance: "unknown", - reasoningProvenance: "unknown", - }, - ]); }), ); }); diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index b865eea4ba80..6b99bf52b1e0 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -161,7 +161,7 @@ function normalizeCodexTokenUsage( ): ThreadTokenUsageSnapshot | undefined { const totalProcessedTokens = usage.total.totalTokens; const usedTokens = usage.last.totalTokens; - if (usedTokens === undefined || usedTokens < 0) { + if (usedTokens === undefined || usedTokens <= 0) { return undefined; } @@ -444,7 +444,6 @@ function runtimeEventBase( return { eventId: event.id, provider: event.provider, - ...(event.providerInstanceId ? { providerInstanceId: event.providerInstanceId } : {}), threadId: canonicalThreadId, createdAt: event.createdAt, ...(event.turnId ? { turnId: event.turnId } : {}), @@ -1016,28 +1015,6 @@ function mapToRuntimeEvents( ...runtimeEventBase(event, canonicalThreadId), payload: { usage: normalizedUsage, - ...(event.turnId - ? { - accounting: [ - { - sourceObservationId: `codex:turn:${event.turnId}`, - sourceKind: "codex.thread-token-usage.last", - model: null, - reasoningLevel: null, - metrics: { - inputTokens: normalizedUsage.lastInputTokens ?? null, - cachedInputTokens: normalizedUsage.lastCachedInputTokens ?? null, - outputTokens: normalizedUsage.lastOutputTokens ?? null, - reasoningOutputTokens: normalizedUsage.lastReasoningOutputTokens ?? null, - totalTokens: normalizedUsage.lastUsedTokens ?? normalizedUsage.usedTokens, - }, - metricsProvenance: "exact" as const, - modelProvenance: "unknown" as const, - reasoningProvenance: "unknown" as const, - }, - ], - } - : {}), }, }, ]; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 28310b7f6b2a..ff21c07a861a 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -22,7 +22,6 @@ import { fixPath } from "./os-jank.ts"; import { websocketRpcRouteLayer } from "./ws.ts"; import * as ExternalLauncher from "./process/externalLauncher.ts"; import { layerConfig as SqlitePersistenceLayerLive } from "./persistence/Layers/Sqlite.ts"; -import { TokenUsageRepositoryLive } from "./persistence/Layers/TokenUsage.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; import { ProviderSessionDirectoryLive } from "./provider/Layers/ProviderSessionDirectory.ts"; @@ -363,7 +362,7 @@ const ProviderRuntimeLayerLive = ProviderSessionReaperLive.pipe( Layer.provideMerge(OrchestrationLayerLive), ); -const RuntimeCoreServicesLive = ReactorLayerLive.pipe( +const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( // Core Services Layer.provideMerge(ServerSettingsLayerLive), Layer.provideMerge(CheckpointingLayerLive), @@ -372,13 +371,9 @@ const RuntimeCoreServicesLive = ReactorLayerLive.pipe( Layer.provideMerge(VcsLayerLive), Layer.provideMerge(ProviderRuntimeLayerLive), Layer.provideMerge(Layer.mergeAll(TerminalLayerLive, PreviewLayerLive)), - Layer.provideMerge(TokenUsageRepositoryLive), Layer.provideMerge(PersistenceLayerLive), Layer.provideMerge(Keybindings.layer), Layer.provideMerge(ProviderRegistryLive), -); - -const RuntimeCoreDependenciesLive = RuntimeCoreServicesLive.pipe( // The instance registry is the new routing keystone — text generation, // adapter lookup, and runtime ingestion all resolve `ProviderInstanceId` // through this layer. Built-in drivers come from `BUILT_IN_DRIVERS`; From 53642b4ae92a6e675ed21ecfe812f08d602270e0 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Sun, 9 Aug 2026 02:04:11 +0530 Subject: [PATCH 11/13] Revert "feat(server): add durable token usage ledger" This reverts commit f7217d2629b1066d06ccb4abe0f2fcf35387a014. --- .../src/persistence/Layers/TokenUsage.test.ts | 175 -------- .../src/persistence/Layers/TokenUsage.ts | 421 ------------------ .../Migrations/035_ProviderTokenUsageFacts.ts | 55 --- .../src/persistence/Services/TokenUsage.ts | 40 -- 4 files changed, 691 deletions(-) delete mode 100644 apps/server/src/persistence/Layers/TokenUsage.test.ts delete mode 100644 apps/server/src/persistence/Layers/TokenUsage.ts delete mode 100644 apps/server/src/persistence/Migrations/035_ProviderTokenUsageFacts.ts delete mode 100644 apps/server/src/persistence/Services/TokenUsage.ts diff --git a/apps/server/src/persistence/Layers/TokenUsage.test.ts b/apps/server/src/persistence/Layers/TokenUsage.test.ts deleted file mode 100644 index d5875d716099..000000000000 --- a/apps/server/src/persistence/Layers/TokenUsage.test.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { assert, it } from "@effect/vitest"; -import { - EventId, - ProviderDriverKind, - ProviderInstanceId, - ThreadId, - TurnId, -} from "@t3tools/contracts"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; - -import { SqlitePersistenceMemory } from "./Sqlite.ts"; -import { TokenUsageRepositoryLive } from "./TokenUsage.ts"; -import { TokenUsageRepository } from "../Services/TokenUsage.ts"; - -const layer = it.layer(TokenUsageRepositoryLive.pipe(Layer.provideMerge(SqlitePersistenceMemory))); - -layer("TokenUsageRepository", (it) => { - it.effect("deduplicates provider observations and buckets them in the requested timezone", () => - Effect.gen(function* () { - const repository = yield* TokenUsageRepository; - const fact = { - provider: ProviderDriverKind.make("codex"), - providerInstanceId: ProviderInstanceId.make("codex"), - runtimeEventId: EventId.make("event-1"), - threadId: ThreadId.make("thread-1"), - turnId: TurnId.make("turn-1"), - occurredAt: "2026-01-02T02:00:00.000Z", - observation: { - sourceObservationId: "codex:turn:turn-1", - sourceKind: "codex.thread-token-usage.last", - model: "gpt-5.4", - reasoningLevel: "high", - metrics: { - inputTokens: 100, - cachedInputTokens: 20, - outputTokens: 50, - reasoningOutputTokens: 10, - totalTokens: 150, - }, - metricsProvenance: "exact" as const, - modelProvenance: "inferred" as const, - reasoningProvenance: "inferred" as const, - }, - }; - - yield* repository.record(fact); - yield* repository.record(fact); - const olderMetrics = { - inputTokens: 90, - cachedInputTokens: 10, - outputTokens: 60, - reasoningOutputTokens: 5, - totalTokens: 150, - }; - yield* repository.record({ - ...fact, - runtimeEventId: EventId.make("event-replayed-older"), - occurredAt: "2026-01-02T01:00:00.000Z", - observation: { - ...fact.observation, - model: "stale-model", - metrics: olderMetrics, - }, - }); - yield* repository.record({ - ...fact, - runtimeEventId: EventId.make("event-richer-attribution"), - occurredAt: "2026-01-02T01:00:00.000Z", - observation: { - ...fact.observation, - model: "gpt-5.4-reported", - modelProvenance: "exact", - metrics: olderMetrics, - }, - }); - - const result = yield* repository.query({ - fromDate: "2026-01-01", - toDate: "2026-01-02", - timeZone: "America/New_York", - }); - - assert.strictEqual(result.days.length, 1); - assert.strictEqual(result.days[0]?.date, "2026-01-01"); - assert.strictEqual(result.days[0]?.metrics.totalTokens, 150); - assert.strictEqual(result.days[0]?.metrics.inputTokens, 100); - assert.strictEqual(result.days[0]?.metrics.outputTokens, 50); - assert.strictEqual(result.days[0]?.breakdown[0]?.model, "gpt-5.4-reported"); - assert.strictEqual(result.days[0]?.breakdown[0]?.modelProvenance, "exact"); - assert.strictEqual(result.days[0]?.breakdown[0]?.reasoningLevel, "high"); - }), - ); - - it.effect("isolates identical source observation ids by provider instance", () => - Effect.gen(function* () { - const repository = yield* TokenUsageRepository; - const observation = { - sourceObservationId: "codex:turn:shared-turn", - sourceKind: "codex.thread-token-usage.last", - model: "gpt-5.4", - reasoningLevel: null, - metrics: { - inputTokens: 10, - cachedInputTokens: 0, - outputTokens: 5, - reasoningOutputTokens: 0, - totalTokens: 15, - }, - metricsProvenance: "exact" as const, - modelProvenance: "exact" as const, - reasoningProvenance: "unknown" as const, - }; - for (const [instanceId, eventId] of [ - ["codex_work", "event-work"], - ["codex_personal", "event-personal"], - ] as const) { - yield* repository.record({ - provider: ProviderDriverKind.make("codex"), - providerInstanceId: ProviderInstanceId.make(instanceId), - runtimeEventId: EventId.make(eventId), - threadId: ThreadId.make("thread-shared"), - turnId: TurnId.make("shared-turn"), - occurredAt: "2026-01-03T12:00:00.000Z", - observation, - }); - } - - const result = yield* repository.query({ - fromDate: "2026-01-03", - toDate: "2026-01-03", - timeZone: "UTC", - }); - assert.strictEqual(result.days[0]?.metrics.totalTokens, 30); - }), - ); - - it.effect("keeps unavailable component metrics unknown", () => - Effect.gen(function* () { - const repository = yield* TokenUsageRepository; - yield* repository.record({ - provider: ProviderDriverKind.make("claudeAgent"), - providerInstanceId: null, - runtimeEventId: EventId.make("event-unknown-components"), - threadId: ThreadId.make("thread-2"), - turnId: TurnId.make("turn-2"), - occurredAt: "2026-02-01T12:00:00.000Z", - observation: { - sourceObservationId: "claude:turn:turn-2", - sourceKind: "claude.result.usage", - model: "claude-opus-4-6", - reasoningLevel: null, - metrics: { - inputTokens: null, - cachedInputTokens: null, - outputTokens: null, - reasoningOutputTokens: null, - totalTokens: 500, - }, - metricsProvenance: "exact", - modelProvenance: "exact", - reasoningProvenance: "unknown", - }, - }); - - const result = yield* repository.query({ - fromDate: "2026-02-01", - toDate: "2026-02-01", - timeZone: "UTC", - }); - assert.strictEqual(result.days[0]?.metrics.totalTokens, 500); - assert.strictEqual(result.days[0]?.metrics.inputTokens, null); - }), - ); -}); diff --git a/apps/server/src/persistence/Layers/TokenUsage.ts b/apps/server/src/persistence/Layers/TokenUsage.ts deleted file mode 100644 index 0a88a20e0102..000000000000 --- a/apps/server/src/persistence/Layers/TokenUsage.ts +++ /dev/null @@ -1,421 +0,0 @@ -import { - type TokenUsageBreakdown, - type TokenUsageDay, - type TokenUsageMetrics, - type TokenUsageProvenance, -} from "@t3tools/contracts"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; -import * as Schema from "effect/Schema"; -import * as SqlClient from "effect/unstable/sql/SqlClient"; -import * as SqlSchema from "effect/unstable/sql/SqlSchema"; - -import { toPersistenceDecodeError, toPersistenceSqlError } from "../Errors.ts"; -import { - RecordTokenUsageFactInput, - TokenUsageRepository, - type TokenUsageRepositoryShape, -} from "../Services/TokenUsage.ts"; - -const QueryBounds = Schema.Struct({ - fromUtc: Schema.String, - toUtc: Schema.String, -}); - -const UsageFactRow = Schema.Struct({ - provider: Schema.String, - occurredAt: Schema.String, - model: Schema.NullOr(Schema.String), - reasoningLevel: Schema.NullOr(Schema.String), - inputTokens: Schema.NullOr(Schema.Number), - cachedInputTokens: Schema.NullOr(Schema.Number), - outputTokens: Schema.NullOr(Schema.Number), - reasoningOutputTokens: Schema.NullOr(Schema.Number), - totalTokens: Schema.NullOr(Schema.Number), - metricsProvenance: Schema.Literals(["exact", "inferred", "unknown"]), - modelProvenance: Schema.Literals(["exact", "inferred", "unknown"]), - reasoningProvenance: Schema.Literals(["exact", "inferred", "unknown"]), -}); -type UsageFactRow = typeof UsageFactRow.Type; - -const TrackingStartRow = Schema.Struct({ trackingStartedAt: Schema.NullOr(Schema.String) }); - -function toPersistenceSqlOrDecodeError(sqlOperation: string, decodeOperation: string) { - return (cause: unknown) => - Schema.isSchemaError(cause) - ? toPersistenceDecodeError(decodeOperation)(cause) - : toPersistenceSqlError(sqlOperation)(cause); -} - -function addUtcDays(calendarDate: string, days: number): string { - const date = new Date(`${calendarDate}T00:00:00.000Z`); - date.setUTCDate(date.getUTCDate() + days); - return date.toISOString(); -} - -function localCalendarDate(occurredAt: string, timeZone: string): string { - const parts = new Intl.DateTimeFormat("en-US", { - timeZone, - year: "numeric", - month: "2-digit", - day: "2-digit", - }).formatToParts(new Date(occurredAt)); - const value = (type: Intl.DateTimeFormatPartTypes) => - parts.find((part) => part.type === type)?.value ?? ""; - return `${value("year")}-${value("month")}-${value("day")}`; -} - -function combineProvenance(values: ReadonlyArray): TokenUsageProvenance { - if (values.some((value) => value === "unknown")) return "unknown"; - if (values.some((value) => value === "inferred")) return "inferred"; - return "exact"; -} - -function sumKnown( - rows: ReadonlyArray, - key: keyof Pick< - UsageFactRow, - "inputTokens" | "cachedInputTokens" | "outputTokens" | "reasoningOutputTokens" | "totalTokens" - >, -): number | null { - let total = 0; - for (const row of rows) { - const value = row[key]; - if (value === null) return null; - total += value; - } - return total; -} - -function aggregateMetrics(rows: ReadonlyArray): TokenUsageMetrics { - return { - inputTokens: sumKnown(rows, "inputTokens"), - cachedInputTokens: sumKnown(rows, "cachedInputTokens"), - outputTokens: sumKnown(rows, "outputTokens"), - reasoningOutputTokens: sumKnown(rows, "reasoningOutputTokens"), - totalTokens: sumKnown(rows, "totalTokens"), - }; -} - -function aggregateRows(rows: ReadonlyArray, date: string): TokenUsageDay { - const byDimension = new Map(); - for (const row of rows) { - const key = JSON.stringify([row.provider, row.model, row.reasoningLevel]); - const current = byDimension.get(key) ?? []; - current.push(row); - byDimension.set(key, current); - } - - const breakdown: TokenUsageBreakdown[] = [...byDimension.values()] - .map((group) => ({ - provider: group[0]!.provider, - model: group[0]!.model, - reasoningLevel: group[0]!.reasoningLevel, - metrics: aggregateMetrics(group), - metricsProvenance: combineProvenance(group.map((row) => row.metricsProvenance)), - modelProvenance: combineProvenance(group.map((row) => row.modelProvenance)), - reasoningProvenance: combineProvenance(group.map((row) => row.reasoningProvenance)), - })) - .toSorted((left, right) => - `${left.provider}:${left.model ?? ""}:${left.reasoningLevel ?? ""}`.localeCompare( - `${right.provider}:${right.model ?? ""}:${right.reasoningLevel ?? ""}`, - ), - ); - - return { - date, - metrics: aggregateMetrics(rows), - provenance: combineProvenance(rows.map((row) => row.metricsProvenance)), - breakdown, - }; -} - -const makeTokenUsageRepository = Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - - const recordFact = SqlSchema.void({ - Request: RecordTokenUsageFactInput, - execute: (input) => { - const providerIdentity = input.providerInstanceId ?? `driver:${input.provider}`; - const { metrics } = input.observation; - return sql` - INSERT INTO provider_token_usage_facts ( - provider_identity, - provider_driver, - provider_instance_id, - source_observation_id, - source_kind, - runtime_event_id, - thread_id, - turn_id, - occurred_at, - model, - reasoning_level, - input_tokens, - cached_input_tokens, - output_tokens, - reasoning_output_tokens, - total_tokens, - metrics_provenance, - model_provenance, - reasoning_provenance - ) VALUES ( - ${providerIdentity}, - ${input.provider}, - ${input.providerInstanceId}, - ${input.observation.sourceObservationId}, - ${input.observation.sourceKind}, - ${input.runtimeEventId}, - ${input.threadId}, - ${input.turnId}, - ${input.occurredAt}, - ${input.observation.model}, - ${input.observation.reasoningLevel}, - ${metrics.inputTokens}, - ${metrics.cachedInputTokens}, - ${metrics.outputTokens}, - ${metrics.reasoningOutputTokens}, - ${metrics.totalTokens}, - ${input.observation.metricsProvenance}, - ${input.observation.modelProvenance}, - ${input.observation.reasoningProvenance} - ) - ON CONFLICT (provider_identity, source_observation_id) - DO UPDATE SET - runtime_event_id = excluded.runtime_event_id, - thread_id = excluded.thread_id, - turn_id = COALESCE(excluded.turn_id, provider_token_usage_facts.turn_id), - occurred_at = MAX(excluded.occurred_at, provider_token_usage_facts.occurred_at), - model = CASE - WHEN excluded.model IS NULL THEN provider_token_usage_facts.model - WHEN provider_token_usage_facts.model IS NULL THEN excluded.model - WHEN CASE excluded.model_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - > CASE provider_token_usage_facts.model_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - THEN excluded.model - ELSE provider_token_usage_facts.model - END, - reasoning_level = CASE - WHEN excluded.reasoning_level IS NULL THEN provider_token_usage_facts.reasoning_level - WHEN provider_token_usage_facts.reasoning_level IS NULL THEN excluded.reasoning_level - WHEN CASE excluded.reasoning_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - > CASE provider_token_usage_facts.reasoning_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - THEN excluded.reasoning_level - ELSE provider_token_usage_facts.reasoning_level - END, - input_tokens = CASE - WHEN ( - CASE excluded.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - > CASE provider_token_usage_facts.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - ) OR ( - excluded.metrics_provenance = provider_token_usage_facts.metrics_provenance - AND excluded.total_tokens IS NOT NULL - AND ( - provider_token_usage_facts.total_tokens IS NULL - OR excluded.total_tokens > provider_token_usage_facts.total_tokens - OR ( - excluded.total_tokens = provider_token_usage_facts.total_tokens - AND excluded.occurred_at > provider_token_usage_facts.occurred_at - ) - ) - ) THEN excluded.input_tokens - ELSE provider_token_usage_facts.input_tokens - END, - cached_input_tokens = CASE - WHEN ( - CASE excluded.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - > CASE provider_token_usage_facts.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - ) OR ( - excluded.metrics_provenance = provider_token_usage_facts.metrics_provenance - AND excluded.total_tokens IS NOT NULL - AND ( - provider_token_usage_facts.total_tokens IS NULL - OR excluded.total_tokens > provider_token_usage_facts.total_tokens - OR ( - excluded.total_tokens = provider_token_usage_facts.total_tokens - AND excluded.occurred_at > provider_token_usage_facts.occurred_at - ) - ) - ) THEN excluded.cached_input_tokens - ELSE provider_token_usage_facts.cached_input_tokens - END, - output_tokens = CASE - WHEN ( - CASE excluded.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - > CASE provider_token_usage_facts.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - ) OR ( - excluded.metrics_provenance = provider_token_usage_facts.metrics_provenance - AND excluded.total_tokens IS NOT NULL - AND ( - provider_token_usage_facts.total_tokens IS NULL - OR excluded.total_tokens > provider_token_usage_facts.total_tokens - OR ( - excluded.total_tokens = provider_token_usage_facts.total_tokens - AND excluded.occurred_at > provider_token_usage_facts.occurred_at - ) - ) - ) THEN excluded.output_tokens - ELSE provider_token_usage_facts.output_tokens - END, - reasoning_output_tokens = CASE - WHEN ( - CASE excluded.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - > CASE provider_token_usage_facts.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - ) OR ( - excluded.metrics_provenance = provider_token_usage_facts.metrics_provenance - AND excluded.total_tokens IS NOT NULL - AND ( - provider_token_usage_facts.total_tokens IS NULL - OR excluded.total_tokens > provider_token_usage_facts.total_tokens - OR ( - excluded.total_tokens = provider_token_usage_facts.total_tokens - AND excluded.occurred_at > provider_token_usage_facts.occurred_at - ) - ) - ) THEN excluded.reasoning_output_tokens - ELSE provider_token_usage_facts.reasoning_output_tokens - END, - total_tokens = CASE - WHEN ( - CASE excluded.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - > CASE provider_token_usage_facts.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - ) OR ( - excluded.metrics_provenance = provider_token_usage_facts.metrics_provenance - AND excluded.total_tokens IS NOT NULL - AND ( - provider_token_usage_facts.total_tokens IS NULL - OR excluded.total_tokens > provider_token_usage_facts.total_tokens - OR ( - excluded.total_tokens = provider_token_usage_facts.total_tokens - AND excluded.occurred_at > provider_token_usage_facts.occurred_at - ) - ) - ) THEN excluded.total_tokens - ELSE provider_token_usage_facts.total_tokens - END, - metrics_provenance = CASE - WHEN CASE excluded.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - > CASE provider_token_usage_facts.metrics_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - THEN excluded.metrics_provenance - ELSE provider_token_usage_facts.metrics_provenance - END, - model_provenance = CASE - WHEN CASE excluded.model_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - > CASE provider_token_usage_facts.model_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - THEN excluded.model_provenance - ELSE provider_token_usage_facts.model_provenance - END, - reasoning_provenance = CASE - WHEN CASE excluded.reasoning_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - > CASE provider_token_usage_facts.reasoning_provenance - WHEN 'exact' THEN 2 WHEN 'inferred' THEN 1 ELSE 0 END - THEN excluded.reasoning_provenance - ELSE provider_token_usage_facts.reasoning_provenance - END - `; - }, - }); - - const listFacts = SqlSchema.findAll({ - Request: QueryBounds, - Result: UsageFactRow, - execute: ({ fromUtc, toUtc }) => sql` - SELECT - provider_driver AS "provider", - occurred_at AS "occurredAt", - model, - reasoning_level AS "reasoningLevel", - input_tokens AS "inputTokens", - cached_input_tokens AS "cachedInputTokens", - output_tokens AS "outputTokens", - reasoning_output_tokens AS "reasoningOutputTokens", - total_tokens AS "totalTokens", - metrics_provenance AS "metricsProvenance", - model_provenance AS "modelProvenance", - reasoning_provenance AS "reasoningProvenance" - FROM provider_token_usage_facts - WHERE occurred_at >= ${fromUtc} - AND occurred_at < ${toUtc} - AND total_tokens IS NOT NULL - ORDER BY occurred_at ASC, id ASC - `, - }); - - const readTrackingStart = SqlSchema.findOne({ - Request: Schema.Struct({}), - Result: TrackingStartRow, - execute: () => sql` - SELECT MIN(occurred_at) AS "trackingStartedAt" - FROM provider_token_usage_facts - WHERE total_tokens IS NOT NULL - `, - }); - - const record: TokenUsageRepositoryShape["record"] = (input) => - recordFact(input).pipe( - Effect.mapError( - toPersistenceSqlOrDecodeError( - "TokenUsageRepository.record:query", - "TokenUsageRepository.record:encodeRequest", - ), - ), - ); - - const query: TokenUsageRepositoryShape["query"] = (input) => - Effect.gen(function* () { - const rows = yield* listFacts({ - fromUtc: addUtcDays(input.fromDate, -2), - toUtc: addUtcDays(input.toDate, 3), - }); - const trackingStart = yield* readTrackingStart({}); - const byDate = new Map(); - for (const row of rows) { - const date = localCalendarDate(row.occurredAt, input.timeZone); - if (date < input.fromDate || date > input.toDate) continue; - const current = byDate.get(date) ?? []; - current.push(row); - byDate.set(date, current); - } - return { - fromDate: input.fromDate, - toDate: input.toDate, - timeZone: input.timeZone, - trackingStartedAt: trackingStart.trackingStartedAt, - days: [...byDate.entries()] - .toSorted(([left], [right]) => left.localeCompare(right)) - .map(([date, dayRows]) => aggregateRows(dayRows, date)), - }; - }).pipe( - Effect.mapError( - toPersistenceSqlOrDecodeError( - "TokenUsageRepository.query:query", - "TokenUsageRepository.query:decodeRows", - ), - ), - ); - - return TokenUsageRepository.of({ record, query }); -}); - -export const TokenUsageRepositoryLive = Layer.effect( - TokenUsageRepository, - makeTokenUsageRepository, -); diff --git a/apps/server/src/persistence/Migrations/035_ProviderTokenUsageFacts.ts b/apps/server/src/persistence/Migrations/035_ProviderTokenUsageFacts.ts deleted file mode 100644 index 811e2d5c2e2a..000000000000 --- a/apps/server/src/persistence/Migrations/035_ProviderTokenUsageFacts.ts +++ /dev/null @@ -1,55 +0,0 @@ -import * as Effect from "effect/Effect"; -import * as SqlClient from "effect/unstable/sql/SqlClient"; - -export default Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - - yield* sql` - CREATE TABLE IF NOT EXISTS provider_token_usage_facts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - provider_identity TEXT NOT NULL, - provider_driver TEXT NOT NULL, - provider_instance_id TEXT, - source_observation_id TEXT NOT NULL, - source_kind TEXT NOT NULL, - runtime_event_id TEXT NOT NULL, - thread_id TEXT NOT NULL, - turn_id TEXT, - occurred_at TEXT NOT NULL, - model TEXT, - reasoning_level TEXT, - input_tokens INTEGER CHECK (input_tokens IS NULL OR input_tokens >= 0), - cached_input_tokens INTEGER CHECK (cached_input_tokens IS NULL OR cached_input_tokens >= 0), - output_tokens INTEGER CHECK (output_tokens IS NULL OR output_tokens >= 0), - reasoning_output_tokens INTEGER CHECK ( - reasoning_output_tokens IS NULL OR reasoning_output_tokens >= 0 - ), - total_tokens INTEGER CHECK (total_tokens IS NULL OR total_tokens >= 0), - metrics_provenance TEXT NOT NULL CHECK ( - metrics_provenance IN ('exact', 'inferred', 'unknown') - ), - model_provenance TEXT NOT NULL CHECK ( - model_provenance IN ('exact', 'inferred', 'unknown') - ), - reasoning_provenance TEXT NOT NULL CHECK ( - reasoning_provenance IN ('exact', 'inferred', 'unknown') - ), - UNIQUE (provider_identity, source_observation_id) - ) - `; - - yield* sql` - CREATE INDEX IF NOT EXISTS idx_provider_token_usage_facts_occurred_at - ON provider_token_usage_facts(occurred_at) - `; - - yield* sql` - CREATE INDEX IF NOT EXISTS idx_provider_token_usage_facts_time_model_reasoning - ON provider_token_usage_facts(occurred_at, model, reasoning_level) - `; - - yield* sql` - CREATE INDEX IF NOT EXISTS idx_provider_token_usage_facts_thread_turn - ON provider_token_usage_facts(thread_id, turn_id) - `; -}); diff --git a/apps/server/src/persistence/Services/TokenUsage.ts b/apps/server/src/persistence/Services/TokenUsage.ts deleted file mode 100644 index 0b9702faf329..000000000000 --- a/apps/server/src/persistence/Services/TokenUsage.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { - EventId, - ProviderDriverKind, - ProviderInstanceId, - ProviderTokenUsageObservation, - ThreadId, - TokenUsageQueryInput, - TokenUsageQueryResult, - TurnId, -} from "@t3tools/contracts"; -import * as Context from "effect/Context"; -import type * as Effect from "effect/Effect"; -import * as Schema from "effect/Schema"; - -import type { ProjectionRepositoryError } from "../Errors.ts"; - -export const RecordTokenUsageFactInput = Schema.Struct({ - provider: ProviderDriverKind, - providerInstanceId: Schema.NullOr(ProviderInstanceId), - runtimeEventId: EventId, - threadId: ThreadId, - turnId: Schema.NullOr(TurnId), - occurredAt: Schema.String, - observation: ProviderTokenUsageObservation, -}); -export type RecordTokenUsageFactInput = typeof RecordTokenUsageFactInput.Type; - -export interface TokenUsageRepositoryShape { - readonly record: ( - input: RecordTokenUsageFactInput, - ) => Effect.Effect; - readonly query: ( - input: TokenUsageQueryInput, - ) => Effect.Effect; -} - -export class TokenUsageRepository extends Context.Service< - TokenUsageRepository, - TokenUsageRepositoryShape ->()("t3/persistence/Services/TokenUsage/TokenUsageRepository") {} From b883c5b88ee69c70554343aa5b3f53436b1dd509 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Sun, 9 Aug 2026 02:04:23 +0530 Subject: [PATCH 12/13] Revert "feat(contracts): define token usage query contracts" This reverts commit a421c409895b3cc6715d6f3bdc3c2281e850b2d8. --- packages/contracts/src/index.ts | 1 - .../contracts/src/providerRuntime.test.ts | 38 --------- packages/contracts/src/providerRuntime.ts | 3 - packages/contracts/src/rpc.ts | 9 --- packages/contracts/src/tokenUsage.test.ts | 71 ---------------- packages/contracts/src/tokenUsage.ts | 80 ------------------- 6 files changed, 202 deletions(-) delete mode 100644 packages/contracts/src/tokenUsage.test.ts delete mode 100644 packages/contracts/src/tokenUsage.ts diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 17e4ec25427a..6181391eca3a 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -11,7 +11,6 @@ export * from "./terminal.ts"; export * from "./provider.ts"; export * from "./providerInstance.ts"; export * from "./providerRuntime.ts"; -export * from "./tokenUsage.ts"; export * from "./model.ts"; export * from "./keybindings.ts"; export * from "./server.ts"; diff --git a/packages/contracts/src/providerRuntime.test.ts b/packages/contracts/src/providerRuntime.test.ts index b1363d9905dc..7563ac155681 100644 --- a/packages/contracts/src/providerRuntime.test.ts +++ b/packages/contracts/src/providerRuntime.test.ts @@ -181,44 +181,6 @@ describe("ProviderRuntimeEvent", () => { expect(parsed.payload.usage.maxTokens).toBe(200000); expect(parsed.payload.usage.usedTokens).toBe(31251); }); - - it("decodes accounting observations separately from context snapshots", () => { - const parsed = decodeRuntimeEvent({ - type: "thread.token-usage.updated", - eventId: "event-token-accounting-1", - provider: "codex", - createdAt: "2026-02-28T00:00:04.000Z", - threadId: "thread-1", - turnId: "turn-1", - payload: { - usage: { usedTokens: 160 }, - accounting: [ - { - sourceObservationId: "codex:turn:turn-1", - sourceKind: "codex.thread-token-usage.last", - model: null, - reasoningLevel: null, - metrics: { - inputTokens: 120, - cachedInputTokens: null, - outputTokens: 40, - reasoningOutputTokens: null, - totalTokens: 160, - }, - metricsProvenance: "exact", - modelProvenance: "unknown", - reasoningProvenance: "unknown", - }, - ], - }, - }); - - expect(parsed.type).toBe("thread.token-usage.updated"); - if (parsed.type !== "thread.token-usage.updated") { - throw new Error("expected thread.token-usage.updated"); - } - expect(parsed.payload.accounting?.[0]?.metrics.totalTokens).toBe(160); - }); }); describe("classifyTaskAgentKind", () => { diff --git a/packages/contracts/src/providerRuntime.ts b/packages/contracts/src/providerRuntime.ts index 48caa41f782e..bd525e6542e2 100644 --- a/packages/contracts/src/providerRuntime.ts +++ b/packages/contracts/src/providerRuntime.ts @@ -14,7 +14,6 @@ import { TurnId, } from "./baseSchemas.ts"; import { ProviderInstanceId, ProviderDriverKind } from "./providerInstance.ts"; -import { ProviderTokenUsageObservation } from "./tokenUsage.ts"; const TrimmedNonEmptyStringSchema = TrimmedNonEmptyString; const UnknownRecordSchema = Schema.Record(Schema.String, Schema.Unknown); @@ -328,7 +327,6 @@ export type ThreadTokenUsageSnapshot = typeof ThreadTokenUsageSnapshot.Type; const ThreadTokenUsageUpdatedPayload = Schema.Struct({ usage: ThreadTokenUsageSnapshot, - accounting: Schema.optional(Schema.Array(ProviderTokenUsageObservation)), }); export type ThreadTokenUsageUpdatedPayload = typeof ThreadTokenUsageUpdatedPayload.Type; @@ -368,7 +366,6 @@ const TurnCompletedPayload = Schema.Struct({ stopReason: Schema.optional(Schema.NullOr(TrimmedNonEmptyStringSchema)), usage: Schema.optional(Schema.Unknown), modelUsage: Schema.optional(UnknownRecordSchema), - accounting: Schema.optional(Schema.Array(ProviderTokenUsageObservation)), totalCostUsd: Schema.optional(Schema.Number), errorMessage: Schema.optional(TrimmedNonEmptyStringSchema), }); diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index d56254168dbf..59255639995e 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -154,7 +154,6 @@ import { } from "./resourceTelemetry.ts"; import { UsageReadError, UsageSummary, UsageSummaryInput } from "./usage.ts"; import { ServerSettings, ServerSettingsError, ServerSettingsPatch } from "./settings.ts"; -import { TokenUsageQueryError, TokenUsageQueryInput, TokenUsageQueryResult } from "./tokenUsage.ts"; import { SourceControlCloneRepositoryInput, SourceControlCloneRepositoryResult, @@ -228,7 +227,6 @@ export const WS_METHODS = { // Server meta serverProbe: "server.probe", serverGetConfig: "server.getConfig", - serverGetTokenUsage: "server.getTokenUsage", serverRefreshProviders: "server.refreshProviders", serverUpdateProvider: "server.updateProvider", serverUpdateServer: "server.updateServer", @@ -295,12 +293,6 @@ export const WsServerGetConfigRpc = Rpc.make(WS_METHODS.serverGetConfig, { error: Schema.Union([KeybindingsConfigError, ServerSettingsError, EnvironmentAuthorizationError]), }); -export const WsServerGetTokenUsageRpc = Rpc.make(WS_METHODS.serverGetTokenUsage, { - payload: TokenUsageQueryInput, - success: TokenUsageQueryResult, - error: Schema.Union([TokenUsageQueryError, EnvironmentAuthorizationError]), -}); - export const WsServerRefreshProvidersRpc = Rpc.make(WS_METHODS.serverRefreshProviders, { payload: Schema.Struct({ /** @@ -821,7 +813,6 @@ export const WsSubscribeResourceTelemetryRpc = Rpc.make(WS_METHODS.subscribeReso export const WsRpcGroup = RpcGroup.make( WsServerProbeRpc, WsServerGetConfigRpc, - WsServerGetTokenUsageRpc, WsServerRefreshProvidersRpc, WsServerUpdateProviderRpc, WsServerUpdateServerRpc, diff --git a/packages/contracts/src/tokenUsage.test.ts b/packages/contracts/src/tokenUsage.test.ts deleted file mode 100644 index d3ff909ac5c3..000000000000 --- a/packages/contracts/src/tokenUsage.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { describe, expect, it } from "vite-plus/test"; -import * as Schema from "effect/Schema"; - -import { - ProviderTokenUsageObservation, - TokenUsageQueryInput, - TokenUsageQueryResult, -} from "./tokenUsage.ts"; - -const decodeObservation = Schema.decodeUnknownSync(ProviderTokenUsageObservation); -const decodeQuery = Schema.decodeUnknownSync(TokenUsageQueryInput); -const decodeResult = Schema.decodeUnknownSync(TokenUsageQueryResult); - -describe("token usage contracts", () => { - it("preserves nullable metrics and provenance", () => { - const observation = decodeObservation({ - sourceObservationId: "codex:turn:turn-1", - sourceKind: "codex.thread-token-usage.last", - model: "gpt-5.4", - reasoningLevel: "high", - metrics: { - inputTokens: 120, - cachedInputTokens: null, - outputTokens: 40, - reasoningOutputTokens: null, - totalTokens: 160, - }, - metricsProvenance: "exact", - modelProvenance: "inferred", - reasoningProvenance: "inferred", - }); - - expect(observation.metrics.cachedInputTokens).toBeNull(); - expect(observation.metricsProvenance).toBe("exact"); - }); - - it("rejects malformed dates and negative counts", () => { - expect(() => - decodeQuery({ fromDate: "2026-1-01", toDate: "2026-01-02", timeZone: "UTC" }), - ).toThrow(); - expect(() => - decodeObservation({ - sourceObservationId: "x", - sourceKind: "test", - model: null, - reasoningLevel: null, - metrics: { - inputTokens: -1, - cachedInputTokens: null, - outputTokens: null, - reasoningOutputTokens: null, - totalTokens: null, - }, - metricsProvenance: "unknown", - modelProvenance: "unknown", - reasoningProvenance: "unknown", - }), - ).toThrow(); - }); - - it("decodes sparse daily results", () => { - const result = decodeResult({ - fromDate: "2026-01-01", - toDate: "2026-01-03", - timeZone: "America/New_York", - trackingStartedAt: null, - days: [], - }); - expect(result.days).toEqual([]); - }); -}); diff --git a/packages/contracts/src/tokenUsage.ts b/packages/contracts/src/tokenUsage.ts deleted file mode 100644 index 4c27aab4c545..000000000000 --- a/packages/contracts/src/tokenUsage.ts +++ /dev/null @@ -1,80 +0,0 @@ -import * as Schema from "effect/Schema"; - -import { NonNegativeInt } from "./baseSchemas.ts"; - -const TrimmedNonEmptyString = Schema.Trimmed.check(Schema.isNonEmpty()); - -export const TokenUsageProvenance = Schema.Literals(["exact", "inferred", "unknown"]); -export type TokenUsageProvenance = typeof TokenUsageProvenance.Type; - -export const TokenUsageMetrics = Schema.Struct({ - inputTokens: Schema.NullOr(NonNegativeInt), - cachedInputTokens: Schema.NullOr(NonNegativeInt), - outputTokens: Schema.NullOr(NonNegativeInt), - reasoningOutputTokens: Schema.NullOr(NonNegativeInt), - totalTokens: Schema.NullOr(NonNegativeInt), -}); -export type TokenUsageMetrics = typeof TokenUsageMetrics.Type; - -/** - * An immutable provider accounting observation. This is deliberately separate - * from context-window telemetry: observations may be summed only after the - * server has deduplicated them by sourceObservationId. - */ -export const ProviderTokenUsageObservation = Schema.Struct({ - sourceObservationId: TrimmedNonEmptyString, - sourceKind: TrimmedNonEmptyString, - model: Schema.NullOr(TrimmedNonEmptyString), - reasoningLevel: Schema.NullOr(TrimmedNonEmptyString), - metrics: TokenUsageMetrics, - metricsProvenance: TokenUsageProvenance, - modelProvenance: TokenUsageProvenance, - reasoningProvenance: TokenUsageProvenance, -}); -export type ProviderTokenUsageObservation = typeof ProviderTokenUsageObservation.Type; - -export const TokenUsageCalendarDate = Schema.String.check(Schema.isPattern(/^\d{4}-\d{2}-\d{2}$/)); -export type TokenUsageCalendarDate = typeof TokenUsageCalendarDate.Type; - -export const TokenUsageQueryInput = Schema.Struct({ - fromDate: TokenUsageCalendarDate, - toDate: TokenUsageCalendarDate, - timeZone: TrimmedNonEmptyString, -}); -export type TokenUsageQueryInput = typeof TokenUsageQueryInput.Type; - -export const TokenUsageBreakdown = Schema.Struct({ - provider: TrimmedNonEmptyString, - model: Schema.NullOr(TrimmedNonEmptyString), - reasoningLevel: Schema.NullOr(TrimmedNonEmptyString), - metrics: TokenUsageMetrics, - metricsProvenance: TokenUsageProvenance, - modelProvenance: TokenUsageProvenance, - reasoningProvenance: TokenUsageProvenance, -}); -export type TokenUsageBreakdown = typeof TokenUsageBreakdown.Type; - -export const TokenUsageDay = Schema.Struct({ - date: TokenUsageCalendarDate, - metrics: TokenUsageMetrics, - provenance: TokenUsageProvenance, - breakdown: Schema.Array(TokenUsageBreakdown), -}); -export type TokenUsageDay = typeof TokenUsageDay.Type; - -export const TokenUsageQueryResult = Schema.Struct({ - fromDate: TokenUsageCalendarDate, - toDate: TokenUsageCalendarDate, - timeZone: TrimmedNonEmptyString, - trackingStartedAt: Schema.NullOr(Schema.String), - days: Schema.Array(TokenUsageDay), -}); -export type TokenUsageQueryResult = typeof TokenUsageQueryResult.Type; - -export class TokenUsageQueryError extends Schema.TaggedErrorClass()( - "TokenUsageQueryError", - { - reason: Schema.Literals(["invalid-range", "invalid-time-zone", "persistence"]), - message: Schema.String, - }, -) {} From 9ee296666f2b03e9b657f99bcb865f760f0ffd21 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan <113882531+Vedant817@users.noreply.github.com> Date: Sun, 9 Aug 2026 02:53:18 +0530 Subject: [PATCH 13/13] feat(usage): include OpenCode activity --- apps/server/src/usage/UsageService.ts | 78 ++++++ .../src/usage/usageOpenCodeReader.test.ts | 155 ++++++++++++ apps/server/src/usage/usageOpenCodeReader.ts | 227 ++++++++++++++++++ apps/web/src/components/usage/UsagePage.tsx | 5 +- .../usage/UsageProviderChart.test.ts | 10 +- .../src/components/usage/usageProviders.ts | 11 +- packages/contracts/src/usage.ts | 5 +- 7 files changed, 480 insertions(+), 11 deletions(-) create mode 100644 apps/server/src/usage/usageOpenCodeReader.test.ts create mode 100644 apps/server/src/usage/usageOpenCodeReader.ts diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 2ad2a729ecb5..781681e26661 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -38,6 +38,7 @@ import * as ServerSettings from "../serverSettings.ts"; import { resolveClaudeHomePath } from "../provider/Drivers/ClaudeHome.ts"; import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts"; import { UsageAggregator } from "./usageAggregation.ts"; +import { readOpenCodeUsage, resolveOpenCodeDataDir } from "./usageOpenCodeReader.ts"; import { parseRateTable, type RateTable } from "./usagePricing.ts"; import { listTranscriptFiles, @@ -128,6 +129,7 @@ export const make = Effect.gen(function* () { const ratesCachePath = path.join(config.stateDir, "usage-model-rates.json"); const scanCachePath = path.join(config.stateDir, "usage-scan-cache.json"); + const openCodeDataDir = resolveOpenCodeDataDir(NodeOS.homedir(), process.env["XDG_DATA_HOME"]); let rates: RateTable = new Map(); let ratesFetchedAtMs: number | null = null; let ratesStatus: UsageSummary["pricing"]["status"] = "unavailable"; @@ -380,6 +382,82 @@ export const make = Effect.gen(function* () { }); } + const openCodeVolumeId = yield* Effect.promise(() => readDirectoryVolumeId(openCodeDataDir)); + const openCodeExists = yield* fileSystem + .exists(openCodeDataDir) + .pipe(Effect.catchCause(() => Effect.succeed(false))); + + if (!openCodeExists) { + sources.push({ + fingerprint: { + hostId, + provider: "opencode", + resolvedHomePath: openCodeDataDir, + volumeId: openCodeVolumeId, + }, + status: "missing", + scannedFiles: 0, + skippedFiles: 0, + malformedRecords: 0, + distinctSessions: 0, + message: "No OpenCode data directory on this environment.", + }); + } else { + const result = yield* Effect.promise(() => readOpenCodeUsage(openCodeDataDir, windowStartMs)); + if (result === null) { + sources.push({ + fingerprint: { + hostId, + provider: "opencode", + resolvedHomePath: openCodeDataDir, + volumeId: openCodeVolumeId, + }, + status: "failed", + scannedFiles: 0, + skippedFiles: 0, + malformedRecords: 0, + distinctSessions: 0, + message: "OpenCode usage storage could not be read.", + }); + } else if (result.storageKind === "missing") { + sources.push({ + fingerprint: { + hostId, + provider: "opencode", + resolvedHomePath: openCodeDataDir, + volumeId: openCodeVolumeId, + }, + status: "missing", + scannedFiles: 0, + skippedFiles: 0, + malformedRecords: 0, + distinctSessions: 0, + message: "No OpenCode message store on this environment.", + }); + } else { + const sessionIds = new Set(); + for (const record of result.records) { + if (aggregator.add(record) && record.sessionId.length > 0) { + sessionIds.add(record.sessionId); + } + } + sources.push({ + fingerprint: { + hostId, + provider: "opencode", + resolvedHomePath: openCodeDataDir, + volumeId: openCodeVolumeId, + }, + status: result.malformedRecords > 0 ? "partial" : "ok", + scannedFiles: result.scannedFiles, + skippedFiles: result.skippedFiles, + malformedRecords: result.malformedRecords, + distinctSessions: sessionIds.size, + message: result.malformedRecords > 0 ? "Some OpenCode messages could not be read." : null, + }); + } + } + const pruned = pruneScanCache(fileCache, { livePaths, walkedRoots, diff --git a/apps/server/src/usage/usageOpenCodeReader.test.ts b/apps/server/src/usage/usageOpenCodeReader.test.ts new file mode 100644 index 000000000000..505b61240384 --- /dev/null +++ b/apps/server/src/usage/usageOpenCodeReader.test.ts @@ -0,0 +1,155 @@ +// @effect-diagnostics nodeBuiltinImport:off +// @effect-diagnostics globalDate:off +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeSqlite from "node:sqlite"; + +import { describe, expect, it } from "@effect/vitest"; + +import { + parseOpenCodeMessageRow, + readOpenCodeUsage, + resolveOpenCodeDataDir, +} from "./usageOpenCodeReader.ts"; + +function assistantMessage(overrides: Record = {}) { + return { + id: "msg_1", + sessionID: "ses_1", + role: "assistant", + modelID: "deepseek-v4-flash-free", + providerID: "opencode", + time: { created: 1_786_211_758_317, completed: 1_786_211_767_010 }, + cost: 0, + tokens: { + input: 137, + output: 851, + reasoning: 230, + cache: { read: 99_200, write: 12 }, + }, + ...overrides, + }; +} + +describe("parseOpenCodeMessageRow", () => { + it("maps OpenCode assistant usage without counting cache or reasoning twice", () => { + const record = parseOpenCodeMessageRow({ + id: "msg_1", + sessionId: "ses_1", + timeCreated: 1_786_211_758_317, + data: JSON.stringify(assistantMessage()), + }); + + expect(record).toEqual({ + provider: "opencode", + timestampMs: 1_786_211_767_010, + model: "deepseek-v4-flash-free", + sessionId: "ses_1", + totals: { + uncachedInputTokens: 137, + cachedInputTokens: 99_200, + cacheCreationTokens: 12, + outputTokens: 851, + reasoningTokens: 230, + }, + reportedCostUsd: 0, + dedupeKey: "opencode:msg_1", + }); + }); + + it("ignores non-assistant, malformed, zero-token, and model-less rows", () => { + const row = (data: unknown) => ({ + id: "msg_1", + sessionId: "ses_1", + timeCreated: 1_786_211_758_317, + data, + }); + + expect(parseOpenCodeMessageRow(row("not json"))).toBeNull(); + expect( + parseOpenCodeMessageRow(row(JSON.stringify(assistantMessage({ role: "user" })))), + ).toBeNull(); + expect( + parseOpenCodeMessageRow( + row(JSON.stringify(assistantMessage({ tokens: { input: 0, output: 0 } }))), + ), + ).toBeNull(); + expect( + parseOpenCodeMessageRow(row(JSON.stringify(assistantMessage({ modelID: "" })))), + ).toBeNull(); + }); +}); + +describe("resolveOpenCodeDataDir", () => { + it("uses XDG_DATA_HOME when configured and the documented default otherwise", () => { + expect(resolveOpenCodeDataDir("/home/dev", "/mnt/data")).toBe( + NodePath.join("/mnt/data", "opencode"), + ); + expect(resolveOpenCodeDataDir("/home/dev", undefined)).toBe( + NodePath.join("/home/dev", ".local", "share", "opencode"), + ); + }); +}); + +describe("readOpenCodeUsage", () => { + it("reads current OpenCode SQLite messages and excludes older rows", async () => { + const dataDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-opencode-usage-")); + const databasePath = NodePath.join(dataDir, "opencode.db"); + const database = new NodeSqlite.DatabaseSync(databasePath); + database.exec( + "CREATE TABLE message (id TEXT PRIMARY KEY, session_id TEXT, time_created INTEGER, data TEXT)", + ); + const insert = database.prepare( + "INSERT INTO message (id, session_id, time_created, data) VALUES (?, ?, ?, ?)", + ); + insert.run("msg_old", "ses_old", 100, JSON.stringify(assistantMessage())); + insert.run( + "msg_new", + "ses_new", + 2_000, + JSON.stringify( + assistantMessage({ + id: "msg_new", + sessionID: "ses_new", + time: { created: 2_000, completed: 2_500 }, + }), + ), + ); + database.close(); + + try { + const result = await readOpenCodeUsage(dataDir, 1_000); + expect(result?.storageKind).toBe("sqlite"); + expect(result?.scannedFiles).toBe(1); + expect(result?.records.map((record) => record.sessionId)).toEqual(["ses_new"]); + } finally { + NodeFS.rmSync(dataDir, { recursive: true, force: true }); + } + }); + + it("falls back to the documented JSON message store when no database exists", async () => { + const dataDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-opencode-usage-")); + const messageDir = NodePath.join(dataDir, "storage", "message", "ses_legacy"); + const now = Date.now(); + NodeFS.mkdirSync(messageDir, { recursive: true }); + NodeFS.writeFileSync( + NodePath.join(messageDir, "msg_legacy.json"), + JSON.stringify( + assistantMessage({ + id: "msg_legacy", + sessionID: "ses_legacy", + time: { created: now, completed: now }, + }), + ), + ); + + try { + const result = await readOpenCodeUsage(dataDir, now - 60_000); + expect(result?.storageKind).toBe("json"); + expect(result?.records.map((record) => record.sessionId)).toEqual(["ses_legacy"]); + } finally { + NodeFS.rmSync(dataDir, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/server/src/usage/usageOpenCodeReader.ts b/apps/server/src/usage/usageOpenCodeReader.ts new file mode 100644 index 000000000000..f129e07e129f --- /dev/null +++ b/apps/server/src/usage/usageOpenCodeReader.ts @@ -0,0 +1,227 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * Reads usage from OpenCode's native data store. + * + * Current OpenCode releases persist messages in `opencode.db`. Older releases + * used `storage/message//.json`; that layout remains a + * fallback only when the database is absent so migrated messages are never + * counted twice. + * + * @module usageOpenCodeReader + */ +import * as NodeFS from "node:fs"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeSqlite from "node:sqlite"; + +import { totalTokens, type UsageRecord } from "./usageTranscripts.ts"; + +export interface OpenCodeMessageRow { + readonly id: unknown; + readonly sessionId: unknown; + readonly timeCreated: unknown; + readonly data: unknown; +} + +export interface OpenCodeUsageReadResult { + readonly storageKind: "sqlite" | "json" | "missing"; + readonly records: readonly UsageRecord[]; + readonly scannedFiles: number; + readonly skippedFiles: number; + readonly malformedRecords: number; +} + +function object(value: unknown): Record | null { + return typeof value === "object" && value !== null ? (value as Record) : null; +} + +function positiveInt(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.trunc(value) : 0; +} + +function timestamp(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? Math.trunc(value) + : null; +} + +function decodeMessageData(data: unknown): Record | null { + if (typeof data !== "string") return object(data); + try { + return object(JSON.parse(data)); + } catch { + return null; + } +} + +/** Maps one OpenCode assistant message to the shared usage representation. */ +export function parseOpenCodeMessageRow(row: OpenCodeMessageRow): UsageRecord | null { + const message = decodeMessageData(row.data); + if (message === null || message["role"] !== "assistant") return null; + + const model = typeof message["modelID"] === "string" ? message["modelID"].trim() : ""; + if (model.length === 0) return null; + + const tokens = object(message["tokens"]); + if (tokens === null) return null; + const cache = object(tokens["cache"]); + const outputTokens = positiveInt(tokens["output"]); + const totals = { + // OpenCode stores uncached input and cache reads/writes as disjoint fields. + uncachedInputTokens: positiveInt(tokens["input"]), + cachedInputTokens: positiveInt(cache?.["read"]), + cacheCreationTokens: positiveInt(cache?.["write"]), + outputTokens, + // Reasoning is included in output, matching the shared usage contract. + reasoningTokens: Math.min(outputTokens, positiveInt(tokens["reasoning"])), + }; + if (totalTokens(totals) === 0) return null; + + const time = object(message["time"]); + const timestampMs = timestamp(time?.["completed"] ?? time?.["created"] ?? row.timeCreated); + if (timestampMs === null) return null; + + const messageId = + typeof message["id"] === "string" ? message["id"] : typeof row.id === "string" ? row.id : ""; + const sessionId = + typeof message["sessionID"] === "string" + ? message["sessionID"] + : typeof row.sessionId === "string" + ? row.sessionId + : ""; + const cost = message["cost"]; + + return { + provider: "opencode", + timestampMs, + model, + sessionId, + totals, + reportedCostUsd: typeof cost === "number" && Number.isFinite(cost) && cost >= 0 ? cost : null, + dedupeKey: messageId.length > 0 ? `opencode:${messageId}` : null, + }; +} + +/** OpenCode follows XDG_DATA_HOME and otherwise uses ~/.local/share/opencode. */ +export function resolveOpenCodeDataDir(homePath: string, xdgDataHome: string | undefined): string { + const dataHome = xdgDataHome?.trim(); + return dataHome + ? NodePath.join(dataHome, "opencode") + : NodePath.join(homePath, ".local", "share", "opencode"); +} + +function readSqliteUsage(databasePath: string, sinceMs: number): OpenCodeUsageReadResult | null { + let database: NodeSqlite.DatabaseSync | undefined; + try { + database = new NodeSqlite.DatabaseSync(databasePath, { readOnly: true, timeout: 5_000 }); + const rows = database + .prepare( + `SELECT id, session_id AS sessionId, time_created AS timeCreated, data + FROM message + WHERE time_created >= ? + AND json_valid(data) = 1 + AND json_extract(data, '$.role') = 'assistant'`, + ) + .all(sinceMs) as unknown as OpenCodeMessageRow[]; + const records: UsageRecord[] = []; + let malformedRecords = 0; + for (const row of rows) { + const record = parseOpenCodeMessageRow(row); + if (record === null) malformedRecords += 1; + else records.push(record); + } + return { + storageKind: "sqlite", + records, + scannedFiles: 1, + skippedFiles: 0, + malformedRecords, + }; + } catch { + return null; + } finally { + database?.close(); + } +} + +async function listLegacyMessages(root: string, sinceMs: number): Promise { + const files: string[] = []; + const walk = async (directory: string): Promise => { + let entries; + try { + entries = await NodeFSP.readdir(directory, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const child = NodePath.join(directory, entry.name); + if (entry.isDirectory()) { + await walk(child); + continue; + } + if (!entry.name.endsWith(".json")) continue; + try { + const stats = await NodeFSP.stat(child); + if (stats.mtimeMs >= sinceMs) files.push(child); + } catch { + // The message rotated while the directory walk was in flight. + } + } + }; + await walk(root); + return files; +} + +async function readLegacyUsage( + messageRoot: string, + sinceMs: number, +): Promise { + const files = await listLegacyMessages(messageRoot, sinceMs); + const records: UsageRecord[] = []; + let skippedFiles = 0; + let malformedRecords = 0; + + for (const filePath of files) { + try { + const data = await NodeFSP.readFile(filePath, "utf8"); + const record = parseOpenCodeMessageRow({ + id: NodePath.basename(filePath, ".json"), + sessionId: NodePath.basename(NodePath.dirname(filePath)), + timeCreated: 0, + data, + }); + if (record === null) skippedFiles += 1; + else records.push(record); + } catch { + malformedRecords += 1; + } + } + + return { + storageKind: "json", + records, + scannedFiles: files.length, + skippedFiles, + malformedRecords, + }; +} + +/** Reads OpenCode usage without opening auth/config files or message parts. */ +export async function readOpenCodeUsage( + dataDir: string, + sinceMs: number, +): Promise { + const databasePath = NodePath.join(dataDir, "opencode.db"); + if (NodeFS.existsSync(databasePath)) return readSqliteUsage(databasePath, sinceMs); + + const messageRoot = NodePath.join(dataDir, "storage", "message"); + if (NodeFS.existsSync(messageRoot)) return readLegacyUsage(messageRoot, sinceMs); + + return { + storageKind: "missing", + records: [], + scannedFiles: 0, + skippedFiles: 0, + malformedRecords: 0, + }; +} diff --git a/apps/web/src/components/usage/UsagePage.tsx b/apps/web/src/components/usage/UsagePage.tsx index 2f3ab4b574c0..96264f191099 100644 --- a/apps/web/src/components/usage/UsagePage.tsx +++ b/apps/web/src/components/usage/UsagePage.tsx @@ -305,7 +305,10 @@ export function UsagePage() { {recentDays.length === 0 ? ( - + No activity in this window. diff --git a/apps/web/src/components/usage/UsageProviderChart.test.ts b/apps/web/src/components/usage/UsageProviderChart.test.ts index a36cd1e88330..fe5e00f60042 100644 --- a/apps/web/src/components/usage/UsageProviderChart.test.ts +++ b/apps/web/src/components/usage/UsageProviderChart.test.ts @@ -47,11 +47,12 @@ describe("buildDayColumns", () => { "2026-08-01", { day: "2026-08-01", - costUsd: 30, - totalTokens: 300, + costUsd: 33, + totalTokens: 330, byProvider: new Map([ ["codex" as const, { costUsd: 10, totalTokens: 100 }], ["claude" as const, { costUsd: 20, totalTokens: 200 }], + ["opencode" as const, { costUsd: 3, totalTokens: 30 }], ]), }, ], @@ -68,12 +69,12 @@ describe("buildDayColumns", () => { ]); it("plots each day on its own", () => { - expect(buildDayColumns(days, byDay, "cost").map((column) => column.total)).toEqual([30, 0, 5]); + expect(buildDayColumns(days, byDay, "cost").map((column) => column.total)).toEqual([33, 0, 5]); }); it("reads the requested metric", () => { expect(buildDayColumns(days, byDay, "tokens").map((column) => column.total)).toEqual([ - 300, 0, 50, + 330, 0, 50, ]); }); @@ -85,6 +86,7 @@ describe("buildDayColumns", () => { expect(first?.bands).toEqual([ { provider: "codex", value: 10 }, { provider: "claude", value: 20 }, + { provider: "opencode", value: 3 }, ]); }); diff --git a/apps/web/src/components/usage/usageProviders.ts b/apps/web/src/components/usage/usageProviders.ts index f8b65877dcf4..f8881f37bcfc 100644 --- a/apps/web/src/components/usage/usageProviders.ts +++ b/apps/web/src/components/usage/usageProviders.ts @@ -1,23 +1,25 @@ import type { UsageProviderKind } from "@t3tools/contracts"; -import { ClaudeAI, type Icon, OpenAI } from "../Icons"; +import { ClaudeAI, type Icon, OpenAI, OpenCodeIcon } from "../Icons"; /** - * Series and table order. The chart layers both providers from a shared zero + * Series and table order. The chart layers all providers from a shared zero * baseline, so this only fixes the reading order of legends, tables and hover * rows; it does not decide which series sits above the other. */ -export const PROVIDER_ORDER: readonly UsageProviderKind[] = ["codex", "claude"]; +export const PROVIDER_ORDER: readonly UsageProviderKind[] = ["codex", "claude", "opencode"]; export const PROVIDER_LABEL: Record = { claude: "Claude Code", codex: "Codex", + opencode: "OpenCode", }; -/** Claude's brand orange against a neutral white for Codex. */ +/** Distinct provider colours used for chart bands and compact share bars. */ export const PROVIDER_COLOR: Record = { claude: "#d97757", codex: "#e6e6e6", + opencode: "#22c55e", }; /** @@ -30,4 +32,5 @@ export const PROVIDER_COLOR: Record = { export const PROVIDER_MARK: Record = { claude: ClaudeAI, codex: OpenAI, + opencode: OpenCodeIcon, }; diff --git a/packages/contracts/src/usage.ts b/packages/contracts/src/usage.ts index 1aa639fe4a00..7f57f3a54286 100644 --- a/packages/contracts/src/usage.ts +++ b/packages/contracts/src/usage.ts @@ -2,7 +2,8 @@ * Usage reporting contract. * * Each environment scans the provider CLIs' own on-disk session transcripts - * (`~/.claude/projects/**\/*.jsonl`, `~/.codex/sessions/**\/*.jsonl`) rather than + * (`~/.claude/projects/**\/*.jsonl`, `~/.codex/sessions/**\/*.jsonl`, and + * `~/.local/share/opencode/opencode.db`) rather than * relying on T3 Code's own orchestration projections, so usage stays complete * even for turns that were never driven through T3 Code. This mirrors the * approach `ccusage` takes. @@ -23,7 +24,7 @@ import { NonNegativeInt, TrimmedNonEmptyString } from "./baseSchemas.ts"; */ export const USAGE_CONTRACT_VERSION = 3 as const; -export const UsageProviderKind = Schema.Literals(["claude", "codex"]); +export const UsageProviderKind = Schema.Literals(["claude", "codex", "opencode"]); export type UsageProviderKind = typeof UsageProviderKind.Type; /**