Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/desktop/src/main/__tests__/onboarding-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,8 @@ describe('createOnboardingService.getSnapshot', () => {
providerLabel: 'Anthropic',
model: 'claude-sonnet-4-5-20250929',
label: 'Claude Sonnet 4.5',
description: 'Balanced Claude model for coding, analysis, agent workflows, and cost control',
knowledgeCutoff: '2025-07-31',
connectionName: 'Anthropic Live',
isDefault: true,
thinkingLevels: ['off'],
Expand Down
4 changes: 3 additions & 1 deletion packages/core/src/__tests__/chat-model-choice.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,16 @@ function connection(overrides: Partial<LlmConnection> = {}): LlmConnection {
};
}

test('projects only enabled chat models with display, default, and thinking metadata', () => {
test('projects enabled chat models with display, models.dev facts, and thinking metadata', () => {
assert.deepEqual(buildChatModelChoices([connection()]), [
{
connectionSlug: 'openai-main',
providerType: 'openai',
providerLabel: 'OpenAI',
model: 'gpt-5.5',
label: 'GPT-5.5',
description: 'Default frontier GPT for coding, computer use, research, and knowledge work',
knowledgeCutoff: '2025-12-01',
connectionName: 'Work API',
isDefault: true,
thinkingLevels: ['off', 'low', 'medium', 'high', 'xhigh'],
Expand Down
25 changes: 25 additions & 0 deletions packages/core/src/__tests__/model-catalog.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,31 @@ test('provider facts override static metadata while missing facts are enriched',
assert.deepEqual(enriched?.capabilities, { reasoning: true, functionCalling: true });
});

test('catalog entries expose generated model facts beyond capabilities', () => {
const [entry] = buildModelCatalogEntries({
providerType: 'anthropic',
defaultModel: 'claude-sonnet-4-5',
models: [{ id: 'claude-sonnet-4-5' }],
modelSource: 'fetched',
});
assert.equal(
entry?.description,
'Balanced Claude model for coding, analysis, agent workflows, and cost control',
);
assert.equal(entry?.knowledgeCutoff, '2025-07-31');
assert.equal(entry?.inputLimit, undefined);
assert.equal(entry?.modalities?.input.includes('pdf'), true);
assert.equal(entry?.structuredOutput, true);
assert.equal(entry?.lastUpdated, '2025-09-29');

const [inputLimited] = buildModelCatalogEntries({
providerType: 'openai',
models: [{ id: 'gpt-5.5-pro' }],
modelSource: 'fetched',
});
assert.equal(inputLimited?.inputLimit, 922_000);
});

test('live inventory blocks missing defaults and preserves higher-priority failures', () => {
const input = {
providerType: 'zai-coding-plan' as const,
Expand Down
33 changes: 33 additions & 0 deletions packages/core/src/__tests__/model-metadata.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ import { describe, it } from 'node:test';
import {
lookupModelMetadata,
openAiAdapterApiProtocol,
resolveModelInputModalities,
resolveModelPdfSupport,
resolveModelVisionSupport,
} from '../model-metadata.js';
import { PROVIDER_DEFAULTS, type ModelInfo, type ProviderType } from '../llm-connections.js';
Expand Down Expand Up @@ -56,6 +58,21 @@ describe('model-metadata vision capability', () => {
);
});

it('preserves synchronized display facts on the Claude subscription path', () => {
const metadata = lookupModelMetadata('claude-subscription', 'claude-sonnet-4-5');
assert.equal(
metadata.description,
'Balanced Claude model for coding, analysis, agent workflows, and cost control',
);
assert.equal(metadata.knowledgeCutoff, '2025-07-31');
assert.equal(metadata.structuredOutput, true);
assert.equal(metadata.lastUpdated, '2025-09-29');
assert.equal(metadata.modalities?.input.includes('pdf'), true);
// Context limits remain access-path-specific and are intentionally not
// copied from the API snapshot onto the subscription route.
assert.equal(metadata.contextWindow, undefined);
});

it('confines the default to the providers that serve Anthropic their own models', () => {
// A claude-prefixed id on somebody else's provider says nothing about what
// is actually behind it, so the default must not travel with the id.
Expand Down Expand Up @@ -116,10 +133,14 @@ describe('model-metadata vision capability', () => {
it('publishes the Kimi Coding Plan K3 limits and effort levels from models.dev', () => {
assert.deepEqual(lookupModelMetadata('kimi-coding-plan', 'k3'), {
displayName: 'Kimi K3',
description:
'Multimodal Kimi model with 1M context and toggleable max-effort thinking for long-horizon agent work',
lifecycle: 'active',
docsUrl: 'https://www.kimi.com/code/docs/en/third-party-tools/other-coding-agents.html',
contextWindow: 1_048_576,
maxOutputTokens: 131_072,
structuredOutput: true,
lastUpdated: '2026-07-16',
capabilities: { vision: true, reasoning: true, functionCalling: true },
thinkingOptions: { efforts: ['low', 'high', 'max'], toggle: true },
modalities: { input: ['text', 'image'], output: ['text'] },
Expand Down Expand Up @@ -256,6 +277,18 @@ describe('resolveModelVisionSupport', () => {
});
});

describe('models.dev extended model facts', () => {
it('keeps PDF input in the generated modality facts', () => {
const metadata = lookupModelMetadata('anthropic', 'claude-sonnet-4-5');
assert.equal(metadata.modalities?.input.includes('pdf'), true);
assert.equal(resolveModelPdfSupport('anthropic', undefined, 'claude-sonnet-4-5'), true);
assert.equal(
resolveModelInputModalities('anthropic', undefined, 'claude-sonnet-4-5').includes('pdf'),
true,
);
});
});

describe('openAiAdapterApiProtocol', () => {
it('routes every gpt-5* family to the Responses wire', () => {
for (const modelId of [
Expand Down
75 changes: 75 additions & 0 deletions packages/core/src/__tests__/models-dev-sync-contract.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import assert from 'node:assert/strict';
import { describe, it } from 'node:test';
import {
GENERATED_MODELS_DEV_DIRECTORY,
GENERATED_MODELS_DEV_METADATA,
GENERATED_MODELS_DEV_PROVIDER_FACTS,
} from '../model-metadata.generated.js';
import { PROVIDER_REGISTRY, type ProviderType } from '../provider-registry.js';
Expand Down Expand Up @@ -62,6 +63,10 @@ const NOT_IN_MODELS_DEV: Readonly<Record<string, string>> = {
'anthropic-compatible': 'user-configured endpoint, no upstream catalog entry',
};

const GENERATED_LIFECYCLES = new Set(['active', 'beta', 'alpha', 'deprecated', 'retired']);
const GENERATED_INPUT_MODALITIES = new Set(['text', 'image', 'audio', 'pdf']);
const GENERATED_OUTPUT_MODALITIES = new Set(['text', 'image', 'audio']);

function expectedSegmentId(providerType: ProviderType): string | undefined {
const facts = GENERATED_MODELS_DEV_PROVIDER_FACTS as Partial<
Record<ProviderType, { id: string }>
Expand Down Expand Up @@ -116,6 +121,76 @@ describe('models.dev sync contract', () => {
assert.deepEqual(gaps, []);
});

it('preserves the extended model facts and only supported modalities', () => {
const gaps: string[] = [];
let inputLimitCount = 0;
let knowledgeCutoffCount = 0;
let structuredOutputCount = 0;
let pdfCount = 0;

for (const [providerType, models] of Object.entries(GENERATED_MODELS_DEV_METADATA)) {
for (const [modelId, metadata] of Object.entries(models)) {
const label = `${providerType}:${modelId}`;
if (metadata.description !== undefined && typeof metadata.description !== 'string') {
gaps.push(`${label} has a non-string description`);
}
if (
metadata.inputLimit !== undefined &&
(typeof metadata.inputLimit !== 'number' || !Number.isFinite(metadata.inputLimit))
) {
gaps.push(`${label} has an invalid inputLimit`);
} else if (metadata.inputLimit !== undefined) {
inputLimitCount += 1;
}
if (
metadata.knowledgeCutoff !== undefined &&
typeof metadata.knowledgeCutoff !== 'string'
) {
gaps.push(`${label} has a non-string knowledgeCutoff`);
} else if (metadata.knowledgeCutoff !== undefined) {
knowledgeCutoffCount += 1;
}
if (
metadata.structuredOutput !== undefined &&
typeof metadata.structuredOutput !== 'boolean'
) {
gaps.push(`${label} has a non-boolean structuredOutput`);
} else if (metadata.structuredOutput !== undefined) {
structuredOutputCount += 1;
}
if (metadata.lastUpdated !== undefined && typeof metadata.lastUpdated !== 'string') {
gaps.push(`${label} has a non-string lastUpdated`);
}
if (metadata.lifecycle !== undefined && !GENERATED_LIFECYCLES.has(metadata.lifecycle)) {
gaps.push(`${label} has an invalid lifecycle ${metadata.lifecycle}`);
}
for (const modality of metadata.modalities?.input ?? []) {
if (!GENERATED_INPUT_MODALITIES.has(modality)) {
gaps.push(`${label} has unsupported input modality ${modality}`);
}
if (modality === 'pdf') pdfCount += 1;
}
for (const modality of metadata.modalities?.output ?? []) {
if (!GENERATED_OUTPUT_MODALITIES.has(modality)) {
gaps.push(`${label} has unsupported output modality ${modality}`);
}
}
}
}

assert.deepEqual(gaps, []);
assert.ok(inputLimitCount > 0, 'input limits must be present in the generated snapshot');
assert.ok(
knowledgeCutoffCount > 0,
'knowledge cutoffs must be present in the generated snapshot',
);
assert.ok(
structuredOutputCount > 0,
'structured output facts must be present in the generated snapshot',
);
assert.ok(pdfCount > 0, 'PDF input support must be preserved in the generated snapshot');
});

it('a whitelisted provider whose base URL matches a directory provider must declare it', () => {
const gaps: string[] = [];
for (const providerType of Object.keys(NOT_IN_MODELS_DEV)) {
Expand Down
4 changes: 4 additions & 0 deletions packages/core/src/chat-model-choice.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ export interface ChatModelChoice {
providerLabel: string;
model: string;
label: string;
description?: string;
knowledgeCutoff?: string;
connectionName?: string;
isDefault: boolean;
thinkingLevels: readonly ThinkingLevel[];
Expand Down Expand Up @@ -64,6 +66,8 @@ export function buildChatModelChoices(connections: readonly LlmConnection[]): Ch
providerLabel: MODEL_MENU_PROVIDER_LABELS[connection.providerType] ?? provider.label,
model: entry.id,
label: entry.displayName?.trim() || entry.id,
...(entry.description !== undefined ? { description: entry.description } : {}),
...(entry.knowledgeCutoff !== undefined ? { knowledgeCutoff: entry.knowledgeCutoff } : {}),
...(provider.authKind === 'oauth_token' ? {} : { connectionName: connection.name }),
isDefault: entry.isDefault,
thinkingLevels: thinkingVariantsForConnection(connection, entry.id),
Expand Down
2 changes: 2 additions & 0 deletions packages/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1550,6 +1550,8 @@ export {
// model-metadata.ts
export {
modelMetadataIdsForProvider,
resolveModelInputModalities,
resolveModelPdfSupport,
resolveModelVisionSupport,
} from './model-metadata.js';
export type {
Expand Down
12 changes: 11 additions & 1 deletion packages/core/src/llm-connections.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,20 @@ export type ConnectionAuth =
export interface ModelInfo {
id: string;
displayName?: string;
/** Short upstream description, when the provider advertises one. */
description?: string;
/** Account-advertised request wire when one provider exposes multiple model protocols. */
apiProtocol?: 'openai-chat' | 'openai-responses' | 'anthropic-messages';
contextWindow?: number;
/** Maximum provider-visible input tokens, when narrower than contextWindow. */
inputLimit?: number;
maxOutputTokens?: number;
/** Knowledge cutoff reported by the provider or static catalog. */
knowledgeCutoff?: string;
/** Whether the model advertises structured JSON output separately from tools. */
structuredOutput?: boolean;
/** Date on which the upstream model facts were last refreshed. */
lastUpdated?: string;
capabilities?: {
chat?: boolean;
vision?: boolean;
Expand All @@ -64,7 +74,7 @@ export interface ModelInfo {
};
/** Multimodal input/output support from provider catalog metadata. */
modalities?: {
input: Array<'text' | 'image' | 'audio'>;
input: Array<'text' | 'image' | 'audio' | 'pdf'>;
output: Array<'text' | 'image' | 'audio'>;
};
}
Expand Down
Loading
Loading