From 061cd4d94ea7b1d29a39e2029da27694b47654f5 Mon Sep 17 00:00:00 2001 From: Alon Gubkin Date: Tue, 22 Sep 2026 04:49:07 -0700 Subject: [PATCH 1/3] feat: expose qualified AI model capabilities --- crates/alien-ai-gateway/src/router/mod.rs | 22 ++++++ crates/alien-core/src/ai_catalog.rs | 72 +++++++++++++++++++ crates/alien-core/src/bin/schema_exporter.rs | 3 + .../ai-gateway/src/__tests__/client.test.ts | 46 +++++++++--- packages/ai-gateway/src/client.ts | 61 +++++++++++++++- packages/core/src/generated/index.ts | 4 ++ .../schemas/modelApiCapabilities.json | 1 + .../schemas/modelCapabilitySupport.json | 1 + packages/core/src/generated/zod/index.ts | 4 ++ .../zod/model-api-capabilities-schema.ts | 35 +++++++++ .../zod/model-capability-support-schema.ts | 13 ++++ 11 files changed, 252 insertions(+), 10 deletions(-) create mode 100644 packages/core/src/generated/schemas/modelApiCapabilities.json create mode 100644 packages/core/src/generated/schemas/modelCapabilitySupport.json create mode 100644 packages/core/src/generated/zod/model-api-capabilities-schema.ts create mode 100644 packages/core/src/generated/zod/model-capability-support-schema.ts diff --git a/crates/alien-ai-gateway/src/router/mod.rs b/crates/alien-ai-gateway/src/router/mod.rs index 3952a7eb55..35c9dbdd88 100644 --- a/crates/alien-ai-gateway/src/router/mod.rs +++ b/crates/alien-ai-gateway/src/router/mod.rs @@ -1156,6 +1156,7 @@ async fn list_models( "object": "model", "provider": model.provider(), "displayName": model.display_name(), + "capabilities": model.capabilities(), }) }) .collect(), @@ -1174,6 +1175,7 @@ async fn list_models( "object": "model", "provider": "anthropic", "displayName": model.display_name(), + "capabilities": ai_catalog::unverified_model_capabilities(), }) }) .collect(), @@ -1195,6 +1197,7 @@ async fn list_models( "object": "model", "provider": "openai", "displayName": model, + "capabilities": ai_catalog::unverified_model_capabilities(), }) }) .collect() @@ -1217,6 +1220,7 @@ async fn list_models( "object": "model", "provider": "databricks", "displayName": model, + "capabilities": ai_catalog::unverified_model_capabilities(), }) }) .collect() @@ -3317,6 +3321,24 @@ mod tests { .expect("byo/gpt-oss-20b entry"); assert_eq!(gpt["provider"], "openai"); assert_eq!(gpt["displayName"], "GPT-OSS 20B"); + let capabilities = gpt["capabilities"].as_array().expect("capability matrix"); + assert_eq!(capabilities.len(), ClientApi::ALL.len()); + let responses = capabilities + .iter() + .find(|entry| entry["api"] == "open-ai-responses") + .expect("Responses capabilities"); + assert_eq!(responses["functionTools"], "supported"); + assert_eq!(responses["serverManagedContinuation"], "supported"); + assert_eq!(responses["statelessReplay"], "unsupported"); + assert_eq!(responses["imageInput"], "unverified"); + assert_eq!(responses["reasoningControls"], "unverified"); + assert_eq!(responses["qualifiedOn"], "2026-09-22"); + let chat = capabilities + .iter() + .find(|entry| entry["api"] == "open-ai-chat-completions") + .expect("Chat Completions capabilities"); + assert_eq!(chat["functionTools"], "unverified"); + assert!(chat.get("qualifiedOn").is_none()); assert_eq!( openai.hits_async().await, 0, diff --git a/crates/alien-core/src/ai_catalog.rs b/crates/alien-core/src/ai_catalog.rs index bb5ce3f65b..0fb9a38f28 100644 --- a/crates/alien-core/src/ai_catalog.rs +++ b/crates/alien-core/src/ai_catalog.rs @@ -23,6 +23,57 @@ pub enum ClientApi { AnthropicMessages, } +/// Whether a model feature has been qualified through one public Gateway API. +/// +/// `Unverified` is deliberately different from `Unsupported`: Alien does not infer support from +/// a model family or from the provider-native protocol used behind the Gateway. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[serde(rename_all = "kebab-case")] +pub enum ModelCapabilitySupport { + Supported, + Unsupported, + Unverified, +} + +/// Qualified model behavior for one public Gateway API. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ModelApiCapabilities { + pub api: ClientApi, + pub function_tools: ModelCapabilitySupport, + pub image_input: ModelCapabilitySupport, + pub reasoning_controls: ModelCapabilitySupport, + pub server_managed_continuation: ModelCapabilitySupport, + pub stateless_replay: ModelCapabilitySupport, + /// Date of the live qualification evidence, or `None` when every fact is unverified. + #[serde(skip_serializing_if = "Option::is_none")] + pub qualified_on: Option, +} + +impl ModelApiCapabilities { + fn unverified(api: ClientApi) -> Self { + Self { + api, + function_tools: ModelCapabilitySupport::Unverified, + image_input: ModelCapabilitySupport::Unverified, + reasoning_controls: ModelCapabilitySupport::Unverified, + server_managed_continuation: ModelCapabilitySupport::Unverified, + stateless_replay: ModelCapabilitySupport::Unverified, + qualified_on: None, + } + } +} + +/// An explicit all-unverified matrix for provider models without live qualification evidence. +pub fn unverified_model_capabilities() -> Vec { + ClientApi::ALL + .into_iter() + .map(ModelApiCapabilities::unverified) + .collect() +} + impl ClientApi { /// Public request protocols accepted for every text-generation model. The /// gateway translates to the model's provider-native protocol when needed. @@ -141,6 +192,27 @@ impl DirectAnthropicModel { } impl CatalogModel { + /// Qualified behavior per public API. Every API is present so callers cannot mistake an + /// omitted record for lack of support. + pub fn capabilities(&self) -> Vec { + let mut capabilities = unverified_model_capabilities(); + + // Live AWS qualification on 2026-09-22. These facts apply only to this exact + // provider/model/API tuple; the Chat Completions and Messages adapters remain unverified. + if self.cloud == Platform::Aws && self.public_id == "gpt-oss-20b" { + let responses = capabilities + .iter_mut() + .find(|entry| entry.api == ClientApi::OpenAiResponses) + .expect("the public API matrix always contains Responses"); + responses.function_tools = ModelCapabilitySupport::Supported; + responses.server_managed_continuation = ModelCapabilitySupport::Supported; + responses.stateless_replay = ModelCapabilitySupport::Unsupported; + responses.qualified_on = Some("2026-09-22".to_string()); + } + + capabilities + } + /// The model's publisher, for grouping in a picker. Derived from the public id, /// so the same public id reports the same provider on every cloud. pub fn provider(&self) -> &'static str { diff --git a/crates/alien-core/src/bin/schema_exporter.rs b/crates/alien-core/src/bin/schema_exporter.rs index 6372b92bd4..7e8cf34e59 100644 --- a/crates/alien-core/src/bin/schema_exporter.rs +++ b/crates/alien-core/src/bin/schema_exporter.rs @@ -1,3 +1,4 @@ +use alien_core::ai_catalog::{ModelApiCapabilities, ModelCapabilitySupport}; use alien_core::sync::{ DynamicContainerHealthCheck, DynamicContainerReport, DynamicContainerStatus, ObservedApplicationImage, ObservedApplicationReport, ObservedApplicationSource, @@ -131,6 +132,8 @@ use utoipa::OpenApi; PostgresOutputs, Ai, AiOutputs, + ModelApiCapabilities, + ModelCapabilitySupport, Queue, QueueOutputs, Sandbox, diff --git a/packages/ai-gateway/src/__tests__/client.test.ts b/packages/ai-gateway/src/__tests__/client.test.ts index 173b4923ee..b0b868df3a 100644 --- a/packages/ai-gateway/src/__tests__/client.test.ts +++ b/packages/ai-gateway/src/__tests__/client.test.ts @@ -17,6 +17,26 @@ const EXTERNAL = JSON.stringify({ service: "external-ai", provider: "openai", ap const GATEWAY_URL = "http://127.0.0.1:41999" +const unverifiedCapabilities = [ + "open-ai-chat-completions", + "open-ai-responses", + "anthropic-messages", +].map(api => ({ + api, + functionTools: "unverified", + imageInput: "unverified", + reasoningControls: "unverified", + serverManagedContinuation: "unverified", + statelessReplay: "unverified", +})) + +const gatewayModel = { + id: "gpt-oss-20b", + provider: "openai", + displayName: "GPT-OSS 20B", + capabilities: unverifiedCapabilities, +} + const stubGateway: Gateway = { startAiGateway: () => Promise.resolve({ url: GATEWAY_URL } as unknown as RawAiGatewayHandle), } @@ -256,24 +276,36 @@ describe("Ai.getAvailableModels", () => { expect(ids).toContain("claude-sonnet-5") expect(ids).toContain("claude-haiku-4-5") expect(ids.some(id => id.startsWith("claude-3-5"))).toBe(false) - // Each entry carries the provider/displayName shape a model picker consumes. - expect(models[0]).toMatchObject({ provider: "anthropic", displayName: models[0]!.id }) + // Each entry carries the same explicit capability shape as a Gateway model. + expect(models[0]).toMatchObject({ + provider: "anthropic", + displayName: models[0]!.id, + capabilities: unverifiedCapabilities, + }) expect(fetchMock).not.toHaveBeenCalled() }) it("fetches the gateway's curated catalog for an ambient binding", async () => { vi.stubEnv("ALIEN_LLM_BINDING", JSON.stringify({ service: "bedrock", region: "us-east-2" })) const fetchMock = stubFetch({ - data: [{ id: "gpt-oss-20b", provider: "openai", displayName: "GPT-OSS 20B" }], + data: [gatewayModel], }) const models = await ai("llm").getAvailableModels() expect(callUrl(fetchMock)).toBe(`${GATEWAY_URL}/llm/v1/models`) - expect(models).toEqual([{ id: "gpt-oss-20b", provider: "openai", displayName: "GPT-OSS 20B" }]) + expect(models).toEqual([gatewayModel]) + }) + + it("rejects a model list whose capability matrix is incomplete", async () => { + vi.stubEnv("ALIEN_LLM_BINDING", JSON.stringify({ service: "bedrock", region: "us-east-2" })) + stubFetch({ data: [{ ...gatewayModel, capabilities: unverifiedCapabilities.slice(1) }] }) + await expect(ai("llm").getAvailableModels()).rejects.toThrow( + "models response entry has an invalid model or capabilities shape", + ) }) it("retries a transient gateway-start failure on a retained instance", async () => { vi.stubEnv("ALIEN_LLM_BINDING", JSON.stringify({ service: "bedrock", region: "us-east-2" })) - stubFetch({ data: [{ id: "gpt-oss-20b", provider: "openai", displayName: "GPT-OSS 20B" }] }) + stubFetch({ data: [gatewayModel] }) const start = vi .fn() .mockRejectedValueOnce(new Error("ambient credential unavailable")) @@ -282,9 +314,7 @@ describe("Ai.getAvailableModels", () => { await expect(llm.getAvailableModels()).rejects.toThrow() // A cached rejection would leave this instance permanently broken; it must retry. - expect(await llm.getAvailableModels()).toEqual([ - { id: "gpt-oss-20b", provider: "openai", displayName: "GPT-OSS 20B" }, - ]) + expect(await llm.getAvailableModels()).toEqual([gatewayModel]) expect(start).toHaveBeenCalledTimes(2) }) }) diff --git a/packages/ai-gateway/src/client.ts b/packages/ai-gateway/src/client.ts index 5712d5d149..7cf3bbe791 100644 --- a/packages/ai-gateway/src/client.ts +++ b/packages/ai-gateway/src/client.ts @@ -47,6 +47,18 @@ export interface ResponseCreateParams { } /** One model the gateway exposes for this binding's cloud. */ +export type ModelCapabilitySupport = "supported" | "unsupported" | "unverified" + +export interface ModelApiCapabilities { + api: "open-ai-chat-completions" | "open-ai-responses" | "anthropic-messages" + functionTools: ModelCapabilitySupport + imageInput: ModelCapabilitySupport + reasoningControls: ModelCapabilitySupport + serverManagedContinuation: ModelCapabilitySupport + statelessReplay: ModelCapabilitySupport + qualifiedOn?: string +} + export interface AiModel { /** The id passed to `chat.completions.create` / `responses.create`. */ id: string @@ -54,6 +66,8 @@ export interface AiModel { provider: string /** A human label for a model picker, e.g. "Claude Opus 4.8". */ displayName: string + /** Qualified behavior for each public Gateway API. */ + capabilities: ModelApiCapabilities[] } /** @@ -113,6 +127,23 @@ function defaultModels(provider: string): string[] { : ["gpt-4o-mini", "gpt-4o"] } +const PUBLIC_AI_APIS: ModelApiCapabilities["api"][] = [ + "open-ai-chat-completions", + "open-ai-responses", + "anthropic-messages", +] + +function unverifiedModelCapabilities(): ModelApiCapabilities[] { + return PUBLIC_AI_APIS.map(api => ({ + api, + functionTools: "unverified", + imageInput: "unverified", + reasoningControls: "unverified", + serverManagedContinuation: "unverified", + statelessReplay: "unverified", + })) +} + /** Resolution shared by `ai()` and `getAiConnection()`. `baseUrl` is the root (no `/v1`); * `apiKey`/`staticModels` are set only for a BYO-key (External) provider. */ export interface ResolvedAiBinding { @@ -145,6 +176,7 @@ export async function resolveAiBinding(gateway: Gateway, name: string): Promise< id, provider: binding.provider, displayName: id, + capabilities: unverifiedModelCapabilities(), })), } } @@ -338,12 +370,13 @@ export class Ai { if ( typeof model?.id !== "string" || typeof model?.provider !== "string" || - typeof model?.displayName !== "string" + typeof model?.displayName !== "string" || + !isModelCapabilities(model.capabilities) ) { throw createUpstreamError( url, response.status, - `models response entry is missing id/provider/displayName: ${JSON.stringify(model)}`, + `models response entry has an invalid model or capabilities shape: ${JSON.stringify(model)}`, ) } } @@ -421,6 +454,30 @@ export class Ai { } } +function isModelCapabilities(value: unknown): value is ModelApiCapabilities[] { + if (!Array.isArray(value) || value.length !== PUBLIC_AI_APIS.length) return false + const support = new Set(["supported", "unsupported", "unverified"]) + const seen = new Set() + for (const entry of value) { + if (!entry || typeof entry !== "object") return false + const capability = entry as Partial + if (!PUBLIC_AI_APIS.includes(capability.api as ModelApiCapabilities["api"])) return false + if (seen.has(capability.api as string)) return false + seen.add(capability.api as string) + if ( + !support.has(capability.functionTools as ModelCapabilitySupport) || + !support.has(capability.imageInput as ModelCapabilitySupport) || + !support.has(capability.reasoningControls as ModelCapabilitySupport) || + !support.has(capability.serverManagedContinuation as ModelCapabilitySupport) || + !support.has(capability.statelessReplay as ModelCapabilitySupport) || + (capability.qualifiedOn !== undefined && typeof capability.qualifiedOn !== "string") + ) { + return false + } + } + return seen.size === PUBLIC_AI_APIS.length +} + // ───────────────────────────────────────────────────────────────────────────── // Entry wiring // ───────────────────────────────────────────────────────────────────────────── diff --git a/packages/core/src/generated/index.ts b/packages/core/src/generated/index.ts index 483572781b..bc5790b317 100644 --- a/packages/core/src/generated/index.ts +++ b/packages/core/src/generated/index.ts @@ -353,6 +353,8 @@ export type { ManagementPermissions } from "./zod/management-permissions-schema. export type { MessagePayload } from "./zod/message-payload-schema.js"; export type { MetricSample } from "./zod/metric-sample-schema.js"; export type { MetricUnit } from "./zod/metric-unit-schema.js"; +export type { ModelApiCapabilities } from "./zod/model-api-capabilities-schema.js"; +export type { ModelCapabilitySupport } from "./zod/model-capability-support-schema.js"; export type { MonitoringConfig } from "./zod/monitoring-config-schema.js"; export type { NetworkHeartbeatData } from "./zod/network-heartbeat-data-schema.js"; export type { NetworkHeartbeatStatus } from "./zod/network-heartbeat-status-schema.js"; @@ -859,6 +861,8 @@ export { ManagementPermissionsSchema } from "./zod/management-permissions-schema export { MessagePayloadSchema } from "./zod/message-payload-schema.js"; export { MetricSampleSchema } from "./zod/metric-sample-schema.js"; export { MetricUnitSchema } from "./zod/metric-unit-schema.js"; +export { ModelApiCapabilitiesSchema } from "./zod/model-api-capabilities-schema.js"; +export { ModelCapabilitySupportSchema } from "./zod/model-capability-support-schema.js"; export { MonitoringConfigSchema } from "./zod/monitoring-config-schema.js"; export { NetworkHeartbeatDataSchema } from "./zod/network-heartbeat-data-schema.js"; export { NetworkHeartbeatStatusSchema } from "./zod/network-heartbeat-status-schema.js"; diff --git a/packages/core/src/generated/schemas/modelApiCapabilities.json b/packages/core/src/generated/schemas/modelApiCapabilities.json new file mode 100644 index 0000000000..33a2268c22 --- /dev/null +++ b/packages/core/src/generated/schemas/modelApiCapabilities.json @@ -0,0 +1 @@ +{"type":"object","description":"Qualified model behavior for one public Gateway API.","required":["api","functionTools","imageInput","reasoningControls","serverManagedContinuation","statelessReplay"],"properties":{"api":{"type":"string","description":"A public API accepted from an application client.","enum":["open-ai-chat-completions","open-ai-responses","anthropic-messages"],"x-readme-ref-name":"ClientApi"},"functionTools":{"type":"string","description":"Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.","enum":["supported","unsupported","unverified"],"x-readme-ref-name":"ModelCapabilitySupport"},"imageInput":{"type":"string","description":"Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.","enum":["supported","unsupported","unverified"],"x-readme-ref-name":"ModelCapabilitySupport"},"qualifiedOn":{"type":["string","null"],"description":"Date of the live qualification evidence, or `None` when every fact is unverified."},"reasoningControls":{"type":"string","description":"Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.","enum":["supported","unsupported","unverified"],"x-readme-ref-name":"ModelCapabilitySupport"},"serverManagedContinuation":{"type":"string","description":"Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.","enum":["supported","unsupported","unverified"],"x-readme-ref-name":"ModelCapabilitySupport"},"statelessReplay":{"type":"string","description":"Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.","enum":["supported","unsupported","unverified"],"x-readme-ref-name":"ModelCapabilitySupport"}},"additionalProperties":false,"x-readme-ref-name":"ModelApiCapabilities"} \ No newline at end of file diff --git a/packages/core/src/generated/schemas/modelCapabilitySupport.json b/packages/core/src/generated/schemas/modelCapabilitySupport.json new file mode 100644 index 0000000000..9d04242e9e --- /dev/null +++ b/packages/core/src/generated/schemas/modelCapabilitySupport.json @@ -0,0 +1 @@ +{"type":"string","description":"Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.","enum":["supported","unsupported","unverified"],"x-readme-ref-name":"ModelCapabilitySupport"} \ No newline at end of file diff --git a/packages/core/src/generated/zod/index.ts b/packages/core/src/generated/zod/index.ts index 6c47a95d83..f9c0fa5934 100644 --- a/packages/core/src/generated/zod/index.ts +++ b/packages/core/src/generated/zod/index.ts @@ -353,6 +353,8 @@ export type { ManagementPermissions } from "./management-permissions-schema.js"; export type { MessagePayload } from "./message-payload-schema.js"; export type { MetricSample } from "./metric-sample-schema.js"; export type { MetricUnit } from "./metric-unit-schema.js"; +export type { ModelApiCapabilities } from "./model-api-capabilities-schema.js"; +export type { ModelCapabilitySupport } from "./model-capability-support-schema.js"; export type { MonitoringConfig } from "./monitoring-config-schema.js"; export type { NetworkHeartbeatData } from "./network-heartbeat-data-schema.js"; export type { NetworkHeartbeatStatus } from "./network-heartbeat-status-schema.js"; @@ -859,6 +861,8 @@ export { ManagementPermissionsSchema } from "./management-permissions-schema.js" export { MessagePayloadSchema } from "./message-payload-schema.js"; export { MetricSampleSchema } from "./metric-sample-schema.js"; export { MetricUnitSchema } from "./metric-unit-schema.js"; +export { ModelApiCapabilitiesSchema } from "./model-api-capabilities-schema.js"; +export { ModelCapabilitySupportSchema } from "./model-capability-support-schema.js"; export { MonitoringConfigSchema } from "./monitoring-config-schema.js"; export { NetworkHeartbeatDataSchema } from "./network-heartbeat-data-schema.js"; export { NetworkHeartbeatStatusSchema } from "./network-heartbeat-status-schema.js"; diff --git a/packages/core/src/generated/zod/model-api-capabilities-schema.ts b/packages/core/src/generated/zod/model-api-capabilities-schema.ts new file mode 100644 index 0000000000..bb019e15a2 --- /dev/null +++ b/packages/core/src/generated/zod/model-api-capabilities-schema.ts @@ -0,0 +1,35 @@ +/** +* Generated by Kubb (https://kubb.dev/). +* Do not edit manually. +*/ + +import * as z from "zod"; +import { ClientApiSchema } from "./client-api-schema.js"; +import { ModelCapabilitySupportSchema } from "./model-capability-support-schema.js"; + +/** + * @description Qualified model behavior for one public Gateway API. + */ +export const ModelApiCapabilitiesSchema = z.object({ + get "api"(){ + return ClientApiSchema.describe("A public API accepted from an application client.") + }, +get "functionTools"(){ + return ModelCapabilitySupportSchema.describe("Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.") + }, +get "imageInput"(){ + return ModelCapabilitySupportSchema.describe("Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.") + }, +"qualifiedOn": z.string().describe("Date of the live qualification evidence, or `None` when every fact is unverified.").nullish(), +get "reasoningControls"(){ + return ModelCapabilitySupportSchema.describe("Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.") + }, +get "serverManagedContinuation"(){ + return ModelCapabilitySupportSchema.describe("Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.") + }, +get "statelessReplay"(){ + return ModelCapabilitySupportSchema.describe("Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.") + } + }).describe("Qualified model behavior for one public Gateway API.") + +export type ModelApiCapabilities = z.infer \ No newline at end of file diff --git a/packages/core/src/generated/zod/model-capability-support-schema.ts b/packages/core/src/generated/zod/model-capability-support-schema.ts new file mode 100644 index 0000000000..940133f823 --- /dev/null +++ b/packages/core/src/generated/zod/model-capability-support-schema.ts @@ -0,0 +1,13 @@ +/** +* Generated by Kubb (https://kubb.dev/). +* Do not edit manually. +*/ + +import * as z from "zod"; + +/** + * @description Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway. + */ +export const ModelCapabilitySupportSchema = z.enum(["supported", "unsupported", "unverified"]).describe("Whether a model feature has been qualified through one public Gateway API.\n\n`Unverified` is deliberately different from `Unsupported`: Alien does not infer support from\na model family or from the provider-native protocol used behind the Gateway.") + +export type ModelCapabilitySupport = z.infer \ No newline at end of file From d646d9370e4c75f32e9bb22b99fce9dc5ac283ea Mon Sep 17 00:00:00 2001 From: Alon Gubkin Date: Fri, 9 Oct 2026 22:02:43 +0300 Subject: [PATCH 2/3] fix: treat an older gateway's missing capability matrix as unverified A Worker image or BYO container can run a gateway binary older than the app's SDK. Rejecting its model list would break getAvailableModels for every such app, so a missing matrix now means no qualification evidence. Co-Authored-By: Claude Opus 5.5 --- .../ai-gateway/src/__tests__/client.test.ts | 9 ++++++- packages/ai-gateway/src/client.ts | 26 ++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/packages/ai-gateway/src/__tests__/client.test.ts b/packages/ai-gateway/src/__tests__/client.test.ts index b0b868df3a..f3a2845760 100644 --- a/packages/ai-gateway/src/__tests__/client.test.ts +++ b/packages/ai-gateway/src/__tests__/client.test.ts @@ -299,10 +299,17 @@ describe("Ai.getAvailableModels", () => { vi.stubEnv("ALIEN_LLM_BINDING", JSON.stringify({ service: "bedrock", region: "us-east-2" })) stubFetch({ data: [{ ...gatewayModel, capabilities: unverifiedCapabilities.slice(1) }] }) await expect(ai("llm").getAvailableModels()).rejects.toThrow( - "models response entry has an invalid model or capabilities shape", + "models response entry has an invalid capabilities matrix", ) }) + it("reports every API as unverified when an older gateway omits the matrix", async () => { + vi.stubEnv("ALIEN_LLM_BINDING", JSON.stringify({ service: "bedrock", region: "us-east-2" })) + const { capabilities: _omitted, ...legacyModel } = gatewayModel + stubFetch({ data: [legacyModel] }) + expect(await ai("llm").getAvailableModels()).toEqual([gatewayModel]) + }) + it("retries a transient gateway-start failure on a retained instance", async () => { vi.stubEnv("ALIEN_LLM_BINDING", JSON.stringify({ service: "bedrock", region: "us-east-2" })) stubFetch({ data: [gatewayModel] }) diff --git a/packages/ai-gateway/src/client.ts b/packages/ai-gateway/src/client.ts index 7cf3bbe791..a60eab2bfa 100644 --- a/packages/ai-gateway/src/client.ts +++ b/packages/ai-gateway/src/client.ts @@ -46,7 +46,7 @@ export interface ResponseCreateParams { [key: string]: unknown } -/** One model the gateway exposes for this binding's cloud. */ +/** Whether a behavior was qualified through one public API. `unverified` is not `unsupported`. */ export type ModelCapabilitySupport = "supported" | "unsupported" | "unverified" export interface ModelApiCapabilities { @@ -59,6 +59,7 @@ export interface ModelApiCapabilities { qualifiedOn?: string } +/** One model the gateway exposes for this binding's cloud. */ export interface AiModel { /** The id passed to `chat.completions.create` / `responses.create`. */ id: string @@ -366,21 +367,32 @@ export class Ai { // The gateway subprocess is a separate trust boundary (it can be a stale or // mismatched binary), so verify each element actually carries the AiModel // shape instead of letting a type assertion hide missing fields. - for (const model of body.data) { + return body.data.map(model => { if ( typeof model?.id !== "string" || typeof model?.provider !== "string" || - typeof model?.displayName !== "string" || - !isModelCapabilities(model.capabilities) + typeof model?.displayName !== "string" ) { throw createUpstreamError( url, response.status, - `models response entry has an invalid model or capabilities shape: ${JSON.stringify(model)}`, + `models response entry is missing id/provider/displayName: ${JSON.stringify(model)}`, ) } - } - return body.data + // A gateway binary older than the capability matrix omits the field. That means no + // qualification evidence, which is exactly "unverified" for every API. + if (model.capabilities === undefined) { + return { ...model, capabilities: unverifiedModelCapabilities() } + } + if (!isModelCapabilities(model.capabilities)) { + throw createUpstreamError( + url, + response.status, + `models response entry has an invalid capabilities matrix: ${JSON.stringify(model)}`, + ) + } + return model + }) } private _chatCompletionsCreate(params: ChatCompletionCreateParams): Promise { From 8ae6372de56f9b470542fdc73c34f39c78f1217e Mon Sep 17 00:00:00 2001 From: Alon Gubkin Date: Fri, 9 Oct 2026 22:14:56 +0300 Subject: [PATCH 3/3] fix: export the model capability types from the AI packages Co-Authored-By: Claude Opus 5.5 --- packages/ai-gateway/src/index.ts | 2 ++ packages/ai-gateway/src/native.ts | 2 ++ packages/sdk/src/index.ts | 2 ++ 3 files changed, 6 insertions(+) diff --git a/packages/ai-gateway/src/index.ts b/packages/ai-gateway/src/index.ts index b8feeff1d5..5fe06f34a7 100644 --- a/packages/ai-gateway/src/index.ts +++ b/packages/ai-gateway/src/index.ts @@ -37,6 +37,8 @@ export type { AiConnection, AiModel, ChatCompletionCreateParams, + ModelApiCapabilities, + ModelCapabilitySupport, ResponseCreateParams, } from "./client.js" export { Ai } from "./client.js" diff --git a/packages/ai-gateway/src/native.ts b/packages/ai-gateway/src/native.ts index 6c9e9a7743..c415f71d2f 100644 --- a/packages/ai-gateway/src/native.ts +++ b/packages/ai-gateway/src/native.ts @@ -41,6 +41,8 @@ export type { AiConnection, AiModel, ChatCompletionCreateParams, + ModelApiCapabilities, + ModelCapabilitySupport, ResponseCreateParams, } from "./client.js" export { Ai } from "./client.js" diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index e56fac1cea..869ce703b0 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -87,6 +87,8 @@ export type { AmbientAiBinding, ChatCompletionCreateParams, ExternalAiBinding, + ModelApiCapabilities, + ModelCapabilitySupport, ResponseCreateParams, } from "@alienplatform/ai-gateway" export {