diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index f95272a73cf..55c0dc9757a 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -507,9 +507,45 @@ MiMo model Command Code serves. solely because the client permits parallel calls. - Accepts Responses `text` controls that are not structured output — `text.verbosity` and `text.format: {"type":"text"}` — without forwarding them. Kiro has no wire field for either, so - they are ignored rather than rejected. Structured output (`text.format` of type `json_schema` - or `json_object`) is still refused: the Kiro wire cannot constrain the response shape, and a - caller expecting JSON would otherwise receive prose. + they are ignored rather than rejected. +- Emulates structured output (Responses `text.format` of type `json_schema` or `json_object`, and + Messages `output_config.format`). The Kiro wire cannot constrain the response shape, so the + adapter advertises a private output tool whose input schema is the requested schema and returns + that tool's input as the response text. A plain-text reply that is already the requested JSON is + accepted as is. Any other reply, including an empty one, gets one bounded retry when the model stopped cleanly (a reply cut by the output-token limit, a content filter or a full context window ends with that stop's own error instead), and if the retry also fails the request ends + with an `invalid_kiro_structured_output` error. A caller expecting JSON never receives prose. + Without `strict: true` the check covers JSON syntax, the root type and the root's required + properties. With `strict: true` the whole value is checked exactly against the schema, and a + strict schema outside this subset is refused before the request is sent: + - `type`, `properties`, `required`, `additionalProperties`, single-schema `items`, `enum`, + `const` and `anyOf`. A `const` or `enum` value that contains a number is refused: the proxy + receives the request already parsed, so a number such as `1.0000000000000001` arrives as `1` + and an exact match could accept a value the caller excluded. + - `$ref` only to `#/$defs/` or `#/definitions/` at the schema root, with `~0` and + `~1` as the only JSON Pointer escapes. References such + as `#`, `#/properties/...` or `#/default`, nested `$defs`, and reference cycles that never + reach a property or item are refused, and so is a `$ref` with an assertion keyword beside it + (drafts disagree on whether such siblings apply), a `$id` below the root, and `$anchor`. + - `minLength`, `maxLength`, `minItems` and `maxItems` as non-negative integers. + - `minimum`, `maximum`, `exclusiveMinimum` and `exclusiveMaximum` as safe integers or decimals + with at most 15 significant digits. The model's numbers are compared exactly as written. This is + a heuristic for the bound itself: a bound written with more digits than a double holds can still + arrive rounded, so a value within about one unit in the last place of the bound may be judged on + the wrong side. + - `multipleOf`, `pattern` and `patternProperties` are refused in strict mode. Without + `strict: true` they are accepted and left to the light checks above. + + Validation work is capped per answer (10,000 schema checks plus 16 per JSON value, 128 schema + levels deep), including `const`/`enum` comparisons and every pass over `required` lists, object + entries, long strings and long numbers. A strict format without a JSON object schema is refused, + and so is a `type` list that repeats a type. An answer that needs more counts as not + matching the format. A strict schema with more than 10,000 nodes, or more than 10,000 values in + its `const`/`enum` literals, is refused. Error messages name the failed check only, never a + property name or path from the answer or the schema. + A Messages `output_config.format` schema always gets the `strict: true` checks, even when it + has optional properties and is therefore forwarded to OpenAI-compatible destinations without + `strict`: Anthropic promises an answer that conforms to the schema. A Messages schema outside + this subset is refused before the request is sent. - Decodes `application/vnd.amazon.eventstream`, reconstructs text/thinking/tool events, detects truncated tool JSON, and estimates usage because the upstream does not return token counts. Decoder teardown initiates cancellation without waiting for transport cleanup to settle, so a diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 585527748fb..52771a93438 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1099,7 +1099,7 @@ "kiro-review-regressions.test.ts": "providers/kiro", "kiro-metering-events.test.ts": "providers/kiro", "kiro-metering-usage.test.ts": "providers/kiro", - "kiro-single-final.test.ts": "providers/kiro", + "kiro-single-final.test.ts": "providers/kiro", "kiro-structured-output.test.ts": "providers/kiro", "kiro-structured-output-strict.test.ts": "providers/kiro", "kiro-stream.test.ts": "providers/kiro", "kiro-transport-parity.test.ts": "providers/kiro", "kiro-usage-quota.test.ts": "providers/kiro", @@ -1671,7 +1671,7 @@ "server-images.test.ts": "server", "server-key-failover-e2e.test.ts": "server", "distributed-key-401.test.ts": "server", "server-jev-combo-e2e.test.ts": "server", "combo-requested-effort.test.ts": "server", - "server-kiro-completion-e2e.test.ts": "server", + "server-kiro-completion-e2e.test.ts": "server", "server-kiro-structured-output-e2e.test.ts": "server", "server-kiro-device-login.test.ts": "server", "server-kiro-oauth-401-replay.test.ts": "server", "server-kiro-refusal-e2e.test.ts": "server", "server-live-existing-call.test.ts": "server", "server-live-frame-log.test.ts": "server", diff --git a/src/adapters/kiro-constants.ts b/src/adapters/kiro-constants.ts index c361614c030..00a7b4c5845 100644 --- a/src/adapters/kiro-constants.ts +++ b/src/adapters/kiro-constants.ts @@ -1,4 +1,13 @@ export const KIRO_COMPLETION_TOOL_NAME = "codex_kiro_final_answer"; +/** + * Private terminal channel for Responses structured output (`text.format` json_schema/json_object). + * + * The CodeWhisperer wire has no schema-constrained response mode, but it does carry tool input + * schemas, and a tool call's input is JSON by construction. A turn that asks for structured output + * therefore advertises this tool INSTEAD of `KIRO_COMPLETION_TOOL_NAME` and relays its input as the + * assistant's text. Reserved in the same collision domain so a client tool can never claim it. + */ +export const KIRO_STRUCTURED_OUTPUT_TOOL_NAME = "codex_kiro_structured_output"; /** * Request-scoped CodeWhisperer service profile for AWS Builder ID accounts. diff --git a/src/adapters/kiro-tools.ts b/src/adapters/kiro-tools.ts index e4e2c11a044..4466ddf7133 100644 --- a/src/adapters/kiro-tools.ts +++ b/src/adapters/kiro-tools.ts @@ -60,25 +60,30 @@ const KIRO_REJECTED_SCHEMA_KEYS = new Set([ // "format"/"pattern" would be deleted. We recurse into the value schemas but keep every name intact. const SCHEMA_MAP_KEYS = new Set(["properties", "$defs", "definitions"]); +// Keys whose values are JSON data rather than schemas. Recursing into them would strip +// keyword-named keys from the data itself and turn `const: {"format":"pdf"}` into `const: {}`, +// silently changing the value a tool argument must equal. +const SCHEMA_LITERAL_KEYS = new Set(["const", "enum", "default", "examples"]); + function sanitizeSchemaMap(value: unknown): unknown { if (!value || typeof value !== "object" || Array.isArray(value)) return sanitizeKiroSchema(value); - const out: Record = {}; - for (const [name, child] of Object.entries(value as Record)) { - out[name] = sanitizeKiroSchema(child); - } - return out; + // fromEntries defines own properties: a property named "__proto__" stays a property instead of + // hitting the prototype setter and vanishing from the advertised schema. + return Object.fromEntries(Object.entries(value as Record).map(([name, child]) => [name, sanitizeKiroSchema(child)])); } -function sanitizeKiroSchema(value: unknown): unknown { +/** Strip the keywords Kiro's tool-spec validator rejects, everywhere in a schema tree. */ +export function sanitizeKiroSchema(value: unknown): unknown { if (Array.isArray(value)) return value.map(sanitizeKiroSchema); if (!value || typeof value !== "object") return value; - const out: Record = {}; + const out: Array<[string, unknown]> = []; for (const [key, child] of Object.entries(value as Record)) { if (KIRO_REJECTED_SCHEMA_KEYS.has(key)) continue; if (key === "required" && Array.isArray(child) && child.length === 0) continue; - out[key] = SCHEMA_MAP_KEYS.has(key) ? sanitizeSchemaMap(child) : sanitizeKiroSchema(child); + if (SCHEMA_LITERAL_KEYS.has(key)) out.push([key, child]); + else out.push([key, SCHEMA_MAP_KEYS.has(key) ? sanitizeSchemaMap(child) : sanitizeKiroSchema(child)]); } - return out; + return Object.fromEntries(out); } function ensureRootObjectType(schema: unknown): Record { diff --git a/src/adapters/kiro-wire.ts b/src/adapters/kiro-wire.ts index 34773814e6d..1897d65671a 100644 --- a/src/adapters/kiro-wire.ts +++ b/src/adapters/kiro-wire.ts @@ -2,7 +2,7 @@ import { createHash, randomUUID } from "node:crypto"; import { hostname, userInfo } from "node:os"; import { normalizeKiroModelId } from "../providers/kiro-models"; import type { OcxParsedRequest } from "../types"; -import { KIRO_COMPLETION_TOOL_NAME } from "./kiro-constants"; +import { KIRO_COMPLETION_TOOL_NAME, KIRO_STRUCTURED_OUTPUT_TOOL_NAME } from "./kiro-constants"; let cachedFp: string | undefined; @@ -77,15 +77,19 @@ export interface KiroToolNameRegistry { readonly nameMap: Map; } -/** One collision domain for advertised tools, replayed calls, and the private completion tool. */ +/** Proxy-owned terminal tools; neither may be claimed by a client tool or a replayed call. */ +const KIRO_RESERVED_TOOL_NAMES: readonly string[] = [KIRO_COMPLETION_TOOL_NAME, KIRO_STRUCTURED_OUTPUT_TOOL_NAME]; + +/** One collision domain for advertised tools, replayed calls, and the private terminal tools. */ export function createKiroToolNameRegistry(): KiroToolNameRegistry { - const used = new Set([KIRO_COMPLETION_TOOL_NAME]); + const used = new Set(KIRO_RESERVED_TOOL_NAMES); const wireToKiro = new Map(); const nameMap = new Map(); return { + /** Return the Kiro-safe alias for a wire tool name, registering it on first use; reserved names are refused. */ alias(wireName: string): string { - if (wireName === KIRO_COMPLETION_TOOL_NAME) { - throw new Error(`Kiro reserves the tool name ${JSON.stringify(KIRO_COMPLETION_TOOL_NAME)}`); + if (KIRO_RESERVED_TOOL_NAMES.includes(wireName)) { + throw new Error(`Kiro reserves the tool name ${JSON.stringify(wireName)}`); } const existing = wireToKiro.get(wireName); if (existing) return existing; diff --git a/src/adapters/kiro/adapter.ts b/src/adapters/kiro/adapter.ts index b4d51979dd2..eaaf4cebcb9 100644 --- a/src/adapters/kiro/adapter.ts +++ b/src/adapters/kiro/adapter.ts @@ -22,6 +22,7 @@ import { fetchKiroWithRetry } from "../kiro-retry"; import { fingerprint, invocationId, osTag } from "../kiro-wire"; import { hasTrailingDeliveredFinalAnswer } from "./conversation"; import { buildKiroPayload } from "./payload"; +import type { KiroStructuredOutputSpec } from "./structured-output"; import { jsonStringSerializedUtf8Bytes, parseKiroStream, @@ -60,6 +61,7 @@ export function createKiroAdapter(provider: OcxProviderConfig): ProviderAdapter let toolNameMap: Map | undefined; let conversationId: string | undefined; let completionMode: KiroCompletionMode = "disabled"; + let structuredOutput: KiroStructuredOutputSpec | undefined; let requestSnapshot: OcxParsedRequest | undefined; let firstRequestBodyBytes = 0; let requestAbortSignal: AbortSignal | undefined; @@ -88,6 +90,10 @@ export function createKiroAdapter(provider: OcxProviderConfig): ProviderAdapter requestOnPhysicalSend?.({ ordinal, ...(recovery ? { recovery } : {}) }); }; + /** + * Build the Kiro request for a parsed turn, optionally forcing the completion mode (the bounded + * retry passes `text_fallback`), and capture the state the stream parser needs. + */ const build = async ( parsed: OcxParsedRequest, forcedCompletionMode?: KiroCompletionMode, @@ -96,6 +102,7 @@ export function createKiroAdapter(provider: OcxProviderConfig): ProviderAdapter nameMap: Map; conversationId: string; completionMode: KiroCompletionMode; + structuredOutput?: KiroStructuredOutputSpec; inputTokens: number; contextInputEstimate: number; }> => { @@ -154,6 +161,7 @@ export function createKiroAdapter(provider: OcxProviderConfig): ProviderAdapter region, requestedModel: parsed.modelId, completionMode: built.completionMode, + structuredOutput: built.structuredOutput !== undefined, bodyBytes: new TextEncoder().encode(body).length, messageCount: kiroPayloadMessages(parsed).length, toolCount: parsed.context.tools?.length ?? 0, @@ -173,6 +181,7 @@ export function createKiroAdapter(provider: OcxProviderConfig): ProviderAdapter nameMap: built.nameMap, conversationId: built.conversationId, completionMode: built.completionMode, + ...(built.structuredOutput ? { structuredOutput: built.structuredOutput } : {}), inputTokens: estimateKiroInputTokens(parsed), contextInputEstimate, }; @@ -297,6 +306,7 @@ export function createKiroAdapter(provider: OcxProviderConfig): ProviderAdapter toolNameMap = built.nameMap; conversationId = built.conversationId; completionMode = built.completionMode; + structuredOutput = built.structuredOutput; // Runtime callbacks belong to the original request, not the retry payload. requestSnapshot = structuredClone(Object.fromEntries( Object.entries(parsed).filter(([, value]) => typeof value !== "function"), @@ -306,6 +316,7 @@ export function createKiroAdapter(provider: OcxProviderConfig): ProviderAdapter return built.request; }, + /** Parse a streaming Kiro response, with the bounded retry wired to this request. */ parseStream(response: Response, budget: TranslatorBudget): AsyncGenerator { return parseKiroStream( response, @@ -318,6 +329,7 @@ export function createKiroAdapter(provider: OcxProviderConfig): ProviderAdapter completionMode, completionMode === "required" ? fallbackFactory : undefined, contextInputEstimate, + structuredOutput, ); }, @@ -345,9 +357,11 @@ export function createKiroAdapter(provider: OcxProviderConfig): ProviderAdapter return safeKiroHttpErrorMessage(status, headers, payloadText); }, - // Kiro always returns an event stream, including for non-streaming Responses requests. Drain - // the decoder into a budget-owned batch so an upstream stream cannot grow this array without - // bound while the caller waits for the complete JSON response. + /** + * Kiro always returns an event stream, including for non-streaming Responses requests. Drain + * the decoder into a budget-owned batch so an upstream stream cannot grow this array without + * bound while the caller waits for the complete JSON response. + */ async parseResponse(response: Response, budget: TranslatorBudget): Promise { const events: AdapterEvent[] = []; try { @@ -362,6 +376,7 @@ export function createKiroAdapter(provider: OcxProviderConfig): ProviderAdapter completionMode, completionMode === "required" ? fallbackFactory : undefined, contextInputEstimate, + structuredOutput, )) { retainTranslatedEvent(e, budget, events.at(-1)); events.push(e); diff --git a/src/adapters/kiro/conversation.ts b/src/adapters/kiro/conversation.ts index 0a6e742e822..f1aa256d6f0 100644 --- a/src/adapters/kiro/conversation.ts +++ b/src/adapters/kiro/conversation.ts @@ -7,6 +7,7 @@ import type { import type { KiroImage } from "../kiro-images"; import type { KiroHistoryEntry, KiroToolResult, KiroToolUse } from "./wire"; +/** Refuse request controls the Kiro wire cannot honour (forced tool choice, service tiers) before a request is built. */ export function validateKiroCapabilities(parsed: OcxParsedRequest): void { const choice = parsed.options.toolChoice; if (choice !== undefined && choice !== "auto" && choice !== "none") { @@ -15,12 +16,14 @@ export function validateKiroCapabilities(parsed: OcxParsedRequest): void { if (parsed.options.serviceTier !== undefined) { throw new Error("Kiro does not support service tiers"); } - // Structured output is a real contract Kiro cannot honour: the wire has no - // schema-constrained response mode, so a caller expecting parseable JSON would receive - // prose and fail downstream. Refuse it. + // Structured output is a real contract the wire has no response mode for, so it is not + // validated here: buildKiroPayload emulates it through the private structured-output tool + // (structured-output.ts), and the stream relays only JSON that tool or the bounded retry + // produced, or fails the turn. It used to be refused at this point, which kept the + // "no prose for a JSON caller" promise by failing every such turn instead. // - // The rest of the Responses `text` object is not that. `text.verbosity` is a length - // preference and `text.format: {type:"text"}` is ordinary prose — the default output + // The rest of the Responses `text` object is not a contract at all. `text.verbosity` is a + // length preference and `text.format: {type:"text"}` is ordinary prose — the default output // mode, which no capability flag governs and every correct client may send. Testing // `_rawBody.text !== undefined` refused those turns for the mere PRESENCE of the key, // the same mistake db040e70f removed one condition earlier where a permissive @@ -30,9 +33,6 @@ export function validateKiroCapabilities(parsed: OcxParsedRequest): void { // buildKiroPayload composes conversationState field by field from `parsed` and never // spreads `_rawBody`, so a tolerated control is dropped by construction. The test // asserts that absence so it stays true. - if (parsed._structuredOutput) { - throw new Error("Kiro does not support Responses structured output"); - } } export type KiroTurn = diff --git a/src/adapters/kiro/payload.ts b/src/adapters/kiro/payload.ts index f23658b21e4..f3221ed37c4 100644 --- a/src/adapters/kiro/payload.ts +++ b/src/adapters/kiro/payload.ts @@ -43,6 +43,13 @@ import { kiroReasoningContent, KIRO_NATIVE_EFFORTS, } from "./reasoning"; +import { + kiroStructuredOutputInstructions, + kiroStructuredOutputRetryMessage, + kiroStructuredOutputSpec, + kiroStructuredOutputTool, + type KiroStructuredOutputSpec, +} from "./structured-output"; import { kiroPayloadMessages, userContentText } from "./usage"; import { kiroToolWireNames, @@ -115,6 +122,7 @@ export function buildKiroPayload( nameMap: Map; conversationId: string; completionMode: KiroCompletionMode; + structuredOutput?: KiroStructuredOutputSpec; } { validateKiroCapabilities(parsed); const modelId = mapModelId(parsed.modelId); @@ -132,11 +140,24 @@ export function buildKiroPayload( // happens before the turn list exists. `forcedCompletionMode` still wins: the fallback retry // passes "text_fallback" explicitly and must not be silently downgraded. const trailingDeliveredAnswer = hasTrailingDeliveredFinalAnswer(kiroPayloadMessages(parsed), parsed); + // Structured output replaces the completion tool as the turn's terminal channel, and it needs + // that channel even on a tool-free turn: ordinary text is never an acceptable final answer for a + // caller that asked for JSON, so the turn is always "required" and the stream validates whatever + // arrives. Both tools at once would offer the model a prose exit from the JSON contract. + // + // It does not yield to `trailingDeliveredAnswer` the way the completion contract does. The + // adapter's `localTerminal` already answers such a turn without sending anything, and a disabled + // mode here would relay unvalidated text if any caller ever built that payload directly. + const structuredOutput = kiroStructuredOutputSpec(parsed); const completionMode: KiroCompletionMode = forcedCompletionMode - ?? (ordinaryTools.length > 0 && !trailingDeliveredAnswer ? "required" : "disabled"); + ?? (structuredOutput || (ordinaryTools.length > 0 && !trailingDeliveredAnswer) ? "required" : "disabled"); + const terminalTool = structuredOutput ? kiroStructuredOutputTool(structuredOutput) : kiroCompletionTool(); + const retryMessage = structuredOutput + ? kiroStructuredOutputRetryMessage(ordinaryTools.length > 0) + : KIRO_COMPLETION_RETRY_MESSAGE; const kiroTools = completionMode === "disabled" ? ordinaryTools - : [...ordinaryTools, kiroCompletionTool()]; + : [...ordinaryTools, terminalTool]; const nameMap = toolContext.nameMap; const systemParts: string[] = []; const injectedChars = { value: 0 }; @@ -176,15 +197,27 @@ export function buildKiroPayload( const codeModeExecName = emittedCodeModeExec && !emittedShellBridge ? emittedAlias(emittedCodeModeExec) : undefined; - const toolCatalogNudge = buildNonOpenAIToolCatalogNudgeFromNames( - kiroToolWireNames(kiroTools), - name => advertisedAlias.get(name) ?? name, - codeModeExecName, - ); + // A tool-free structured-output turn advertises only its output channel. The catalog nudge is a + // work-tool contract (shell, file search, result counting) and would describe tools that do not + // exist, so it is reserved for turns that carry real tools. + const toolCatalogNudge = structuredOutput && ordinaryTools.length === 0 + ? undefined + : buildNonOpenAIToolCatalogNudgeFromNames( + kiroToolWireNames(kiroTools), + name => advertisedAlias.get(name) ?? name, + codeModeExecName, + ); const boundedNudge = toolCatalogNudge ? boundedInjectedInstruction(toolCatalogNudge, injectedChars) : undefined; if (boundedNudge) systemParts.push(boundedNudge); if (completionMode !== "disabled") { - const boundedCompletion = boundedInjectedInstruction(KIRO_COMPLETION_INSTRUCTIONS, injectedChars); + const contract = structuredOutput + ? kiroStructuredOutputInstructions( + structuredOutput, + ordinaryTools.length > 0, + MAX_KIRO_INJECTED_INSTRUCTION_CHARS - injectedChars.value, + ) + : KIRO_COMPLETION_INSTRUCTIONS; + const boundedCompletion = boundedInjectedInstruction(contract, injectedChars); if (boundedCompletion) systemParts.push(boundedCompletion); } const systemPrefix = systemParts.length > 0 ? `${systemParts.join("\n\n")}\n\n` : ""; @@ -364,7 +397,7 @@ export function buildKiroPayload( // structurally valid, but carrying no instruction to resume. const trailing = turns.at(-1); if (trailing?.kind === "assistant") { - const resumeText = completionMode === "text_fallback" ? KIRO_COMPLETION_RETRY_MESSAGE : KIRO_CONTINUATION_MESSAGE; + const resumeText = completionMode === "text_fallback" ? retryMessage : KIRO_CONTINUATION_MESSAGE; turns.push({ kind: "user", content: trailing.finalAnswer ? KIRO_ANSWER_DELIVERED_MESSAGE : resumeText, @@ -425,8 +458,8 @@ export function buildKiroPayload( // Never append the retry instruction onto the answer-delivered acknowledgement: it exists // precisely to avoid asking a finished turn for another completion call, and appending here // would reinstate the loop it prevents. - if (currentUim.content !== KIRO_COMPLETION_RETRY_MESSAGE && !answerDeliveredAck) { - currentUim.content = appendTurnText(currentUim.content, KIRO_COMPLETION_RETRY_MESSAGE); + if (currentUim.content !== retryMessage && !answerDeliveredAck) { + currentUim.content = appendTurnText(currentUim.content, retryMessage); } } else if ( !currentUim.userInputMessageContext?.toolResults @@ -464,5 +497,5 @@ export function buildKiroPayload( } } if (profileArn) payload.profileArn = profileArn; - return { payload, nameMap, conversationId, completionMode }; + return { payload, nameMap, conversationId, completionMode, ...(structuredOutput ? { structuredOutput } : {}) }; } diff --git a/src/adapters/kiro/stream.ts b/src/adapters/kiro/stream.ts index 72501ecaa6b..b0f7db6a02a 100644 --- a/src/adapters/kiro/stream.ts +++ b/src/adapters/kiro/stream.ts @@ -6,7 +6,11 @@ import { } from "../../lib/translator-budget"; import type { AdapterEvent, OcxUsage } from "../../types"; import { recordKiroCalibration, rekeyKiroCalibration } from "../kiro-calibration"; -import { KIRO_COMPLETION_TOOL_NAME, type KiroCompletionMode } from "../kiro-constants"; +import { + KIRO_COMPLETION_TOOL_NAME, + KIRO_STRUCTURED_OUTPUT_TOOL_NAME, + type KiroCompletionMode, +} from "../kiro-constants"; import { classifyKiroEventError, classifyKiroHttpError, @@ -22,6 +26,16 @@ import { isCompleteKiroToolInput, kiroTruncationErrorMessage } from "../kiro-tru import { isValidKiroConversationId } from "../kiro-wire"; import { readDisplaySafeErrorPayloadText } from "../upstream-http-error"; import { tagKiroReasoningBlob } from "./reasoning"; +import { + isKiroStructuredCleanStop, + KIRO_STRUCTURED_EMPTY_FAILURE, + KIRO_STRUCTURED_TEXT_FAILURE, + kiroStructuredOutputFailure, + kiroStructuredStopFailure, + parseKiroStructuredText, + parseKiroStructuredToolInput, + type KiroStructuredOutputSpec, +} from "./structured-output"; import { estimateKiroTokens, kiroUpstreamContextWindow } from "./usage"; // Stream parsing (shared by parseStream + parseResponse) @@ -258,6 +272,10 @@ export function isRetryableKiroStreamCatchError(err: unknown, emittedOutput: boo /** Native clean-stop reason eligible for bounded private-completion validation. */ const KIRO_END_TURN_STOP_REASON = "END_TURN"; +/** + * Run one attempt through `parseKiroAttemptEvents` while holding commentary until completion is + * validated, and hand the held events back to the caller so a bounded retry can release or drop them. + */ async function* parseKiroAttempt( response: Response, budget: TranslatorBudget, @@ -271,6 +289,7 @@ async function* parseKiroAttempt( /** True when an earlier attempt has output that must survive a failed retry. */ priorEmittedOutput = false, priorAttempt?: KiroAttemptResult, + structuredOutput?: KiroStructuredOutputSpec, ): AsyncGenerator { // Hold commentary through completion validation; tools and failures still release progress. const deferred: AdapterEvent[] = []; @@ -302,7 +321,13 @@ async function* parseKiroAttempt( contextInputEstimate, priorEmittedOutput, priorAttempt, + structuredOutput, ); + // Held text that no terminal claimed is prose. On an ordinary turn that is still real progress + // and is released before a failure; for a structured-output caller it is exactly the wrong-format + // answer the contract exists to withhold, so only its non-text events are released. A real tool + // call has already flushed its leading commentary by the time anything reaches these drains. + const dropHeldProse = structuredOutput !== undefined; let handedOff = false; try { const result = yield* attempt; @@ -315,8 +340,8 @@ async function* parseKiroAttempt( if (staged && !result.needsFallback) { recordKiroCalibration(staged.conversationId, staged.estimated, staged.charged); } - if (priorAttempt) yield* priorAttempt.drainDeferred(); - if (!result.needsFallback) yield* drainDeferred(); + if (priorAttempt) yield* priorAttempt.drainDeferred(dropHeldProse); + if (!result.needsFallback) yield* drainDeferred(dropHeldProse); handedOff = true; return { ...result, drainDeferred, @@ -348,6 +373,7 @@ async function* parseKiroAttemptEvents( contextInputEstimate?: number, priorEmittedOutput = false, priorAttempt?: KiroAttemptResult, + structuredOutput?: KiroStructuredOutputSpec, ): AsyncGenerator { const emptyResult = (): KiroAttemptParseResult => ({ assistantText: "", sawReasoning: false }); // Every early return below is a failure path that stages nothing; only the completion path @@ -377,6 +403,12 @@ async function* parseKiroAttemptEvents( let sawRealTool = false; let completionAnswer: string | undefined; let completionCalls = 0; + // A structured-output call whose input failed validation on the FIRST attempt. It earns the one + // bounded retry instead of failing the turn, because the likely fault (a missing required key, a + // wrong root type) is one the model can correct when told; the retry itself gets no second one. + let structuredRetry = false; + // Structured output replaces the completion tool as the turn's terminal channel (payload.ts). + const terminalToolName = structuredOutput ? KIRO_STRUCTURED_OUTPUT_TOOL_NAME : KIRO_COMPLETION_TOOL_NAME; let authoritativeUsage: OcxUsage | undefined; let providerCredits: number | undefined; let stopReason: string | undefined; @@ -445,6 +477,7 @@ async function* parseKiroAttemptEvents( }; }; + /** The terminal for a classified upstream error frame; retryable only while nothing has been emitted. */ const classifiedTerminal = (failure: KiroErrorClassification): AdapterEvent => { // Upstream exception/error frames can arrive after commentary was already staged (and will be // flushed before this terminal is yielded). Replaying after that content would duplicate it. @@ -470,8 +503,16 @@ async function* parseKiroAttemptEvents( }; }; + /** + * The terminal for a protocol violation. A malformed private-tool answer on the bounded retry is + * a retryable incomplete for ordinary turns and a non-retryable failure for structured output; + * everything else is a non-retryable 502. + */ const protocolTerminal = (message: string, malformedCompletion = false): AdapterEvent => { if (mode === "text_fallback" && malformedCompletion) { + // The bounded retry was the structured contract's second ask; a malformed answer on it ends + // the turn with the non-retryable structured failure, never a replayable incomplete. + if (structuredOutput) return kiroStructuredOutputFailure(message, usage()); return retryableKiroIncomplete( "malformed_kiro_completion", message, @@ -500,13 +541,23 @@ async function* parseKiroAttemptEvents( console.warn("[kiro] dropped the private final-answer call; the turn also carried a real tool call"); }; + /** + * Mark a tool call as the turn's private terminal tool or an ordinary client tool, and return a + * terminal when the call names a reserved tool this turn did not advertise or completion is disabled. + */ const classifyTool = ( tool: { id: string; name: string; chunks: string[]; completion: boolean }, ): AdapterEvent | undefined => { - if (tool.name !== KIRO_COMPLETION_TOOL_NAME) { + if (tool.name !== KIRO_COMPLETION_TOOL_NAME && tool.name !== KIRO_STRUCTURED_OUTPUT_TOOL_NAME) { tool.completion = false; return undefined; } + // Both private names are reserved against client tools (kiro-wire.ts), so a call to the one + // this turn did not advertise can only be the model inventing it. Relaying it as a client + // tool would hand the client a call to a tool it never declared. + if (tool.name !== terminalToolName) { + return protocolTerminal("Kiro returned a reserved private tool this turn did not advertise"); + } if (mode === "disabled") { return protocolTerminal("Kiro returned the reserved private final-answer tool while explicit completion was disabled"); } @@ -538,6 +589,11 @@ async function* parseKiroAttemptEvents( return [{ type: "heartbeat" }]; }; + /** + * Account for one content event and route it by completion mode: emitted directly when completion + * is disabled, held until validation in `required` mode, and retained for the bounded retry's + * decision in `text_fallback` mode. + */ const stage = (event: AdapterEvent): AdapterEvent[] => { if (event.type === "text_delta") { const nextAssistantTextBytes = appendedUtf8Bytes(assistantText, assistantTextBytes, event.text); @@ -584,7 +640,12 @@ async function* parseKiroAttemptEvents( return mode === "required" ? defer(event) : [event]; }; + /** Parse the private terminal tool's input into the answer text: the validated JSON for structured output, otherwise the completion tool's `answer`. */ const parseCompletion = (chunks: string[]): string | Error => { + if (structuredOutput) { + const parsed = parseKiroStructuredToolInput(structuredOutput, chunks.join("")); + return "json" in parsed ? parsed.json : new Error(parsed.error); + } const raw = chunks.join("").trim(); let value: unknown; try { @@ -602,6 +663,10 @@ async function* parseKiroAttemptEvents( return answer; }; + /** + * Close the open tool call: record a private terminal answer (or schedule its bounded retry), or + * emit a real client tool call. Returns a terminal when the call cannot be accepted. + */ const flushOpen = (): { events: AdapterEvent[]; terminal?: AdapterEvent } => { if (!open) return { events: [] }; const tool = open; @@ -609,6 +674,14 @@ async function* parseKiroAttemptEvents( closeOpenCall(); const input = tool.chunks.join(""); if (!isCompleteKiroToolInput(input)) { + // A truncated structured answer is the same correctable fault as one that fails to parse: it + // earns the bounded retry, decided with the stop reason at the end of the attempt. The retry + // itself (text_fallback) ends non-retryably through protocolTerminal. + if (structuredOutput && mode === "required" && tool.completion) { + completionCalls++; + structuredRetry = true; + return { events: [] }; + } return { events: [], terminal: protocolTerminal(kiroTruncationErrorMessage("incomplete tool input JSON"), tool.completion) }; } if (tool.completion) { @@ -617,6 +690,11 @@ async function* parseKiroAttemptEvents( return { events: [], terminal: protocolTerminal("Kiro returned more than one private final-answer tool call", true) }; } const answer = parseCompletion(tool.chunks); + if (answer instanceof Error && structuredOutput) { + if (mode !== "required") return { events: [], terminal: kiroStructuredOutputFailure(answer.message, usage()) }; + structuredRetry = true; + return { events: [] }; + } if (answer instanceof Error) return { events: [], terminal: protocolTerminal(answer.message, true) }; if (sawRealTool) { noteDroppedCompletion(); @@ -796,7 +874,11 @@ async function* parseKiroAttemptEvents( } if (open) { const input = open.chunks.join(""); - if (!isCompleteKiroToolInput(input)) { + // A truncated structured answer still open at end of stream goes through flushOpen like one + // closed by a tool-stop frame: the bounded retry, decided with the stop reason below. Every + // other open call keeps this guard's terminal. + const structuredTruncation = structuredOutput !== undefined && mode === "required" && open.completion; + if (!structuredTruncation && !isCompleteKiroToolInput(input)) { const privateTool = open.completion; open = null; return { @@ -862,6 +944,7 @@ async function* parseKiroAttemptEvents( sawRealTool, completionCalls, nativeCompletionStop, + ...(structuredOutput ? { structuredOutput: true, structuredRetry } : {}), ...(stopReason !== undefined ? { stopReason } : {}), assistantChars: assistantText.length, }); @@ -876,8 +959,10 @@ async function* parseKiroAttemptEvents( // the first attempt's held progress is released rather than superseded (retry text normally // replaces it), and the filtered retry text follows as the commentary it was staged as. if (completionAnswer === undefined && !sawRealTool && isKiroContentFilterStop(normalizedStopReason)) { - if (priorAttempt) yield* priorAttempt.drainDeferred(false); - yield* emitRetained(fallbackEvents); + // A structured-output caller never receives prose: text from either attempt is dropped and + // only their non-text events and the filter terminal remain. + if (priorAttempt) yield* priorAttempt.drainDeferred(structuredOutput !== undefined); + yield* structuredOutput ? consumeSupersededByCompletion(fallbackEvents) : emitRetained(fallbackEvents); return { assistantText, sawReasoning, @@ -885,7 +970,11 @@ async function* parseKiroAttemptEvents( }; } if (priorAttempt) { - yield* priorAttempt.drainDeferred(completionAnswer !== undefined || (sawText && !sawRealTool)); + // Structured output supersedes first-attempt prose on every outcome but a real tool call: + // success replaces it, and failure must not relay it (see `dropHeldProse`). + yield* priorAttempt.drainDeferred(structuredOutput + ? !sawRealTool + : completionAnswer !== undefined || (sawText && !sawRealTool)); } if (completionAnswer !== undefined) { yield* consumeSupersededByCompletion(fallbackEvents); @@ -904,6 +993,25 @@ async function* parseKiroAttemptEvents( terminal: { type: "done", usage: finalUsage, endTurn: false, ...(finalProviderState ? { providerState: finalProviderState } : {}) }, }; } + if (sawText && structuredOutput) { + // Same clean-stop rule as the first attempt: text a MAX_TOKENS cut ended is not an answer, + // even when its prefix parses. + const cleanStop = isKiroStructuredCleanStop(normalizedStopReason); + const json = cleanStop ? parseKiroStructuredText(structuredOutput, assistantText) : undefined; + yield* consumeSupersededByCompletion(fallbackEvents); + if (json === undefined) { + const message = !cleanStop && normalizedStopReason + ? kiroStructuredStopFailure(normalizedStopReason) + : KIRO_STRUCTURED_TEXT_FAILURE; + return { assistantText, sawReasoning, terminal: kiroStructuredOutputFailure(message, finalUsage) }; + } + yield { type: "text_delta", text: json, phase: "final_answer" }; + return { + assistantText, + sawReasoning, + terminal: { type: "done", usage: finalUsage, endTurn: true, ...(finalProviderState ? { providerState: finalProviderState } : {}) }, + }; + } if (sawText) { for (const event of fallbackEvents) { try { @@ -919,6 +1027,13 @@ async function* parseKiroAttemptEvents( terminal: { type: "done", usage: finalUsage, endTurn: true, ...(finalProviderState ? { providerState: finalProviderState } : {}) }, }; } + if (structuredOutput) { + // The model was already asked twice inside this turn; an empty or reasoning-only retry ends + // the structured contract with its non-retryable failure, never a retryable empty fallback + // that would let the proxy replay the whole turn. + yield* consumeSupersededByCompletion(fallbackEvents); + return { assistantText, sawReasoning, terminal: kiroStructuredOutputFailure(KIRO_STRUCTURED_EMPTY_FAILURE, finalUsage) }; + } yield* emitRetained(fallbackEvents); return { assistantText, @@ -962,6 +1077,34 @@ async function* parseKiroAttemptEvents( terminal: { type: "done", usage: finalUsage, endTurn: false, ...(finalProviderState ? { providerState: finalProviderState } : {}) }, }; } + if (mode === "required" && structuredOutput) { + // Any stop reason other than a clean one falls through to its own handling below: a failed + // structured answer under MAX_TOKENS, a content filter or an exhausted context window earns no + // retry, and a MAX_TOKENS cut is never mistaken for an answer even when its prefix parses. + const cleanStop = isKiroStructuredCleanStop(normalizedStopReason); + if (structuredRetry && cleanStop) { + return { assistantText, sawReasoning, needsFallback: true, usage: finalUsage, providerState: finalProviderState }; + } + // A model that wrote the requested JSON as text, and stopped cleanly, has answered: accept it + // without a second paid request. + const json = sawText && cleanStop && completionCalls === 0 + ? parseKiroStructuredText(structuredOutput, assistantText) + : undefined; + if (json !== undefined) { + yield* consumeSupersededByCompletion(deferred); + yield { type: "text_delta", text: json, phase: "final_answer" }; + return { + assistantText, + sawReasoning, + terminal: { type: "done", usage: finalUsage, endTurn: true, ...(finalProviderState ? { providerState: finalProviderState } : {}) }, + }; + } + // A clean stop without an accepted answer, prose or nothing at all, earns the one bounded + // retry: an empty END_TURN is as recoverable as prose, and the caller still needs JSON. + if (cleanStop) { + return { assistantText, sawReasoning, needsFallback: true, usage: finalUsage, providerState: finalProviderState }; + } + } if (mode === "required" && nativeCompletionStop) { return { assistantText, @@ -1101,6 +1244,10 @@ async function* parseKiroAttemptEvents( } } +/** + * Parse a Kiro response into adapter events, running at most one bounded retry through + * `fallbackFactory` when the first attempt ends without a validated answer. + */ export async function* parseKiroStream( response: Response, budget: TranslatorBudget, @@ -1112,8 +1259,11 @@ export async function* parseKiroStream( completionMode: KiroCompletionMode = "disabled", fallbackFactory?: KiroFallbackFactory, contextInputEstimate?: number, + structuredOutput?: KiroStructuredOutputSpec, ): AsyncGenerator { const contextWindowState: KiroContextWindowState = { value: contextWindow }; + // See `dropHeldProse` in parseKiroAttempt: a structured-output caller never receives held prose. + const dropHeldProse = structuredOutput !== undefined; const firstResult = yield* parseKiroAttempt( response, budget, @@ -1125,6 +1275,8 @@ export async function* parseKiroStream( conversationId, contextInputEstimate, false, + undefined, + structuredOutput, ); try { if (!firstResult.needsFallback) { @@ -1132,7 +1284,7 @@ export async function* parseKiroStream( return; } if (!fallbackFactory) { - yield* firstResult.drainDeferred(); + yield* firstResult.drainDeferred(dropHeldProse); yield retryableKiroIncomplete( "uncompleted_kiro_response", "Kiro produced progress without an explicit final answer and no bounded retry transport was available", @@ -1157,7 +1309,7 @@ export async function* parseKiroStream( budget, ); } catch (err) { - yield* firstResult.drainDeferred(); + yield* firstResult.drainDeferred(dropHeldProse); firstAssistantText = ""; firstResult.assistantText = ""; firstResult.releaseRetained(); @@ -1189,7 +1341,7 @@ export async function* parseKiroStream( firstResult.releaseCollectors(); fallback.releaseRequestBody?.(); if (!fallback.response.ok) { - yield* firstResult.drainDeferred(); + yield* firstResult.drainDeferred(dropHeldProse); const payload = await readDisplaySafeErrorPayloadText(fallback.response, fallback.abortSignal); const failure = classifyKiroHttpError(fallback.response.status, fallback.response.headers, payload); yield { @@ -1217,6 +1369,7 @@ export async function* parseKiroStream( // Failed validation will release the held first-attempt progress. priorEmittedOutput, firstResult, + structuredOutput, ); try { if (!secondResult.terminal) { diff --git a/src/adapters/kiro/structured-output-json.ts b/src/adapters/kiro/structured-output-json.ts new file mode 100644 index 00000000000..5dbe486e064 --- /dev/null +++ b/src/adapters/kiro/structured-output-json.ts @@ -0,0 +1,191 @@ +/** + * Lexeme-preserving JSON reading and exact decimal arithmetic for Kiro strict structured output. + * + * Strict validation has to judge the bytes it relays. `JSON.parse` rounds every number to a double, + * so `9007199254740993` would compare equal to a `const` of `9007199254740992` and + * `10.0000000000000001` would pass `maximum: 10`, while the client receives the original text. + * This reader keeps each number's lexeme, and the decimal helpers compare and divide those lexemes + * exactly, without converting them to doubles. + */ + +/** One parsed JSON value with its byte span in the source text. */ +export type KiroJsonValue = { start: number; end: number } & ( + | { kind: "null" } + | { kind: "boolean"; value: boolean } + | { kind: "number"; lexeme: string } + | { kind: "string"; value: string } + | { kind: "array"; items: KiroJsonValue[] } + | { kind: "object"; entries: Map } +); + +/** A parsed document and how many values it holds, which sizes the validation budget. */ +export interface KiroJsonDocument { + value: KiroJsonValue; + nodes: number; +} + +/** Deepest array/object nesting the reader accepts; deeper values are reported as non-conforming. */ +export const KIRO_JSON_MAX_NESTING = 512; + +const WHITESPACE = new Set([" ", "\t", "\n", "\r"]); +const NUMBER = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/y; + +/** + * Read `text` into a lexeme-preserving tree, or undefined when it is not JSON or nests deeper than + * {@link KIRO_JSON_MAX_NESTING}. Duplicate object keys keep the last value, as `JSON.parse` does. + */ +export function readKiroJson(text: string): KiroJsonDocument | undefined { + let pos = 0; + let nodes = 0; + const skip = (): void => { + while (pos < text.length && WHITESPACE.has(text[pos]!)) pos++; + }; + const readString = (): string => { + const start = pos; + pos++; + while (pos < text.length && text[pos] !== '"') pos += text[pos] === "\\" ? 2 : 1; + if (pos >= text.length) throw new SyntaxError("unterminated string"); + pos++; + return JSON.parse(text.slice(start, pos)) as string; + }; + const read = (depth: number): KiroJsonValue => { + if (depth > KIRO_JSON_MAX_NESTING) throw new RangeError("nesting"); + skip(); + nodes++; + const start = pos; + const ch = text[pos]; + if (ch === '"') { + const value = readString(); + return { kind: "string", value, start, end: pos }; + } + if (ch === "[" || ch === "{") { + pos++; + skip(); + const close = ch === "[" ? "]" : "}"; + const items: KiroJsonValue[] = []; + const entries = new Map(); + if (text[pos] === close) pos++; + else { + for (;;) { + if (ch === "[") items.push(read(depth + 1)); + else { + if (text[pos] !== '"') throw new SyntaxError("expected a key"); + const key = readString(); + skip(); + if (text[pos++] !== ":") throw new SyntaxError("expected a colon"); + entries.set(key, read(depth + 1)); + } + skip(); + const next = text[pos++]; + if (next === close) break; + if (next !== ",") throw new SyntaxError("expected a comma"); + skip(); + } + } + return ch === "[" + ? { kind: "array", items, start, end: pos } + : { kind: "object", entries, start, end: pos }; + } + for (const [word, value] of [["true", true], ["false", false], ["null", null]] as const) { + if (text.startsWith(word, pos)) { + pos += word.length; + return value === null ? { kind: "null", start, end: pos } : { kind: "boolean", value, start, end: pos }; + } + } + NUMBER.lastIndex = pos; + const match = NUMBER.exec(text); + if (!match) throw new SyntaxError("unexpected token"); + pos += match[0].length; + return { kind: "number", lexeme: match[0], start, end: pos }; + }; + try { + const value = read(0); + skip(); + return pos === text.length ? { value, nodes } : undefined; + } catch { + return undefined; + } +} + +/** + * An exact decimal: `sign × 0.digits × 10^exponent`, with `digits` free of leading and trailing + * zeros. Zero has sign 0 and empty digits. The exponent is a bigint because a lexeme such as + * `1e99999999999999999999` is valid JSON. + */ +export interface KiroDecimal { + sign: -1 | 0 | 1; + digits: string; + exponent: bigint; +} + +const DECIMAL = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/; + +/** + * Largest exponent magnitude kept exactly. A model-written exponent can be millions of digits long, + * and converting it to a bigint costs more than linear time, so a longer one saturates here. No + * lexeme within the answer size limits has as many digits, so a saturated value still compares, + * and tests as an integer, exactly as its true value would against any schema bound. + */ +const EXPONENT_LIMIT = 10n ** 15n; + +/** The exponent text as a bigint, saturated at {@link EXPONENT_LIMIT}; linear in its length. */ +function boundedExponent(exp: string): bigint { + const negative = exp.startsWith("-"); + let start = exp.startsWith("-") || exp.startsWith("+") ? 1 : 0; + while (start < exp.length - 1 && exp.charCodeAt(start) === 48) start += 1; + const magnitude = exp.length - start > 15 ? EXPONENT_LIMIT : BigInt(exp.slice(start)); + return negative ? -magnitude : magnitude; +} + +/** The exact value of a JSON number lexeme (or of `String(n)` for a finite double). */ +export function kiroDecimal(lexeme: string): KiroDecimal { + const match = DECIMAL.exec(lexeme); + if (!match) throw new Error(`not a JSON number: ${lexeme}`); + const [, minus, whole = "", fraction = "", exp = "0"] = match; + const all = whole + fraction; + const lead = all.search(/[1-9]/); + if (lead < 0) return { sign: 0, digits: "", exponent: 0n }; + // One backward scan for trailing zeros: `/0+$/` retries a long zero run at every position. + let end = all.length; + while (end > lead && all.charCodeAt(end - 1) === 48) end -= 1; + return { + sign: minus ? -1 : 1, + digits: all.slice(lead, end), + exponent: BigInt(whole.length - lead) + boundedExponent(exp), + }; +} + +/** -1, 0 or 1 as `a` is below, equal to or above `b`. */ +export function compareKiroDecimal(a: KiroDecimal, b: KiroDecimal): number { + if (a.sign !== b.sign) return a.sign < b.sign ? -1 : 1; + if (a.sign === 0) return 0; + let magnitude = 0; + if (a.exponent !== b.exponent) magnitude = a.exponent < b.exponent ? -1 : 1; + // Without trailing zeros, digit strings order as their fractions do: "12" < "123" < "13". + else if (a.digits !== b.digits) magnitude = a.digits < b.digits ? -1 : 1; + return magnitude * a.sign; +} + +/** Whether the decimal is a whole number. */ +export function isKiroDecimalInteger(value: KiroDecimal): boolean { + return value.sign === 0 || BigInt(value.digits.length) <= value.exponent; +} + +/** `base^power mod modulus` by square-and-multiply, so a huge exponent costs only its bit length. */ +function modPow(base: bigint, power: bigint, modulus: bigint): bigint { + let result = 1n % modulus; + let factor = base % modulus; + for (let rest = power; rest > 0n; rest >>= 1n) { + if (rest & 1n) result = (result * factor) % modulus; + factor = (factor * factor) % modulus; + } + return result; +} + +/** Whether the decimal is an integer multiple of the positive integer `divisor`. */ +export function isKiroDecimalMultipleOf(value: KiroDecimal, divisor: bigint): boolean { + if (value.sign === 0) return true; + if (!isKiroDecimalInteger(value)) return false; + const zeros = value.exponent - BigInt(value.digits.length); + return (BigInt(value.digits) % divisor) * modPow(10n, zeros, divisor) % divisor === 0n; +} diff --git a/src/adapters/kiro/structured-output-schema.ts b/src/adapters/kiro/structured-output-schema.ts new file mode 100644 index 00000000000..7dc1159605f --- /dev/null +++ b/src/adapters/kiro/structured-output-schema.ts @@ -0,0 +1,542 @@ +/** + * Full validation for strict structured output (`strict: true`) on Kiro. + * + * A strict caller is promised a value that conforms to the whole schema, which the light root + * checks in structured-output.ts cannot keep: `{"title":42,"extra":true}` passes a root `type` and + * `required` check against a schema that wants a string title and forbids extra keys. Kiro's wire + * cannot constrain decoding, so the emulation enforces the contract afterwards instead. + * + * Every schema this module accepts is validated exactly and in bounded work; anything else is + * refused before a request is sent, because relaying a value the proxy could not check would break + * the promise strict mode makes. The supported subset: + * + * - `type` (a known type or a non-empty array of them), `properties`, `required`, + * `additionalProperties` (boolean or schema), `items` (one schema), `enum`, `const`, `anyOf`. + * A `const` or `enum` literal that contains a number is refused: the request reaches this module + * already parsed, so a literal such as `1.0000000000000001` arrives as `1`, and an exact equality + * check would accept a value the caller excluded. + * - `$ref` only as `#/$defs/` or `#/definitions/`, naming a definition in the root's + * `$defs`/`definitions`. `$defs`/`definitions` are accepted only at the root. Root-relative + * references (`#`, `#/properties/...`), references into annotations and remote references are + * refused, and so is a cycle of references and `anyOf` branches that consumes none of the value. + * - `minLength`, `maxLength`, `minItems`, `maxItems` as non-negative safe integers. + * - `minimum`, `maximum`, `exclusiveMinimum`, `exclusiveMaximum` as finite numbers that are either + * safe integers or non-integers with at most 15 significant digits. This is a heuristic: a bound + * written with more digits than a double holds can still arrive rounded, so a value within about + * one unit in the last place of the bound may be judged on the wrong side. The model's numbers + * are compared against their original text exactly. + * - `multipleOf`, `pattern` and `patternProperties` are refused. `multipleOf` is exact equality on a + * possibly rounded number, and a regular expression evaluated against model output can take + * super-linear time on the request path. + * + * Annotations (`title`, `description`, `default`, ...) carry no assertion and are ignored. + * + * Work is bounded per value: one validation may visit at most + * {@link KIRO_STRICT_BASE_VISITS} schema nodes plus {@link KIRO_STRICT_VISITS_PER_VALUE_NODE} per JSON + * value in the answer, and nest at most {@link KIRO_STRICT_MAX_DEPTH} schema levels deep across + * references, `anyOf` branches, properties and items. A value that needs more is reported as + * non-conforming, so a recursive `anyOf` cannot branch exponentially and a deep value cannot + * exhaust the stack. + */ +import { + compareKiroDecimal, + isKiroDecimalInteger, + kiroDecimal, + type KiroDecimal, + type KiroJsonValue, +} from "./structured-output-json"; + +type JsonRecord = Record; + +/** Schema-node visits every validation may spend regardless of the value's size. */ +export const KIRO_STRICT_BASE_VISITS = 10_000; +/** Extra schema-node visits granted per JSON value in the answer. */ +export const KIRO_STRICT_VISITS_PER_VALUE_NODE = 16; +/** One depth limit across references, anyOf branches, properties and items. */ +export const KIRO_STRICT_MAX_DEPTH = 128; +/** Deepest schema nesting accepted at the boundary. */ +const MAX_SCHEMA_DEPTH = 64; +/** Schema nodes accepted at the boundary; preparation work is linear in this. */ +const MAX_SCHEMA_NODES = 10_000; +/** JSON values accepted across every `const`/`enum` literal of one schema. */ +const MAX_LITERAL_NODES = 10_000; + +const KNOWN_TYPES = new Set(["object", "array", "string", "number", "integer", "boolean", "null"]); +const ANNOTATIONS = new Set([ + "title", "description", "default", "examples", "$schema", "$comment", "deprecated", "readOnly", "writeOnly", +]); +const COUNTS = ["minLength", "maxLength", "minItems", "maxItems"] as const; +const BOUNDS = ["minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum"] as const; +const REF = /^#\/(\$defs|definitions)\/([^/]+)$/; + +/** One schema position, prepared once at the boundary. */ +interface StrictNode { + never?: true; + ref?: StrictNode; + types?: string[]; + enumValues?: unknown[]; + constValue?: { value: unknown }; + anyOf?: StrictNode[]; + minLength?: number; + maxLength?: number; + minItems?: number; + maxItems?: number; + minimum?: KiroDecimal; + maximum?: KiroDecimal; + exclusiveMinimum?: KiroDecimal; + exclusiveMaximum?: KiroDecimal; + items?: StrictNode; + required?: string[]; + properties?: Map; + closed?: true; + additional?: StrictNode; +} + +/** A plain JSON object: not null and not an array. */ +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +/** + * JSON Schema keyword names a refusal may name. Any other key is the caller's own text and is not + * echoed: a refusal names a keyword only from this fixed vocabulary. + */ +const NAMEABLE_KEYWORDS = new Set([ + "schema", "type", "properties", "required", "additionalProperties", "items", "enum", "const", "anyOf", + "oneOf", "allOf", "not", "if", "then", "else", "$ref", "$defs", "definitions", "$anchor", "$dynamicRef", + "$dynamicAnchor", "$id", "$recursiveRef", "$recursiveAnchor", "multipleOf", "pattern", "patternProperties", + "minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "minLength", "maxLength", "minItems", + "maxItems", "minProperties", "maxProperties", "uniqueItems", "contains", "minContains", "maxContains", + "prefixItems", "additionalItems", "unevaluatedItems", "unevaluatedProperties", "propertyNames", + "dependentRequired", "dependentSchemas", "dependencies", "format", "contentEncoding", + "contentMediaType", "contentSchema", +]); + +class Unsupported extends Error { + readonly keyword: string | undefined; + constructor(keyword: string) { + super("unsupported strict schema keyword"); + this.keyword = NAMEABLE_KEYWORDS.has(keyword) ? keyword : undefined; + } +} + +/** Whether a schema number still states what the caller wrote after JSON.parse rounded it. */ +function isFaithfulNumber(value: unknown): value is number { + if (typeof value !== "number" || !Number.isFinite(value)) return false; + if (Number.isInteger(value)) return Number.isSafeInteger(value); + return kiroDecimal(String(value)).digits.length <= 15; +} + +/** + * The JSON values in a `const`/`enum` literal, or -1 when it contains a number. Iterative, so a deep + * literal is safe; it stops counting at `limit`. + */ +function literalNodes(literal: unknown, limit: number): number { + const stack = [literal]; + let count = 0; + while (stack.length > 0 && count <= limit) { + const item = stack.pop(); + count += 1; + if (typeof item === "number") return -1; + if (Array.isArray(item)) for (const child of item) stack.push(child); + else if (isRecord(item)) for (const child of Object.values(item)) stack.push(child); + } + return count; +} + +/** A prepared strict schema: the root node of its validation graph. */ +interface PreparedSchema { + root: StrictNode; +} + +/** Build the validation graph for `schema`, or throw {@link Unsupported} naming the first keyword it cannot enforce. */ +function prepare(schema: JsonRecord): PreparedSchema { + const definitions = new Map(); + const pending: Array<[StrictNode, unknown]> = []; + for (const section of ["$defs", "definitions"] as const) { + if (!Object.hasOwn(schema, section)) continue; + const defs = schema[section]; + if (!isRecord(defs)) throw new Unsupported(section); + for (const [name, def] of Object.entries(defs)) { + const node: StrictNode = {}; + definitions.set(`${section}/${name}`, node); + pending.push([node, def]); + } + } + const nodes: StrictNode[] = []; + let literalBudget = MAX_LITERAL_NODES; + /** Charge a literal against the schema-wide cap; a number or an oversized literal is refused. */ + const admitLiteral = (literal: unknown, key: string): void => { + const count = literalNodes(literal, literalBudget); + if (count < 0 || count > literalBudget) throw new Unsupported(key); + literalBudget -= count; + }; + + const resolve = (ref: unknown): StrictNode => { + if (typeof ref !== "string") throw new Unsupported("$ref"); + // Percent-decode the whole fragment before reading pointer segments: an encoded "/" is a + // segment separator, so `#/$defs/a%2Fb` points into definition `a`, which is outside the subset. + let decoded: string; + try { + decoded = decodeURIComponent(ref); + } catch { + throw new Unsupported("$ref"); + } + const match = REF.exec(decoded); + if (!match) throw new Unsupported("$ref"); + // JSON Pointer defines only `~0` and `~1`; any other `~` makes the pointer invalid. + if (/~(?![01])/.test(match[2]!)) throw new Unsupported("$ref"); + const name = match[2]!.replaceAll("~1", "/").replaceAll("~0", "~"); + const target = definitions.get(`${match[1]}/${name}`); + if (!target) throw new Unsupported("$ref"); + return target; + }; + + const build = (raw: unknown, depth: number, keyword: string, node: StrictNode = {}, root = false): StrictNode => { + if (depth > MAX_SCHEMA_DEPTH || nodes.length >= MAX_SCHEMA_NODES) throw new Unsupported(keyword); + nodes.push(node); + if (raw === false) node.never = true; + if (typeof raw === "boolean") return node; + if (!isRecord(raw)) throw new Unsupported("schema"); + // Drafts disagree on whether keywords beside `$ref` apply (draft-07 ignores them, 2019-09 and + // later apply them), so a `$ref` with an assertion beside it has no single meaning to enforce. + if (Object.hasOwn(raw, "$ref") + && Object.keys(raw).some(key => key !== "$ref" && key !== "$defs" && key !== "definitions" && !ANNOTATIONS.has(key))) { + throw new Unsupported("$ref"); + } + for (const [key, arg] of Object.entries(raw)) { + if (ANNOTATIONS.has(key)) continue; + switch (key) { + case "$defs": + case "definitions": + if (!root) throw new Unsupported(key); + break; + // A nested `$id` starts a new resource that `#/...` references would resolve against, and + // anchors name targets this resolver does not follow; only a root `$id` (same document) is kept. + case "$id": + if (!root) throw new Unsupported(key); + break; + case "$anchor": + case "$dynamicAnchor": + throw new Unsupported(key); + case "type": { + const types = Array.isArray(arg) ? arg : [arg]; + // Distinct known types only, so a type list is at most seven entries long. + if (types.length === 0 || new Set(types).size !== types.length + || !types.every(t => typeof t === "string" && KNOWN_TYPES.has(t))) throw new Unsupported(key); + node.types = types as string[]; + break; + } + case "properties": + if (!isRecord(arg)) throw new Unsupported(key); + node.properties = new Map(Object.entries(arg).map(([name, child]) => [name, build(child, depth + 1, key)])); + break; + case "required": + if (!(Array.isArray(arg) && arg.every(k => typeof k === "string"))) throw new Unsupported(key); + node.required = arg as string[]; + break; + case "additionalProperties": + if (arg === false) node.closed = true; + else if (arg !== true) node.additional = build(arg, depth + 1, key); + break; + case "items": + if (Array.isArray(arg)) throw new Unsupported(key); + node.items = build(arg, depth + 1, key); + break; + case "enum": + if (!Array.isArray(arg)) throw new Unsupported(key); + admitLiteral(arg, key); + node.enumValues = arg; + break; + case "const": + admitLiteral(arg, key); + node.constValue = { value: arg }; + break; + case "anyOf": + if (!Array.isArray(arg) || arg.length === 0) throw new Unsupported(key); + node.anyOf = arg.map(branch => build(branch, depth + 1, key)); + break; + case "$ref": + node.ref = resolve(arg); + break; + default: + if ((COUNTS as readonly string[]).includes(key)) { + if (!(Number.isSafeInteger(arg) && (arg as number) >= 0)) throw new Unsupported(key); + node[key as (typeof COUNTS)[number]] = arg as number; + } else if ((BOUNDS as readonly string[]).includes(key)) { + if (!isFaithfulNumber(arg)) throw new Unsupported(key); + node[key as (typeof BOUNDS)[number]] = kiroDecimal(String(arg)); + } else throw new Unsupported(key); + } + } + return node; + }; + + for (const [node, def] of pending) build(def, 1, "$ref", node); + const root = build(schema, 0, "schema", {}, true); + assertNoEmptyCycle(nodes); + return { root }; +} + +/** + * Refuse a cycle of `$ref` and `anyOf` edges: following it consumes none of the value, so it can + * never decide anything. Recursion through `properties`, `items` or `additionalProperties` is fine. + */ +function assertNoEmptyCycle(nodes: StrictNode[]): void { + const state = new Map(); + // Each frame builds its node's edge list once, so the walk is linear in nodes plus edges. + const edges = (node: StrictNode): StrictNode[] => [...(node.ref ? [node.ref] : []), ...(node.anyOf ?? [])]; + for (const start of nodes) { + if (state.has(start)) continue; + const stack: Array<[StrictNode, StrictNode[], number]> = [[start, edges(start), 0]]; + state.set(start, 1); + while (stack.length > 0) { + const frame = stack[stack.length - 1]!; + const next = frame[1][frame[2]++]; + if (next === undefined) { + state.set(frame[0], 2); + stack.pop(); + } else if (state.get(next) === 1) throw new Unsupported("$ref"); + else if (!state.has(next)) { + state.set(next, 1); + stack.push([next, edges(next), 0]); + } + } + } +} + +const prepared = new WeakMap(); + +/** The prepared graph for `schema`, built once per schema object. */ +function preparedFor(schema: JsonRecord): PreparedSchema { + let plan = prepared.get(schema); + if (!plan) { + plan = prepare(schema); + prepared.set(schema, plan); + } + return plan; +} + +/** Throw the structured-output refusal when a strict schema needs a check this emulation does not have. */ +export function assertKiroStrictSchemaSupported(schema: JsonRecord): void { + try { + preparedFor(schema); + } catch (error) { + if (!(error instanceof Unsupported)) throw error; + throw new Error(error.keyword === undefined + ? "Kiro does not support Responses structured output: the strict schema uses a keyword that cannot be enforced" + : `Kiro does not support Responses structured output: strict schema keyword ${JSON.stringify(error.keyword)} cannot be enforced`); + } +} + +/** Whether a parsed JSON value has the given JSON Schema type. */ +/** A number value's decimal, parsed once per value however many nodes inspect it. */ +const decimals = new WeakMap(); +function decimalOf(value: KiroJsonValue & { kind: "number" }): KiroDecimal { + let decimal = decimals.get(value); + if (!decimal) { + decimal = kiroDecimal(value.lexeme); + decimals.set(value, decimal); + } + return decimal; +} + +function hasType(value: KiroJsonValue, type: string): boolean { + switch (type) { + case "object": return value.kind === "object"; + case "array": return value.kind === "array"; + case "string": return value.kind === "string"; + case "number": return value.kind === "number"; + case "integer": return value.kind === "number" && isKiroDecimalInteger(decimalOf(value)); + case "boolean": return value.kind === "boolean"; + case "null": return value.kind === "null"; + default: return false; + } +} + +/** Whether the model's value equals a schema literal, comparing numbers exactly by their decimal value. */ +function literalEquals(literal: unknown, value: KiroJsonValue, charge: (cost: number) => void): boolean { + // Every comparison step is charged, so one large literal referenced from many places cannot + // multiply work past the validation budget. Long strings cost in proportion to their length. + charge(1); + switch (value.kind) { + case "null": return literal === null; + case "boolean": return literal === value.value; + case "string": + charge(Math.floor(value.value.length / 256)); + return literal === value.value; + case "number": + return typeof literal === "number" + && compareKiroDecimal(kiroDecimal(String(literal)), kiroDecimal(value.lexeme)) === 0; + case "array": + return Array.isArray(literal) && literal.length === value.items.length + && value.items.every((item, i) => literalEquals(literal[i], item, charge)); + case "object": { + if (!isRecord(literal)) return false; + const keys = Object.keys(literal); + charge(keys.length); + return keys.length === value.entries.size + && keys.every(key => value.entries.has(key) && literalEquals(literal[key], value.entries.get(key)!, charge)); + } + } +} + +/** Code points in a string value, counted once per value however many nodes check its length. */ +const codePointLengths = new WeakMap(); +function codePointLength(value: KiroJsonValue & { kind: "string" }): number { + let length = codePointLengths.get(value); + if (length === undefined) { + length = 0; + for (const _ of value.value) length += 1; + codePointLengths.set(value, length); + } + return length; +} + +class OverBudget extends Error { + constructor(readonly problem: string) { + super(problem); + } +} + +/** The outcome of one strict validation, with the schema-node visits it spent. */ +export interface KiroStrictValidation { + problem?: string; + visits: number; + budget: number; +} + +/** + * Validate `value` (holding `valueNodes` JSON values) against the strict `schema`. + * + * Problems are fixed categories. They never quote a property name, a path or any other text from + * the model's answer or the caller's schema, because they reach the caller and diagnostics. + */ +export function validateKiroStrictValue(schema: JsonRecord, value: KiroJsonValue, valueNodes: number): KiroStrictValidation { + const { root } = preparedFor(schema); + const budget = KIRO_STRICT_BASE_VISITS + KIRO_STRICT_VISITS_PER_VALUE_NODE * valueNodes; + let visits = 0; + const spend = (cost = 1): void => { + visits += cost; + if (visits > budget) throw new OverBudget("validating the value needs more work than strict validation allows"); + }; + + const check = (node: StrictNode, value: KiroJsonValue, depth: number): string | undefined => { + spend(); + if (depth > KIRO_STRICT_MAX_DEPTH) throw new OverBudget("the value nests deeper than strict validation allows"); + if (node.never) return "a value is present where none is allowed"; + if (node.ref) { + const problem = check(node.ref, value, depth + 1); + if (problem) return problem; + } + if (node.types) { + // An integer check walks the number's digits; long lexemes are charged in proportion. + if (value.kind === "number" && node.types.includes("integer")) spend(Math.floor(value.lexeme.length / 256)); + if (!node.types.some(type => hasType(value, type))) return `expected a JSON ${node.types.join(" or ")}`; + } + if (node.enumValues && !node.enumValues.some(option => literalEquals(option, value, spend))) return "expected one of the enum values"; + if (node.constValue && !literalEquals(node.constValue.value, value, spend)) return "expected the const value"; + if (node.anyOf && !node.anyOf.some(branch => check(branch, value, depth + 1) === undefined)) { + return "matched no anyOf branch"; + } + if (value.kind === "string") { + if (node.minLength !== undefined || node.maxLength !== undefined) { + // Counting code points walks the string, so repeated checks of one long string are charged, + // and only a node with a length constraint counts at all. + spend(Math.floor(value.value.length / 256)); + const length = codePointLength(value); + if (node.minLength !== undefined && length < node.minLength) return "a string is shorter than minLength"; + if (node.maxLength !== undefined && length > node.maxLength) return "a string is longer than maxLength"; + } + } + if (value.kind === "number" && (node.minimum || node.maximum || node.exclusiveMinimum || node.exclusiveMaximum)) { + // Bound comparisons walk the digits too: parsed once per value, charged on every use. + spend(Math.floor(value.lexeme.length / 256)); + const number = decimalOf(value); + if (node.minimum && compareKiroDecimal(number, node.minimum) < 0) return "a number is below minimum"; + if (node.maximum && compareKiroDecimal(number, node.maximum) > 0) return "a number is above maximum"; + if (node.exclusiveMinimum && compareKiroDecimal(number, node.exclusiveMinimum) <= 0) return "a number is not above exclusiveMinimum"; + if (node.exclusiveMaximum && compareKiroDecimal(number, node.exclusiveMaximum) >= 0) return "a number is not below exclusiveMaximum"; + } + if (value.kind === "array") { + if (node.minItems !== undefined && value.items.length < node.minItems) return "an array has fewer than minItems"; + if (node.maxItems !== undefined && value.items.length > node.maxItems) return "an array has more than maxItems"; + if (node.items) { + for (const item of value.items) { + const problem = check(node.items, item, depth + 1); + if (problem) return problem; + } + } + } + if (value.kind === "object") { + // Each pass over `required` and over the entries is charged, so a failing anyOf that re-checks + // one large object or one long `required` list stays within the budget. + spend(node.required?.length ?? 0); + if (node.required?.some(key => !value.entries.has(key))) return "an object is missing a required property"; + spend(value.entries.size); + for (const [key, child] of value.entries) { + const declared = node.properties?.get(key); + if (!declared && node.closed) return "an object has a property the schema does not allow"; + const childNode = declared ?? node.additional; + if (!childNode) continue; + const problem = check(childNode, child, depth + 1); + if (problem) return problem; + } + } + return undefined; + }; + + try { + const problem = check(root, value, 0); + return { ...(problem !== undefined ? { problem } : {}), visits, budget }; + } catch (error) { + if (!(error instanceof OverBudget)) throw error; + return { problem: error.problem, visits, budget }; + } +} + +/** The first way `value` breaks the strict `schema`, or undefined when it conforms. */ +export function kiroStrictSchemaProblem(schema: JsonRecord, value: KiroJsonValue, valueNodes: number): string | undefined { + return validateKiroStrictValue(schema, value, valueNodes).problem; +} + +/** Keys whose values are JSON data, never schemas, so a `$ref`-named key inside them is left alone. */ +const LITERAL_KEYS = new Set(["const", "enum", "default", "examples"]); +/** Keys whose values map names to schemas; a name such as "const" is not a keyword there. */ +const SCHEMA_MAPS = new Set(["properties", "$defs", "definitions"]); + +/** + * Point every local `$ref` at the original root after the schema moves under `properties.value`. + * + * `#` and root-relative pointers such as `#/items` now resolve against the wrapper, so they gain + * the `/properties/value` prefix. `#/$defs/...` and `#/definitions/...` keep their target because + * the wrapper hoists the definitions to its own root. + */ +export function rewriteKiroWrappedRefs(schema: unknown, root = true): unknown { + if (Array.isArray(schema)) return schema.map(item => rewriteKiroWrappedRefs(item, false)); + if (!isRecord(schema)) return schema; + // A nested `$id` starts its own resource: references inside it are relative to that resource, + // which moves along with it, so they stay as written. + if (!root && typeof schema.$id === "string") return schema; + // fromEntries defines own properties, so a key such as "__proto__" stays a key. + return Object.fromEntries(Object.entries(schema).map(([key, child]): [string, unknown] => { + if (SCHEMA_MAPS.has(key) && isRecord(child)) { + return [key, Object.fromEntries(Object.entries(child).map(([name, sub]) => [name, rewriteKiroWrappedRefs(sub, false)]))]; + } + if (LITERAL_KEYS.has(key)) return [key, child]; + if (key === "$ref" && typeof child === "string") return [key, movedRef(child)]; + return [key, rewriteKiroWrappedRefs(child, false)]; + })); +} + +/** One reference rewritten for the wrapped document. */ +function movedRef(ref: string): string { + // Judge the decoded pointer: `#%2Fitems` is root-relative and `#/%24defs/item` names the hoisted + // `$defs`, exactly like their unencoded forms. Prefixing the raw remainder keeps its encoding. + let decoded = ref; + try { decoded = decodeURIComponent(ref); } catch { /* an undecodable pointer is judged as written */ } + if (decoded !== "#" && !decoded.startsWith("#/")) return ref; + // The definition containers themselves, and anything under them, stay at the wrapper root. + if (["#/$defs", "#/definitions"].some(container => decoded === container || decoded.startsWith(`${container}/`))) return ref; + return `#/properties/value${ref.slice(1)}`; +} diff --git a/src/adapters/kiro/structured-output.ts b/src/adapters/kiro/structured-output.ts new file mode 100644 index 00000000000..eb68d03611b --- /dev/null +++ b/src/adapters/kiro/structured-output.ts @@ -0,0 +1,367 @@ +import type { AdapterEvent, OcxParsedRequest, OcxUsage } from "../../types"; +import { KIRO_STRUCTURED_OUTPUT_TOOL_NAME } from "../kiro-constants"; +import { sanitizeKiroSchema } from "../kiro-tools"; +import { readKiroJson } from "./structured-output-json"; +import { assertKiroStrictSchemaSupported, kiroStrictSchemaProblem, rewriteKiroWrappedRefs } from "./structured-output-schema"; + +/** + * Structured output on Kiro, emulated through a private terminal tool. + * + * The CodeWhisperer wire has no schema-constrained response mode, which is why this adapter used to + * refuse `text.format` json_schema/json_object outright: a caller that expects parseable JSON and + * receives prose fails downstream. Refusing kept that promise but broke every caller that only asks + * for a small JSON answer — T3 Code titles each thread through a Messages `output_config.format` + * request, and Claude Code's own structured-output calls take the same path. + * + * The wire does carry tool input schemas, and a tool call's input is JSON by construction. So a + * structured-output turn advertises one tool whose input schema IS the caller's format, tells the + * model to answer by calling it, and relays the call's input as the assistant's text. The promise + * is unchanged: the client receives JSON that parsed and passed the checks below, or an error — + * never prose. A non-strict format gets light checks (root type and the root's `required` keys), + * the same position the Cursor prompt fallback takes. A `strict: true` format is validated against + * the whole schema (structured-output-schema.ts), and a strict schema that validator cannot enforce + * is refused before sending. + */ +export interface KiroStructuredOutputSpec { + /** The caller's schema exactly as sent; absent when only "a JSON object" was requested. */ + schema?: Record; + /** True when the caller asked for `strict: true`, so the value must conform to the whole schema. */ + strict?: boolean; + /** The tool input schema actually advertised: sanitized, and wrapped when `wrapped` is true. */ + toolSchema: Record; + /** + * True when the answer rides the tool input's `value` property instead of BEING the input. + * + * Kiro, like Bedrock, accepts only an object-typed tool input root and refuses a root + * anyOf/oneOf/allOf. kiro-tools flattens such roots for ordinary tools, which is harmless for an + * argument bag but would silently change the meaning of a caller's output contract — a root + * `anyOf` would become "any of these properties". Wrapping keeps the schema intact one level down. + */ + wrapped: boolean; + name?: string; + description?: string; +} + +type JsonRecord = Record; + +/** A plain JSON object: not null and not an array. */ +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +/** An object-typed root without root composition can be the tool input schema as it stands. */ +function isPlainObjectRoot(schema: JsonRecord): boolean { + return schema.type === "object" + && !["anyOf", "oneOf", "allOf", "$ref"].some(key => Object.hasOwn(schema, key)); +} + +/** + * The structured-output contract this turn carries, or undefined for an ordinary turn. + * + * `options.textFormat` is the source; `_structuredOutput` alone (no format object) still means the + * caller expects JSON, so it is honoured as the weakest form of the contract: a JSON object. + */ +export function kiroStructuredOutputSpec(parsed: OcxParsedRequest): KiroStructuredOutputSpec | undefined { + const format = parsed.options.textFormat; + if (!format && !parsed._structuredOutput) return undefined; + const meta = { + ...(format?.type === "json_schema" && format.name ? { name: format.name } : {}), + ...(format?.type === "json_schema" && format.description ? { description: format.description } : {}), + }; + const schema = format?.type === "json_schema" && isRecord(format.schema) ? format.schema : undefined; + // A Messages schema is a conformance promise even when its translation states `strict: false`. + const strict = format?.type === "json_schema" && (format.strict === true || parsed._structuredOutputConformance === true); + // A strict format promises conformance to its schema; without an object schema (absent, or a + // boolean such as `false`) there is nothing this emulation can enforce, so it is refused. + if (!schema && strict) { + throw new Error("Kiro does not support Responses structured output: a strict format needs a JSON object schema"); + } + if (!schema) return { toolSchema: { type: "object" }, wrapped: false, ...meta }; + if (strict) assertKiroStrictSchemaSupported(schema); + const contract = { schema, ...(strict ? { strict } : {}) }; + const sanitized = sanitizeKiroSchema(schema) as JsonRecord; + if (isPlainObjectRoot(schema)) return { ...contract, toolSchema: sanitized, wrapped: false, ...meta }; + // `#/$defs/...` references resolve against the document root, which is now the wrapper. Hoist the + // definitions with them so a referenced schema still resolves after it moves under `value`, and + // re-point `#` and other root-relative references at `#/properties/value`, the original root. + // A root `$id` stays at the wrapper root with the hoisted definitions, so references keep the + // same base; moved under `value` it would start a new resource. + const { $defs, definitions, $id, ...inner } = rewriteKiroWrappedRefs(sanitized) as JsonRecord; + return { + ...contract, + wrapped: true, + toolSchema: { + ...($id !== undefined ? { $id } : {}), + type: "object", + properties: { value: inner }, + required: ["value"], + ...($defs !== undefined ? { $defs } : {}), + ...(definitions !== undefined ? { definitions } : {}), + }, + ...meta, + }; +} + +/** The private terminal tool advertised for a structured-output turn, with the caller's (sanitized, possibly wrapped) schema as its input schema. */ +export function kiroStructuredOutputTool(spec: KiroStructuredOutputSpec): Record { + const delivery = spec.wrapped + ? "Put the complete final JSON value in `value`." + : "The tool input itself is the complete final JSON answer."; + return { + toolSpecification: { + name: KIRO_STRUCTURED_OUTPUT_TOOL_NAME, + // Same reasoning as the completion tool's description (payload.ts): this is the surface the + // model reads while choosing between tools, so it states terminality and the obligation here + // rather than leaving them to the prose contract alone. + description: `Terminal structured-output channel, not an ordinary work tool. The caller requires the final answer as JSON in exactly the format of this tool's input schema. When the task is complete, call this tool exactly once instead of writing the answer as ordinary assistant text. ${delivery} The call is complete when issued: it ends the turn, returns no tool result, and no text or tool call may follow it.`, + inputSchema: { json: spec.toolSchema }, + }, + }; +} + +/** + * The prose half of the structured-output contract, at most `maxChars` long. + * + * The caller's schema is restated verbatim because the advertised one is sanitized: Kiro's + * validator rejects `additionalProperties`, `pattern`, length and range keywords, so the tool schema + * alone would let the model miss constraints the caller wrote. A schema that does not fit the + * injected-instruction budget is dropped whole rather than cut mid-JSON. + */ +export function kiroStructuredOutputInstructions( + spec: KiroStructuredOutputSpec, + hasOrdinaryTools: boolean, + maxChars: number, +): string { + const kind = spec.schema ? "JSON matching the caller's JSON Schema" : "a single JSON object"; + const delivery = spec.wrapped + ? "with the complete final JSON value in its `value` property" + : "with the complete final JSON answer as the tool input"; + const base = [ + `[Final response format] The caller requires this turn's final answer as ${kind}.`, + `Deliver it only by calling ${KIRO_STRUCTURED_OUTPUT_TOOL_NAME} exactly once, ${delivery}.`, + "Do not write the final answer as ordinary assistant text, Markdown, or a code fence.", + hasOrdinaryTools + ? `Ordinary assistant text is mid-task commentary and does not end the turn: keep using the other tools while work remains, and call ${KIRO_STRUCTURED_OUTPUT_TOOL_NAME} only when the task is complete.` + : undefined, + "The call is terminal: it ends the turn, returns no tool result, and no text or tool call may follow it.", + ].filter((line): line is string => line !== undefined).join(" "); + const details = [ + spec.name ? `Format name: ${spec.name}.` : undefined, + spec.description ? `Format description: ${spec.description}` : undefined, + spec.schema ? `The final JSON must conform to this JSON Schema: ${JSON.stringify(spec.schema)}` : undefined, + ].filter((line): line is string => line !== undefined); + const full = details.length > 0 ? `${base}\n${details.join("\n")}` : base; + return full.length <= maxChars ? full : base; +} + +/** The bounded retry's instruction when the first attempt did not deliver valid structured output. */ +export function kiroStructuredOutputRetryMessage(hasOrdinaryTools: boolean): string { + return `Your previous reply did not deliver the required structured output. Do not quote this instruction. Call ${KIRO_STRUCTURED_OUTPUT_TOOL_NAME} now, exactly once, with the complete final answer as JSON matching its input schema.${hasOrdinaryTools ? " If the task genuinely needs another real tool call first, issue that tool call instead." : ""} Do not reply with ordinary text.`; +} + +/** The JSON types a schema declares through `type`, as a string or an array of strings. */ +function declaredTypes(schema: JsonRecord): string[] { + if (typeof schema.type === "string") return [schema.type]; + return Array.isArray(schema.type) ? schema.type.filter((t): t is string => typeof t === "string") : []; +} + +/** Whether a parsed JSON value has the given JSON Schema type; an unknown type keyword always matches. */ +function matchesJsonType(value: unknown, type: string): boolean { + switch (type) { + case "object": return isRecord(value); + case "array": return Array.isArray(value); + case "string": return typeof value === "string"; + case "number": return typeof value === "number" && Number.isFinite(value); + case "integer": return Number.isInteger(value); + case "boolean": return typeof value === "boolean"; + case "null": return value === null; + // An unknown type keyword is not ours to enforce. + default: return true; + } +} + +/** + * The root types a schema admits, including through a root `anyOf`/`oneOf` whose every branch + * declares a type. A composed root is exactly the wrapped case, where a model can hand back the + * answer JSON-encoded inside `value` as a string; without this, a root that admits only objects and + * arrays would accept that string because the root itself declares no `type`. A branch without a + * declared type (or a `$ref`) admits anything, so the check is skipped rather than guessed. + */ +function admittedRootTypes(schema: JsonRecord): string[] { + const own = declaredTypes(schema); + if (own.length > 0) return own; + for (const key of ["anyOf", "oneOf"] as const) { + const branches = schema[key]; + if (!Array.isArray(branches) || branches.length === 0) continue; + const perBranch = branches.map(branch => (isRecord(branch) ? declaredTypes(branch) : [])); + if (perBranch.some(types => types.length === 0)) return []; + return [...new Set(perBranch.flat())]; + } + return []; +} + +/** The first reason `value` breaks the light contract, or undefined when it passes. */ +function structuredValueProblem(spec: KiroStructuredOutputSpec, value: unknown): string | undefined { + const types = spec.schema ? admittedRootTypes(spec.schema) : ["object"]; + if (types.length > 0 && !types.some(type => matchesJsonType(value, type))) { + return `expected a JSON ${types.join(" or ")} at the root`; + } + const required = spec.schema?.required; + if (isRecord(value) && Array.isArray(required)) { + const missing = required.find(key => typeof key === "string" && !Object.hasOwn(value, key)); + // No property name: diagnostics carry fixed categories, never schema or answer text. + if (missing !== undefined) return "the root is missing a required property"; + } + return undefined; +} + +/** + * Strict validation of JSON `text` that already parsed: the JSON text to relay, or the problem. + * + * The strict validator reads the text again keeping every number's lexeme, so bounds, `const` and + * `enum` judge the exact bytes the client will receive. The wrapped form relays the original bytes + * of `value`, which keeps integers beyond 2^53 intact there too. + */ +function strictStructuredJson(schema: JsonRecord, text: string, wrapped: boolean): { json: string } | { problem: string } { + const document = readKiroJson(text); + if (!document) return { problem: "the value nests deeper than strict validation allows" }; + const root = document.value; + const value = wrapped && root.kind === "object" ? root.entries.get("value") : root; + if (!value) return { problem: "missing `value`" }; + const problem = kiroStrictSchemaProblem(schema, value, document.nodes); + if (problem) return { problem }; + return { json: wrapped ? text.slice(value.start, value.end) : text }; +} + +const FAILURE_PREFIX = "Kiro structured output did not match the requested format"; + +/** + * Validate the structured-output tool's input and return the JSON text to relay. + * + * Both forms relay the model's own bytes rather than a re-serialization, so integers beyond 2^53, + * numbers past the double range and key order survive exactly as written. The wrapped form relays + * the original bytes of `value`. + */ +export function parseKiroStructuredToolInput( + spec: KiroStructuredOutputSpec, + raw: string, +): { json: string } | { error: string } { + const text = raw.trim() || "{}"; + let input: unknown; + try { + input = JSON.parse(text); + } catch { + return { error: "Kiro returned invalid JSON for the structured-output tool" }; + } + if (!isRecord(input)) return { error: "Kiro returned a non-object input for the structured-output tool" }; + if (spec.wrapped && !Object.hasOwn(input, "value")) { + return { error: `${FAILURE_PREFIX}: missing \`value\`` }; + } + const value = spec.wrapped ? input.value : input; + if (spec.strict && spec.schema) { + const strict = strictStructuredJson(spec.schema, text, spec.wrapped); + return "json" in strict ? strict : { error: `${FAILURE_PREFIX}: ${strict.problem}` }; + } + const problem = structuredValueProblem(spec, value); + if (problem) return { error: `${FAILURE_PREFIX}: ${problem}` }; + if (!spec.wrapped) return { json: text }; + const wrappedText = wrappedValueText(text); + return wrappedText === undefined + ? { error: `${FAILURE_PREFIX}: the value nests deeper than the proxy can relay exactly` } + : { json: wrappedText }; +} + +/** + * The original bytes of the wrapper's `value`, or undefined past the reader's nesting limit. Such a + * value is refused rather than re-serialized, because re-serializing rewrites large numbers. + */ +function wrappedValueText(text: string): string | undefined { + const root = readKiroJson(text)?.value; + const span = root?.kind === "object" ? root.entries.get("value") : undefined; + return span ? text.slice(span.start, span.end) : undefined; +} + +const WHOLE_FENCE = /^```[A-Za-z0-9_-]*[ \t]*\r?\n([\s\S]*?)\r?\n?```$/; + +/** + * Accept a plain-text answer only when it already IS the requested JSON. + * + * A model that ignores the tool and writes the JSON as text has still answered correctly, and + * spending a second paid request to have it repeat the same bytes through a tool would be waste. + * One whole-message code fence is unwrapped because that is the most common way models present + * JSON; anything else — prose around the JSON, two fenced blocks, trailing commentary — is not JSON + * and is refused, which sends the turn to the bounded retry or to an error. + */ +export function parseKiroStructuredText(spec: KiroStructuredOutputSpec, text: string): string | undefined { + const trimmed = text.trim(); + const candidate = WHOLE_FENCE.exec(trimmed)?.[1]?.trim() ?? trimmed; + if (!candidate) return undefined; + let value: unknown; + try { + value = JSON.parse(candidate); + } catch { + return undefined; + } + if (spec.strict && spec.schema) { + // A text answer is the value itself, even when the tool input would have wrapped it. + return "json" in strictStructuredJson(spec.schema, candidate, false) ? candidate : undefined; + } + return structuredValueProblem(spec, value) === undefined ? candidate : undefined; +} + +/** + * Whether Kiro stopped the way a finished answer stops: no stop reason, END_TURN or STOP_SEQUENCE. + * + * Every other stop reason (MAX_TOKENS, a content filter, an exhausted context window) has already + * ended the inference upstream. Text cut by one of them is never accepted as the requested JSON, + * even when its prefix parses, and a failed structured answer under one of them earns no retry: + * the retry would hit the same limit or verdict, so the stop's own terminal handling runs instead. + */ +export function isKiroStructuredCleanStop(normalizedStopReason: string | undefined): boolean { + return normalizedStopReason === undefined + || normalizedStopReason === "END_TURN" + || normalizedStopReason === "STOP_SEQUENCE"; +} + +/** Message for a bounded retry that answered in text that is not the requested JSON. */ +export const KIRO_STRUCTURED_TEXT_FAILURE = `${FAILURE_PREFIX}: the model answered with text that is not the requested JSON`; + +/** Message for a bounded retry that ended without text, a tool call, or a structured answer. */ +export const KIRO_STRUCTURED_EMPTY_FAILURE = `${FAILURE_PREFIX}: the bounded retry produced no answer`; + +/** Stop reasons Kiro documents; only these are named in a diagnostic. */ +const KNOWN_STOP_REASONS = new Set([ + "MAX_TOKENS", "MODEL_CONTEXT_WINDOW_EXCEEDED", "MALFORMED_TOOL_USE", "MALFORMED_MODEL_OUTPUT", + "TOOL_USE", "CONTENT_FILTERED", "GUARDRAIL_INTERVENED", +]); + +/** + * Message for a bounded retry whose text was cut by a stop that does not end a finished answer. The + * stop reason comes from upstream, so an unrecognized one is not echoed. + */ +export function kiroStructuredStopFailure(normalizedStopReason: string): string { + const stop = KNOWN_STOP_REASONS.has(normalizedStopReason) ? ` with ${normalizedStopReason}` : ""; + return `${FAILURE_PREFIX}: the bounded retry stopped${stop} before a complete answer`; +} + +/** + * Terminal for a turn that, after its one bounded retry, still produced no valid structured output. + * + * A hard error with `retryable: false` rather than the retryable `incomplete` the completion + * contract uses for a malformed answer: the model has already been asked twice inside this turn, so + * the proxy must not replay the whole turn on its own. The status is the 502 every other Kiro + * model-output protocol error uses, which a client's own retry policy may still act on (Claude + * outbound reports it as `overloaded_error`); that choice stays with the client. The distinct code + * lets a caller tell this apart from a transport failure. + */ +export function kiroStructuredOutputFailure(message: string, usage: OcxUsage): AdapterEvent { + return { + type: "error", + message, + status: 502, + errorType: "upstream_error", + code: "invalid_kiro_structured_output", + retryable: false, + usage, + }; +} diff --git a/src/server/responses/request-prepare.ts b/src/server/responses/request-prepare.ts index 49c7d12ace6..9d373415c04 100644 --- a/src/server/responses/request-prepare.ts +++ b/src/server/responses/request-prepare.ts @@ -515,6 +515,7 @@ export async function prepareResponsesRequest( parsed._nativeReasoningMint = options.nativeReasoningMint; parsed._anthropicServerTools = options.anthropicServerTools; parsed._anthropicServerReplay = options.anthropicServerReplay; + if (parsed.options.textFormat?.type === "json_schema") parsed._structuredOutputConformance = true; } parsed._promptCacheKeyIsSharedCohort = options.promptCacheKeyIsSharedCohort; // The body may have been rebuilt since the inbound observation (previous-response diff --git a/src/server/responses/response-effects.ts b/src/server/responses/response-effects.ts index 199a8beb36a..75bdd29158b 100644 --- a/src/server/responses/response-effects.ts +++ b/src/server/responses/response-effects.ts @@ -106,8 +106,9 @@ export function createResponsesEffects( // The compaction turn is a plain prose summary; a surviving structured-output format // would force schema-constrained JSON into the synthetic compaction item. The flag and // the raw `text` control go too: the key-mode openai-responses adapter builds from - // _rawBody, so a surviving format there would still reach the upstream. (The Kiro - // guard no longer reads _rawBody.text; it refuses structured output only.) + // _rawBody, so a surviving format there would still reach the upstream. (Kiro reads + // options.textFormat, never _rawBody.text, and would emulate a surviving format through + // its structured-output tool, so it must go here too.) delete parsed.options.textFormat; delete parsed._structuredOutput; if (parsed._rawBody && typeof parsed._rawBody === "object") { diff --git a/src/types/request.ts b/src/types/request.ts index 352acaa972c..c5423f1a568 100644 --- a/src/types/request.ts +++ b/src/types/request.ts @@ -180,6 +180,13 @@ export interface OcxParsedRequest { * answer/"Sources:" text can't bleed into and corrupt the model's schema-constrained output. */ _structuredOutput?: boolean; + /** + * True for a Claude Messages `output_config.format` json_schema. Anthropic promises an answer + * that conforms to that schema, whatever `strict` the Responses translation states for + * OpenAI-compatible destinations, so an emulating adapter (Kiro) holds it to full conformance. + * Process-local; never forwarded. + */ + _structuredOutputConformance?: boolean; /** * True when the input carried `{type:"compaction_trigger"}` — Codex remote compaction v2 asking * this turn to produce a `{type:"compaction"}` output item. Routed adapters can't natively; @@ -414,7 +421,7 @@ export interface OcxRequestOptions { * upstream wire has an equivalent. The openai-chat adapter re-nests it as chat * `response_format`, the exact inverse of responseFormatToText in src/chat/inbound.ts. * The native passthrough ignores it (it forwards `_rawBody.text` verbatim) and Kiro - * keeps rejecting structured output via `_structuredOutput`. + * emulates it through a private structured-output tool (src/adapters/kiro/structured-output.ts). */ textFormat?: { type: "json_schema" | "json_object"; diff --git a/structure/INDEX.md b/structure/INDEX.md index 6309ec0f448..f1d0a5bb8c3 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -107,7 +107,7 @@ Per-vendor contracts and the adapter authority that constructs them. | [`providers/openai-signed-in-preservation.md`](providers/openai-signed-in-preservation.md) | Opt-in effective priority bands that spend other pool accounts before the signed-in identity, its identity observation and management API. | | [`providers/cursor.md`](providers/cursor.md) | Cursor native exec, parameterized models, checkpoints, and active-context usage. | | [`providers/google.md`](providers/google.md) | Gemini thought-text, response parts, thought-signature replay, and adjacency repair. | -| [`providers/kiro.md`](providers/kiro.md) | Kiro parallel-tool hints, Responses text controls, and reasoning round-trip. | +| [`providers/kiro.md`](providers/kiro.md) | Kiro parallel-tool hints, Responses text controls, structured-output emulation, and reasoning round-trip. | | [`providers/xai-grok.md`](providers/xai-grok.md) | Grok Build contract parity and hardening. | | [`providers/lm-studio-chat.md`](providers/lm-studio-chat.md) | Preset-scoped single-leading-system normalization, configuration and editor policy. | | [`providers/chat-compat.md`](providers/chat-compat.md) | Cross-vendor Chat Completions behavior: reasoning, tool results, structured output, parallel tools. | @@ -213,6 +213,7 @@ A source area can be described by more than one doc, because these docs are orga | `src/adapters/anthropic-tool-catalog.ts` | [`transports/anthropic-thinking-recovery.md`](transports/anthropic-thinking-recovery.md) | | `src/adapters/base.ts` | [`providers/completed-tool-blocks.md`](providers/completed-tool-blocks.md) | | `src/adapters/dsh-account.ts` | [`providers/dsh-account.md`](providers/dsh-account.md) | +| `src/adapters/kiro/` | [`providers/kiro.md`](providers/kiro.md) | | `src/adapters/openai-chat.ts` | [`providers/completed-tool-blocks.md`](providers/completed-tool-blocks.md)
[`providers/mimo-bare-calls.md`](providers/mimo-bare-calls.md) | | `src/adapters/openai-chat/` | [`providers/lm-studio-chat.md`](providers/lm-studio-chat.md)
[`providers/mimo-bare-calls.md`](providers/mimo-bare-calls.md) | | `src/adapters/openai-chat/completed-tool-blocks.ts` | [`providers/completed-tool-blocks.md`](providers/completed-tool-blocks.md) | diff --git a/structure/decisions/ADR-8071-kiro-structured-output-emulation.md b/structure/decisions/ADR-8071-kiro-structured-output-emulation.md new file mode 100644 index 00000000000..8da48990f62 --- /dev/null +++ b/structure/decisions/ADR-8071-kiro-structured-output-emulation.md @@ -0,0 +1,12 @@ +# ADR-8071 — decision recorded under "Kiro structured output" + +- Contract owner: [providers/kiro.md](../providers/kiro.md#kiro-structured-output) + +## Decision record + +- 목적과 의도: Let Kiro-routed turns that request structured output succeed with valid JSON instead of being refused, while keeping the guarantee that a caller expecting JSON never receives prose. +- 기존 구현 및 제약 조건: `validateKiroCapabilities` threw on `_structuredOutput`, so every json_schema/json_object turn failed locally with HTTP 400 and sendCount 0. T3 Code titles each thread through a Messages `output_config.format` request and Claude Code issues its own structured-output calls, so no Kiro thread got a title. The CodeWhisperer wire has no schema-constrained response mode, but it carries tool input schemas, and the adapter already had a private terminal tool, `codex_kiro_final_answer`, with a bounded `text_fallback` retry. +- 검토한 주요 대안: Keep refusing; prompt-only emulation like `src/adapters/cursor/structured-output.ts`, relaying whatever text arrives; add a structured-output tool beside the completion tool; replace the completion tool with a structured-output tool whose input schema is the caller's schema, validate its input, and accept JSON-only text with one bounded retry. +- 선택한 방식: Replace the completion tool with `codex_kiro_structured_output`, force completion mode `required`, relay the validated tool input as final-answer text, accept clean-stop text that is itself the JSON, retry once on prose or an invalid input, and fail with `invalid_kiro_structured_output` after that. Held prose is dropped on every structured-output path. +- 다른 대안 대신 이 방식을 선택한 이유: A tool input is JSON by construction and its schema is the caller's format, which a prompt alone does not give; relaying unvalidated text would break the promise the refusal existed to keep. Advertising both tools would offer the model a prose exit. Accepting JSON-only text avoids paying a second request for a correct answer delivered without the tool. +- 장점, 단점 및 영향: Structured-output turns on Kiro now succeed, and failures stay explicit errors. Non-strict validation is light (parse, root type, root `required` keys), so a value that violates a deeper schema constraint can still pass. A `strict: true` format is validated against the whole schema instead, because strict promises full conformance; a strict schema outside the subset listed in the contract owner is refused rather than relayed unchecked. The subset is narrowed so every accepted schema is checked exactly and in bounded work: `$ref` only to root `$defs`/`definitions` with empty reference cycles refused; one global visit budget and depth limit per value, where exceeding either is non-conforming (chosen over a static bound because recursive schemas have none); numbers compared by their original text; numeric bounds limited to safe integers and decimals of at most 15 significant digits, a heuristic that can still misjudge a value within about one ulp of a bound the caller wrote with more digits; and numeric `const`/`enum` literals, `multipleOf`, `pattern` and `patternProperties` refused. The request reaches the adapter already parsed, so exact numeric equality cannot be checked against what the caller wrote, and a regular expression evaluated against model output can block the event loop; refusing them was chosen over a pattern-shape heuristic, which cannot rule out super-linear matching. A general JSON Schema engine was rejected because it would add a dependency to the request path and still need the same bounds. Refusing every strict format was rejected because OpenAI SDK helpers and Codex output schemas send `strict: true` by default, which would have refused most callers again. A turn can cost two upstream requests. Kiro's acceptance of every caller schema shape as a tool input schema is verified only by sanitization, not live. diff --git a/structure/manifest.json b/structure/manifest.json index b0b8c747127..2ce2d59c8d7 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -812,8 +812,9 @@ "path": "providers/kiro.md", "tier": 4, "title": "Kiro Provider", - "scope": "Kiro parallel-tool hints, Responses text controls, and reasoning round-trip.", + "scope": "Kiro parallel-tool hints, Responses text controls, structured-output emulation, and reasoning round-trip.", "documents": [ + "src/adapters/kiro/", "src/responses/" ] }, diff --git a/structure/providers/kiro.md b/structure/providers/kiro.md index c5ee47e2f62..9cf75311e29 100644 --- a/structure/providers/kiro.md +++ b/structure/providers/kiro.md @@ -139,15 +139,94 @@ Kiro shares the Responses freeform restoration boundary in `src/responses/apply-patch-envelope.ts`: contractual `input` wrappers are unwrapped, while alternate field and outer-fence recovery is limited to unambiguous bare or `default.`-prefixed `exec` and `apply_patch` bodies. -Kiro refuses structured output and tolerates every other Responses `text` member. `text.format` -of type `json_schema` or `json_object` is a contract the CodeWhisperer wire cannot honour, so the -adapter rejects it rather than returning prose to a caller expecting JSON. `text.verbosity` and -`text.format: {"type":"text"}` are preferences, not contracts; they are accepted and dropped, -because `buildKiroPayload` composes `conversationState` from parsed fields and never forwards the -raw body. +`text.verbosity` and `text.format: {"type":"text"}` are preferences, not contracts; they are +accepted and dropped, because `buildKiroPayload` composes `conversationState` from parsed fields +and never forwards the raw body. > Decision record: [ADR-0061](../decisions/ADR-0061-kiro-responses-text-controls.md) +### Kiro structured output + +`text.format` of type `json_schema` or `json_object` (`options.textFormat`; on `/v1/messages` it +comes from `output_config.format`) is a contract the CodeWhisperer wire has no response mode for. +`src/adapters/kiro/structured-output.ts` emulates it with the private terminal tool +`codex_kiro_structured_output`, reserved against client tools in `src/adapters/kiro-wire.ts`: + +- The tool's input schema is the caller's schema, sanitized like every Kiro tool schema. A root that + is not a plain `type: "object"` (array, scalar, root `anyOf`/`oneOf`/`allOf`/`$ref`) is wrapped + as `{value: }` with `$defs`/`definitions` hoisted and every other local `$ref` (`#`, + `#/items`, ...) re-pointed under `#/properties/value`, rather than flattened the way + `src/adapters/kiro-tools.ts` flattens ordinary argument schemas. `json_object` advertises + `{type: "object"}`. Sanitization copies `const`, `enum`, `default` and `examples` verbatim, + because they hold data whose keys only look like keywords. +- It replaces `codex_kiro_final_answer` as the terminal channel and forces completion mode + `required` even on a tool-free turn. Client tools are still advertised ahead of it; a tool-free + turn carries no tool-catalog nudge. The prose contract restates the caller's unsanitized schema + when it fits the injected-instruction budget, and is otherwise sent without it. +- The stream relays a valid call's input as one `final_answer` text delta and surfaces no tool + call. The direct form relays the model's bytes; the wrapped form relays the original bytes of + `value`, so numbers past 2^53 or the double range are never rewritten. A value past the reader's + 512-level nesting limit is refused rather than re-serialized. + Without `strict: true`, validation is light: the input parses, the root matches the schema's declared `type` (or, for a root `anyOf`/`oneOf` whose branches all declare one, one of the branch types), and the + root's `required` keys are present. With `strict: true`, `src/adapters/kiro/structured-output-schema.ts` + prepares the original schema into a validation graph once, at the boundary, and validates the whole + value exactly and in bounded work. A strict format whose schema is missing or not a JSON object + (such as `false`) is refused. Anything outside this subset is refused before anything is sent, + with the same refusal structured output had before emulation: + (A Messages `output_config.format` schema counts as strict regardless of the `strict` its + Responses translation states: `src/server/responses/request-prepare.ts` sets the process-local + `_structuredOutputConformance` for Anthropic inbound requests.) + - `type` (a known type or a non-empty array of them) with no repeated type, `properties`, `required`, + `additionalProperties` (boolean or schema), single-schema `items`, `enum`, `const`, `anyOf`. + A `const` or `enum` literal that contains a number is refused: the request arrives parsed, so + `1.0000000000000001` is already `1`, and exact equality could accept a value the caller excluded. + - `$ref` only as `#/$defs/` or `#/definitions/` naming a root definition, with no + JSON Pointer escape other than `~0`/`~1`; the + fragment is percent-decoded before its segments are read, so an encoded `/` is a separator; + `$defs`/`definitions` only at the root. Root-relative references (`#`, `#/properties/...`), + references into annotations (`#/default`), remote references, and a cycle of `$ref`/`anyOf` + edges that consumes none of the value are refused. Recursion through properties or items is allowed. + A `$ref` beside an assertion keyword is refused, because drafts disagree on whether those + siblings apply; annotations and root `$defs`/`definitions` beside it are fine. A `$id` below the + root (a new resource that references would resolve against) and `$anchor`/`$dynamicAnchor` are refused. + - `minLength`, `maxLength`, `minItems`, `maxItems` as non-negative safe integers. + - `minimum`, `maximum`, `exclusiveMinimum`, `exclusiveMaximum` must be a safe integer or a + non-integer with at most 15 significant digits. This is a heuristic: a bound written with more + digits than a double holds can still arrive rounded, so a value within about one unit in the last + place of the bound may be judged on the wrong side. The model's numbers are compared by their + original text with exact decimal arithmetic (`src/adapters/kiro/structured-output-json.ts`). + - `multipleOf`, `pattern` and `patternProperties` are refused: `multipleOf` is exact equality on + a possibly rounded number, and a regular expression run against model output can take + super-linear time on the request path. + - Annotations (`title`, `description`, `default`, `examples`, ...) are ignored. + + One validation visits at most 10,000 schema nodes plus 16 per JSON value in the answer, and one + depth limit of 128 schema levels spans references, `anyOf` branches, properties and items; the + reader stops at 512 levels of nesting. A value that needs more is non-conforming (it takes the + bounded retry like any invalid answer), so a recursive `anyOf` cannot branch exponentially and a + deep value cannot exhaust the stack. `const`/`enum` comparisons, each pass over a `required` + list or an object's entries, string length checks and number comparisons (each number parsed once) are charged to the same budget, long strings + in proportion to their length. Preparation is bounded too: a schema with more than + 10,000 nodes, or more than 10,000 JSON values across its `const`/`enum` literals, is refused, and + the reference-cycle check is linear in nodes and edges. Validation problems and refusals are fixed + categories: they never quote a path, a property name, or a schema key outside the JSON Schema + vocabulary, because they reach the caller. +- A clean-stop text answer (`END_TURN`, `STOP_SEQUENCE`, or no stop reason) that is itself the + JSON, optionally in one whole-message code fence, is accepted without a retry. Prose, an empty clean response, or a tool + input that fails validation (including truncated JSON), takes the one bounded `text_fallback` retry with a structured-output + retry instruction, but only after a clean stop: under `MAX_TOKENS`, a content filter or an exhausted context window the stop's own terminal applies, because a retry would hit the same limit or verdict. The retry accepts JSON text only after a clean stop too. If the retry yields neither a valid call nor JSON text, including an empty or reasoning-only retry, the turn ends with a + non-retryable 502 `invalid_kiro_structured_output` error. +- Held prose never reaches a structured-output caller: every drain of held events drops text and + releases only non-text events. Commentary that precedes a real client tool call is released with + that call, as on ordinary turns. +- A call to whichever private tool the turn did not advertise is a protocol error, never a client + tool call. + +Coverage: `tests/providers/kiro/kiro-structured-output.test.ts`, `tests/providers/kiro/kiro-structured-output-strict.test.ts` and +`tests/server/server-kiro-structured-output-e2e.test.ts`. + +> Decision record: [ADR-8071](../decisions/ADR-8071-kiro-structured-output-emulation.md) + ## Bounded fallback HTTP errors Tool-enabled turns in `src/adapters/kiro/stream.ts` hold ordinary text through the one diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index f95d8088f47..10b0bebf434 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -962,7 +962,7 @@ "kiro-remote-image.test.ts": "providers/kiro", "kiro-retry.test.ts": "providers/kiro", "kiro-review-regressions.test.ts": "providers/kiro", "kiro-metering-events.test.ts": "providers/kiro", "kiro-metering-usage.test.ts": "providers/kiro", "kiro-completion-with-real-tools.test.ts": "providers/kiro", - "kiro-single-final.test.ts": "providers/kiro", + "kiro-single-final.test.ts": "providers/kiro", "kiro-structured-output.test.ts": "providers/kiro", "kiro-structured-output-strict.test.ts": "providers/kiro", "kiro-stream.test.ts": "providers/kiro", "kiro-transport-parity.test.ts": "providers/kiro", "kiro-usage-quota.test.ts": "providers/kiro", "kiro-usage-restart.test.ts": "providers/kiro", "kiro-windows-cli-db-path.test.ts": "providers/kiro", "kiro-windows-cli-executable-path.test.ts": "providers/kiro", @@ -1331,7 +1331,7 @@ "server-images-bodyless-content-length.test.ts": "server", "server-images-pool-admission.test.ts": "server", "server-images.test.ts": "server", "server-key-failover-e2e.test.ts": "server", "distributed-key-401.test.ts": "server", "server-jev-combo-e2e.test.ts": "server", "server-kiro-completion-e2e.test.ts": "server", - "combo-requested-effort.test.ts": "server", + "combo-requested-effort.test.ts": "server", "server-kiro-structured-output-e2e.test.ts": "server", "server-kiro-device-login.test.ts": "server", "server-kiro-oauth-401-replay.test.ts": "server", "server-kiro-refusal-e2e.test.ts": "server", "server-kiro-usage-log-e2e.test.ts": "server", "server-live-existing-call.test.ts": "server", "server-live-close.test.ts": "server", diff --git a/tests/providers/kiro/kiro-adapter.test.ts b/tests/providers/kiro/kiro-adapter.test.ts index a55bcae4aa0..aee5513f886 100644 --- a/tests/providers/kiro/kiro-adapter.test.ts +++ b/tests/providers/kiro/kiro-adapter.test.ts @@ -1467,11 +1467,6 @@ describe("kiro adapter — buildRequest", () => { } as OcxParsedRequest)).rejects.toThrow(/Kiro (supports only|does not support)/); } - await expect(createKiroAdapter(provider).buildRequest({ - ...parsedWith([{ role: "user", content: "hi" }], [bashTool]), - _structuredOutput: true, - } as OcxParsedRequest)).rejects.toThrow("Kiro does not support Responses structured output"); - const none = { ...parsedWith([{ role: "user", content: "hi" }], [bashTool]), options: { toolChoice: "none" } } as OcxParsedRequest; const current = JSON.parse((await createKiroAdapter(provider).buildRequest(none)).body).conversationState.currentMessage.userInputMessage; expect(current.userInputMessageContext?.tools).toBeUndefined(); @@ -1530,22 +1525,7 @@ describe("kiro adapter — buildRequest", () => { } }); - test("still refuses genuine structured output", async () => { - for (const text of [ - { format: { type: "json_schema", name: "answer", schema: { type: "object" } } }, - { format: { type: "json_object" } }, - ]) { - const parsed = parseRequest({ - model: "kiro/claude-haiku-4.5", - input: "test", - stream: true, - text, - } as never); - expect(parsed._structuredOutput).toBe(true); - await expect(createKiroAdapter(provider).buildRequest(parsed)) - .rejects.toThrow("Kiro does not support Responses structured output"); - } - }); + // Genuine structured output is emulated, not refused: kiro-structured-output.test.ts owns it. test("accepts Codex's permissive parallel-tool hint while keeping the Kiro wire serialized", async () => { const parsed = parseRequest({ diff --git a/tests/providers/kiro/kiro-structured-output-strict.test.ts b/tests/providers/kiro/kiro-structured-output-strict.test.ts new file mode 100644 index 00000000000..68f5e7d4a6d --- /dev/null +++ b/tests/providers/kiro/kiro-structured-output-strict.test.ts @@ -0,0 +1,514 @@ +import { describe, expect, test } from "bun:test"; +import { sanitizeKiroSchema } from "../../../src/adapters/kiro-tools"; +import { kiroStructuredOutputSpec, kiroStructuredStopFailure, parseKiroStructuredText, parseKiroStructuredToolInput } from "../../../src/adapters/kiro/structured-output"; +import { kiroDecimal, readKiroJson } from "../../../src/adapters/kiro/structured-output-json"; +import { validateKiroStrictValue } from "../../../src/adapters/kiro/structured-output-schema"; +import { parseRequest } from "../../../src/responses/parser"; +import type { OcxParsedRequest } from "../../../src/types"; + +// Strict structured output (`strict: true`) promises the caller a value that conforms to the whole +// schema, so the emulation validates the full OpenAI strict subset before relaying a value, and +// refuses a strict schema that uses a keyword it cannot check. Non-strict formats keep the light +// root checks covered in kiro-structured-output.test.ts. + +/** A parsed Responses request whose `text.format` is a json_schema with `schema` and `strict`. */ +function request(schema: Record, strict = true): OcxParsedRequest { + return parseRequest({ + model: "kiro/claude-opus-5.5", + input: "Answer in JSON", + stream: true, + text: { format: { type: "json_schema", name: "response", strict, schema } }, + } as never); +} +const FAIL = "Kiro structured output did not match the requested format"; +const titleSchema = { + type: "object", + properties: { title: { type: "string" } }, + required: ["title"], + additionalProperties: false, +}; + +describe("Kiro strict structured output validation", () => { + const spec = kiroStructuredOutputSpec(request(titleSchema))!; + + test("a wrong property type and an extra property are refused under strict", () => { + expect(parseKiroStructuredToolInput(spec, '{"title":42,"extra":true}')).toEqual({ + error: `${FAIL}: expected a JSON string`, + }); + expect(parseKiroStructuredToolInput(spec, '{"title":"x","extra":true}')).toEqual({ + error: `${FAIL}: an object has a property the schema does not allow`, + }); + expect(parseKiroStructuredText(spec, '{"title":"x","extra":1}')).toBeUndefined(); + expect(parseKiroStructuredToolInput(spec, '{"title":"x"}')).toEqual({ json: '{"title":"x"}' }); + }); + + test("the same value stays accepted when the format is not strict", () => { + const loose = kiroStructuredOutputSpec(request(titleSchema, false))!; + expect(parseKiroStructuredToolInput(loose, '{"title":42,"extra":true}')).toEqual({ json: '{"title":42,"extra":true}' }); + }); + + const rich = { + type: "object", + properties: { + kind: { enum: ["a", "b"] }, + version: { const: "v2" }, + note: { type: ["string", "null"] }, + tags: { type: "array", items: { type: "string" } }, + item: { $ref: "#/$defs/item" }, + choice: { anyOf: [{ type: "integer" }, { type: "object", properties: { x: { type: "boolean" } }, required: ["x"], additionalProperties: false }] }, + }, + required: ["kind", "version", "note", "tags", "item", "choice"], + additionalProperties: false, + $defs: { item: { type: "object", properties: { id: { type: "string" } }, required: ["id"], additionalProperties: false } }, + }; + const richSpec = kiroStructuredOutputSpec(request(rich))!; + const good = { kind: "a", version: "v2", note: null, tags: ["t"], item: { id: "i" }, choice: { x: true } }; + test.each([ + [good, undefined], + [{ ...good, choice: 3 }, undefined], + [{ ...good, kind: "c" }, "expected one of the enum values"], + [{ ...good, version: "v3" }, "expected the const value"], + [{ ...good, note: 1 }, "expected a JSON string or null"], + [{ ...good, tags: ["t", 1] }, "expected a JSON string"], + [{ ...good, item: { id: "i", more: 1 } }, "an object has a property the schema does not allow"], + [{ ...good, choice: 1.5 }, "matched no anyOf branch"], + [{ ...good, tags: undefined }, "an object is missing a required property"], + ])("%j -> %j", (value, problem) => { + const raw = JSON.stringify(value); + expect(parseKiroStructuredToolInput(richSpec, raw)).toEqual(problem === undefined ? { json: raw } : { error: `${FAIL}: ${problem}` }); + }); + + test("a strict schema using a keyword the emulation cannot check is refused before sending", () => { + expect(() => kiroStructuredOutputSpec(request({ ...titleSchema, properties: { title: { type: "string", format: "email" } } }))) + .toThrow('Kiro does not support Responses structured output: strict schema keyword "format" cannot be enforced'); + expect(() => kiroStructuredOutputSpec(request({ oneOf: [{ type: "string" }, { type: "integer" }] }))) + .toThrow('strict schema keyword "oneOf" cannot be enforced'); + // The same keywords stay acceptable without strict, where only the light checks apply. + expect(kiroStructuredOutputSpec(request({ oneOf: [{ type: "string" }] }, false))?.wrapped).toBe(true); + }); +}); + +describe("Kiro structured-output wrapping keeps references on the original root", () => { + const tree = { anyOf: [{ type: "string" }, { type: "array", items: { $ref: "#" } }] }; + + test("a root self-reference is rewritten to the wrapped root without strict", () => { + const spec = kiroStructuredOutputSpec(request(tree, false))!; + expect(spec.wrapped).toBe(true); + const value = (spec.toolSchema.properties as Record).value; + expect(value.anyOf[1].items.$ref).toBe("#/properties/value"); + expect(spec.schema).toEqual(tree); + }); + + test("root-relative pointers move with the root, and hoisted definitions keep theirs", () => { + const schema = { + type: "array", + items: { $ref: "#/$defs/node" }, + $defs: { node: { type: "object", properties: { kids: { $ref: "#" }, peer: { $ref: "#/items" } } } }, + }; + const spec = kiroStructuredOutputSpec(request(schema, false))!; + const tool = spec.toolSchema as Record; + expect(tool.properties.value.items.$ref).toBe("#/$defs/node"); + expect(tool.$defs.node.properties.kids.$ref).toBe("#/properties/value"); + expect(tool.$defs.node.properties.peer.$ref).toBe("#/properties/value/items"); + }); + + test("a strict recursive schema validates through a definition", () => { + const spec = kiroStructuredOutputSpec(request({ + $ref: "#/$defs/tree", + $defs: { tree: { anyOf: [{ type: "string" }, { type: "array", items: { $ref: "#/$defs/tree" } }] } }, + }))!; + expect(parseKiroStructuredToolInput(spec, '{"value":["a",["b"]]}')).toEqual({ json: '["a",["b"]]' }); + expect(parseKiroStructuredToolInput(spec, '{"value":["a",[1]]}')).toEqual({ + error: `${FAIL}: matched no anyOf branch`, + }); + }); +}); + +const REFUSED = "Kiro does not support Responses structured output: strict schema keyword"; + +describe("Kiro strict validation stays bounded", () => { + // Three branches, two of them identical recursive arrays: an unbounded walk tries both at every + // level, so a value 40 arrays deep that fails at the bottom costs about 2^40 visits. + const branching = { + $ref: "#/$defs/n", + $defs: { n: { anyOf: [{ type: "null" }, { type: "array", items: { $ref: "#/$defs/n" } }, { type: "array", items: { $ref: "#/$defs/n" } }] } }, + }; + const nested = (depth: number, leaf: string): string => `${"[".repeat(depth)}${leaf}${"]".repeat(depth)}`; + + test("a recursive anyOf stops at the global visit budget instead of branching exponentially", () => { + const spec = kiroStructuredOutputSpec(request(branching))!; + const document = readKiroJson(nested(40, "1"))!; + const result = validateKiroStrictValue(branching, document.value, document.nodes); + expect(result.problem).toContain("needs more work than strict validation allows"); + expect(result.visits).toBeLessThanOrEqual(result.budget + 1); + expect(parseKiroStructuredToolInput(spec, `{"value":${nested(40, "1")}}`)).toEqual({ + error: expect.stringContaining("needs more work than strict validation allows"), + }); + expect(parseKiroStructuredToolInput(spec, `{"value":${nested(40, "null")}}`)).toEqual({ json: nested(40, "null") }); + }); + + test("one depth limit spans items and properties, so a deep value cannot exhaust the stack", () => { + const list = { $ref: "#/$defs/l", $defs: { l: { type: "array", items: { $ref: "#/$defs/l" } } } }; + const spec = kiroStructuredOutputSpec(request(list))!; + const tooDeep = { error: expect.stringMatching(/^Kiro structured output did not match the requested format: the value nests deeper than strict validation allows$/) }; + // 200 levels pass the reader but not the validator; 100,000 levels stop at the reader. + for (const depth of [200, 100_000]) { + expect(parseKiroStructuredToolInput(spec, `{"value":${nested(depth, "")}}`)).toEqual(tooDeep); + } + expect(parseKiroStructuredToolInput(spec, `{"value":${nested(20, "")}}`)).toEqual({ json: nested(20, "") }); + // Alternating properties and items share the same counter, which used to restart at each. + const chain = { $defs: { n: { type: "object", properties: { k: { type: "array", items: { $ref: "#/$defs/n" } } } } }, $ref: "#/$defs/n" }; + const chainSpec = kiroStructuredOutputSpec(request(chain))!; + const deep = `${'{"k":['.repeat(150)}${"]}".repeat(150)}`; + expect(parseKiroStructuredToolInput(chainSpec, `{"value":${deep}}`)).toEqual(tooDeep); + }); +}); + +describe("Kiro strict references", () => { + test.each([ + ["#/default", { type: "object", default: { format: "email" }, properties: { a: { $ref: "#/default" } } }], + ["#", { type: "object", properties: { a: { $ref: "#" } } }], + ["#/properties/b", { type: "object", properties: { a: { $ref: "#/properties/b" }, b: { type: "string" } } }], + ["a nested $defs", { type: "object", properties: { a: { $defs: { x: { type: "string" } } } } }], + ["a missing definition", { type: "object", properties: { a: { $ref: "#/$defs/missing" } } }], + ["a ref-only cycle", { $ref: "#/$defs/a", $defs: { a: { $ref: "#/$defs/b" }, b: { anyOf: [{ $ref: "#/$defs/a" }] } } }], + ])("%s is refused before sending", (_label, schema) => { + expect(() => kiroStructuredOutputSpec(request(schema as Record))).toThrow(REFUSED); + }); + + test("an annotation's content is never validated through a reference", () => { + // The pre-fix walker resolved #/default and enforced `format` nowhere, relaying any string. + expect(() => kiroStructuredOutputSpec(request({ type: "object", default: { format: "email" }, properties: { a: { $ref: "#/default" } } }))) + .toThrow(`${REFUSED} "$ref" cannot be enforced`); + }); +}); + +describe("Kiro strict numbers", () => { + // The request reaches the adapter already parsed, so a schema number written with more digits than + // a double holds arrives rounded: 1.0000000000000001 is 1 by then. Exact-equality keywords on + // numbers are therefore refused in strict mode. + test.each([ + ["const", { const: 5 }], + ["const", { const: JSON.parse("1.0000000000000001") }], + ["const", { const: { limit: 1 } }], + ["enum", { enum: [1, 2] }], + ["enum", { enum: ["a", [3]] }], + ])("a numeric %s literal is refused before sending", (keyword, property) => { + expect(() => kiroStructuredOutputSpec(request({ type: "object", properties: { n: property } }))) + .toThrow(`${REFUSED} "${keyword}" cannot be enforced`); + }); + + test("const and enum without numbers are still enforced", () => { + const spec = kiroStructuredOutputSpec(request({ + type: "object", + properties: { kind: { const: "pdf" }, mode: { enum: ["a", null, true, { x: "y" }] } }, + additionalProperties: false, + }))!; + expect(parseKiroStructuredToolInput(spec, '{"kind":"pdf","mode":{"x":"y"}}')).toEqual({ json: '{"kind":"pdf","mode":{"x":"y"}}' }); + expect(parseKiroStructuredToolInput(spec, '{"kind":"png"}')).toEqual({ error: `${FAIL}: expected the const value` }); + expect(parseKiroStructuredToolInput(spec, '{"mode":"b"}')).toEqual({ error: `${FAIL}: expected one of the enum values` }); + }); + + test("bounds accept safe integers and decimals up to 15 significant digits, and refuse past that edge", () => { + for (const maximum of [100, -5, 0.123456789012345, 9007199254740991]) { + expect(() => kiroStructuredOutputSpec(request({ type: "number", maximum }))).not.toThrow(); + } + for (const maximum of [0.1234567890123456, 9007199254740992, 1e300]) { + expect(() => kiroStructuredOutputSpec(request({ type: "number", maximum }))) + .toThrow(`${REFUSED} "maximum" cannot be enforced`); + } + }); + + test("bounds judge the exact lexeme the client receives", () => { + const spec = kiroStructuredOutputSpec(request({ + type: "object", + properties: { m: { type: "number", maximum: 10 }, i: { type: "integer", minimum: 0 } }, + additionalProperties: false, + }))!; + expect(parseKiroStructuredToolInput(spec, '{"m":10.0}')).toEqual({ json: '{"m":10.0}' }); + expect(parseKiroStructuredToolInput(spec, '{"m":10.0000000000000001}')).toEqual({ error: `${FAIL}: a number is above maximum` }); + expect(parseKiroStructuredToolInput(spec, '{"i":-0.0000000000000001}')).toEqual({ error: `${FAIL}: expected a JSON integer` }); + expect(parseKiroStructuredToolInput(spec, '{"i":1.0000000000000001}')).toEqual({ error: `${FAIL}: expected a JSON integer` }); + expect(parseKiroStructuredText(spec, '{"m":1e400}')).toBeUndefined(); + }); + + test("the wrapped form relays the original bytes of value", () => { + const spec = kiroStructuredOutputSpec(request({ type: "integer" }))!; + expect(spec.wrapped).toBe(true); + expect(parseKiroStructuredToolInput(spec, '{"value":12345678901234567890}')).toEqual({ json: "12345678901234567890" }); + }); +}); + +describe("Kiro strict multipleOf and regular expressions are refused", () => { + test.each([ + ["multipleOf", { type: "number", multipleOf: 3 }], + ["multipleOf", { type: "number", multipleOf: 0.1 }], + ["pattern", { type: "string", pattern: "^[a-z]+$" }], + ["pattern", { type: "string", pattern: "(a+)+$" }], + ["patternProperties", { type: "object", patternProperties: { "^x": { type: "string" } } }], + ])("strict %s is refused before sending", (keyword, schema) => { + expect(() => kiroStructuredOutputSpec(request(schema))).toThrow(`${REFUSED} "${keyword}" cannot be enforced`); + }); + + test("non-strict formats still accept them, with the light root checks", () => { + for (const schema of [ + { type: "number", multipleOf: 3 }, + { type: "string", pattern: "(a+)+$" }, + { type: "object", properties: { n: { const: 5 } } }, + ]) { + expect(() => kiroStructuredOutputSpec(request(schema, false))).not.toThrow(); + } + }); +}); + +describe("Kiro schema sanitization keeps literal values", () => { + test("const, enum and default are data, so keyword-named keys inside them survive", () => { + const schema = { + type: "object", + properties: { + kind: { const: { format: "pdf" } }, + mode: { enum: [{ pattern: "x" }, "plain"] }, + opts: { type: "object", default: { additionalProperties: 1 } }, + name: { type: "string", format: "email" }, + }, + }; + expect(sanitizeKiroSchema(schema)).toEqual({ + type: "object", + properties: { + kind: { const: { format: "pdf" } }, + mode: { enum: [{ pattern: "x" }, "plain"] }, + opts: { type: "object", default: { additionalProperties: 1 } }, + name: { type: "string" }, + }, + }); + }); +}); + +describe("Kiro strict schema hardening", () => { + test("a $ref beside an assertion has no single meaning and is refused; annotations and root $defs are fine", () => { + const defs = { $defs: { n: { type: "integer" } } }; + expect(() => kiroStructuredOutputSpec(request({ type: "object", properties: { a: { $ref: "#/$defs/n", type: "string" } }, ...defs }))) + .toThrow(REFUSED + ' "$ref" cannot be enforced'); + expect(() => kiroStructuredOutputSpec(request({ type: "object", properties: { a: { $ref: "#/$defs/n", description: "count" } }, ...defs }))).not.toThrow(); + expect(() => kiroStructuredOutputSpec(request({ $ref: "#/$defs/n", ...defs }))).not.toThrow(); + }); + + test("a schema past the node cap is refused quickly, and a wide accepted anyOf prepares in linear time", () => { + const wide = (n: number) => ({ type: "object", properties: { a: { anyOf: Array.from({ length: n }, () => true) } } }); + expect(() => kiroStructuredOutputSpec(request(wide(64_000)))).toThrow(REFUSED); + expect(() => kiroStructuredOutputSpec(request(wide(9_000)))).not.toThrow(); + }, 30_000); + + test("literal comparisons are charged to the validation budget", () => { + const big = Array.from({ length: 5_000 }, () => "x"); + const schema = { + type: "object", + properties: { a: { anyOf: Array.from({ length: 2_000 }, () => ({ $ref: "#/$defs/big" })) } }, + $defs: { big: { const: big } }, + }; + const raw = JSON.stringify({ a: [...big.slice(1), "y"] }); + const document = readKiroJson(raw)!; + const result = validateKiroStrictValue(schema, document.value, document.nodes); + expect(result.problem).toContain("needs more work than strict validation allows"); + expect(result.visits).toBeLessThanOrEqual(result.budget + 1); + }, 30_000); + + test("a literal larger than the schema-wide cap is refused before sending", () => { + expect(() => kiroStructuredOutputSpec(request({ type: "object", properties: { a: { const: Array.from({ length: 20_000 }, () => "x") } } }))) + .toThrow(REFUSED + ' "const" cannot be enforced'); + }); + + test("diagnostics and refusals carry no schema or answer text", () => { + const spec = kiroStructuredOutputSpec(request(titleSchema))!; + const result = parseKiroStructuredToolInput(spec, '{"title":"t","secret-model-key-123":1}'); + expect(result).toEqual({ error: FAIL + ": an object has a property the schema does not allow" }); + let message = ""; + try { kiroStructuredOutputSpec(request({ type: "object", "caller-key-xyz": true })); } catch (error) { message = String(error); } + expect(message).toContain("uses a keyword that cannot be enforced"); + expect(message).not.toContain("caller-key-xyz"); + }); +}); + +describe("Kiro schema sanitization keeps a __proto__ property", () => { + test("a property or definition named __proto__ stays an own key", () => { + const schema = JSON.parse('{"type":"object","properties":{"__proto__":{"type":"string"}},"required":["__proto__"],"$defs":{"__proto__":{"type":"string"}}}'); + const out = sanitizeKiroSchema(schema) as { properties: object; $defs: object }; + expect(Object.keys(out.properties)).toEqual(["__proto__"]); + expect(Object.keys(out.$defs)).toEqual(["__proto__"]); + expect(JSON.stringify(out)).toContain('"properties":{"__proto__":{"type":"string"}}'); + }); +}); + +describe("Kiro strict budget covers every per-node loop", () => { + test("a long required list re-checked through failing anyOf branches stays within the budget", () => { + const required = Array.from({ length: 5_000 }, (_, i) => "k" + i); + const schema = { + type: "object", + properties: { a: { anyOf: Array.from({ length: 2_000 }, () => ({ $ref: "#/$defs/obj" })) } }, + $defs: { obj: { type: "object", required: [...required.slice(1), "missing"] } }, + }; + const raw = JSON.stringify({ a: Object.fromEntries(required.map(key => [key, 1])) }); + const document = readKiroJson(raw)!; + const result = validateKiroStrictValue(schema, document.value, document.nodes); + expect(result.problem).toContain("needs more work than strict validation allows"); + }, 30_000); + + test("a percent-encoded separator in $ref is a pointer segment, not part of a definition name", () => { + const schema = { + type: "object", + properties: { v: { $ref: "#/$defs/a%2Fproperties%2Fb" } }, + $defs: { a: { type: "object", properties: { b: { type: "string" } } }, "a/properties/b": { type: "number" } }, + }; + expect(() => kiroStructuredOutputSpec(request(schema))).toThrow(REFUSED + ' "$ref" cannot be enforced'); + const named = { type: "object", properties: { v: { $ref: "#/$defs/x~1y" } }, $defs: { "x/y": { type: "number" } } }; + const spec = kiroStructuredOutputSpec(request(named))!; + expect(parseKiroStructuredToolInput(spec, '{"v":1}')).toEqual({ json: '{"v":1}' }); + expect(parseKiroStructuredToolInput(spec, '{"v":"s"}')).toEqual({ error: FAIL + ": expected a JSON number" }); + }); +}); + +describe("Kiro strict repeated string work and resource identifiers", () => { + test("a long string re-checked by many length-constrained branches stays fast and bounded", () => { + const schema = { type: "object", properties: { s: { anyOf: Array.from({ length: 9_000 }, () => ({ maxLength: 0 })) } } }; + const raw = JSON.stringify({ s: "x".repeat(1_000_000) }); + const document = readKiroJson(raw)!; + const result = validateKiroStrictValue(schema, document.value, document.nodes); + expect(result.problem).toBeDefined(); + }, 30_000); + + test("branches without a length constraint never walk the string", () => { + const schema = { type: "object", properties: { s: { anyOf: [...Array.from({ length: 9_000 }, () => ({ type: "number" })), { type: "string" }] } } }; + const raw = JSON.stringify({ s: "x".repeat(1_000_000) }); + const document = readKiroJson(raw)!; + expect(validateKiroStrictValue(schema, document.value, document.nodes).problem).toBeUndefined(); + }, 30_000); + + test("a nested $id or an anchor is refused; a root $id is kept", () => { + const defs = { $defs: { x: { type: "string" } } }; + expect(() => kiroStructuredOutputSpec(request({ type: "object", properties: { a: { $id: "https://example.com/inner", type: "string" } }, ...defs }))) + .toThrow(REFUSED + ' "$id" cannot be enforced'); + expect(() => kiroStructuredOutputSpec(request({ type: "object", properties: { a: { $anchor: "here", type: "string" } } }))) + .toThrow(REFUSED + ' "$anchor" cannot be enforced'); + expect(() => kiroStructuredOutputSpec(request({ $id: "https://example.com/root", type: "object", properties: { a: { $ref: "#/$defs/x" } }, ...defs }))) + .not.toThrow(); + }); +}); + +describe("Kiro strict numbers, types and wrapping edges", () => { + test("a long number lexeme re-checked by many bounded branches is charged and stays within the budget", () => { + const schema = { type: "object", properties: { n: { anyOf: Array.from({ length: 1_000 }, () => ({ maximum: 0 })) } } }; + const raw = '{"n":1e' + "9".repeat(100_000) + "}"; + const document = readKiroJson(raw)!; + const result = validateKiroStrictValue(schema, document.value, document.nodes); + expect(result.problem).toContain("needs more work than strict validation allows"); + expect(result.visits).toBeLessThanOrEqual(result.budget + 1_000); + }, 30_000); + + test("an integer type check on a long lexeme is charged the same way", () => { + const schema = { type: "object", properties: { n: { anyOf: Array.from({ length: 1_000 }, () => ({ type: ["integer", "null"], maximum: -1 })) } } }; + const raw = '{"n":' + "1".repeat(100_000) + "}"; + const document = readKiroJson(raw)!; + expect(validateKiroStrictValue(schema, document.value, document.nodes).problem).toContain("needs more work than strict validation allows"); + }, 30_000); + + test("a type list with a repeated type is refused, so type lists stay short", () => { + expect(() => kiroStructuredOutputSpec(request({ type: ["string", "string"] }))).toThrow(REFUSED + ' "type" cannot be enforced'); + }); + + test("a strict format without an object schema is refused instead of becoming an unrestricted object", () => { + const strictFalse = parseRequest({ + model: "kiro/claude-opus-5.5", input: "x", stream: true, + text: { format: { type: "json_schema", name: "response", strict: true, schema: false } }, + } as never); + expect(() => kiroStructuredOutputSpec(strictFalse)).toThrow("a strict format needs a JSON object schema"); + const looseFalse = parseRequest({ + model: "kiro/claude-opus-5.5", input: "x", stream: true, + text: { format: { type: "json_schema", name: "response", strict: false, schema: false } }, + } as never); + expect(kiroStructuredOutputSpec(looseFalse)).toMatchObject({ toolSchema: { type: "object" }, wrapped: false }); + }); + + test("wrapping keeps an encoded $defs reference and a root $id at the wrapper root", () => { + const spec = kiroStructuredOutputSpec(request({ + $id: "https://example.com/root", + type: "array", + items: { $ref: "#/%24defs/item" }, + $defs: { item: { type: "string" } }, + }))!; + expect(spec.wrapped).toBe(true); + const tool = spec.toolSchema as { $id?: string; $defs?: object; properties: { value: { $id?: string; items: { $ref: string } } } }; + expect(tool.$id).toBe("https://example.com/root"); + expect(tool.properties.value.$id).toBeUndefined(); + expect(tool.properties.value.items.$ref).toBe("#/%24defs/item"); + expect(tool.$defs).toEqual({ item: { type: "string" } }); + }); + + test("a wrapped answer past the reader's nesting limit is refused rather than re-serialized", () => { + const spec = kiroStructuredOutputSpec(request({ type: "array" }, false))!; + expect(spec.wrapped).toBe(true); + const deep = "[".repeat(520) + "9007199254740993" + "]".repeat(520); + expect(parseKiroStructuredToolInput(spec, '{"value":' + deep + "}")) + .toEqual({ error: FAIL + ": the value nests deeper than the proxy can relay exactly" }); + }); +}); + +describe("Kiro decimal and wrapped reference edges", () => { + test("a long run of zeros inside a number is normalized in linear time", () => { + const lexeme = "1" + "0".repeat(200_000) + "1"; + const decimal = kiroDecimal(lexeme); + expect(decimal.digits.length).toBe(200_002); + expect(kiroDecimal("1" + "0".repeat(200_000)).digits).toBe("1"); + }, 30_000); + + test("an encoded root-relative reference is re-pointed under value when wrapping", () => { + const spec = kiroStructuredOutputSpec(request({ type: "array", items: { anyOf: [{ type: "string" }, { $ref: "#%2Fitems" }] } }, false))!; + expect(spec.wrapped).toBe(true); + const tool = spec.toolSchema as { properties: { value: { items: { anyOf: Array<{ $ref?: string }> } } } }; + expect(tool.properties.value.items.anyOf[1]!.$ref).toBe("#/properties/value%2Fitems"); + }); + + test("references inside a nested $id resource are left relative to that resource", () => { + const spec = kiroStructuredOutputSpec(request({ + type: "array", + items: { $id: "https://example.com/inner", type: "object", properties: { x: { type: "string" }, y: { $ref: "#/properties/x" } } }, + }, false))!; + const tool = spec.toolSchema as { properties: { value: { items: { properties: { y: { $ref: string } } } } } }; + expect(tool.properties.value.items.properties.y.$ref).toBe("#/properties/x"); + }); + + test("an exponent millions of digits long saturates, and bounds still judge the number correctly", () => { + const huge = "1e" + "9".repeat(2_000_000); + expect(kiroDecimal(huge).exponent).toBe(10n ** 15n + 1n); + expect(kiroDecimal("-1e-" + "9".repeat(2_000_000)).exponent).toBe(1n - 10n ** 15n); + const over = readKiroJson(huge)!; + expect(validateKiroStrictValue({ type: "number", maximum: 100 }, over.value, over.nodes).problem).toBeDefined(); + expect(validateKiroStrictValue({ type: "integer" }, over.value, over.nodes).problem).toBeUndefined(); + const tiny = readKiroJson("1e-" + "9".repeat(2_000_000))!; + expect(validateKiroStrictValue({ type: "number", minimum: 0, maximum: 1 }, tiny.value, tiny.nodes).problem).toBeUndefined(); + expect(validateKiroStrictValue({ type: "integer" }, tiny.value, tiny.nodes).problem).toBeDefined(); + }, 30_000); + + test("wrapping keeps references to the definition containers themselves at the wrapper root", () => { + for (const ref of ["#/$defs", "#/%24defs", "#/definitions", "#%2F%24defs"]) { + const spec = kiroStructuredOutputSpec(request({ type: "array", items: { $ref: ref }, $defs: { not: false } }, false))!; + const tool = spec.toolSchema as { properties: { value: { items: { $ref: string } } } }; + expect(tool.properties.value.items.$ref).toBe(ref); + } + }); + + test("a strict $ref with a JSON Pointer escape other than ~0 or ~1 is refused", () => { + const schema = (ref: string, name: string) => ({ + type: "object", properties: { a: { $ref: ref } }, required: ["a"], additionalProperties: false, $defs: { [name]: { type: "string" } }, + }); + expect(() => kiroStructuredOutputSpec(request(schema("#/$defs/a~2b", "a~2b")))).toThrow(REFUSED + ' "$ref" cannot be enforced'); + expect(() => kiroStructuredOutputSpec(request(schema("#/$defs/a~", "a~")))).toThrow(REFUSED + ' "$ref" cannot be enforced'); + expect(kiroStructuredOutputSpec(request(schema("#/$defs/a~0b", "a~b")))?.strict).toBe(true); + }); + + test("a retry stop diagnostic names only stop reasons Kiro documents", () => { + expect(kiroStructuredStopFailure("MAX_TOKENS")).toContain("stopped with MAX_TOKENS"); + const unknown = kiroStructuredStopFailure("UPSTREAM_SUPPLIED_TEXT"); + expect(unknown).not.toContain("UPSTREAM_SUPPLIED_TEXT"); + expect(unknown).toContain("stopped before a complete answer"); + }); +}); diff --git a/tests/providers/kiro/kiro-structured-output.test.ts b/tests/providers/kiro/kiro-structured-output.test.ts new file mode 100644 index 00000000000..970b4d78044 --- /dev/null +++ b/tests/providers/kiro/kiro-structured-output.test.ts @@ -0,0 +1,452 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { createKiroAdapter } from "../../../src/adapters/kiro"; +import { + KIRO_COMPLETION_TOOL_NAME, + KIRO_STRUCTURED_OUTPUT_TOOL_NAME, +} from "../../../src/adapters/kiro-constants"; +import { resetKiroThrottleStateForTests } from "../../../src/adapters/kiro-retry"; +import { parseKiroStructuredText, parseKiroStructuredToolInput, kiroStructuredOutputSpec } from "../../../src/adapters/kiro/structured-output"; +import { encodeMessage } from "../../../src/lib/eventstream-decoder"; +import { createTranslatorBudget, releaseTranslatedEvent } from "../../../src/lib/translator-budget"; +import { parseRequest } from "../../../src/responses/parser"; +import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig } from "../../../src/types"; + +// Kiro has no schema-constrained response mode, so structured output is emulated through a private +// terminal tool whose input schema is the caller's format (src/adapters/kiro/structured-output.ts). +// The contract under test: the client receives JSON that tool (or a JSON-only text answer) +// produced, or an error — never prose — and an ordinary turn is untouched. + +const provider: OcxProviderConfig = { + adapter: "kiro", baseUrl: "https://runtime.us-east-1.kiro.dev", apiKey: "ksk_test", +}; +const titleSchema = { + type: "object", + properties: { title: { type: "string" } }, + required: ["title"], + additionalProperties: false, +}; +const bash = { type: "function", name: "bash", description: "Run a command", parameters: { type: "object" } }; + +/** A parsed Responses request asking for structured output in `format` (the T3 Code title schema by default). */ +function structuredRequest(format: Record = { type: "json_schema", name: "response", schema: titleSchema }, tools?: unknown[]): OcxParsedRequest { + return parseRequest({ + model: "kiro/claude-opus-5.5", + input: "Title this thread: fix the Kiro structured output bug", + stream: true, + text: { format }, + ...(tools ? { tools } : {}), + } as never); +} + +interface KiroTool { toolSpecification: { name: string; description: string; inputSchema: { json: Record } } } +/** Build the Kiro request for `parsed` and return its current user message and advertised tools. */ +async function payloadOf(parsed: OcxParsedRequest) { + const body = JSON.parse((await createKiroAdapter(provider).buildRequest(parsed)).body); + const current = body.conversationState.currentMessage.userInputMessage; + const tools = (current.userInputMessageContext?.tools ?? []) as KiroTool[]; + return { current, tools, names: tools.map(tool => tool.toolSpecification.name) }; +} + +const frame = (type: string, payload: object) => encodeMessage( + { ":message-type": "event", ":event-type": type }, + new TextEncoder().encode(JSON.stringify(payload)), +); +const text = (content: string) => frame("assistantResponseEvent", { content }); +const stop = (stopReason: string) => frame("metadataEvent", { stopReason }); +/** The frames of one complete Kiro tool call: its input, then its stop frame. */ +function tool(name: string, input: unknown, id = "call-1"): Uint8Array[] { + return [ + frame("toolUseEvent", { name, toolUseId: id, input: typeof input === "string" ? input : JSON.stringify(input) }), + frame("toolUseEvent", { name, toolUseId: id, stop: true }), + ]; +} +const structured = (input: unknown) => tool(KIRO_STRUCTURED_OUTPUT_TOOL_NAME, input); +/** A Kiro event-stream response that replays the given frames in order. */ +function response(frames: Uint8Array[]): Response { + return new Response(new ReadableStream({ + /** Enqueue every frame, then close the stream. */ + start(controller) { + for (const value of frames) controller.enqueue(value); + controller.close(); + }, + })); +} +afterEach(resetKiroThrottleStateForTests); + +/** Run a streaming turn whose upstream attempts answer with `attempts` in order. */ +async function run(input: OcxParsedRequest, ...attempts: Uint8Array[][]) { + return runWith(input, false, ...attempts); +} + +/** + * Run one turn through the adapter, streaming or buffered, with each upstream attempt answering from + * `attempts`. Returns the client events, every request body, the text deltas and the terminal event. + */ +async function runWith(input: OcxParsedRequest, buffered: boolean, ...attempts: Uint8Array[][]) { + const adapter = createKiroAdapter(provider); + const budget = createTranslatorBudget(); + const events: AdapterEvent[] = []; + const bodies: string[] = []; + try { + const request = await adapter.buildRequest(input); + const upstream = await adapter.fetchResponse!(request, { + /** Record each request body and answer it with the next scripted attempt. */ + executor: (async (_url, init) => { + bodies.push(String(init?.body)); + const frames = attempts[bodies.length - 1]; + if (!frames) throw new Error("unexpected extra Kiro attempt"); + return response(frames); + }) as typeof fetch, + }); + if (buffered) events.push(...await adapter.parseResponse!(upstream, budget)); + else for await (const event of adapter.parseStream(upstream, budget)) events.push(event); + if (buffered) for (const event of events) releaseTranslatedEvent(event, budget); + expect(budget.snapshot().currentBytes).toBe(0); + const texts = events.filter(event => event.type === "text_delta"); + return { events, bodies, texts, terminal: events.at(-1) }; + } finally { + budget.dispose(); + } +} + +describe("Kiro structured-output payload", () => { + test("a Responses json_schema turn is no longer refused and advertises only the structured tool", async () => { + const parsed = structuredRequest(); + expect(parsed._structuredOutput).toBe(true); + const { current, tools, names } = await payloadOf(parsed); + // The structured tool REPLACES the completion tool: both would offer the model a prose exit. + expect(names).toEqual([KIRO_STRUCTURED_OUTPUT_TOOL_NAME]); + // Sanitized the way every Kiro tool schema is: additionalProperties is a validator reject. + expect(tools[0].toolSpecification.inputSchema.json).toEqual({ + type: "object", properties: { title: { type: "string" } }, required: ["title"], + }); + // ...so the caller's original constraints are restated verbatim in the prose contract. + expect(current.content).toContain(`calling ${KIRO_STRUCTURED_OUTPUT_TOOL_NAME} exactly once`); + expect(current.content).toContain(JSON.stringify(titleSchema)); + // A tool-free turn gets no work-tool catalog nudge and no completion contract. + expect(current.content).not.toContain("Tool contract:"); + expect(current.content).not.toContain(KIRO_COMPLETION_TOOL_NAME); + }); + + test("ordinary client tools stay ahead of the structured tool, still without the completion tool", async () => { + const { current, names } = await payloadOf(structuredRequest(undefined, [bash])); + expect(names).toEqual(["bash", KIRO_STRUCTURED_OUTPUT_TOOL_NAME]); + expect(current.content).toContain("Tool contract:"); + expect(current.content).toContain("Ordinary assistant text is mid-task commentary"); + expect(current.content).not.toContain(KIRO_COMPLETION_TOOL_NAME); + }); + + test("a non-object or composed root is wrapped under `value`, with definitions hoisted", async () => { + const schema = { + anyOf: [{ $ref: "#/$defs/item" }, { type: "array", items: { $ref: "#/$defs/item" } }], + $defs: { item: { type: "string", maxLength: 10 } }, + }; + const { tools } = await payloadOf(structuredRequest({ type: "json_schema", name: "items", schema })); + expect(tools[0].toolSpecification.inputSchema.json).toEqual({ + type: "object", + properties: { value: { anyOf: [{ $ref: "#/$defs/item" }, { type: "array", items: { $ref: "#/$defs/item" } }] } }, + required: ["value"], + $defs: { item: { type: "string" } }, + }); + }); + + test("json_object asks for any JSON object", async () => { + const { tools } = await payloadOf(structuredRequest({ type: "json_object" })); + expect(tools.map(entry => entry.toolSpecification.inputSchema.json)).toEqual([{ type: "object" }]); + }); + + test("a plain-text format and an ordinary turn keep the completion tool contract", async () => { + for (const parsed of [ + parseRequest({ model: "kiro/claude-opus-5.5", input: "hi", stream: true, text: { format: { type: "text" } }, tools: [bash] } as never), + parseRequest({ model: "kiro/claude-opus-5.5", input: "hi", stream: true, tools: [bash] } as never), + ]) { + const { names } = await payloadOf(parsed); + expect(names).toEqual(["bash", KIRO_COMPLETION_TOOL_NAME]); + } + }); + + test("a client tool may not claim the structured-output name", async () => { + const parsed = structuredRequest(undefined, [{ ...bash, name: KIRO_STRUCTURED_OUTPUT_TOOL_NAME }]); + await expect(createKiroAdapter(provider).buildRequest(parsed)).rejects.toThrow("reserves the tool name"); + }); +}); + +describe("Kiro structured-output stream", () => { + for (const buffered of [false, true]) { + test(`a structured tool call is relayed as the JSON answer, not as a tool call (buffered=${buffered})`, async () => { + const { events, bodies, texts, terminal } = await runWith( + structuredRequest(), buffered, + [text("Here is a title."), ...structured({ title: "Fix Kiro structured output" })], + ); + expect(texts).toEqual([{ type: "text_delta", text: '{"title":"Fix Kiro structured output"}', phase: "final_answer" }]); + expect(events.some(event => event.type.startsWith("tool_call"))).toBe(false); + expect(terminal).toMatchObject({ type: "done", endTurn: true }); + expect(bodies).toHaveLength(1); + }); + } + + test.each([ + ["bare JSON", '{"title":"Fix Kiro"}', '{"title":"Fix Kiro"}'], + ["one whole fenced block", '```json\n{"title":"Fix Kiro"}\n```', '{"title":"Fix Kiro"}'], + ])("a clean-stop text answer that IS the JSON is accepted without a retry (%s)", async (_label, answer, expected) => { + const { bodies, texts, terminal } = await run(structuredRequest(), [text(answer), stop("END_TURN")]); + expect(texts).toEqual([{ type: "text_delta", text: expected, phase: "final_answer" }]); + expect(terminal).toMatchObject({ type: "done", endTurn: true }); + expect(bodies).toHaveLength(1); + }); + + test("prose earns one bounded retry; the prose itself never reaches the client", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), + [text("Sure! A good title would be: Fix Kiro."), stop("END_TURN")], + structured({ title: "Fix Kiro" }), + ); + expect(bodies).toHaveLength(2); + expect(bodies[1]).toContain("did not deliver the required structured output"); + expect(bodies[1]).toContain(KIRO_STRUCTURED_OUTPUT_TOOL_NAME); + expect(texts).toEqual([{ type: "text_delta", text: '{"title":"Fix Kiro"}', phase: "final_answer" }]); + expect(terminal).toMatchObject({ type: "done", endTurn: true }); + }); + + test("a JSON text answer on the retry is accepted too", async () => { + const { bodies, texts } = await run(structuredRequest(), [text("Thinking about it.")], [text('{"title":"Fix Kiro"}')]); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([{ type: "text_delta", text: '{"title":"Fix Kiro"}', phase: "final_answer" }]); + }); + + test("prose twice is a clear non-retryable error with no text relayed", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), [text("A title: Fix Kiro.")], [text("Fix Kiro structured output")], + ); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([]); + expect(terminal).toMatchObject({ type: "error", code: "invalid_kiro_structured_output", retryable: false, status: 502 }); + }); + + test("a tool input missing a required key is retried once, then accepted", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), structured({ name: "wrong key" }), structured({ title: "Fix Kiro" }), + ); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([{ type: "text_delta", text: '{"title":"Fix Kiro"}', phase: "final_answer" }]); + expect(terminal).toMatchObject({ type: "done", endTurn: true }); + }); + + test("an invalid tool input on the retry fails the turn instead of relaying it", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), structured({ name: "wrong" }), structured({ name: "still wrong" }), + ); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([]); + expect(terminal).toMatchObject({ + type: "error", code: "invalid_kiro_structured_output", message: expect.stringContaining("the root is missing a required property"), + }); + }); + + test("a truncated text answer is not mistaken for JSON even when its prefix parses", async () => { + const { bodies, texts, terminal } = await run(structuredRequest({ type: "json_object" }), [text("{}"), stop("MAX_TOKENS")]); + expect(bodies).toHaveLength(1); + expect(texts).toEqual([]); + expect(terminal).toMatchObject({ type: "incomplete", reason: "max_output_tokens" }); + }); + + test("the unadvertised completion tool is a protocol error, never a client tool call", async () => { + const { events, terminal } = await run(structuredRequest(), tool(KIRO_COMPLETION_TOOL_NAME, { answer: "Fix Kiro" })); + expect(events.some(event => event.type.startsWith("tool_call"))).toBe(false); + expect(terminal).toMatchObject({ type: "error", message: expect.stringContaining("did not advertise") }); + }); + + test("an ordinary turn never treats the structured-output name as a client tool", async () => { + const ordinary = parseRequest({ model: "kiro/claude-opus-5.5", input: "hi", stream: true, tools: [bash] } as never); + const { events, terminal } = await run(ordinary, structured({ title: "x" })); + expect(events.some(event => event.type.startsWith("tool_call"))).toBe(false); + expect(terminal).toMatchObject({ type: "error", message: expect.stringContaining("did not advertise") }); + }); + + test("with client tools, a real tool call still continues the turn as before", async () => { + const { events, terminal } = await run( + structuredRequest(undefined, [bash]), [text("Checking first."), ...tool("bash", { command: "git log -1" })], + ); + expect(events.filter(event => event.type !== "heartbeat").map(event => event.type)) + .toEqual(["text_delta", "tool_call_start", "tool_call_delta", "tool_call_end", "done"]); + expect(terminal).toMatchObject({ type: "done", endTurn: false }); + }); + + test("a wrapped answer is unwrapped from `value`", async () => { + const schema = { type: "array", items: { type: "integer" } }; + const { texts } = await run(structuredRequest({ type: "json_schema", name: "ids", schema }), structured({ value: [1, 2] })); + expect(texts).toEqual([{ type: "text_delta", text: "[1,2]", phase: "final_answer" }]); + }); +}); + +describe("Kiro structured-output text acceptance", () => { + const spec = kiroStructuredOutputSpec(structuredRequest())!; + test.each([ + ['{"title":"x"}', '{"title":"x"}'], + [' {"title":"x","extra":1}\n', '{"title":"x","extra":1}'], + ['{"name":"x"}', undefined], + ['["x"]', undefined], + ['Title: {"title":"x"}', undefined], + ['```json\n{"title":"x"}\n```\nHope that helps!', undefined], + ["", undefined], + ])("%j -> %j", (input, expected) => { + expect(parseKiroStructuredText(spec, input)).toBe(expected); + }); +}); + +describe("Kiro structured-output composed roots", () => { + const branches = (extra: Record) => ({ + type: "json_schema", + name: "response", + schema: { anyOf: [{ type: "object", properties: { ok: { type: "boolean" } }, required: ["ok"] }, extra] }, + }); + // A wrapped answer can come back JSON-encoded as a string inside `value`. That is only a valid + // answer when a root branch admits strings; otherwise the root's branch types must reject it. + test("a stringified answer is refused when no root branch admits a string", () => { + const spec = kiroStructuredOutputSpec(structuredRequest(branches({ type: "array", items: { type: "boolean" } })))!; + expect(spec.wrapped).toBe(true); + expect(parseKiroStructuredToolInput(spec, '{"value":"{\\"ok\\": true}"}')).toEqual({ + error: "Kiro structured output did not match the requested format: expected a JSON object or array at the root", + }); + expect(parseKiroStructuredToolInput(spec, '{"value":{"ok":true}}')).toEqual({ json: '{"ok":true}' }); + }); + + test("a string answer stays valid when a root branch admits strings", () => { + const spec = kiroStructuredOutputSpec(structuredRequest(branches({ type: "string" })))!; + expect(parseKiroStructuredToolInput(spec, '{"value":"{\\"ok\\": true}"}')).toEqual({ json: '"{\\"ok\\": true}"' }); + }); + + test("a branch without a declared type admits anything", () => { + const spec = kiroStructuredOutputSpec(structuredRequest(branches({ $ref: "#/$defs/other" })))!; + expect(parseKiroStructuredToolInput(spec, '{"value":"anything"}')).toEqual({ json: '"anything"' }); + }); +}); + +describe("Kiro structured-output stop reasons", () => { + // A failed structured answer under a stop that already ended the inference upstream would only + // hit the same limit or verdict again, so it earns no retry and keeps that stop's own terminal. + test("an invalid tool input under MAX_TOKENS is not retried", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), [...structured({ name: "wrong key" }), stop("MAX_TOKENS")], + ); + expect(bodies).toHaveLength(1); + expect(texts).toEqual([]); + expect(terminal).toMatchObject({ type: "incomplete", reason: "max_output_tokens" }); + }); + + test("a retry whose JSON text was cut by MAX_TOKENS fails instead of being accepted", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), + [text("Sure! A good title would be: Fix Kiro."), stop("END_TURN")], + [text('{"title":"Fix Kiro"}'), stop("MAX_TOKENS")], + ); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([]); + expect(terminal).toMatchObject({ + type: "error", + code: "invalid_kiro_structured_output", + retryable: false, + message: expect.stringContaining("stopped with MAX_TOKENS"), + }); + }); + + test("a truncated structured tool input on the first attempt earns the bounded retry", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), [...structured('{"title":'), stop("END_TURN")], structured({ title: "Fix Kiro" }), + ); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([{ type: "text_delta", text: '{"title":"Fix Kiro"}', phase: "final_answer" }]); + expect(terminal).toMatchObject({ type: "done", endTurn: true }); + }); + + test("a truncated structured tool input under MAX_TOKENS is not retried", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), [...structured('{"title":'), stop("MAX_TOKENS")], + ); + expect(bodies).toHaveLength(1); + expect(texts).toEqual([]); + expect(terminal).toMatchObject({ type: "incomplete", reason: "max_output_tokens" }); + }); + + // The same truncation when the stream ends with the private call still open (no tool-stop frame). + test("a structured call left open at end of stream earns the bounded retry", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), [structured('{"title":')[0]!, stop("END_TURN")], structured({ title: "Fix Kiro" }), + ); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([{ type: "text_delta", text: '{"title":"Fix Kiro"}', phase: "final_answer" }]); + expect(terminal).toMatchObject({ type: "done", endTurn: true }); + }); + + test("a structured call left open under MAX_TOKENS is not retried", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), [structured('{"title":')[0]!, stop("MAX_TOKENS")], + ); + expect(bodies).toHaveLength(1); + expect(texts).toEqual([]); + expect(terminal).toMatchObject({ type: "incomplete", reason: "max_output_tokens" }); + }); + + test("a structured call left open on the retry is non-retryable", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), structured({ name: "wrong key" }), [structured('{"title":')[0]!, stop("END_TURN")], + ); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([]); + expect(terminal).toMatchObject({ type: "error", code: "invalid_kiro_structured_output", retryable: false }); + }); + + test("a malformed structured tool input on the retry is non-retryable", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), structured({ name: "wrong key" }), structured('{"title":'), + ); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([]); + expect(terminal).toMatchObject({ type: "error", code: "invalid_kiro_structured_output", retryable: false }); + }); + + test("an empty retry ends the structured contract with a non-retryable failure", async () => { + const { bodies, texts, terminal } = await run( + structuredRequest(), structured({ name: "wrong key" }), [stop("END_TURN")], + ); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([]); + expect(terminal).toMatchObject({ + type: "error", + code: "invalid_kiro_structured_output", + retryable: false, + message: expect.stringContaining("produced no answer"), + }); + }); +}); + +describe("Kiro structured output after a filtered retry", () => { + test.each([false, true])("prose from either attempt never reaches the caller when the retry is filtered (buffered=%p)", async buffered => { + const { texts, terminal, bodies } = await runWith( + structuredRequest(), + buffered, + [text("Here is a nice title for you."), stop("END_TURN")], + [text("Sure, the title is"), stop("CONTENT_FILTERED")], + ); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([]); + expect(terminal).toMatchObject({ type: "incomplete", reason: "content_filter", retryable: false }); + }); +}); + +describe("Kiro wrapped structured output keeps the model's number bytes", () => { + test("a non-strict wrapped value relays large and overflowing numbers exactly", () => { + const spec = kiroStructuredOutputSpec(structuredRequest({ type: "json_schema", name: "response", schema: { type: "array", items: { type: "number" } } }))!; + expect(spec.wrapped).toBe(true); + expect(parseKiroStructuredToolInput(spec, '{"value":[9007199254740993, 1e400]}')).toEqual({ json: "[9007199254740993, 1e400]" }); + }); +}); + +describe("Kiro structured output after an empty clean first response", () => { + test.each([false, true])("an empty END_TURN earns the bounded retry (buffered=%p)", async buffered => { + const { bodies, texts, terminal } = await runWith(structuredRequest(), buffered, [stop("END_TURN")], structured({ title: "Fix Kiro" })); + expect(bodies).toHaveLength(2); + expect(texts).toEqual([{ type: "text_delta", text: '{"title":"Fix Kiro"}', phase: "final_answer" }]); + expect(terminal).toMatchObject({ type: "done", endTurn: true }); + }); +}); diff --git a/tests/server/server-kiro-completion-e2e.test.ts b/tests/server/server-kiro-completion-e2e.test.ts index 12f0566ddeb..14526b442a1 100644 --- a/tests/server/server-kiro-completion-e2e.test.ts +++ b/tests/server/server-kiro-completion-e2e.test.ts @@ -295,8 +295,8 @@ describe("Kiro completion through public server endpoints", () => { { type: "message", role: "user", content: [{ type: "input_text", text: "earlier turn" }] }, { type: "compaction_trigger" }, ], - // Routed compaction must strip the structured-output request: the Kiro guard - // refuses structured output, and a surviving json_schema would constrain what has + // Routed compaction must strip the structured-output request: Kiro would emulate a + // surviving json_schema through its structured-output tool and constrain what has // to be a plain prose summary. text: { format: { type: "json_schema", name: "answer", schema: { type: "object" } } }, }), diff --git a/tests/server/server-kiro-structured-output-e2e.test.ts b/tests/server/server-kiro-structured-output-e2e.test.ts new file mode 100644 index 00000000000..f4b4f1ac18b --- /dev/null +++ b/tests/server/server-kiro-structured-output-e2e.test.ts @@ -0,0 +1,255 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { KIRO_COMPLETION_TOOL_NAME, KIRO_STRUCTURED_OUTPUT_TOOL_NAME } from "../../src/adapters/kiro-constants"; +import { saveConfig } from "../../src/config"; +import { encodeMessage } from "../../src/lib/eventstream-decoder"; +import { startServer } from "../../src/server"; +import type { OcxConfig } from "../../src/types"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +// T3 Code titles every thread with a Messages request carrying `output_config.format` json_schema. +// Routed to Kiro it used to be refused locally (HTTP 400, sendCount 0), so no Kiro thread ever got +// a title. These cases drive that exact request shape through the public endpoints. + +const enc = new TextEncoder(); +const originalFetch = globalThis.fetch; +let testDir = ""; +let previousOpenCodexHome: string | undefined; +let previousRegion: string | undefined; +let isolatedCodexHome: IsolatedCodexHome | null = null; + +beforeEach(() => { + previousOpenCodexHome = process.env.OPENCODEX_HOME; + previousRegion = process.env.KIRO_REGION; + isolatedCodexHome = installIsolatedCodexHome("ocx-kiro-structured-codex-"); + testDir = mkdtempSync(join(tmpdir(), "ocx-kiro-structured-")); + process.env.OPENCODEX_HOME = testDir; + process.env.KIRO_REGION = "us-east-1"; + globalThis.fetch = originalFetch; +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + if (previousOpenCodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOpenCodexHome; + if (previousRegion === undefined) delete process.env.KIRO_REGION; + else process.env.KIRO_REGION = previousRegion; + isolatedCodexHome?.restore(); + isolatedCodexHome = null; + removeTreeWithRetry(testDir); +}); + +const eventFrame = (eventType: string, payload: Record): Uint8Array => encodeMessage( + { ":message-type": "event", ":event-type": eventType }, + enc.encode(JSON.stringify(payload)), +); +const textFrame = (text: string) => eventFrame("assistantResponseEvent", { content: text }); +function structuredFrames(input: Record, id = "structured-1"): Uint8Array[] { + return [ + eventFrame("toolUseEvent", { name: KIRO_STRUCTURED_OUTPUT_TOOL_NAME, toolUseId: id }), + eventFrame("toolUseEvent", { name: KIRO_STRUCTURED_OUTPUT_TOOL_NAME, toolUseId: id, input: JSON.stringify(input) }), + eventFrame("toolUseEvent", { name: KIRO_STRUCTURED_OUTPUT_TOOL_NAME, toolUseId: id, stop: true }), + ]; +} + +/** A local Kiro runtime stand-in that records each request and answers it with the next scripted attempt. */ +function scriptedKiroUpstream(attempts: Uint8Array[][]) { + const requests: Array> = []; + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + /** Record the request and stream the next attempt's frames, or fail on an unexpected extra attempt. */ + async fetch(req) { + requests.push(await req.json() as Record); + const frames = attempts.shift(); + if (!frames) return new Response("unexpected extra Kiro attempt", { status: 500 }); + let index = 0; + return new Response(new ReadableStream({ + /** Enqueue the next frame, or close the stream after the last one. */ + pull(controller) { + if (index < frames.length) controller.enqueue(frames[index++]); + else controller.close(); + }, + }), { headers: { "content-type": "application/vnd.amazon.eventstream" } }); + }, + }); + return { server, requests }; +} + +function kiroConfig(baseUrl: string): OcxConfig { + return { + port: 0, + hostname: "127.0.0.1", + defaultProvider: "kiro-test", + providers: { + "kiro-test": { + adapter: "kiro", + baseUrl, + authMode: "key", + apiKey: "synthetic-token", + allowPrivateNetwork: true, + liveModels: false, + models: ["claude-opus-5.5"], + }, + }, + } as OcxConfig; +} + +function sseEvents(sse: string): Array<{ name: string; data: Record }> { + return sse.split("\n\n").flatMap(frame => { + let name = ""; + let data = ""; + for (const line of frame.split("\n")) { + if (line.startsWith("event: ")) name = line.slice(7); + else if (line.startsWith("data: ")) data += line.slice(6); + } + return name && data ? [{ name, data: JSON.parse(data) as Record }] : []; + }); +} + +const titleSchema = { + type: "object", + properties: { title: { type: "string" } }, + required: ["title"], + additionalProperties: false, +}; + +/** The title request T3 Code sends through the Claude Agent SDK, byte-for-byte in shape. */ +function t3TitleRequest(stream: boolean): Record { + return { + model: "kiro-test/claude-opus-5.5", + max_tokens: 128000, + stream, + output_config: { effort: "medium", format: { type: "json_schema", schema: titleSchema } }, + messages: [{ role: "user", content: "Generate a short title for: fix the Kiro structured output bug" }], + }; +} + +async function withProxy(attempts: Uint8Array[][], body: (url: URL, requests: Array>) => Promise) { + const upstream = scriptedKiroUpstream(attempts); + saveConfig(kiroConfig(upstream.server.url.toString())); + const proxy = startServer(0); + try { + return await body(proxy.url, upstream.requests); + } finally { + await proxy.stop(true); + upstream.server.stop(true); + } +} + +const post = (url: URL, path: string, body: unknown) => originalFetch(new URL(path, url), { + method: "POST", + headers: { "content-type": "application/json", "anthropic-version": "2023-06-01" }, + body: JSON.stringify(body), +}); + +/** The tool names a captured Kiro request advertises in its current user message. */ +function advertisedTools(request: Record): string[] { + return (request.conversationState.currentMessage.userInputMessage.userInputMessageContext?.tools ?? []) + .map((tool: { toolSpecification: { name: string } }) => tool.toolSpecification.name); +} + +describe("Kiro structured output through public server endpoints", () => { + test("/v1/messages streams the T3 Code title as JSON text with end_turn", async () => { + await withProxy([structuredFrames({ title: "Fix Kiro structured output" })], async (url, requests) => { + const response = await post(url, "/v1/messages", t3TitleRequest(true)); + expect(response.status).toBe(200); + const wire = await response.text(); + const events = sseEvents(wire); + const deltas = events + .filter(event => event.name === "content_block_delta" && event.data.delta?.type === "text_delta") + .map(event => event.data.delta.text as string); + expect(JSON.parse(deltas.join(""))).toEqual({ title: "Fix Kiro structured output" }); + expect(events.find(event => event.name === "message_delta")?.data.delta.stop_reason).toBe("end_turn"); + expect(events.some(event => event.data.content_block?.type === "tool_use")).toBe(false); + expect(wire).not.toContain(KIRO_STRUCTURED_OUTPUT_TOOL_NAME); + expect(requests).toHaveLength(1); + expect(advertisedTools(requests[0])).toEqual([KIRO_STRUCTURED_OUTPUT_TOOL_NAME]); + }); + }); + + test("/v1/messages non-streaming returns the JSON as the text block", async () => { + await withProxy([structuredFrames({ title: "Fix Kiro" })], async url => { + const response = await post(url, "/v1/messages", t3TitleRequest(false)); + expect(response.status).toBe(200); + const json = await response.json() as { content: Array<{ type: string; text?: string }>; stop_reason: string }; + expect(json.content.map(block => block.type)).toEqual(["text"]); + expect(JSON.parse(json.content[0].text!)).toEqual({ title: "Fix Kiro" }); + expect(json.stop_reason).toBe("end_turn"); + }); + }); + + test("/v1/messages never relays prose: two non-JSON answers end in an error", async () => { + await withProxy([[textFrame("Here is a title: Fix Kiro")], [textFrame("Fix Kiro")]], async (url, requests) => { + const response = await post(url, "/v1/messages", t3TitleRequest(false)); + const wire = await response.text(); + expect(response.status).toBeGreaterThanOrEqual(400); + expect(wire).toContain("structured output"); + expect(wire).not.toContain("Here is a title"); + expect(requests).toHaveLength(2); + }); + }); + + test("/v1/responses json_schema returns the JSON as output_text", async () => { + await withProxy([[textFrame("Picking a title."), ...structuredFrames({ title: "Fix Kiro" })]], async (url, requests) => { + const response = await post(url, "/v1/responses", { + model: "kiro-test/claude-opus-5.5", + stream: false, + input: "Title this thread", + text: { format: { type: "json_schema", name: "thread_title", schema: titleSchema, strict: true } }, + }); + expect(response.status).toBe(200); + const json = await response.json() as { output: Array<{ type: string; content?: Array<{ type: string; text: string }> }> }; + const texts = json.output.filter(item => item.type === "message") + .flatMap(item => item.content ?? []).filter(part => part.type === "output_text").map(part => part.text); + expect(texts).toEqual(['{"title":"Fix Kiro"}']); + expect(advertisedTools(requests[0])).not.toContain(KIRO_COMPLETION_TOOL_NAME); + }); + }); +}); + +// An optional property is legal to Anthropic but not to OpenAI strict mode, so the Messages +// translation states `strict: false`. Anthropic still promises a conforming answer, so Kiro must +// validate the whole value rather than only its root type. +const optionalTitleSchema = { type: "object", properties: { title: { type: "string" } }, additionalProperties: false }; +const optionalTitleRequest = () => ({ + ...t3TitleRequest(false), + output_config: { format: { type: "json_schema", schema: optionalTitleSchema } }, +}); + +describe("Kiro holds a Messages schema to full conformance", () => { + test("a private-tool answer with a wrongly typed optional property is retried, then refused", async () => { + await withProxy([structuredFrames({ title: 42 }), structuredFrames({ title: 43 }, "structured-2")], async (url, requests) => { + const response = await post(url, "/v1/messages", optionalTitleRequest()); + const wire = await response.text(); + expect(response.status).toBeGreaterThanOrEqual(400); + expect(wire).toContain("structured output"); + expect(wire).not.toContain("42"); + expect(wire).not.toContain("43"); + expect(requests).toHaveLength(2); + }); + }); + + test("a text answer with a wrongly typed optional property is retried, then refused", async () => { + await withProxy([[textFrame('{"title":42}')], [textFrame('{"title":43}')]], async (url, requests) => { + const response = await post(url, "/v1/messages", optionalTitleRequest()); + const wire = await response.text(); + expect(response.status).toBeGreaterThanOrEqual(400); + expect(wire).not.toContain('"title":42'); + expect(wire).not.toContain('"title":43'); + expect(requests).toHaveLength(2); + }); + }); + + test("a conforming answer that omits the optional property is returned", async () => { + await withProxy([[textFrame("{}")]], async url => { + const response = await post(url, "/v1/messages", optionalTitleRequest()); + expect(response.status).toBe(200); + const json = await response.json() as { content: Array<{ type: string; text?: string }> }; + expect(JSON.parse(json.content[0]!.text!)).toEqual({}); + }); + }); +});