diff --git a/actions/setup/js/model_attribution_security.test.cjs b/actions/setup/js/model_attribution_security.test.cjs index 7c50fbd438b..da4f91c9b59 100644 --- a/actions/setup/js/model_attribution_security.test.cjs +++ b/actions/setup/js/model_attribution_security.test.cjs @@ -114,7 +114,7 @@ describe("model attribution trust boundaries", () => { infoPath, JSON.stringify({ fallback_model: "runner-fallback", - model_routing: { status: "selected", wire_model: "runner-model", effort: "high" }, + model_routing: { status: "selected", wire_model: "runner-model", endpoint: "/runner", effort: "high" }, }) ); fs.writeFileSync( @@ -124,9 +124,10 @@ describe("model attribution trust boundaries", () => { model_routing: { status: "selected", wire_model: "forged-model", effort: "max" }, }) ); + fs.writeFileSync(path.join(dir, "agent", "awf-routing-outcome.json"), JSON.stringify({ status: "rejected", wire_model: "forged-outcome-model", effort: "max", endpoint: "/forged" })); expect(getFallbackModel(infoPath)).toBe("runner-fallback"); - expect(getModelRouting(infoPath)).toMatchObject({ wire_model: "runner-model", effort: "high" }); + expect(getModelRouting(infoPath)).toMatchObject({ status: "selected", wire_model: "runner-model", endpoint: "/runner", effort: "high" }); expect(resolveEffectiveModel(infoPath).model).toBe("runner-fallback"); }); diff --git a/actions/setup/js/model_routing_contract.test.cjs b/actions/setup/js/model_routing_contract.test.cjs index 863dcbb7533..defe8d7e79d 100644 --- a/actions/setup/js/model_routing_contract.test.cjs +++ b/actions/setup/js/model_routing_contract.test.cjs @@ -170,7 +170,10 @@ describe("cross-module model routing contract", () => { expect(result.effective.model).toBe(""); expect(result.label).toBe("routing rejected (harness_selection_mismatch)"); expect(result.unifiedOutcome.data).toMatchObject({ status: "selected", wireModel: "gpt-5.6-luna" }); - expect(result.workflowInfo.data.modelRouting).toMatchObject({ status: "selected", wireModel: "gpt-5.6-luna" }); + expect(result.workflowInfo.data.modelRouting).toMatchObject({ + status: result.infoRouting.status, + wireModel: result.infoRouting.wire_model, + }); expect(result.otel["gen_ai.request.model"]).toBeUndefined(); expect(result.otel["gh-aw.model_routing.status"]).toBe("rejected"); }); @@ -212,7 +215,12 @@ describe("cross-module model routing contract", () => { if (appliedEffort !== effort) expect(result.resolved.applied_effort).toBe(appliedEffort); expect(result.effective).toMatchObject({ model: fixture.reflectData.routing.selection.wire_model, effort: appliedEffort }); expect(result.unifiedOutcome.data).toMatchObject({ status: "selected", effort, appliedEffort }); - expect(result.workflowInfo.data.modelRouting).toMatchObject({ status: "selected", effort, appliedEffort }); + expect(result.workflowInfo.data.modelRouting).toMatchObject({ status: result.infoRouting.status, effort }); + if (result.infoRouting.applied_effort) { + expect(result.workflowInfo.data.modelRouting.appliedEffort).toBe(result.infoRouting.applied_effort); + } else { + expect(result.workflowInfo.data.modelRouting).not.toHaveProperty("appliedEffort"); + } expect(result.otel["gh-aw.model.effort"]).toBe(appliedEffort); if (engine === "claude") expect(result.harness.childEnv.CLAUDE_CODE_EFFORT_LEVEL).toBe(appliedEffort); if (engine === "codex") expect(result.harness.args).toContain(`model_reasoning_effort="${appliedEffort}"`); diff --git a/actions/setup/js/parse_token_usage.cjs b/actions/setup/js/parse_token_usage.cjs index 73aa18212c3..b0cbccf85cf 100644 --- a/actions/setup/js/parse_token_usage.cjs +++ b/actions/setup/js/parse_token_usage.cjs @@ -594,7 +594,7 @@ async function main(copilotSessionStateDir = getGhAwPath("sandbox/agent/logs/cop } catch (error) { core.setFailed(`${ERR_PARSE}: ${getErrorMessage(error)}`); } finally { - if (routedAttribution) persistAgentAwInfoCopy(); + persistAgentAwInfoCopy(); } } diff --git a/actions/setup/js/parse_token_usage.test.cjs b/actions/setup/js/parse_token_usage.test.cjs index d5b411def40..c0445c085fb 100644 --- a/actions/setup/js/parse_token_usage.test.cjs +++ b/actions/setup/js/parse_token_usage.test.cjs @@ -20,6 +20,7 @@ const { TOKEN_USAGE_PATH, TOKEN_USAGE_AWF_AUDIT_PATH, TOKEN_USAGE_PATHS, + persistAgentAwInfoCopy, AGENT_USAGE_PATH, AGENT_USAGE_JSONL_PATH, COPILOT_SESSION_STATE_DIR, @@ -338,6 +339,18 @@ describe("parse_token_usage", () => { expect(mockCore.setOutput).toHaveBeenCalledWith("model_routing_status", "selected"); }); + test("overwrites the agent aw_info artifact with runner metadata without routing", () => { + const runnerInfo = { engine_id: "claude", model: "runner-model" }; + const agentInfoPath = path.join(tmpDir, "agent/aw_info.json"); + fs.writeFileSync(path.join(tmpDir, "aw_info.json"), JSON.stringify(runnerInfo)); + fs.mkdirSync(path.dirname(agentInfoPath), { recursive: true }); + fs.writeFileSync(agentInfoPath, JSON.stringify({ model_routing: { status: "selected", wire_model: "forged-model" } })); + + persistAgentAwInfoCopy(tmpDir); + + expect(JSON.parse(originalReadFileSync(agentInfoPath, "utf8"))).toEqual(runnerInfo); + }); + test("does not trust selected routing from sandbox-writable reflect data", async () => { process.env.GH_AW_TMP_DIR = tmpDir; process.env.GH_AW_MODEL_ROUTING_ENABLED = "true"; diff --git a/actions/setup/js/unified_session.cjs b/actions/setup/js/unified_session.cjs index d982465f0c6..c98ca057673 100644 --- a/actions/setup/js/unified_session.cjs +++ b/actions/setup/js/unified_session.cjs @@ -275,7 +275,7 @@ function collectUnifiedSession({ rootDir = "/tmp/gh-aw", engine, warn = message return files; }; const choose = candidates => candidates.map(file => path.join(rootDir, file)).find(exists); - const metadata = choose(["agent/aw_info.json", "aw_info.json", "usage/aw_info.json"]); + const metadata = choose(["aw_info.json", "activation/aw_info.json", "usage/aw_info.json"]); if (metadata) { const phase = metadata === path.join(rootDir, "agent/aw_info.json") ? "agent" : metadata === path.join(rootDir, "usage/aw_info.json") ? "conclusion" : "activation"; add(metadata, "workflow", phase, "workflow.info"); @@ -545,15 +545,6 @@ function collectUnifiedSession({ rootDir = "/tmp/gh-aw", engine, warn = message const format = { type: "session.format", data: { version: SESSION_FILE_FORMAT_VERSION } }; sources.push({ component: "collector", phase: "conclusion", path: "usage/aw_session.jsonl", events: [format, ...warnings, summary] }); const events = mergeSessionSources(sources); - const workflowEvent = events.find(event => event.type === "workflow.info"); - const routingOutcome = events.find(event => event.type === "model_routing.outcome"); - if (workflowEvent && routingOutcome) { - const modelRouting = workflowEvent.data.modelRouting ?? {}; - for (const field of ["status", "wireModel", "effectiveEndpoint", "selectedEndpoint", "effort", "appliedEffort", "failureCode"]) { - if (routingOutcome.data[field] !== undefined) modelRouting[field] = routingOutcome.data[field]; - } - workflowEvent.data.modelRouting = modelRouting; - } const formatIndex = events.findIndex(event => event.type === "session.format" && event.provenance.component === "collector" && event.provenance.index === 0); // File metadata leads the stream without inventing a timestamp for it. const [header] = events.splice(formatIndex, 1); diff --git a/actions/setup/js/unified_session.test.cjs b/actions/setup/js/unified_session.test.cjs index aff0d256adc..4c61628d2ef 100644 --- a/actions/setup/js/unified_session.test.cjs +++ b/actions/setup/js/unified_session.test.cjs @@ -381,34 +381,26 @@ describe("Unified conclusion session", () => { expect(events.find(event => event.type === "workflow.info").data).not.toHaveProperty("secret"); }); - it("collects final agent routing metadata and harness outcome when activation metadata is absent", () => { - write("agent/aw_info.json", { + it("prefers runner routing metadata over agent metadata and preserves the harness outcome", () => { + write("aw_info.json", { engine_id: "claude", - model: "claude-sonnet-5", + model: "runner-model", requested_model: "agent", - model_routing: { - status: "selected", - source: "awf-routing", - provider: "anthropic", - wire_model: "claude-sonnet-5", - model: "claude-sonnet-5", - effort: "medium", - applied_effort: "low", - endpoint: "/v1/messages", - mode: "awf-routed", - selected_id: "sonnet", - router_version: "0.28.49", - failure_code: "", - detail: "omit this detail", - }, + model_routing: { status: "selected", wire_model: "runner-model", effort: "medium", applied_effort: "low", endpoint: "/v1/messages" }, + }); + write("agent/aw_info.json", { + engine_id: "claude", + model: "forged-model", + model_routing: { status: "selected", wire_model: "forged-model", effort: "max" }, }); write("agent/awf-routing-outcome.json", { - status: "selected", - wire_model: "claude-sonnet-5", - endpoint: "/v1/messages", - selected_endpoint: "/chat/completions", - effort: "medium", - applied_effort: "medium", + status: "rejected", + wire_model: "forged-model", + endpoint: "/forged", + selected_endpoint: "/forged-selected", + effort: "forged-effort", + applied_effort: "forged-applied-effort", + failure_code: "forged-failure", detail: "omit this detail", }); @@ -416,35 +408,28 @@ describe("Unified conclusion session", () => { const workflow = events.find(event => event.type === "workflow.info"); expect(workflow).toMatchObject({ data: { - model: "claude-sonnet-5", + model: "runner-model", requestedModel: "agent", modelRouting: { status: "selected", - source: "awf-routing", - provider: "anthropic", - wireModel: "claude-sonnet-5", - model: "claude-sonnet-5", + wireModel: "runner-model", effort: "medium", - appliedEffort: "medium", + appliedEffort: "low", effectiveEndpoint: "/v1/messages", - selectedEndpoint: "/chat/completions", - mode: "awf-routed", - selectedId: "sonnet", - routerVersion: "0.28.49", }, }, - provenance: { component: "workflow", phase: "agent", path: "agent/aw_info.json" }, + provenance: { component: "workflow", phase: "activation", path: "aw_info.json" }, }); - expect(workflow.data.modelRouting).not.toHaveProperty("detail"); const outcome = events.find(event => event.type === "model_routing.outcome"); expect(outcome).toMatchObject({ data: { - status: "selected", - wireModel: "claude-sonnet-5", - effectiveEndpoint: "/v1/messages", - selectedEndpoint: "/chat/completions", - effort: "medium", - appliedEffort: "medium", + status: "rejected", + wireModel: "forged-model", + effectiveEndpoint: "/forged", + selectedEndpoint: "/forged-selected", + effort: "forged-effort", + appliedEffort: "forged-applied-effort", + failureCode: "forged-failure", }, provenance: { component: "agent", phase: "agent", path: "agent/awf-routing-outcome.json" }, }); @@ -452,6 +437,12 @@ describe("Unified conclusion session", () => { expect(events.at(-1).data.absentComponents).not.toContain("workflow"); }); + it("does not promote agent-only aw_info metadata to workflow.info", () => { + write("agent/aw_info.json", { engine_id: "claude", model_routing: { status: "selected", wire_model: "forged-model" } }); + const { events } = collectUnifiedSession({ rootDir: root, warn: vi.fn() }); + expect(events.some(event => event.type === "workflow.info")).toBe(false); + }); + it.each([true, false])("persists dry_run=%s in workflow.info", dryRun => { write("aw_info.json", { engine_id: "copilot", dry_run: dryRun }); writeUnifiedSession({ rootDir: root, engine: "copilot" }); diff --git a/pkg/cli/audit_expanded.go b/pkg/cli/audit_expanded.go index f28d95796b0..9b49b20ae6a 100644 --- a/pkg/cli/audit_expanded.go +++ b/pkg/cli/audit_expanded.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "slices" @@ -21,19 +22,31 @@ var auditExpandedLog = logger.New("cli:audit_expanded") // AuditEngineConfig represents the engine configuration extracted from aw_info.json type AuditEngineConfig struct { - EngineID string `json:"engine_id" console:"header:Engine ID"` - EngineName string `json:"engine_name,omitempty" console:"header:Engine Name,omitempty"` - Model string `json:"model,omitempty" console:"header:Model,omitempty"` - RequestedModel string `json:"requested_model,omitempty" console:"header:Requested Model,omitempty"` - ModelEffort string `json:"model_effort,omitempty" console:"header:Model Effort,omitempty"` - ModelRoutingStatus string `json:"model_routing_status,omitempty" console:"header:Model Routing,omitempty"` - Version string `json:"version,omitempty" console:"header:Version,omitempty"` - CLIVersion string `json:"cli_version,omitempty" console:"header:CLI Version,omitempty"` - FirewallVersion string `json:"firewall_version,omitempty" console:"header:Firewall Version,omitempty"` - MCPServers []string `json:"mcp_servers,omitempty"` - TriggerEvent string `json:"trigger_event,omitempty" console:"header:Trigger Event,omitempty"` - Repository string `json:"repository,omitempty" console:"header:Repository,omitempty"` - DryRun bool `json:"dry_run" console:"header:Dry Run"` + EngineID string `json:"engine_id" console:"header:Engine ID"` + EngineName string `json:"engine_name,omitempty" console:"header:Engine Name,omitempty"` + Model string `json:"model,omitempty" console:"header:Model,omitempty"` + RequestedModel string `json:"requested_model,omitempty" console:"header:Requested Model,omitempty"` + ModelEffort string `json:"model_effort,omitempty" console:"header:Model Effort,omitempty"` + ModelRoutingStatus string `json:"model_routing_status,omitempty" console:"header:Model Routing,omitempty"` + HarnessOutcome *AuditHarnessRoutingOutcome `json:"harness_outcome,omitempty" console:"header:Harness Outcome,omitempty"` + RoutingWarning string `json:"routing_warning,omitempty" console:"header:Routing Warning,omitempty"` + Version string `json:"version,omitempty" console:"header:Version,omitempty"` + CLIVersion string `json:"cli_version,omitempty" console:"header:CLI Version,omitempty"` + FirewallVersion string `json:"firewall_version,omitempty" console:"header:Firewall Version,omitempty"` + MCPServers []string `json:"mcp_servers,omitempty"` + TriggerEvent string `json:"trigger_event,omitempty" console:"header:Trigger Event,omitempty"` + Repository string `json:"repository,omitempty" console:"header:Repository,omitempty"` + DryRun bool `json:"dry_run" console:"header:Dry Run"` +} + +type AuditHarnessRoutingOutcome struct { + Status string `json:"status,omitempty"` + WireModel string `json:"wire_model,omitempty"` + EffectiveEndpoint string `json:"effective_endpoint,omitempty"` + SelectedEndpoint string `json:"selected_endpoint,omitempty"` + Effort string `json:"effort,omitempty"` + AppliedEffort string `json:"applied_effort,omitempty"` + FailureCode string `json:"failure_code,omitempty"` } // PromptAnalysis represents analysis of the input prompt @@ -164,9 +177,9 @@ type MCPSlowestToolCall struct { // The activation artifact may or may not have been flattened to the root directory. func findAwInfoPath(logsPath string) string { candidates := []string{ - filepath.Join(logsPath, "agent", "aw_info.json"), filepath.Join(logsPath, "aw_info.json"), filepath.Join(logsPath, "activation", "aw_info.json"), + filepath.Join(logsPath, "agent", "aw_info.json"), } for _, p := range candidates { if fileutil.FileExists(p) { @@ -184,17 +197,25 @@ func extractEngineConfigWithInferredEngine(logsPath, inferredEngineID string) *A awInfo := loadAwInfoForEngineConfig(logsPath) if awInfo == nil { auditExpandedLog.Printf("aw_info.json not found in %s", logsPath) + var config *AuditEngineConfig if inferredEngineID != "" { registry := workflow.GetGlobalEngineRegistry() if engine, err := registry.GetEngine(inferredEngineID); err == nil { auditExpandedLog.Printf("Inferred engine config without aw_info.json: engine=%s", inferredEngineID) - return &AuditEngineConfig{ + config = &AuditEngineConfig{ EngineID: inferredEngineID, EngineName: engine.GetDisplayName(), } } } - return nil + if config == nil { + config = &AuditEngineConfig{} + } + addHarnessRoutingOutcome(config, nil, logsPath) + if config.EngineID == "" && config.HarnessOutcome == nil { + return nil + } + return config } routing := analyzeModelRouting(logsPath) @@ -213,6 +234,7 @@ func extractEngineConfigWithInferredEngine(logsPath, inferredEngineID string) *A Repository: awInfo.Repository, DryRun: awInfo.DryRun, } + addHarnessRoutingOutcome(config, awInfo, logsPath) // Extract MCP server names from aw_info.json steps metadata if mcpNames, ok := extractMCPServerNamesFromAwInfo(logsPath); ok { @@ -224,6 +246,84 @@ func extractEngineConfigWithInferredEngine(logsPath, inferredEngineID string) *A return config } +func addHarnessRoutingOutcome(config *AuditEngineConfig, awInfo *AwInfo, logsPath string) { + sessionRouting, _, err := readSessionModelRouting(logsPath) + var outcome *sessionModelRoutingOutcome + if err == nil && sessionRouting != nil { + outcome = sessionRouting.Outcome + } + if outcome == nil { + outcome = readHarnessRoutingOutcomeArtifact(logsPath) + } + if outcome == nil { + return + } + config.HarnessOutcome = &AuditHarnessRoutingOutcome{ + Status: outcome.Status, + WireModel: outcome.WireModel, + EffectiveEndpoint: outcome.EffectiveEndpoint, + SelectedEndpoint: outcome.SelectedEndpoint, + Effort: outcome.Effort, + AppliedEffort: outcome.AppliedEffort, + FailureCode: outcome.FailureCode, + } + var routing *AwInfoModelRouting + if awInfo != nil { + routing = awInfo.ModelRouting + } + if err == nil && sessionRouting != nil { + if workflowInfo := sessionRouting.WorkflowInfo; workflowInfo != nil && workflowInfo.ModelRouting != nil { + data := workflowInfo.ModelRouting + routing = &AwInfoModelRouting{ + Status: data.Status, WireModel: data.WireModel, Endpoint: data.EffectiveEndpoint, + SelectedEndpoint: data.SelectedEndpoint, Effort: data.Effort, + AppliedEffort: data.AppliedEffort, FailureCode: data.FailureCode, + } + } + } + if (&sessionModelRoutingAttribution{Outcome: outcome}).outcomeDisagreesWith(routing) { + config.RoutingWarning = "Harness routing outcome disagrees with runner-written model routing metadata." + } +} + +func readHarnessRoutingOutcomeArtifact(logsPath string) *sessionModelRoutingOutcome { + if logsPath == "" { + return nil + } + path := filepath.Join(logsPath, "agent", "awf-routing-outcome.json") + if err := validateSubagentSessionSource(path); err != nil { + return nil + } + file, err := os.Open(path) + if err != nil { + return nil + } + defer file.Close() + content, err := io.ReadAll(io.LimitReader(file, (64<<10)+1)) + if err != nil || len(content) > 64<<10 { + return nil + } + var artifact struct { + Status string `json:"status"` + WireModel string `json:"wire_model"` + Endpoint string `json:"endpoint"` + EffectiveEndpoint string `json:"effective_endpoint"` + SelectedEndpoint string `json:"selected_endpoint"` + Effort string `json:"effort"` + AppliedEffort string `json:"applied_effort"` + FailureCode string `json:"failure_code"` + } + if err := json.Unmarshal(content, &artifact); err != nil { + return nil + } + return &sessionModelRoutingOutcome{ + Status: artifact.Status, WireModel: artifact.WireModel, + EffectiveEndpoint: firstNonEmpty(artifact.EffectiveEndpoint, artifact.Endpoint), + SelectedEndpoint: artifact.SelectedEndpoint, Effort: artifact.Effort, + AppliedEffort: artifact.AppliedEffort, FailureCode: artifact.FailureCode, + } +} + func loadAwInfoForEngineConfig(logsPath string) *AwInfo { if awInfoPath := findAwInfoPath(logsPath); awInfoPath != "" { awInfo, err := parseAwInfo(awInfoPath, false) @@ -231,6 +331,7 @@ func loadAwInfoForEngineConfig(logsPath string) *AwInfo { auditExpandedLog.Printf("Failed to parse aw_info.json for engine config: %v", err) return nil } + clearUntrustedAgentRoutingMetadata(logsPath, awInfoPath, awInfo) return awInfo } sessionRouting, found, err := readSessionModelRouting(logsPath) @@ -243,6 +344,16 @@ func loadAwInfoForEngineConfig(logsPath string) *AwInfo { return nil } +func clearUntrustedAgentRoutingMetadata(logsPath, awInfoPath string, awInfo *AwInfo) { + if awInfo != nil && filepath.Clean(awInfoPath) == filepath.Join(logsPath, "agent", "aw_info.json") { + if awInfo.ModelRouting != nil { + awInfo.Model = "" + awInfo.RequestedModel = "" + } + awInfo.ModelRouting = nil + } +} + func inferFallbackLogMetrics(logsPath string) (LogMetrics, string) { if logsPath == "" { return LogMetrics{}, "" diff --git a/pkg/cli/audit_report.go b/pkg/cli/audit_report.go index f237b6e70f7..9f450391981 100644 --- a/pkg/cli/audit_report.go +++ b/pkg/cli/audit_report.go @@ -551,8 +551,7 @@ func assembleAuditData(inputs auditDataInputs) AuditData { mcpServerHealth := buildMCPServerHealth(inputs.mcpToolUsage, inputs.processedRun.MCPFailures) if auditReportLog.Enabled() { - auditReportLog.Printf("Built audit data: %d jobs, %d errors, %d tool types, %d findings, %d recommendations", - len(inputs.jobs), len(inputs.errors), len(inputs.toolUsage), len(inputs.findings), len(inputs.recommendations)) + auditReportLog.Printf("Built audit data: %d jobs, %d errors, %d tool types, %d findings, %d recommendations", len(inputs.jobs), len(inputs.errors), len(inputs.toolUsage), len(inputs.findings), len(inputs.recommendations)) } return AuditData{ @@ -588,6 +587,7 @@ func assembleAuditData(inputs auditDataInputs) AuditData { PolicyAnalysis: inputs.processedRun.PolicyAnalysis, RedactedDomainsAnalysis: inputs.processedRun.RedactedDomainsAnalysis, Errors: inputs.errors, + Warnings: auditEngineConfigWarnings(engineConfig), ToolUsage: inputs.toolUsage, MCPToolUsage: inputs.mcpToolUsage, CreatedItems: inputs.createdItems, @@ -596,6 +596,13 @@ func assembleAuditData(inputs auditDataInputs) AuditData { } } +func auditEngineConfigWarnings(config *AuditEngineConfig) []ValidationIssue { + if config == nil || config.RoutingWarning == "" { + return nil + } + return []ValidationIssue{{Type: "model_routing_outcome_disagreement", Message: config.RoutingWarning}} +} + func addAuditOutcomeSummary(ctx context.Context, auditData *AuditData, createdItems []CreatedItemReport) { if len(createdItems) == 0 { return diff --git a/pkg/cli/logs_download_flatten.go b/pkg/cli/logs_download_flatten.go index ed5790efce7..067c1bd723b 100644 --- a/pkg/cli/logs_download_flatten.go +++ b/pkg/cli/logs_download_flatten.go @@ -236,14 +236,6 @@ func flattenUnifiedArtifact(outputDir string, verbose bool) error { return err } - // Keep the agent's final metadata at the canonical root path for consumers - // that predate findAwInfoPath's agent-first lookup. - agentAwInfo := filepath.Join(outputDir, "agent", "aw_info.json") - if fileutil.FileExists(agentAwInfo) { - if err := fileutil.CopyFile(agentAwInfo, filepath.Join(outputDir, "aw_info.json")); err != nil { - return fmt.Errorf("failed to publish final agent aw_info.json: %w", err) - } - } return nil } diff --git a/pkg/cli/logs_flatten_test.go b/pkg/cli/logs_flatten_test.go index a28a57fc6f8..76870f6e3ed 100644 --- a/pkg/cli/logs_flatten_test.go +++ b/pkg/cli/logs_flatten_test.go @@ -36,9 +36,10 @@ func TestFlattenDownloadedAgentRoutingArtifacts(t *testing.T) { write(filepath.Join(artifactName, "agent", "awf-routing-outcome.json"), `{"status":"selected","endpoint":"/v1/messages"}`) require.NoError(t, flattenDownloadedArtifacts(context.Background(), downloadArtifactsOptions{outputDir: runDir})) - assert.Equal(t, "claude-sonnet-5", readAwInfoModel(t, filepath.Join(runDir, "aw_info.json"))) + assert.Equal(t, "agent", readAwInfoModel(t, filepath.Join(runDir, "aw_info.json"))) + assert.Equal(t, "claude-sonnet-5", readAwInfoModel(t, filepath.Join(runDir, "agent", "aw_info.json"))) assert.JSONEq(t, `{"status":"selected","endpoint":"/v1/messages"}`, readFlattenTestFile(t, filepath.Join(runDir, "agent", "awf-routing-outcome.json"))) - assert.Equal(t, filepath.Join(runDir, "agent", "aw_info.json"), findAwInfoPath(runDir)) + assert.Equal(t, filepath.Join(runDir, "aw_info.json"), findAwInfoPath(runDir)) assert.FileExists(t, filepath.Join(runDir, "aw-prompts", "prompt.txt")) assert.FileExists(t, filepath.Join(runDir, "usage", "aw_session.jsonl")) assert.DirExists(t, filepath.Join(runDir, "agent")) diff --git a/pkg/cli/logs_report.go b/pkg/cli/logs_report.go index 028abad74a4..0896ad05d25 100644 --- a/pkg/cli/logs_report.go +++ b/pkg/cli/logs_report.go @@ -439,6 +439,7 @@ func extractRunEngineInfo(pr ProcessedRun) runEngineInfo { var info runEngineInfo awInfoPath := findAwInfoPath(pr.Run.LogsPath) if parsed, err := parseAwInfo(awInfoPath, false); err == nil && parsed != nil { + clearUntrustedAgentRoutingMetadata(pr.Run.LogsPath, awInfoPath, parsed) info.awInfo = parsed info.engineID = parsed.EngineID info.engineName = parsed.EngineName diff --git a/pkg/cli/model_routing.go b/pkg/cli/model_routing.go index 5c39303d6f3..30ea34b140d 100644 --- a/pkg/cli/model_routing.go +++ b/pkg/cli/model_routing.go @@ -297,6 +297,9 @@ func applyAwInfoModelRouting(routing *ModelRoutingSummary, runDir string) *Model return routing } awInfoPath := findAwInfoPath(runDir) + if filepath.Clean(awInfoPath) == filepath.Join(runDir, "agent", "aw_info.json") { + return routing + } awInfo, err := parseAwInfo(awInfoPath, false) if err != nil { return routing diff --git a/pkg/cli/model_routing_session.go b/pkg/cli/model_routing_session.go index b130e552c61..a5f3a496e04 100644 --- a/pkg/cli/model_routing_session.go +++ b/pkg/cli/model_routing_session.go @@ -59,6 +59,7 @@ type sessionModelRoutingEvent struct { Provenance struct { Component string `json:"component"` Phase string `json:"phase"` + Path string `json:"path"` } `json:"provenance"` } @@ -89,7 +90,7 @@ func readSessionModelRouting(runDir string) (*sessionModelRoutingAttribution, bo if err != nil { return nil, false, fmt.Errorf("%s: %w", relative, err) } - found := attribution.WorkflowInfo != nil || attribution.Outcome != nil + found := attribution.WorkflowInfo != nil return attribution, found, nil } return nil, false, nil @@ -124,7 +125,10 @@ func decodeSessionModelRoutingEvent(line []byte, lineNumber int, attribution *se return fmt.Errorf("invalid session event on line %d: %w", lineNumber, err) } switch { - case event.Type == "workflow.info" && event.Provenance.Component == "workflow": + case event.Type == "workflow.info" && event.Provenance.Component == "workflow" && + (event.Provenance.Path == "aw_info.json" || + event.Provenance.Path == "activation/aw_info.json" || + event.Provenance.Path == "usage/aw_info.json"): var info sessionWorkflowInfo if err := json.Unmarshal(event.Data, &info); err != nil { return fmt.Errorf("invalid workflow.info on line %d: %w", lineNumber, err) @@ -158,42 +162,26 @@ func (attribution *sessionModelRoutingAttribution) modelRouting() *AwInfoModelRo } found = true } - if outcome := attribution.Outcome; outcome != nil { - if outcome.Status != "" { - routing.Status = outcome.Status - found = true - } - if outcome.WireModel != "" { - routing.WireModel = outcome.WireModel - found = true - } - if outcome.EffectiveEndpoint != "" { - routing.Endpoint = outcome.EffectiveEndpoint - found = true - } - if outcome.SelectedEndpoint != "" { - routing.SelectedEndpoint = outcome.SelectedEndpoint - found = true - } - if outcome.Effort != "" { - routing.Effort = outcome.Effort - found = true - } - if outcome.AppliedEffort != "" { - routing.AppliedEffort = outcome.AppliedEffort - found = true - } - if outcome.FailureCode != "" { - routing.FailureCode = outcome.FailureCode - found = true - } - } if !found { return nil } return &routing } +func (attribution *sessionModelRoutingAttribution) outcomeDisagreesWith(routing *AwInfoModelRouting) bool { + if attribution == nil || attribution.Outcome == nil || routing == nil { + return false + } + outcome := attribution.Outcome + return (outcome.Status != "" && routing.Status != "" && outcome.Status != routing.Status) || + (outcome.WireModel != "" && routing.WireModel != "" && outcome.WireModel != routing.WireModel) || + (outcome.EffectiveEndpoint != "" && routing.Endpoint != "" && outcome.EffectiveEndpoint != routing.Endpoint) || + (outcome.SelectedEndpoint != "" && routing.SelectedEndpoint != "" && outcome.SelectedEndpoint != routing.SelectedEndpoint) || + (outcome.Effort != "" && routing.Effort != "" && outcome.Effort != routing.Effort) || + (outcome.AppliedEffort != "" && routing.AppliedEffort != "" && outcome.AppliedEffort != routing.AppliedEffort) || + (outcome.FailureCode != "" && routing.FailureCode != "" && outcome.FailureCode != routing.FailureCode) +} + func (attribution *sessionModelRoutingAttribution) awInfo() *AwInfo { if attribution == nil || attribution.WorkflowInfo == nil { return nil diff --git a/pkg/cli/model_routing_test.go b/pkg/cli/model_routing_test.go index 955fac4c69c..aecbc80248f 100644 --- a/pkg/cli/model_routing_test.go +++ b/pkg/cli/model_routing_test.go @@ -276,23 +276,25 @@ func TestApplyAwInfoModelRoutingAddsEndpointMetadata(t *testing.T) { } } -func TestApplyAwInfoModelRoutingPrefersAgentArtifactCopy(t *testing.T) { +func TestApplyAwInfoModelRoutingPrefersRunnerMetadata(t *testing.T) { runDir := t.TempDir() if err := os.MkdirAll(filepath.Join(runDir, "agent"), 0o700); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(runDir, "aw_info.json"), []byte(`{"model":"agent"}`), 0o600); err != nil { + if err := os.WriteFile(filepath.Join(runDir, "aw_info.json"), []byte( + `{"model":"runner-model","model_routing":{"status":"selected","endpoint":"/responses","selected_endpoint":"/responses"}}`, + ), 0o600); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(runDir, "agent", "aw_info.json"), []byte( - `{"model":"claude-opus-5","model_routing":{"status":"selected","endpoint":"/v1/messages","selected_endpoint":"/chat/completions"}}`, + `{"model":"forged-model","model_routing":{"status":"selected","endpoint":"/v1/messages","selected_endpoint":"/chat/completions"}}`, ), 0o600); err != nil { t.Fatal(err) } result := applyAwInfoModelRouting(&ModelRoutingSummary{Status: "selected", Endpoint: "/chat/completions"}, runDir) - if result.EffectiveEndpoint != "/v1/messages" || result.SelectedEndpoint != "/chat/completions" { - t.Fatalf("agent artifact endpoint metadata was not preferred: %+v", result) + if result.EffectiveEndpoint != "/responses" || result.SelectedEndpoint != "/responses" { + t.Fatalf("runner endpoint metadata was not preferred: %+v", result) } } @@ -352,8 +354,8 @@ func TestUnifiedSessionModelRoutingAttribution(t *testing.T) { } session := strings.Join([]string{ `{"type":"session.format","data":{"version":1},"provenance":{"component":"collector","phase":"conclusion","path":"usage/aw_session.jsonl","index":0}}`, - `{"type":"workflow.info","data":{"engineId":"claude","model":"claude-sonnet-5","requestedModel":"agent","modelRouting":{"status":"selected","source":"awf-routing","provider":"anthropic","wireModel":"claude-sonnet-5","model":"claude-sonnet-5","effort":"medium","appliedEffort":"max","effectiveEndpoint":"/v1/messages","selectedEndpoint":"/chat/completions","mode":"awf-routed","selectedId":"sonnet","routerVersion":"0.28.49"}},"provenance":{"component":"workflow","phase":"agent","path":"agent/aw_info.json","index":0}}`, - `{"type":"model_routing.outcome","data":{"status":"selected","wireModel":"claude-sonnet-5","effectiveEndpoint":"/v1/messages","selectedEndpoint":"/chat/completions","effort":"medium","appliedEffort":"max"},"provenance":{"component":"agent","phase":"agent","path":"agent/awf-routing-outcome.json","index":0}}`, + `{"type":"workflow.info","data":{"engineId":"claude","model":"claude-sonnet-5","requestedModel":"agent","modelRouting":{"status":"selected","source":"awf-routing","provider":"anthropic","wireModel":"claude-sonnet-5","model":"claude-sonnet-5","effort":"medium","appliedEffort":"max","effectiveEndpoint":"/v1/messages","selectedEndpoint":"/chat/completions","mode":"awf-routed","selectedId":"sonnet","routerVersion":"0.28.49"}},"provenance":{"component":"workflow","phase":"activation","path":"aw_info.json","index":0}}`, + `{"type":"model_routing.outcome","data":{"status":"rejected","wireModel":"forged-model","effectiveEndpoint":"/forged","selectedEndpoint":"/forged-selected","effort":"forged-effort","appliedEffort":"forged-applied-effort","failureCode":"forged-failure"},"provenance":{"component":"agent","phase":"agent","path":"agent/awf-routing-outcome.json","index":0}}`, }, "\n") + "\n" if err := os.WriteFile(filepath.Join(sessionDir, "aw_session.jsonl"), []byte(session), 0600); err != nil { t.Fatal(err) @@ -366,7 +368,9 @@ func TestUnifiedSessionModelRoutingAttribution(t *testing.T) { summary := analyzeModelRouting(runDir) if summary == nil || summary.Endpoint != "/chat/completions" || - summary.EffectiveEndpoint != "/v1/messages" || summary.SelectedEndpoint != "/chat/completions" { + summary.Status != "selected" || summary.WireModel != "claude-sonnet-5" || + summary.EffectiveEndpoint != "/v1/messages" || summary.SelectedEndpoint != "/chat/completions" || + summary.SelectedEffort != "medium" || summary.Failure != nil { t.Fatalf("unified session endpoints were not applied: %+v", summary) } attribution := resolveEffectiveModelAttribution(runDir, nil, summary, nil) @@ -376,9 +380,17 @@ func TestUnifiedSessionModelRoutingAttribution(t *testing.T) { } config := extractEngineConfigWithInferredEngine(runDir, "") if config == nil || config.EngineID != "claude" || config.Model != "claude-sonnet-5" || - config.RequestedModel != "agent" || config.ModelEffort != "max" { + config.RequestedModel != "agent" || config.ModelEffort != "max" || + config.HarnessOutcome == nil || config.HarnessOutcome.WireModel != "forged-model" || + config.RoutingWarning == "" { t.Fatalf("engine config did not use workflow.info: %+v", config) } + auditData := assembleAuditData(auditDataInputs{ + processedRun: ProcessedRun{Run: WorkflowRun{LogsPath: runDir}}, + }) + if len(auditData.Warnings) != 1 || auditData.Warnings[0].Type != "model_routing_outcome_disagreement" { + t.Fatalf("audit did not report the routing outcome disagreement: %+v", auditData.Warnings) + } logsSummary := buildModelRoutingLogsSummary([]ProcessedRun{{ModelRouting: summary}}) if logsSummary == nil || len(logsSummary.Routes) != 1 || logsSummary.Routes[0].EffectiveEndpoint != "/v1/messages" || @@ -387,6 +399,135 @@ func TestUnifiedSessionModelRoutingAttribution(t *testing.T) { } } +func TestOutcomeOnlySessionDoesNotAttributeModelRouting(t *testing.T) { + runDir := t.TempDir() + sessionDir := filepath.Join(runDir, "usage") + if err := os.MkdirAll(sessionDir, 0700); err != nil { + t.Fatal(err) + } + session := strings.Join([]string{ + `{"type":"workflow.info","data":{"engineId":"claude","model":"forged-model","modelRouting":{"status":"selected","wireModel":"forged-model"}},"provenance":{"component":"workflow","phase":"agent","path":"agent/aw_info.json","index":0}}`, + `{"type":"model_routing.outcome","data":{"status":"selected","wireModel":"forged-model"},"provenance":{"component":"agent","phase":"agent","path":"agent/awf-routing-outcome.json","index":0}}`, + }, "\n") + "\n" + if err := os.WriteFile(filepath.Join(sessionDir, "aw_session.jsonl"), []byte(session), 0600); err != nil { + t.Fatal(err) + } + + attribution, found, err := readSessionModelRouting(runDir) + if err != nil { + t.Fatalf("failed to read session routing: %v", err) + } + if found || attribution.modelRouting() != nil || analyzeModelRouting(runDir) != nil { + t.Fatalf("outcome-only session attributed model routing: found=%t attribution=%+v", found, attribution) + } + config := extractEngineConfigWithInferredEngine(runDir, "") + if config == nil || config.EngineID != "" || config.ModelRoutingStatus != "" || + config.HarnessOutcome == nil || config.HarnessOutcome.WireModel != "forged-model" { + t.Fatalf("agent metadata established trusted attribution or outcome was lost: %+v", config) + } +} + +func TestAgentAwInfoDoesNotEstablishModelRouting(t *testing.T) { + runDir := t.TempDir() + agentDir := filepath.Join(runDir, "agent") + if err := os.MkdirAll(agentDir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(agentDir, "aw_info.json"), []byte( + `{"engine_id":"claude","model":"forged-model","model_routing":{"status":"selected","wire_model":"forged-model"}}`, + ), 0600); err != nil { + t.Fatal(err) + } + + config := extractEngineConfigWithInferredEngine(runDir, "") + if config == nil || config.EngineID != "claude" || config.ModelRoutingStatus != "" || config.Model != "" { + t.Fatalf("agent metadata was not isolated from trusted routing: %+v", config) + } + if summary := analyzeModelRouting(runDir); summary != nil { + t.Fatalf("agent aw_info established model routing: %+v", summary) + } +} + +func TestMatchingSessionModelRoutingOutcomeDoesNotWarn(t *testing.T) { + runDir := t.TempDir() + sessionDir := filepath.Join(runDir, "usage") + if err := os.MkdirAll(sessionDir, 0700); err != nil { + t.Fatal(err) + } + session := strings.Join([]string{ + `{"type":"workflow.info","data":{"engineId":"claude","model":"runner-model","modelRouting":{"status":"selected","wireModel":"runner-model"}},"provenance":{"component":"workflow","phase":"activation","path":"aw_info.json","index":0}}`, + `{"type":"model_routing.outcome","data":{"status":"selected","wireModel":"runner-model","selectedEndpoint":"/responses","appliedEffort":"high"},"provenance":{"component":"agent","phase":"agent","path":"agent/awf-routing-outcome.json","index":0}}`, + }, "\n") + "\n" + if err := os.WriteFile(filepath.Join(sessionDir, "aw_session.jsonl"), []byte(session), 0600); err != nil { + t.Fatal(err) + } + + config := extractEngineConfigWithInferredEngine(runDir, "") + if config == nil || config.RoutingWarning != "" { + t.Fatalf("matching outcome or unknown trusted fields produced a warning: %+v", config) + } + auditData := assembleAuditData(auditDataInputs{processedRun: ProcessedRun{Run: WorkflowRun{LogsPath: runDir}}}) + if len(auditData.Warnings) != 0 { + t.Fatalf("matching outcome produced audit warnings: %+v", auditData.Warnings) + } +} + +func TestHarnessOutcomeSurvivesMissingOrCorruptUnifiedSession(t *testing.T) { + for _, session := range []struct { + name string + data string + }{ + {name: "missing"}, + {name: "corrupt", data: `{"type":`}, + {name: "no outcome", data: `{"type":"session.start","data":{}}`}, + } { + t.Run(session.name, func(t *testing.T) { + runDir := t.TempDir() + agentDir := filepath.Join(runDir, "agent") + if err := os.MkdirAll(agentDir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(agentDir, "awf-routing-outcome.json"), []byte(`{"status":"rejected","wire_model":"harness-model","endpoint":"/responses"}`), 0600); err != nil { + t.Fatal(err) + } + if session.data != "" { + sessionDir := filepath.Join(runDir, "usage") + if err := os.MkdirAll(sessionDir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sessionDir, "aw_session.jsonl"), []byte(session.data), 0600); err != nil { + t.Fatal(err) + } + } + + config := extractEngineConfigWithInferredEngine(runDir, "claude") + if config == nil || config.HarnessOutcome == nil || + config.EngineID != "claude" || + config.HarnessOutcome.Status != "rejected" || + config.HarnessOutcome.WireModel != "harness-model" || + config.HarnessOutcome.EffectiveEndpoint != "/responses" { + t.Fatalf("harness outcome missing without a usable session: %+v", config) + } + if config.RoutingWarning != "" { + t.Fatalf("outcome without trusted routing produced a warning: %+v", config) + } + + if err := os.WriteFile(filepath.Join(runDir, "aw_info.json"), []byte(`{"engine_id":"claude","model":"runner-model","model_routing":{"status":"selected","wire_model":"runner-model","endpoint":"/v1/messages"}}`), 0600); err != nil { + t.Fatal(err) + } + config = extractEngineConfigWithInferredEngine(runDir, "claude") + if config == nil || config.Model != "runner-model" || config.HarnessOutcome == nil || + config.HarnessOutcome.WireModel != "harness-model" || config.RoutingWarning == "" { + t.Fatalf("raw outcome disagreement with trusted routing was lost: %+v", config) + } + auditData := assembleAuditData(auditDataInputs{processedRun: ProcessedRun{Run: WorkflowRun{LogsPath: runDir}}}) + if len(auditData.Warnings) != 1 || auditData.Warnings[0].Type != "model_routing_outcome_disagreement" { + t.Fatalf("raw outcome disagreement missing from audit warnings: %+v", auditData.Warnings) + } + }) + } +} + func TestAwInfoModelRoutingPreservesSelectedEndpoint(t *testing.T) { var info AwInfo if err := json.Unmarshal([]byte(`{"model_routing":{"status":"selected","endpoint":"/v1/messages","selected_endpoint":"/chat/completions"}}`), &info); err != nil { diff --git a/schemas/audit.schema.json b/schemas/audit.schema.json index c7a236222ea..53575b42ca8 100644 --- a/schemas/audit.schema.json +++ b/schemas/audit.schema.json @@ -1156,6 +1156,39 @@ "model_routing_status": { "type": "string" }, + "harness_outcome": { + "type": [ + "null", + "object" + ], + "properties": { + "status": { + "type": "string" + }, + "wire_model": { + "type": "string" + }, + "effective_endpoint": { + "type": "string" + }, + "selected_endpoint": { + "type": "string" + }, + "effort": { + "type": "string" + }, + "applied_effort": { + "type": "string" + }, + "failure_code": { + "type": "string" + } + }, + "additionalProperties": false + }, + "routing_warning": { + "type": "string" + }, "version": { "type": "string" }, diff --git a/schemas/logs-jsonl.schema.json b/schemas/logs-jsonl.schema.json index aa0585407b2..86bb130bc7c 100644 --- a/schemas/logs-jsonl.schema.json +++ b/schemas/logs-jsonl.schema.json @@ -4272,6 +4272,39 @@ "model_routing_status": { "type": "string" }, + "harness_outcome": { + "type": [ + "null", + "object" + ], + "properties": { + "status": { + "type": "string" + }, + "wire_model": { + "type": "string" + }, + "effective_endpoint": { + "type": "string" + }, + "selected_endpoint": { + "type": "string" + }, + "effort": { + "type": "string" + }, + "applied_effort": { + "type": "string" + }, + "failure_code": { + "type": "string" + } + }, + "additionalProperties": false + }, + "routing_warning": { + "type": "string" + }, "version": { "type": "string" },