From 6753d95340d2a841693bf2a31ae3d3e3c4ed241a Mon Sep 17 00:00:00 2001 From: JeffPengCoder Date: Fri, 6 Nov 2026 11:44:14 +0800 Subject: [PATCH 1/3] Improve missing-metric onset evidence with bounded history search Refs #103. Preserve range boundaries and source timestamps, add a bounded read-only last-seen investigation, and wire its skill, permissions, Helm assets, and synthetic regression coverage. Signed-off-by: JeffPengCoder --- .../extensions/automation-context.ts | 1 + .../extensions/observability/index.ts | 69 ++++-- .../observability/metric-history.ts | 173 +++++++++++++++ .../observability/openclaw.plugin.json | 1 + .../skills/observability/SKILL.md | 62 ++++++ .../templates/openclaw-seed-configmap.yaml | 2 + helm/mosaic-stack/templates/openclaw.yaml | 1 + .../tests/automation-context.test.mjs | 2 + helm/mosaic-stack/tests/chart-render.test.mjs | 14 ++ .../fixtures/metric-history-prometheus.yml | 6 + .../tests/fixtures/metric-history.mjs | 21 ++ helm/mosaic-stack/tests/metric-history.md | 91 ++++++++ .../tests/metric-history.test.mjs | 197 ++++++++++++++++++ .../tests/observability-history.test.mjs | 181 ++++++++++++++++ 14 files changed, 806 insertions(+), 15 deletions(-) create mode 100644 helm/mosaic-stack/files/openclaw-seed/extensions/observability/metric-history.ts create mode 100644 helm/mosaic-stack/tests/fixtures/metric-history-prometheus.yml create mode 100644 helm/mosaic-stack/tests/fixtures/metric-history.mjs create mode 100644 helm/mosaic-stack/tests/metric-history.md create mode 100644 helm/mosaic-stack/tests/metric-history.test.mjs create mode 100644 helm/mosaic-stack/tests/observability-history.test.mjs diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/automation-context.ts b/helm/mosaic-stack/files/openclaw-seed/extensions/automation-context.ts index cc970398..06ea42f2 100644 --- a/helm/mosaic-stack/files/openclaw-seed/extensions/automation-context.ts +++ b/helm/mosaic-stack/files/openclaw-seed/extensions/automation-context.ts @@ -73,6 +73,7 @@ const READ_ONLY_TOOLS = new Set([ "observability_metric_names", "observability_query", "observability_range_query", + "observability_metric_last_seen", "observability_log_labels", "observability_logs", "pdf", diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts index 60cbbb06..2bfb0af8 100644 --- a/helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts +++ b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts @@ -4,6 +4,7 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; import { registerLokiTools } from "./loki.ts"; import { registerGrafanaTools } from "./grafana.ts"; +import { metricHistoryTool } from "./metric-history.ts"; type PrometheusVectorSample = { metric: Record; @@ -80,6 +81,23 @@ function parsePromValue(value: string) { return Number.isFinite(parsed) ? parsed : value; } +function unixTimeIso(value: unknown) { + if (typeof value !== "number" || !Number.isFinite(value)) return null; + const date = new Date(value * 1000); + return Number.isFinite(date.getTime()) ? date.toISOString() : null; +} + +function timestampValues(rawParams: Record) { + if (rawParams.valueFormat === undefined) return false; + if (rawParams.valueFormat !== "unix_seconds") throw new Error("valueFormat must be unix_seconds when query values represent timestamps"); + return true; +} + +function compactSample(timestamp: number, rawValue: string, valuesAreTimestamps: boolean) { + const value = parsePromValue(rawValue); + return { timestamp, value, ...(valuesAreTimestamps ? { valueIso: unixTimeIso(value) } : {}) }; +} + async function prometheusFetch(baseUrl: string, path: string, params?: Record) { const url = new URL(`${baseUrl}${path}`); for (const [key, value] of Object.entries(params || {})) { @@ -99,7 +117,7 @@ async function prometheusFetch(baseUrl: string, path: string, params?: Record + values[Math.floor(index * (values.length - 1) / (maxPoints - 1))]); return { metric: series.metric, pointCount: values.length, - values: values - .filter((_value, index) => index % stride === 0) - .slice(0, maxPoints) - .map(([timestamp, value]) => ({ timestamp, value: parsePromValue(value) })), + returnedPointCount: selected.length, + downsampled: selected.length < values.length, + firstReturnedTimestamp: values[0]?.[0] ?? null, + lastReturnedTimestamp: values.at(-1)?.[0] ?? null, + firstReturnedTimestampIso: unixTimeIso(values[0]?.[0]), + lastReturnedTimestampIso: unixTimeIso(values.at(-1)?.[0]), + values: selected.map(([timestamp, value]) => compactSample(timestamp, value, valuesAreTimestamps)), }; }), truncated: result.length > maxSeries, }; } -async function query(baseUrl: string, promql: string, time?: string, maxSeries = 50) { +async function query(baseUrl: string, promql: string, time?: string, maxSeries = 50, valuesAreTimestamps = false) { const payload = await prometheusFetch(baseUrl, "/api/v1/query", { query: promql, ...(time ? { time } : {}), }); - return compactResult(payload, maxSeries, 1); + return compactResult(payload, maxSeries, 1, valuesAreTimestamps); } async function queryRange(baseUrl: string, promql: string, rawParams: Record) { + const valuesAreTimestamps = timestampValues(rawParams); const rangeMinutes = boundedNumber(rawParams.rangeMinutes, 60, 1, 10080); const stepSeconds = boundedNumber(rawParams.stepSeconds, 60, 5, 3600); const maxSeries = Math.trunc(boundedNumber(rawParams.maxSeries, 20, 1, 200)); @@ -155,12 +179,23 @@ async function queryRange(baseUrl: string, promql: string, rawParams: Record series.value || series.values?.length); return { query: promql, start, end, + startIso: unixTimeIso(start), + endIso: unixTimeIso(end), stepSeconds, - ...compactResult(payload, maxSeries, maxPoints), + ...compactResult(payload, maxSeries, maxPoints, valuesAreTimestamps), + historyEvidence: { + status: hasSamples ? "samples_returned" : "no_samples_in_window", + incidentOnsetDetermined: false, + timestampMeaning: "query_evaluation_time", + guidance: hasSamples + ? "Query boundaries are not incident onset times. To date a gap, find the last present and first missing observations for the same source, then refine that interval. Evaluation timestamps may differ from source sample timestamps; query timestamp(metric) with valueFormat: unix_seconds to obtain source sample times as valueIso. Use returned ISO times rather than mentally converting Unix timestamps. Downsampling can hide intermediate gaps." + : "No samples in this query window does not establish when data stopped. For a missing metric, call observability_metric_last_seen with the same metric and exact source labels to search earlier raw samples. If unavailable, search earlier windows for that source. If history or investigation limits prevent this, report the onset as unknown and state the checked window, not an incident start time.", + }, }; } @@ -180,6 +215,7 @@ export default definePluginEntry({ registerLokiTools(api.pluginConfig, registerTool); const baseUrl = resolveBaseUrl(api.pluginConfig); if (!baseUrl) return; + registerTool(metricHistoryTool(baseUrl)); registerTool({ name: "observability_metric_names", @@ -222,6 +258,7 @@ export default definePluginEntry({ properties: { query: { type: "string", description: "PromQL or MetricsQL instant query supported by the configured backend." }, time: { type: "string", description: "Optional RFC3339 or Unix timestamp." }, + valueFormat: { type: "string", enum: ["unix_seconds"], description: "Only for query values that represent Unix timestamps, e.g. timestamp(metric): add valueIso without changing the numeric value. Omit for ordinary metrics, durations, and counts." }, maxSeries: { type: "number", description: "Maximum returned series." }, }, }, @@ -233,6 +270,7 @@ export default definePluginEntry({ promql, stringParam(rawParams.time), Math.trunc(boundedNumber(rawParams.maxSeries, 50, 1, 500)), + timestampValues(rawParams), ); return jsonToolResult({ source: baseUrl, query: promql, ...result }); }, @@ -241,17 +279,18 @@ export default definePluginEntry({ registerTool({ name: "observability_range_query", label: "Metrics Range Query", - description: "Run a historical PromQL or MetricsQL range query.", + description: "Run a historical PromQL or MetricsQL range query. Query-window boundaries are not incident onset times; trace missing data backward to a last-present/first-missing interval.", parameters: { type: "object", additionalProperties: false, required: ["query"], properties: { query: { type: "string", description: "PromQL or MetricsQL range query supported by the configured backend." }, - rangeMinutes: { type: "number", description: "Lookback window in minutes. Default 60." }, + rangeMinutes: { type: "number", description: "Lookback window in minutes. Default 60; clamped to 1–10080 (7 days). Use explicit start/end for older windows." }, stepSeconds: { type: "number", description: "Prometheus step in seconds. Default 60." }, start: { type: "string", description: "Optional explicit start time." }, end: { type: "string", description: "Optional explicit end time." }, + valueFormat: { type: "string", enum: ["unix_seconds"], description: "Only for query values that represent Unix timestamps, e.g. timestamp(metric): add valueIso without changing the numeric value. Omit for ordinary metrics, durations, and counts." }, maxSeries: { type: "number", description: "Maximum returned series." }, maxPoints: { type: "number", description: "Maximum returned points per series after downsampling." }, }, diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/observability/metric-history.ts b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/metric-history.ts new file mode 100644 index 00000000..0f4d1e1d --- /dev/null +++ b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/metric-history.ts @@ -0,0 +1,173 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; + +type Params = Record; +type Limits = { maxRequests?: number; timeoutMs?: number; maxResponseBytes?: number }; +type Window = { start: number; end: number; startIso: string; endIso: string; query: string; status: string; seriesCount?: number; sampleCount?: number }; +const iso = (seconds: number) => new Date(seconds * 1000).toISOString(); +function elapsedDuration(seconds: number) { + const whole = Math.floor(seconds); + return `${Math.floor(whole / 86400)} days, ${Math.floor(whole % 86400 / 3600)} hours, ${Math.floor(whole % 3600 / 60)} minutes, ${whole % 60} seconds`; +} + +function number(value: unknown, fallback: number, min: number, max: number, name: string) { + if (value === undefined) return fallback; + if (typeof value !== "number" || !Number.isFinite(value) || value < min || value > max) { + throw new Error(`${name} must be between ${min} and ${max}`); + } + return value; +} + +function selection(params: Params) { + const metric = params.metric; + if (typeof metric !== "string" || !/^[a-zA-Z_:][a-zA-Z0-9_:]*$/.test(metric) || metric.length > 256) { + throw new Error("metric must be a plain Prometheus metric name, not a PromQL expression"); + } + if (!params.labels || typeof params.labels !== "object" || Array.isArray(params.labels)) { + throw new Error("labels must contain exact source label matches"); + } + const labels = Object.entries(params.labels); + if (!labels.length || labels.length > 20 || labels.some(([key, value]) => + !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key) || key === "__name__" || + typeof value !== "string" || !value.length || value.length > 512)) { + throw new Error("Provide 1–20 nonempty exact label values; __name__ is selected by metric"); + } + return { metric, labels: Object.fromEntries(labels) as Record, + selector: `${metric}{${labels.sort(([a], [b]) => a.localeCompare(b)).map(([key, value]) => `${key}=${JSON.stringify(value)}`).join(",")}}` }; +} + +async function request(url: URL, timeoutMs: number, maxBytes: number, fetcher: typeof fetch) { + const response = await fetcher(url, { method: "GET", redirect: "error", signal: AbortSignal.timeout(Math.max(1, Math.ceil(timeoutMs))) }); + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`backend_http_${response.status}`); + } + const chunks: Uint8Array[] = []; + let bytes = 0; + for await (const chunk of response.body!) { + bytes += chunk.length; + if (bytes > maxBytes) throw new Error("response_size_limit"); + chunks.push(chunk); + } + let payload; + try { payload = JSON.parse(Buffer.concat(chunks).toString("utf8")); } + catch { throw new Error("invalid_backend_json"); } + if (payload.status !== "success") throw new Error("backend_query_failed"); + if (payload.warnings?.length || payload.isPartial === true) throw new Error("backend_partial_response_warning"); + if (payload.data?.resultType !== "matrix" || !Array.isArray(payload.data.result)) throw new Error("expected_raw_range_vector"); + return payload.data.result; +} + +/** Walk adjacent raw range vectors, newest first. Unlike query_range this + * returns original sample times without lookback reuse or step downsampling. */ +export async function inspectMetricHistory(baseUrl: string, params: Params, fetcher = fetch, limits: Limits = {}) { + const selected = selection(params); + const lookbackHours = number(params.lookbackHours, 168, 1, 720, "lookbackHours"); + const expectedInterval = params.expectedIntervalSeconds === undefined ? null + : number(params.expectedIntervalSeconds, 60, 1, 86400, "expectedIntervalSeconds"); + const parsedEnd = params.end === undefined ? Date.now() : typeof params.end === "string" ? Date.parse(params.end) : NaN; + if (!Number.isFinite(parsedEnd)) throw new Error("end must be an ISO timestamp"); + const end = Math.floor(parsedEnd / 1000); + const start = end - Math.ceil(lookbackHours * 3600); + const windows: Window[] = []; + const maxRequests = limits.maxRequests ?? 32; + const deadline = performance.now() + (limits.timeoutMs ?? 20000); + const base = { + source: baseUrl, selector: selected.selector, requestedStart: start, requestedEnd: end, + requestedStartIso: iso(start), requestedEndIso: iso(end), expectedIntervalSeconds: expectedInterval, + timestampMeaning: "original_source_sample_time", intervalSemantics: "(start,end]", + }; + let cursor = end; + function result(status: string, extra: Params = {}) { + return { ...base, status, searchedStart: cursor, searchedEnd: end, searchedStartIso: iso(cursor), searchedEndIso: iso(end), + searchedDurationHours: (end - cursor) / 3600, maxSupportedLookbackHours: 720, + coverageSummary: `Completed windows cover (${iso(cursor)}, ${iso(end)}]: ${(end - cursor) / 3600} hours actually checked, within a maximum requested budget of ${lookbackHours} hours.`, + backendRetentionKnown: false, windows, ...extra }; + } + for (let count = 0; cursor > start; count++) { + if (count >= maxRequests || performance.now() >= deadline) { + return result("incomplete", { reason: "search_budget_exhausted", conclusion: "The search limit was reached. Only completed windows are evidence; the incident onset remains unknown." }); + } + const seconds = Math.min(cursor - start, count === 0 ? 3600 : count === 1 ? 21600 : 86400); + const window: Window = { start: cursor - seconds, end: cursor, startIso: iso(cursor - seconds), endIso: iso(cursor), + query: `${selected.selector}[${seconds}s]`, status: "pending" }; + windows.push(window); + const url = new URL(`${baseUrl.replace(/\/+$/, "")}/api/v1/query`); + url.searchParams.set("query", window.query); + url.searchParams.set("time", String(cursor)); + let series; + try { + series = await request(url, Math.min(5000, deadline - performance.now()), limits.maxResponseBytes ?? 2 * 1024 * 1024, fetcher); + for (const item of series) { + if (!item.metric || item.metric.__name__ !== selected.metric || + Object.entries(selected.labels).some(([key, value]) => item.metric[key] !== value)) throw new Error("unexpected_source_labels"); + if (item.histograms?.length) throw new Error("native_histograms_not_supported"); + if (!Array.isArray(item.values)) throw new Error("invalid_sample_array"); + let previous = -Infinity; + for (const sample of item.values) { + if (!Array.isArray(sample) || sample.length !== 2 || typeof sample[0] !== "number" || !Number.isFinite(sample[0]) || + sample[0] <= window.start || sample[0] > window.end || sample[0] <= previous || typeof sample[1] !== "string" || + !/^(?:[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?|[+-]?Inf|NaN)$/.test(sample[1])) { + throw new Error("invalid_source_sample"); + } + previous = sample[0]; + } + } + } catch (error) { + window.status = "failed"; + // Do not echo an arbitrary backend body, URL, or transport credential. + const message = error instanceof Error ? error.message : "request_failed"; + const reason = /^(backend_http_\d+|response_size_limit|invalid_backend_json|backend_query_failed|backend_partial_response_warning|expected_raw_range_vector|unexpected_source_labels|native_histograms_not_supported|invalid_sample_array|invalid_source_sample)$/.test(message) ? message : "request_failed_or_timed_out"; + return result("incomplete", { reason, conclusion: "The search could not be completed. An error or partial response is not evidence of missing samples; incident onset is unknown." }); + } + window.seriesCount = series.length; + window.sampleCount = series.reduce((sum: number, item: { values: unknown[] }) => sum + item.values.length, 0); + window.status = "complete"; + cursor = window.start; + const matches = series.filter((item: { values: unknown[] }) => item.values.length > 0); + if (matches.length > 1) { + return result("ambiguous_series", { seriesCount: matches.length, candidateLabels: matches.slice(0, 20).map((item: { metric: Params }) => item.metric), + candidatesTruncated: matches.length > 20, conclusion: "The selector matches multiple series. Refine exact source labels; do not combine their last-seen times into one incident onset." }); + } + if (!matches.length) continue; + const match = matches[0]; + const [timestamp, rawValue] = match.values.at(-1) as [number, string]; + const parsedValue = Number(rawValue); + const ageSeconds = end - timestamp; + const elapsedSinceLastSample = elapsedDuration(ageSeconds); + const finite = Number.isFinite(parsedValue); + const lastSample = { timestamp, iso: iso(timestamp), value: finite ? parsedValue : rawValue, finite, labels: match.metric }; + const firstExpectedMissing = expectedInterval !== null && ageSeconds >= expectedInterval + ? { timestamp: timestamp + expectedInterval, iso: iso(timestamp + expectedInterval), assumption: "Provided sampling interval; not a measured physical fault time." } : null; + const status = firstExpectedMissing ? "data_gap" : expectedInterval === null ? "sample_found" : "recent_sample"; + const conclusion = `Last actual source sample: ${lastSample.iso} (value ${rawValue}). No newer raw sample was found through ${iso(end)}. Elapsed since that sample: ${elapsedSinceLastSample}; this is not the duration of the searched window.` + + (firstExpectedMissing ? ` With the provided ${expectedInterval}-second interval, the first missing expected sample is ${firstExpectedMissing.iso}. This is a data-gap estimate, not a proven network or collector failure time.` + : expectedInterval === null ? " Sampling cadence is unknown; do not invent a first missed sample or physical fault time." + : " The latest sample is within the provided sampling interval; a zero value is not missing data.") + + (finite ? "" : " The sample is non-finite, so presence does not establish a healthy metric."); + return result(status, { lastSample, ageSeconds, elapsedSinceLastSample, noNewerSamplesThrough: iso(end), firstExpectedMissing, conclusion }); + } + return result("not_found_in_checked_history", { lastSample: null, firstExpectedMissing: null, + conclusion: `No matching source samples were found in (${iso(start)}, ${iso(end)}]. It is unknown whether this exact series ever existed. The onset is unknown, not established to predate this window. Neither query boundary is an incident start. These ${lookbackHours} hours are the checked query budget, not backend retention or the tool's maximum. Older samples may be unavailable. Do not infer an outage duration from empty history.` }); +} + +export function metricHistoryTool(baseUrl: string) { + return { + name: "observability_metric_last_seen", + label: "Metric Last Source Sample", + description: "Find when a specific missing metric last reported. Automatically searches earlier raw sample windows and returns the last real source timestamp, not a query-window boundary or lookback evaluation time. Use for missing-data onset questions, including follow-ups after empty queries. Does not date threshold breaches or previous recovered gaps. Requires a backend supporting raw Prometheus range vectors; fails closed otherwise. Read-only and bounded.", + parameters: { type: "object", additionalProperties: false, required: ["metric", "labels"], properties: { + metric: { type: "string", description: "Discovered plain metric name, without functions or selectors." }, + labels: { type: "object", additionalProperties: { type: "string" }, minProperties: 1, maxProperties: 20, description: "Exact source label matches from the user's request or discovered series, e.g. Hostname and gpu. Do not broaden source scope." }, + end: { type: "string", description: "Investigation time as ISO 8601. Defaults to now." }, + lookbackHours: { type: "number", minimum: 1, maximum: 720, description: "Normally omit to search up to 168 hours (7 days). Override only for an explicit user history bound or to extend an exhausted search, up to 720 hours. An earlier empty chart window is not a history bound. This is not a retention claim." }, + expectedIntervalSeconds: { type: "number", minimum: 1, maximum: 86400, description: "Supply the sampling interval when independently known from the user or environment configuration. Enables a first-missed-expected-sample estimate; never an exact physical fault time. Omit when unknown." }, + } }, + async execute(_id: string, params: Params) { + const output = await inspectMetricHistory(baseUrl, params); + return { content: [{ type: "text", text: JSON.stringify(output, null, 2) }], details: output }; + }, + }; +} diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/observability/openclaw.plugin.json b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/openclaw.plugin.json index b03b2a08..de9dfa02 100644 --- a/helm/mosaic-stack/files/openclaw-seed/extensions/observability/openclaw.plugin.json +++ b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/openclaw.plugin.json @@ -5,6 +5,7 @@ "observability_metric_names", "observability_query", "observability_range_query", + "observability_metric_last_seen", "dashboard_list", "dashboard_create", "dashboard_open", diff --git a/helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md b/helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md index 84aafe4b..02055e18 100644 --- a/helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md +++ b/helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md @@ -26,6 +26,7 @@ Prefer these tools over shell `curl` when answering questions about historical c - `observability_metric_names` lists available Prometheus or VictoriaMetrics metrics. - `observability_query` runs an instant PromQL or MetricsQL query supported by the configured backend. - `observability_range_query` runs a historical range query. +- `observability_metric_last_seen` searches backward for a missing metric's last original source sample. - `dashboard_list` discovers Grafana datasource names, types, and UIDs and searches dashboards. - `dashboard_create` validates explicit panel PromQL, MetricsQL or LogQL, creates a Grafana dashboard, and opens it in the UI Grafana tab. - `dashboard_open` opens an existing Grafana dashboard UID in the UI Grafana tab. @@ -57,6 +58,67 @@ When the user asks for a dashboard, provide explicit panel queries, validate the appropriate source queries first, then create the dashboard. Do not claim dashboard success unless `dashboard_create` returns `created: true`. +## Incident onset and missing metrics + +A query's `start` and `end` delimit the investigation window, not the incident. +An empty window proves only that the query returned no matching samples there; +never report its start as the outage onset. Distinguish missing samples, stale +samples, zero values, and explicit unhealthy values. Metrics alone do not prove +that a collector or network failed. + +For a missing metric, call `observability_metric_last_seen` with the discovered +metric and exact source labels, including on a follow-up after an empty recent +query. Actually perform the historical search; suggesting that the user widen +the window does not answer when the data stopped. Keep the source fixed. Supply +`expectedIntervalSeconds` only when the sampling cadence is independently known. +Use a cadence already supplied by the user or environment; do not call it +unknown merely because the history response does not discover it. +The tool searches adjacent raw-sample windows automatically, with a 7-day default +and a 30-day maximum. Normally omit `lookbackHours`: an earlier empty chart window +does not limit an unrestricted onset follow-up. Respect an explicit user time +boundary via `end` and `lookbackHours` (use generic queries for windows shorter +than one hour); these limits are not guarantees of backend retention. + +Use `lastSample.iso` as the last observed source time. `firstExpectedMissing.iso` +is conditional on the supplied cadence, not a measured physical fault time. +`recent_sample` with value zero is not missing data. `ambiguous_series` requires +more exact source labels, not combining GPUs. For `incomplete` or +`not_found_in_checked_history`, report the checked range and unknown onset; +errors, partial responses, and empty history do not establish retention or an +outage boundary. A found sample proves presence, not that its value was healthy. +Report `searchedStartIso`/`searchedEndIso` and `searchedDurationHours` as actual +coverage; the requested budget may be longer than the completed search. +Use `elapsedSinceLastSample` for the age of a found sample; searched coverage +includes earlier present data and is not the outage duration. + +This tool requires raw Prometheus range-vector support; a backend returning +evaluated vectors instead fails closed. If it is unavailable, or the question +concerns a threshold breach or an older recovered gap, use historical queries +to bracket the last present/healthy and first missing/unhealthy observations. +For wholly empty or abnormal windows, query earlier or progressively wider +windows (hours, then days), keeping the source fixed. `rangeMinutes` supports +10080 minutes (7 days); explicit ISO `start`/`end` can query older windows. + +After locating a candidate transition, narrow the interval and query at finer +resolution. Range-result timestamps are evaluation times; Prometheus lookback +can reuse an older sample. For a discovered selector, `timestamp()` +exposes the source sample timestamp as the value. Distinguish that value from +its evaluation timestamp. Apply `timestamp` directly to the original selector, +before aggregation. Set `valueFormat: "unix_seconds"` for that query to obtain +the source time as `valueIso`; use returned ISO fields instead of mentally +converting epoch seconds. Do not use this format for ordinary metric values, +counts, or durations. Account for scrape cadence, lookback, and reporting +delay. Check `downsampled`, `pointCount`, and `truncated`: reduced points or +omitted series cannot establish a precise onset or continuous absence. + +Report the last successful observation, the first missing/unhealthy observation, +the resulting interval and its uncertainty, the source selector, and windows +checked. If available history or query/time limits never reveal an earlier good +point, report the onset as unknown, state the evidence gap, and do not invent +a precise time. Without evidence that this exact series previously existed, +do not even claim that an outage predates the checked window. Do not infer +backend retention merely from an empty response. + ## Historical logs When Loki is configured, use `observability_log_labels` to discover label names diff --git a/helm/mosaic-stack/templates/openclaw-seed-configmap.yaml b/helm/mosaic-stack/templates/openclaw-seed-configmap.yaml index 5d64d617..b75445de 100644 --- a/helm/mosaic-stack/templates/openclaw-seed-configmap.yaml +++ b/helm/mosaic-stack/templates/openclaw-seed-configmap.yaml @@ -145,6 +145,8 @@ data: {{ .Files.Get "files/openclaw-seed/extensions/observability/index.ts" | indent 4 }} observability.loki.ts: |- {{ .Files.Get "files/openclaw-seed/extensions/observability/loki.ts" | indent 4 }} + observability.metric-history.ts: |- +{{ .Files.Get "files/openclaw-seed/extensions/observability/metric-history.ts" | indent 4 }} observability.grafana.ts: |- {{ .Files.Get "files/openclaw-seed/extensions/observability/grafana.ts" | indent 4 }} observability.openclaw.plugin.json: |- diff --git a/helm/mosaic-stack/templates/openclaw.yaml b/helm/mosaic-stack/templates/openclaw.yaml index c408f061..9cd8cfca 100644 --- a/helm/mosaic-stack/templates/openclaw.yaml +++ b/helm/mosaic-stack/templates/openclaw.yaml @@ -259,6 +259,7 @@ spec: cp /seed/observability.SKILL.md /home/node/.openclaw/workspace/skills/observability/SKILL.md cp /seed/observability.index.ts /home/node/.openclaw/extensions/observability/index.ts cp /seed/observability.loki.ts /home/node/.openclaw/extensions/observability/loki.ts + cp /seed/observability.metric-history.ts /home/node/.openclaw/extensions/observability/metric-history.ts cp /seed/observability.grafana.ts /home/node/.openclaw/extensions/observability/grafana.ts cp /seed/observability.openclaw.plugin.json /home/node/.openclaw/extensions/observability/openclaw.plugin.json cp /seed/observability.package.json /home/node/.openclaw/extensions/observability/package.json diff --git a/helm/mosaic-stack/tests/automation-context.test.mjs b/helm/mosaic-stack/tests/automation-context.test.mjs index 5af38931..c7f890e3 100644 --- a/helm/mosaic-stack/tests/automation-context.test.mjs +++ b/helm/mosaic-stack/tests/automation-context.test.mjs @@ -132,6 +132,8 @@ test("separates sandbox execution from external tool permissions", () => { assert.equal(toolRequiresEdit("hardware_analyze_dut"), false); assert.equal(toolRequiresEdit("bcm_execute_cmsh"), false); assert.equal(toolRequiresEdit("observability_query"), false); + assert.equal(toolAccess("observability_metric_last_seen"), "read"); + assert.equal(toolRequiresEdit("observability_metric_last_seen"), false); assert.equal(toolRequiresEdit("third_party_delete_cluster"), true); assert.equal(toolAccess("third_party_delete_cluster"), "unknown"); assert.equal(toolRequiresEdit("exec", { command: "nvidia-smi -L" }), false); diff --git a/helm/mosaic-stack/tests/chart-render.test.mjs b/helm/mosaic-stack/tests/chart-render.test.mjs index df89e5e7..4eed5bec 100644 --- a/helm/mosaic-stack/tests/chart-render.test.mjs +++ b/helm/mosaic-stack/tests/chart-render.test.mjs @@ -79,6 +79,20 @@ function render(...args) { }); } +test("ships the metric-history module and disables it with observability", () => { + const seed = render("--show-only", "templates/openclaw-seed-configmap.yaml"); + const runtime = render("--show-only", "templates/openclaw.yaml"); + assert.equal(configMapBlock(seed, "observability.metric-history.ts").trim(), + readFileSync(new URL("../files/openclaw-seed/extensions/observability/metric-history.ts", import.meta.url), "utf8").trim()); + assert.match(configMapBlock(seed, "observability.index.ts"), /from "\.\/metric-history\.ts"/); + assert.ok(JSON.parse(configMapBlock(seed, "observability.openclaw.plugin.json")).contracts.tools.includes("observability_metric_last_seen")); + assert.match(runtime, /cp \/seed\/observability\.metric-history\.ts \/home\/node\/\.openclaw\/extensions\/observability\/metric-history\.ts/); + for (const template of ["openclaw", "openclaw-seed-configmap"]) { + const disabled = render("--set", "modules.observability.enabled=false", "--show-only", `templates/${template}.yaml`); + assert.doesNotMatch(disabled, /observability\.metric-history\.ts/); + } +}); + function renderDiagnostics(...args) { return render( "--set", "modules.diagnostics.enabled=true", diff --git a/helm/mosaic-stack/tests/fixtures/metric-history-prometheus.yml b/helm/mosaic-stack/tests/fixtures/metric-history-prometheus.yml new file mode 100644 index 00000000..7c055665 --- /dev/null +++ b/helm/mosaic-stack/tests/fixtures/metric-history-prometheus.yml @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +global: + scrape_interval: 60s +scrape_configs: [] diff --git a/helm/mosaic-stack/tests/fixtures/metric-history.mjs b/helm/mosaic-stack/tests/fixtures/metric-history.mjs new file mode 100644 index 00000000..62121008 --- /dev/null +++ b/helm/mosaic-stack/tests/fixtures/metric-history.mjs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Print the complete synthetic fixture as OpenMetrics for promtool ingestion. +// No production endpoints, credentials, or scrape targets are used. +const end = Date.parse("2026-11-06T00:00:00Z") / 1000; +const start = end - 8 * 86400; +const cases = [ + ["fixture-gpu-a", end - 3 * 86400 - 60, 42], + ["fixture-gpu-b", null, 42], + ["fixture-gpu-c", end, 0], + ["fixture-gpu-d", end - 2 * 3600 - 60, 23], +]; +console.log("# HELP DCGM_FI_DEV_GPU_UTIL Synthetic GPU utilization for issue 103; not production data."); +console.log("# TYPE DCGM_FI_DEV_GPU_UTIL gauge"); +for (const [host, last, value] of cases) { + for (let time = start; last !== null && time <= last; time += 60) { + console.log(`DCGM_FI_DEV_GPU_UTIL{Hostname="${host}",gpu="0"} ${value} ${time}`); + } +} +console.log("# EOF"); diff --git a/helm/mosaic-stack/tests/metric-history.md b/helm/mosaic-stack/tests/metric-history.md new file mode 100644 index 00000000..33e2ff7d --- /dev/null +++ b/helm/mosaic-stack/tests/metric-history.md @@ -0,0 +1,91 @@ +# Missing metric history regression + +The `observability_metric_last_seen` tool searches adjacent original-sample +windows, newest first. An empty recent query does not date an incident. Unlike +`query_range`, a raw range selector sent to the Prometheus instant query API +returns source timestamps without lookback reuse or step downsampling. + +The tool keeps the metric and exact label selection fixed. It checks one hour, +the preceding six hours, then adjacent 24-hour windows, stopping at the newest +sample or the requested history bound. Defaults are seven days, 32 requests, +20 seconds total, five seconds per request, and 2 MiB per response. The maximum +requested history is 30 days. Errors, partial responses, native histograms, or +non-matrix responses fail closed. Multiple matching series require more exact +labels; the tool does not combine their last-seen times. A zero-valued sample +is present, and a non-finite sample does not establish health. + +Only an independently known sampling interval permits a first-missed-expected +sample estimate. This is not an exact physical network or collector failure +time. Actual searched coverage and the requested budget are separate. This +coverage is not outage duration: `elapsedSinceLastSample` formats the age of +the original sample without requiring the model to subtract epoch timestamps. +An entirely empty history does not prove that the series ever existed or that +an outage started before the checked window. This +tool does not locate threshold breaches or previous gaps that have recovered. +Backends that substitute evaluated vectors for raw range vectors are not +supported by this tool; the existing generic query tools remain available. + +## Unit tests + +From the repository root, with Node.js 24 or newer: + +```bash +node --test helm/mosaic-stack/tests/metric-history.test.mjs \ + helm/mosaic-stack/tests/observability-history.test.mjs \ + helm/mosaic-stack/tests/automation-context.test.mjs +``` + +These cover continuous window boundaries, exact source time, zero/non-finite +values, missing cadence, empty history, bounded work, ambiguous sources, escaped +labels, rejected input, transport errors, partial responses, malformed results, +and registration/permission controls. Helm rendering tests additionally check +ConfigMap contents, runtime copying, the plugin contract, and module disabling. + +## Real Prometheus fixture + +Use Prometheus and promtool from the same release. The fixture was tested with +Prometheus 3.16.0. Put those binaries on `PATH`. This imports only generated +samples into a fresh temporary TSDB and does not scrape any live target. +Run from the repository root: + +```bash +metric_history_dir=$(mktemp -d) +node helm/mosaic-stack/tests/fixtures/metric-history.mjs > "$metric_history_dir/samples.openmetrics" +promtool tsdb create-blocks-from openmetrics \ + "$metric_history_dir/samples.openmetrics" "$metric_history_dir/tsdb" +prometheus \ + --config.file=helm/mosaic-stack/tests/fixtures/metric-history-prometheus.yml \ + --storage.tsdb.path="$metric_history_dir/tsdb" \ + --storage.tsdb.retention.time=10000d \ + --web.listen-address=127.0.0.1:19093 +``` + +Keep that process running. In another terminal at the same repository root: + +```bash +MOSAIC_HISTORY_TEST_URL=http://127.0.0.1:19093 \ + node --test helm/mosaic-stack/tests/metric-history.test.mjs +``` + +Stop the foreground Prometheus with Ctrl-C after testing. The temporary data +can be retained for inspection. Do not import blocks into a running TSDB. + +All fixture samples are generated at 60-second intervals starting +`2026-10-29T00:00:00Z`, with investigation time `2026-11-06T00:00:00Z`. +No stale markers are inserted, so real Prometheus lookback behavior is exercised. + +| Scenario | Hostname | GPU | Value | Sample count | Last original sample UTC | First expected missing sample UTC | +| --- | --- | --- | ---: | ---: | --- | --- | +| Three-day gap | fixture-gpu-a | 0 | 42 | 7200 | 2026-11-02T23:59:00Z | 2026-11-03T00:00:00Z | +| No available history | fixture-gpu-b | 0 | No values | 0 | Unknown | Unknown | +| Healthy presence at zero | fixture-gpu-c | 0 | 0 | 11521 | 2026-11-06T00:00:00Z | None | +| Two-hour gap | fixture-gpu-d | 0 | 23 | 11400 | 2026-11-05T21:59:00Z | 2026-11-05T22:00:00Z | + +The generator emits every one of the 30121 samples. It can also support +model/tool replay tests. Passing the deterministic tool tests does not prove +that an agent selected the skill, invoked the tool, or completed the answer; +those require separate native-runtime behavior tests. + +References: [Prometheus range selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#range-vector-selectors), +[HTTP instant queries and result formats](https://prometheus.io/docs/prometheus/latest/querying/api/), +and [Mosaic issue 103](https://github.com/NVIDIA/AI-Factory-Operations-Agent/issues/103). diff --git a/helm/mosaic-stack/tests/metric-history.test.mjs b/helm/mosaic-stack/tests/metric-history.test.mjs new file mode 100644 index 00000000..d01d540e --- /dev/null +++ b/helm/mosaic-stack/tests/metric-history.test.mjs @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { inspectMetricHistory, metricHistoryTool } from "../files/openclaw-seed/extensions/observability/metric-history.ts"; + +const end = Date.parse("2026-11-06T00:00:00Z") / 1000; +const iso = seconds => new Date(seconds * 1000).toISOString(); +const labels = { __name__: "DCGM_FI_DEV_GPU_UTIL", Hostname: "fixture-gpu-a", gpu: "0" }; +const params = { metric: labels.__name__, labels: { Hostname: labels.Hostname, gpu: labels.gpu }, end: iso(end), expectedIntervalSeconds: 60 }; +const endpoint = "https://metrics.example/select/17/prometheus/"; +const payload = (result, extra = {}) => ({ status: "success", data: { resultType: "matrix", result }, ...extra }); +const response = body => new Response(JSON.stringify(body), { headers: { "content-type": "application/json" } }); + +function backend(samples, metric = labels) { + const requests = []; + return { requests, fetch: async (url, options) => { + assert.equal(options.method, "GET"); + assert.equal(options.redirect, "error"); + assert.ok(options.signal instanceof AbortSignal); + assert.equal(url.pathname, "/select/17/prometheus/api/v1/query"); + const stop = Number(url.searchParams.get("time")); + const seconds = Number(url.searchParams.get("query").match(/\[(\d+)s\]$/)[1]); + requests.push({ query: url.searchParams.get("query"), start: stop - seconds, end: stop }); + const values = samples.filter(([ts]) => ts > stop - seconds && ts <= stop); + return response(payload(values.length ? [{ metric, values }] : [])); + } }; +} + +test("finds a three-day gap in adjacent raw windows, preserving exact sample time and scope", async () => { + const last = end - 3 * 86400 - 60; + const mock = backend([[last - 60, "42"], [last, "42"]]); + const result = await inspectMetricHistory(endpoint, params, mock.fetch); + assert.equal(result.status, "data_gap"); + assert.deepEqual(result.lastSample, { timestamp: last, iso: "2026-11-02T23:59:00.000Z", value: 42, finite: true, labels }); + assert.equal(result.firstExpectedMissing.iso, "2026-11-03T00:00:00.000Z"); + assert.equal(result.ageSeconds, 259260); + assert.equal(result.elapsedSinceLastSample, "3 days, 0 hours, 1 minutes, 0 seconds"); + assert.equal(result.backendRetentionKnown, false); + assert.equal(result.timestampMeaning, "original_source_sample_time"); + assert.equal(mock.requests.length, 5); + assert.deepEqual(mock.requests.map(r => r.end - r.start), [3600, 21600, 86400, 86400, 86400]); + assert.ok(mock.requests.every(r => r.query.startsWith('DCGM_FI_DEV_GPU_UTIL{gpu="0",Hostname="fixture-gpu-a"}['))); + for (let i = 1; i < mock.requests.length; i++) assert.equal(mock.requests[i].end, mock.requests[i - 1].start); + assert.equal(result.searchedStart, mock.requests.at(-1).start); + assert.equal(result.searchedEnd, end); + assert.equal(result.searchedDurationHours, 79); + assert.match(result.coverageSummary, /79 hours actually checked.*168 hours/); + assert.deepEqual(result.windows.map(w => w.sampleCount), [0, 0, 0, 0, 2]); +}); + +test("a sample exactly on a window boundary is found once in the next (start,end] window", async () => { + const mock = backend([[end - 3600, "0"]]); + const result = await inspectMetricHistory(endpoint, params, mock.fetch); + assert.equal(result.lastSample.timestamp, end - 3600); + assert.deepEqual(result.windows.map(w => w.sampleCount), [0, 1]); +}); + +test("zero is present, unknown cadence is not fabricated, and non-finite presence is not health", async () => { + for (const [raw, finite, value] of [["0", true, 0], ["NaN", false, "NaN"], ["+Inf", false, "+Inf"]]) { + const mock = backend([[end - 30, raw]]); + const result = await inspectMetricHistory(endpoint, params, mock.fetch); + assert.equal(result.status, "recent_sample"); + assert.equal(result.lastSample.finite, finite); + assert.equal(result.lastSample.value, value); + assert.equal(result.firstExpectedMissing, null); + if (!finite) assert.match(result.conclusion, /does not establish a healthy metric/); + } + const result = await inspectMetricHistory(endpoint, { ...params, expectedIntervalSeconds: undefined }, backend([[end - 9000, "1"]]).fetch); + assert.equal(result.status, "sample_found"); + assert.equal(result.firstExpectedMissing, null); + assert.equal(result.expectedIntervalSeconds, null); +}); + +test("empty history, a caller time bound, and a budget exhaustion are distinct evidence", async () => { + for (const hours of [1, 2, 168, 720]) { + const mock = backend([]); + const result = await inspectMetricHistory(endpoint, { ...params, lookbackHours: hours }, mock.fetch); + assert.equal(result.status, "not_found_in_checked_history"); + assert.equal(result.searchedStart, end - hours * 3600); + assert.equal(result.lastSample, null); + assert.equal(result.firstExpectedMissing, null); + assert.equal(result.backendRetentionKnown, false); + assert.equal(result.maxSupportedLookbackHours, 720); + assert.match(result.conclusion, /unknown whether this exact series ever existed/); + assert.match(result.conclusion, /not established to predate this window/); + assert.ok(mock.requests.length <= 32); + assert.match(result.conclusion, /onset is unknown/); + } + const mock = backend([]); + const limited = await inspectMetricHistory(endpoint, params, mock.fetch, { maxRequests: 2 }); + assert.equal(limited.status, "incomplete"); + assert.equal(limited.reason, "search_budget_exhausted"); + assert.equal(limited.searchedStart, end - 7 * 3600); + assert.equal(mock.requests.length, 2); + const noTime = await inspectMetricHistory(endpoint, params, mock.fetch, { timeoutMs: 0 }); + assert.equal(noTime.status, "incomplete"); + assert.equal(noTime.searchedStart, end); + assert.equal(mock.requests.length, 2); +}); + +test("multiple series require disambiguation, not a combined outage time", async () => { + const series = Array.from({ length: 21 }, (_, i) => ({ metric: { ...labels, instance: `collector-${i}` }, values: [[end - 30, "0"]] })); + const result = await inspectMetricHistory(endpoint, params, async () => response(payload(series))); + assert.equal(result.status, "ambiguous_series"); + assert.equal(result.seriesCount, 21); + assert.equal(result.candidateLabels.length, 20); + assert.equal(result.candidatesTruncated, true); + assert.equal(result.lastSample, undefined); +}); + +test("label values are escaped and arbitrary PromQL or broad source selections are rejected before I/O", async () => { + const Hostname = 'host"\\\n) or up{a="b'; + const mock = backend([[end, "1"]], { ...labels, Hostname }); + const result = await inspectMetricHistory(endpoint, { ...params, labels: { Hostname, gpu: "0" } }, mock.fetch); + assert.equal(result.status, "recent_sample"); + assert.equal(mock.requests[0].query, `DCGM_FI_DEV_GPU_UTIL{gpu="0",Hostname=${JSON.stringify(Hostname)}}[3600s]`); + for (const invalid of [ + { metric: "sum(up)" }, { metric: "up[1d]" }, { labels: {} }, { labels: { Hostname: "" } }, + { labels: { "bad-key": "x" } }, { labels: { __name__: "up" } }, { labels: { Hostname: 1 } }, + { labels: [] }, { end: "not-a-date" }, { lookbackHours: 721 }, { lookbackHours: 0 }, + { lookbackHours: NaN }, { expectedIntervalSeconds: 0 }, + ]) await assert.rejects(inspectMetricHistory(endpoint, { ...params, ...invalid }, async () => { assert.fail("invalid input reached network"); })); +}); + +const sample = { metric: labels, values: [[end - 60, "1"]] }; +for (const [name, body, reason] of [ + ["warnings", payload([], { warnings: ["partial response"] }), "backend_partial_response_warning"], + ["partial flag", payload([], { isPartial: true }), "backend_partial_response_warning"], + ["backend error", { status: "error", error: "SECRET" }, "backend_query_failed"], + ["evaluated vector", { status: "success", data: { resultType: "vector", result: [] } }, "expected_raw_range_vector"], + ["wrong source", payload([{ ...sample, metric: { ...labels, gpu: "1" } }]), "unexpected_source_labels"], + ["wrong metric", payload([{ ...sample, metric: { ...labels, __name__: "up" } }]), "unexpected_source_labels"], + ["native histogram", payload([{ ...sample, histograms: [[end, {}]] }]), "native_histograms_not_supported"], + ["missing values", payload([{ metric: labels }]), "invalid_sample_array"], + ["out of window", payload([{ ...sample, values: [[end + 1, "1"]] }]), "invalid_source_sample"], + ["duplicate time", payload([{ ...sample, values: [[end, "1"], [end, "2"]] }]), "invalid_source_sample"], + ["unordered time", payload([{ ...sample, values: [[end, "1"], [end - 30, "2"]] }]), "invalid_source_sample"], + ["empty value", payload([{ ...sample, values: [[end, ""]] }]), "invalid_source_sample"], +]) test(`${name} fails closed and does not count a failed window as searched absence`, async () => { + let count = 0; + const result = await inspectMetricHistory(endpoint, params, async () => response(++count === 1 ? payload([]) : body)); + assert.equal(result.status, "incomplete"); + assert.equal(result.reason, reason); + assert.equal(result.searchedStart, end - 3600); + assert.deepEqual(result.windows.map(w => w.status), ["complete", "failed"]); + assert.equal(result.lastSample, undefined); + assert.ok(!JSON.stringify(result).includes("SECRET")); +}); + +test("transport, timeout, invalid JSON, oversized bodies and HTTP errors are not empty history", async () => { + for (const [fetcher, limits, reason] of [ + [async () => { throw new Error("SECRET transport detail"); }, {}, "request_failed_or_timed_out"], + [async () => new Response("SECRET invalid json"), {}, "invalid_backend_json"], + [async () => new Response("SECRET", { status: 503 }), {}, "backend_http_503"], + [async () => response(payload([sample])), { maxResponseBytes: 10 }, "response_size_limit"], + [(_url, { signal }) => new Promise((_resolve, reject) => { + const timer = setTimeout(() => reject(new Error("test timeout fallback")), 1000); + signal.addEventListener("abort", () => { clearTimeout(timer); reject(signal.reason); }, { once: true }); + }), { timeoutMs: 10 }, "request_failed_or_timed_out"], + ]) { + const result = await inspectMetricHistory(endpoint, params, fetcher, limits); + assert.equal(result.status, "incomplete"); + assert.equal(result.reason, reason); + assert.equal(result.searchedStart, end); + assert.ok(!JSON.stringify(result).includes("SECRET")); + } +}); + +test("tool schema and declared contract expose only bounded read-only investigation arguments", () => { + const tool = metricHistoryTool(endpoint); + assert.equal(tool.name, "observability_metric_last_seen"); + assert.deepEqual(tool.parameters.required, ["metric", "labels"]); + assert.equal(tool.parameters.additionalProperties, false); + assert.equal(tool.parameters.properties.lookbackHours.maximum, 720); + const manifest = JSON.parse(readFileSync(new URL("../files/openclaw-seed/extensions/observability/openclaw.plugin.json", import.meta.url))); + assert.ok(manifest.contracts.tools.includes(tool.name)); +}); + +test("real Prometheus source timestamps (opt in with MOSAIC_HISTORY_TEST_URL for the documented fixture)", { + skip: !process.env.MOSAIC_HISTORY_TEST_URL, +}, async () => { + for (const [host, status, last, missing, value] of [ + ["fixture-gpu-a", "data_gap", "2026-11-02T23:59:00.000Z", "2026-11-03T00:00:00.000Z", 42], + ["fixture-gpu-b", "not_found_in_checked_history", null, null, null], + ["fixture-gpu-c", "recent_sample", "2026-11-06T00:00:00.000Z", null, 0], + ["fixture-gpu-d", "data_gap", "2026-11-05T21:59:00.000Z", "2026-11-05T22:00:00.000Z", 23], + ]) { + const result = await inspectMetricHistory(process.env.MOSAIC_HISTORY_TEST_URL, { ...params, labels: { Hostname: host, gpu: "0" } }); + assert.equal(result.status, status, host); + assert.equal(result.lastSample?.iso ?? null, last, host); + assert.equal(result.firstExpectedMissing?.iso ?? null, missing, host); + assert.equal(result.lastSample?.value ?? null, value, host); + } +}); diff --git a/helm/mosaic-stack/tests/observability-history.test.mjs b/helm/mosaic-stack/tests/observability-history.test.mjs new file mode 100644 index 00000000..f384fb93 --- /dev/null +++ b/helm/mosaic-stack/tests/observability-history.test.mjs @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { registerHooks } from "node:module"; +import test from "node:test"; + +const hooks = registerHooks({ resolve(specifier, context, next) { + return specifier === "openclaw/plugin-sdk/plugin-entry" + ? { url: "data:text/javascript,export const definePluginEntry = x => x;", shortCircuit: true } + : next(specifier, context); +} }); +const { default: plugin } = await import("../files/openclaw-seed/extensions/observability/index.ts"); +hooks.deregister(); + +async function fixture(t, respond) { + const requests = []; + const server = createServer((req, res) => { + const url = new URL(req.url, "http://localhost"); + requests.push({ method: req.method, url }); + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ status: "success", data: respond(url) })); + }).listen(0, "127.0.0.1"); + await new Promise(resolve => server.once("listening", resolve)); + t.after(() => { server.closeAllConnections(); server.close(); }); + const tools = {}; + plugin.register({ + pluginConfig: { prometheusUrl: `http://127.0.0.1:${server.address().port}`, grafanaEnabled: false }, + registerTool: tool => { tools[tool.name] = async params => (await tool.execute("history-test", params)).details; }, + }); + return { tools, requests }; +} + +const end = Date.parse("2026-11-06T00:00:00Z") / 1000; +const lastPresent = end - 3 * 86400; +const selector = 'DCGM_FI_DEV_GPU_UTIL{Hostname="fixture-gpu-a",gpu="0"}'; +const labels = { __name__: "DCGM_FI_DEV_GPU_UTIL", Hostname: "fixture-gpu-a", gpu: "0" }; +const iso = seconds => new Date(seconds * 1000).toISOString(); + +test("last-seen tool obeys the same endpoint and tool enablement controls as other metrics tools", () => { + for (const [config, expected] of [ + [{}, false], + [{ prometheusUrl: "https://metrics.example" }, true], + [{ prometheusUrl: "https://metrics.example", disabledTools: ["observability_metric_last_seen"] }, false], + [{ prometheusUrl: "https://metrics.example", enabledTools: ["observability_query"] }, false], + [{ prometheusUrl: "https://metrics.example", enabledTools: ["observability_metric_last_seen"] }, true], + ]) { + const names = []; + plugin.register({ pluginConfig: { ...config, grafanaEnabled: false }, registerTool: tool => names.push(tool.name) }); + assert.equal(names.includes("observability_metric_last_seen"), expected); + } +}); + +test("three-day outage: an empty recent window is not an incident start; older data remains queryable", async t => { + const { tools, requests } = await fixture(t, url => { + const values = []; + const start = Number(url.searchParams.get("start")); + const stop = Number(url.searchParams.get("end")); + const step = Number(url.searchParams.get("step")); + for (let ts = start; ts <= Math.min(stop, lastPresent); ts += step) values.push([ts, "0"]); + return { resultType: "matrix", result: values.length ? [{ metric: labels, values }] : [] }; + }); + for (const rangeMinutes of [60, 180, 1440]) { + const result = await tools.observability_range_query({ query: selector, end: iso(end), rangeMinutes }); + assert.deepEqual(result.result, []); + assert.equal(result.start, end - rangeMinutes * 60); + assert.equal(result.historyEvidence.status, "no_samples_in_window"); + assert.equal(result.historyEvidence.incidentOnsetDetermined, false); + } + const older = await tools.observability_range_query({ query: selector, end: iso(end), rangeMinutes: 5760, maxPoints: 10 }); + assert.equal(older.historyEvidence.status, "samples_returned"); + assert.equal(older.historyEvidence.incidentOnsetDetermined, false); + assert.equal(older.result[0].pointCount, 1441); + assert.equal(older.result[0].returnedPointCount, 10); + assert.equal(older.result[0].downsampled, true); + assert.deepEqual(older.result[0].values.at(-1), { timestamp: lastPresent, value: 0 }); + assert.ok(requests.every(r => r.method === "GET" && r.url.searchParams.get("query") === selector)); +}); + +test("compaction retains both boundary points and discloses per-series sampling and series truncation", async t => { + const values = Array.from({ length: 242 }, (_, i) => [end - (241 - i) * 60, String(i)]); + const { tools } = await fixture(t, () => ({ resultType: "matrix", result: [ + { metric: labels, values }, + { metric: { ...labels, gpu: "1" }, values: values.slice(0, 4) }, + ] })); + const result = await tools.observability_range_query({ query: selector, end: iso(end), rangeMinutes: 240, maxPoints: 240, maxSeries: 1 }); + assert.equal(result.seriesCount, 2); + assert.equal(result.truncated, true); + assert.equal(result.result.length, 1); + const series = result.result[0]; + assert.deepEqual(series.values.at(-1), { timestamp: end, value: 241 }); + assert.equal(series.pointCount, 242); + assert.equal(series.returnedPointCount, 240); + assert.equal(series.downsampled, true); + assert.deepEqual(series.values[0], { timestamp: values[0][0], value: 0 }); + assert.equal(series.firstReturnedTimestamp, values[0][0]); + assert.equal(series.lastReturnedTimestamp, end); + assert.equal(series.lastReturnedTimestampIso, "2026-11-06T00:00:00.000Z"); + assert.equal(result.endIso, "2026-11-06T00:00:00.000Z"); + assert.equal(result.historyEvidence.timestampMeaning, "query_evaluation_time"); +}); + +test("sample values, including zero and NaN, are evidence rather than inferred health or outage timestamps", async t => { + const values = [[end - 120, "0"], [end - 60, "NaN"], [end, "1"]]; + const { tools } = await fixture(t, () => ({ resultType: "matrix", result: [{ metric: labels, values }] })); + const result = await tools.observability_range_query({ query: selector, end: iso(end) }); + assert.deepEqual(result.result[0].values, [ + { timestamp: end - 120, value: 0 }, { timestamp: end - 60, value: "NaN" }, { timestamp: end, value: 1 }, + ]); + assert.equal(result.result[0].downsampled, false); + assert.equal(result.result[0].returnedPointCount, 3); + assert.equal(result.historyEvidence.incidentOnsetDetermined, false); +}); + +test("no retained samples stays unknown even across seven days or an explicit older window", async t => { + const { tools, requests } = await fixture(t, () => ({ resultType: "matrix", result: [] })); + for (const params of [ + { end: iso(end), rangeMinutes: 10080 }, + { start: "2026-10-01T00:00:00Z", end: "2026-10-02T00:00:00Z" }, + ]) { + const result = await tools.observability_range_query({ query: selector, ...params }); + assert.equal(result.historyEvidence.status, "no_samples_in_window"); + assert.equal(result.historyEvidence.incidentOnsetDetermined, false); + assert.deepEqual(result.result, []); + } + assert.equal(Number(requests[1].url.searchParams.get("start")), Date.parse("2026-10-01T00:00:00Z") / 1000); +}); + +test("empty series are not mistaken for returned sample evidence", async t => { + const { tools } = await fixture(t, () => ({ resultType: "matrix", result: [{ metric: labels, values: [] }] })); + const result = await tools.observability_range_query({ query: selector, end: iso(end) }); + assert.equal(result.historyEvidence.status, "no_samples_in_window"); + assert.equal(result.result[0].firstReturnedTimestamp, null); + assert.equal(result.result[0].lastReturnedTimestamp, null); + assert.equal(result.result[0].returnedPointCount, 0); +}); + +test("instant query output remains compatible", async t => { + const { tools } = await fixture(t, () => ({ resultType: "vector", result: [{ metric: labels, value: [end, "0"] }] })); + const result = await tools.observability_query({ query: selector, time: iso(end) }); + assert.deepEqual(result.result, [{ metric: labels, value: { timestamp: end, value: 0 } }]); + assert.equal(result.historyEvidence, undefined); +}); + +test("timestamp values expose source time separately from evaluation time, only when explicitly requested", async t => { + const sourceTime = lastPresent - 60; + const evaluationTime = lastPresent + 180; + const { tools, requests } = await fixture(t, url => ({ + resultType: url.pathname.endsWith("query_range") ? "matrix" : "vector", + result: [{ metric: labels, ...(url.pathname.endsWith("query_range") + ? { values: [[evaluationTime, String(sourceTime)]] } + : { value: [evaluationTime, String(sourceTime)] }) }], + })); + const query = `timestamp(${selector})`; + const instant = await tools.observability_query({ query, time: iso(evaluationTime), valueFormat: "unix_seconds" }); + assert.deepEqual(instant.result[0].value, { + timestamp: evaluationTime, value: sourceTime, valueIso: "2026-11-02T23:59:00.000Z", + }); + const range = await tools.observability_range_query({ query, start: iso(lastPresent), end: iso(evaluationTime), valueFormat: "unix_seconds" }); + assert.equal(range.result[0].lastReturnedTimestampIso, "2026-11-03T00:03:00.000Z"); + assert.deepEqual(range.result[0].values[0], instant.result[0].value); + const plain = await tools.observability_query({ query, time: iso(evaluationTime) }); + assert.equal(plain.result[0].value.valueIso, undefined); + assert.equal(plain.result[0].value.value, sourceTime); + assert.ok(requests.every(request => !request.url.searchParams.has("valueFormat"))); + const count = requests.length; + for (const tool of [tools.observability_query, tools.observability_range_query]) { + await assert.rejects(tool({ query, valueFormat: "seconds" }), /valueFormat/); + } + assert.equal(requests.length, count); +}); + +test("non-finite or out-of-range timestamp values are not converted to fabricated dates", async t => { + const { tools } = await fixture(t, () => ({ resultType: "matrix", result: [{ metric: labels, values: [[end - 60, "NaN"], [end, "1e100"]] }] })); + const result = await tools.observability_range_query({ query: `timestamp(${selector})`, end: iso(end), valueFormat: "unix_seconds" }); + assert.deepEqual(result.result[0].values, [ + { timestamp: end - 60, value: "NaN", valueIso: null }, + { timestamp: end, value: 1e100, valueIso: null }, + ]); +}); From f169d388b8d344adf14fa7a4c31de3d2b9815d69 Mon Sep 17 00:00:00 2001 From: JeffPengCoder Date: Mon, 9 Nov 2026 22:42:05 +0800 Subject: [PATCH 2/3] Add matched model replay for missing-metric investigations Signed-off-by: JeffPengCoder --- .../tests/fixtures/metric-history-cases.mjs | 22 ++ .../tests/fixtures/metric-history.mjs | 11 +- .../tests/metric-history-replay.md | 186 ++++++++++ .../tests/metric-history-replay.mjs | 340 ++++++++++++++++++ .../tests/metric-history-replay.test.mjs | 203 +++++++++++ helm/mosaic-stack/tests/metric-history.md | 3 +- 6 files changed, 755 insertions(+), 10 deletions(-) create mode 100644 helm/mosaic-stack/tests/fixtures/metric-history-cases.mjs create mode 100644 helm/mosaic-stack/tests/metric-history-replay.md create mode 100644 helm/mosaic-stack/tests/metric-history-replay.mjs create mode 100644 helm/mosaic-stack/tests/metric-history-replay.test.mjs diff --git a/helm/mosaic-stack/tests/fixtures/metric-history-cases.mjs b/helm/mosaic-stack/tests/fixtures/metric-history-cases.mjs new file mode 100644 index 00000000..dd3d2b6e --- /dev/null +++ b/helm/mosaic-stack/tests/fixtures/metric-history-cases.mjs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const metric = "DCGM_FI_DEV_GPU_UTIL"; +export const end = Date.parse("2026-11-06T00:00:00Z") / 1000; +export const start = end - 8 * 86400; +export const interval = 60; +export const cases = [ + { id: "three-day-gap", host: "fixture-gpu-a", last: end - 3 * 86400 - 60, missing: end - 3 * 86400, value: 42, count: 7200 }, + { id: "history-unavailable", host: "fixture-gpu-b", last: null, missing: null, value: 42, count: 0 }, + { id: "healthy-zero", host: "fixture-gpu-c", last: end, missing: null, value: 0, count: 11521 }, + { id: "two-hour-gap", host: "fixture-gpu-d", last: end - 2 * 3600 - 60, missing: end - 2 * 3600, value: 23, count: 11400 }, +]; +export const replayCases = [...cases, { ...cases[0], id: "three-day-gap-followup", followup: true }]; + +// Ground truth belongs to the evaluator. Only host, metric, current time and +// independently known scrape cadence are supplied to the model. +export function userPrompt(scenario) { + if (scenario.followup) return `请检查 ${scenario.host} 的 GPU 0 在最近两小时的 ${metric} 数据。`; + if (scenario.id === "healthy-zero") return `请检查 ${scenario.host} 的 GPU 0,指标是 ${metric}。最近的值一直是 0,这是否说明采集已经中断了?请查数据回答,不要更改配置。`; + return `请检查 ${scenario.host} 的 GPU 0 状态指标 ${metric}。最近图上没有数据了,请判断大约是从什么时候开始缺失的,并给出时间依据。不要更改配置。`; +} diff --git a/helm/mosaic-stack/tests/fixtures/metric-history.mjs b/helm/mosaic-stack/tests/fixtures/metric-history.mjs index 62121008..ef415c49 100644 --- a/helm/mosaic-stack/tests/fixtures/metric-history.mjs +++ b/helm/mosaic-stack/tests/fixtures/metric-history.mjs @@ -3,17 +3,10 @@ // Print the complete synthetic fixture as OpenMetrics for promtool ingestion. // No production endpoints, credentials, or scrape targets are used. -const end = Date.parse("2026-11-06T00:00:00Z") / 1000; -const start = end - 8 * 86400; -const cases = [ - ["fixture-gpu-a", end - 3 * 86400 - 60, 42], - ["fixture-gpu-b", null, 42], - ["fixture-gpu-c", end, 0], - ["fixture-gpu-d", end - 2 * 3600 - 60, 23], -]; +import { start, cases } from "./metric-history-cases.mjs"; console.log("# HELP DCGM_FI_DEV_GPU_UTIL Synthetic GPU utilization for issue 103; not production data."); console.log("# TYPE DCGM_FI_DEV_GPU_UTIL gauge"); -for (const [host, last, value] of cases) { +for (const { host, last, value } of cases) { for (let time = start; last !== null && time <= last; time += 60) { console.log(`DCGM_FI_DEV_GPU_UTIL{Hostname="${host}",gpu="0"} ${value} ${time}`); } diff --git a/helm/mosaic-stack/tests/metric-history-replay.md b/helm/mosaic-stack/tests/metric-history-replay.md new file mode 100644 index 00000000..b4697aca --- /dev/null +++ b/helm/mosaic-stack/tests/metric-history-replay.md @@ -0,0 +1,186 @@ +# Backtest missing-metric investigation changes + +This runner compares the observability plugin and skill from two repository +revisions against the same model, prompts, original metric samples and inference +settings. It addresses the missing-metric incident-onset cases in +[issue 103](https://github.com/NVIDIA/AI-Factory-Operations-Agent/issues/103). +Use the customer's exact model and serving configuration when available. A +different model, precision, reasoning setting or parser is a separate experiment. + +This is an isolated model/tool replay. It loads the real observability plugin +and explicitly supplies its skill, stubbing only the OpenClaw SDK registration +wrapper. It does not test native OpenClaw skill selection, complete runtime +prompts, session routing, approvals or the UI. Those need a separate deployment +test. No score or model behavior is asserted by adding this runner. + +## Prepare the fixture and revisions + +Use Node.js 24 or newer and the running local Prometheus fixture documented in +[metric-history.md](metric-history.md#real-prometheus-fixture). The backend must +be loopback-only. The runner validates every timestamp, value and label in the +30,121 generated samples before calling the model. It will reject partial or +different data. Do not point it at a live metrics backend. + +From a checkout containing this runner, choose the two commits to compare and +create separate worktrees. Replace the two commit placeholders before running: + +```bash +metric_replay_dir=$(mktemp -d) +metric_replay_runner="$PWD/helm/mosaic-stack/tests/metric-history-replay.mjs" +metric_replay_base='' +metric_replay_candidate='' +git worktree add --detach "$metric_replay_dir/base-source" "$metric_replay_base" +git worktree add --detach "$metric_replay_dir/candidate-source" "$metric_replay_candidate" +``` + +Keep the runner itself fixed for both sides, including when the baseline +predates this runner. The selected source revision supplies the plugin's tools, +schemas and skill. Both revisions must have the observability plugin and +`observability_range_query`; the baseline need not have +`observability_metric_last_seen`. + +The fixture fixes investigation time at `2026-11-06T00:00:00Z`. Each case tells +the model the metric, exact hostname/GPU and independently known 60-second +sampling cadence. Ground truth is saved with the results, never in the model +prompt. All five cases are included by default: + +| Case ID | Hostname | Original sample count | Last original sample UTC | First expected missing sample UTC | Additional conversation context | +| --- | --- | ---: | --- | --- | --- | +| `three-day-gap` | `fixture-gpu-a` | 7200 | 2026-11-02T23:59:00Z | 2026-11-03T00:00:00Z | None | +| `history-unavailable` | `fixture-gpu-b` | 0 | Unknown | Unknown | None | +| `healthy-zero` | `fixture-gpu-c` | 11521 | 2026-11-06T00:00:00Z | None | None | +| `two-hour-gap` | `fixture-gpu-d` | 11400 | 2026-11-05T21:59:00Z | 2026-11-05T22:00:00Z | None | +| `three-day-gap-followup` | `fixture-gpu-a` | 7200 (same series as first case) | 2026-11-02T23:59:00Z | 2026-11-03T00:00:00Z | A real empty two-hour range query, followed by the user's onset question | + +The follow-up obtains its seeded tool result from the selected revision and +fixture. A query failure cannot be converted into a claim of empty history. + +## Freeze the model configuration and run both sides + +Set the model ID to the value accepted by your server's `/v1/models` endpoint. +Also record the weights revision, precision, serving image/digest, tokenizer, +chat template, reasoning/tool parsers, context limit and cache configuration. +An API model name alone does not establish those properties. The two revision +labels below should identify that saved serving configuration; `unrecorded` +defaults are allowed for exploratory runs but limit reproducibility. + +This is an example of **explicit** settings, not a universal model recipe. +Choose temperature, sampling and reasoning settings for the selected deployment +and keep them identical on both sides. The options file accepts only `top_p`, +`reasoning_effort` and `chat_template_kwargs`; omit unsupported options. + +```bash +metric_replay_model_url='http://127.0.0.1:18003/v1' +metric_replay_model='' +metric_replay_model_revision='' +metric_replay_server_revision='' +cat > "$metric_replay_dir/model-options.json" <<'JSON' +{ + "top_p": 0.95, + "chat_template_kwargs": { "enable_thinking": false } +} +JSON + +for metric_replay_side in base candidate; do + node "$metric_replay_runner" run \ + --source "$metric_replay_dir/$metric_replay_side-source" \ + --prometheus http://127.0.0.1:19093 \ + --model-url "$metric_replay_model_url" \ + --model "$metric_replay_model" \ + --model-revision "$metric_replay_model_revision" \ + --server-revision "$metric_replay_server_revision" \ + --temperature 1 \ + --model-options "$metric_replay_dir/model-options.json" \ + --repetitions 3 --max-turns 24 --max-tokens 16384 \ + --timeout-ms 180000 --cache normal \ + --output "$metric_replay_dir/$metric_replay_side-results" +done + +node "$metric_replay_runner" compare \ + "$metric_replay_dir/base-results" \ + "$metric_replay_dir/candidate-results" \ + > "$metric_replay_dir/comparison.json" +``` + +This runs 5 cases × 3 repetitions × 2 revisions, 30 agent replays in total, +serially. Each repetition uses its number as the seed; a seed does not guarantee +bitwise determinism across servers. Output directories must not already exist. +Use `--case three-day-gap-followup --repetitions 1` for an initial smoke check, +in separate directories. Keep smoke runs separate from formal comparisons. + +Defaults are temperature 0, 3 repetitions, 24 model turns, 2,200 generated tokens +per turn and 180 seconds per model request. Reasoning models may need a larger +output budget, as shown above. Exhausted budgets and errors remain in the result; +they are not silently retried or excluded. A suite exits nonzero if any answer +is unfinished or makes no model-selected tool call. Exit zero still does not +mean that the answers are correct. + +For a protected model endpoint, inject `MOSAIC_REPLAY_API_KEY` into the process +environment from your secret manager. Authorization headers are not recorded; +credential-bearing URLs are rejected. Do not put credentials in model options. +Only synthetic fixture data, selected source prompts and tool results are sent +to the selected model endpoint. Confirm it is an appropriate destination. + +Normal cache behavior is the default. `--cache isolated` adds a fresh +`cache_salt` to each model request for servers supporting that option. Treat +this as a separate diagnostic, match it on both sides, and do not substitute +its results for normal-cache measurements. + +## Review every answer against its evidence + +The comparison rejects different model IDs, model/server revision labels, +fixture hashes, runner/suite hashes, Node versions, sampling settings, budgets, +cache modes or case/repetition coverage. It retains every before/after row, +including failures, full answers, token-usage records and elapsed time. + +Execution observations and factual accuracy are distinct. `completed` means +the server ended a nonempty final answer with `stop`; `investigated` means the +model requested at least one tool. Both can be true for a wrong answer. Every +row initially has `answerReview: pending` and requires this factual review: + +| Case | Required finding | Claims that fail review | +| --- | --- | --- | +| Three-day gap | Cite the last original sample at 23:59 UTC on November 2 and, using the known cadence, the first expected missing sample at 00:00 UTC on November 3. | Dating onset from a recent query boundary; treating an evaluated/lookback timestamp as the original sample; contradicting those dates elsewhere in the answer. | +| No available history | Say onset is unknown from the checked evidence; report actual search coverage and any gaps. | Asserting prior existence, backend retention, or an outage predating the search without evidence; reporting the requested budget as completed coverage. | +| Healthy presence at zero | Recognize zero as a present sample, including the current sample. | Reporting missing data because the value is zero; treating presence alone as proof of complete GPU/host health. | +| Two-hour gap | Cite the last original sample at 21:59 UTC and first expected missing sample at 22:00 UTC on November 5. | Confusing Prometheus lookback reuse or the query start with the original last-present/first-missing interval. | +| Follow-up | Recover the same three-day evidence after the empty two-hour query; answer the follow-up with a supported time basis. | Treating the two-hour window as outage duration or onset; losing the source selection or repeating an unsupported earlier claim. | + +For both gap cases and the follow-up, the first missed expected sample is an +estimate based on cadence, not proof of the exact physical failure time or its +cause. Review the whole answer, including suggested queries and summaries, for +contradictions. Do not require a particular tool name: a different valid query +strategy can establish the same facts. + +Write a separate review record with one row per case/repetition/revision, +verdict (`pass`, `fail`, or `not_assessed`), the exact claim, supporting tool +evidence, and the reason. Report the full denominator, all excluded or failed +runs, and each baseline-to-candidate transition. Keep unknown/unavailable +measurements distinct from zero. Do not edit the original transcripts to fit +the review or score truncated answers as successful completions. + +## Retained artifacts and checks + +Each suite keeps `manifest.json`, full `fixture-preflight.json`, a snapshot of +every loaded observability TypeScript source file and the skill, and +`summaries.json`. Each case/repetition keeps initial messages, actual tool +schemas, every full model request/response, backend request/response, tool +result, conversation transcript and final answer. Follow-ups also save the +seeded query/result. Failures save `error.json`. Source hashes identify the +actual loaded files even if the checkout is dirty. + +Keep these records and the serving manifest together in the external result +directory. Do not commit model weights, endpoint credentials or deployment +logs. Review artifacts for deployment-specific information before sharing. + +The runner's own tests use a small HTTP model stub and generated metrics; they +are not evidence of real LLM accuracy: + +```bash +node --test helm/mosaic-stack/tests/metric-history-replay.test.mjs +``` + +They cover fixture integrity, hidden ground truth, matched configuration, +full evidence capture, authentication separation, genuine follow-up context, +transport failures, missing investigation and exhausted budgets. These tests +are included by the existing CI test glob; real model runs remain opt-in. diff --git a/helm/mosaic-stack/tests/metric-history-replay.mjs b/helm/mosaic-stack/tests/metric-history-replay.mjs new file mode 100644 index 00000000..0451ac8c --- /dev/null +++ b/helm/mosaic-stack/tests/metric-history-replay.mjs @@ -0,0 +1,340 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Isolated model/tool replay, deliberately separate from native OpenClaw tests. +import assert from "node:assert/strict"; +import { createHash, randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { appendFileSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { registerHooks } from "node:module"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { performance } from "node:perf_hooks"; +import { cases, replayCases, end, start, interval, metric, userPrompt } from "./fixtures/metric-history-cases.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const extension = "helm/mosaic-stack/files/openclaw-seed/extensions/observability"; +const skillPath = "helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md"; +const NativeDate = Date; +const nativeFetch = globalThis.fetch; +const sha = value => createHash("sha256").update(value).digest("hex"); +const json = (path, value) => writeFileSync(path, JSON.stringify(value, null, 2) + "\n"); +const log = (path, value) => appendFileSync(path, JSON.stringify(value) + "\n"); +const iso = seconds => new NativeDate(seconds * 1000).toISOString(); +const textResult = result => result.content?.filter(item => item.type === "text").map(item => item.text).join("\n") ?? JSON.stringify(result); +const integer = (value, fallback, max, name) => { + const result = value === undefined ? fallback : Number(value); + if (!Number.isSafeInteger(result) || result < 1 || result > max) throw new Error(`${name} must be an integer from 1 to ${max}`); + return result; +}; + +export function endpoint(value, name, loopback = false) { + const url = new URL(value); + if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash) { + throw new Error(`${name} must be HTTP(S) without credentials, query, or fragment`); + } + if (loopback && !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname)) { + throw new Error(`${name} must address the local synthetic fixture, not production`); + } + return url.toString().replace(/\/+$/, ""); +} + +export function parseOptions(argv) { + const options = {}; + const allowed = new Set(["source", "prometheus", "model-url", "model", "output", "case", "repetitions", "max-turns", "max-tokens", "temperature", "timeout-ms", "cache", "model-options", "model-revision", "server-revision"]); + for (let i = 0; i < argv.length; i += 2) { + const key = argv[i]?.replace(/^--/, ""); + if (!argv[i]?.startsWith("--") || !allowed.has(key) || argv[i + 1] === undefined || key in options) { + throw new Error(`Invalid or repeated option: ${argv[i]}`); + } + options[key] = argv[i + 1]; + } + for (const key of ["source", "prometheus", "model-url", "model", "output"]) { + if (!options[key]?.trim()) throw new Error(`--${key} is required`); + } + const selected = options.case && options.case !== "all" ? options.case.split(",") : replayCases.map(item => item.id); + if (new Set(selected).size !== selected.length || selected.some(id => !replayCases.some(item => item.id === id))) throw new Error("Unknown or repeated case"); + const cache = options.cache ?? "normal"; + if (!["normal", "isolated"].includes(cache)) throw new Error("--cache must be normal or isolated"); + const temperature = Number(options.temperature ?? 0); + if (!Number.isFinite(temperature) || temperature < 0 || temperature > 2) throw new Error("--temperature must be from 0 to 2"); + const modelOptions = options["model-options"] ? JSON.parse(readFileSync(options["model-options"], "utf8")) : {}; + if (!modelOptions || Array.isArray(modelOptions) || typeof modelOptions !== "object" || + Object.keys(modelOptions).some(key => !["chat_template_kwargs", "reasoning_effort", "top_p"].includes(key))) { + throw new Error("Model options may contain only chat_template_kwargs, reasoning_effort and top_p"); + } + if (modelOptions.top_p !== undefined && (typeof modelOptions.top_p !== "number" || !Number.isFinite(modelOptions.top_p) || modelOptions.top_p <= 0 || modelOptions.top_p > 1)) throw new Error("top_p must be from 0 (exclusive) to 1"); + if (modelOptions.chat_template_kwargs !== undefined && (!modelOptions.chat_template_kwargs || Array.isArray(modelOptions.chat_template_kwargs) || typeof modelOptions.chat_template_kwargs !== "object")) throw new Error("chat_template_kwargs must be an object"); + if (modelOptions.reasoning_effort !== undefined && typeof modelOptions.reasoning_effort !== "string") throw new Error("reasoning_effort must be a string"); + return { + source: resolve(options.source), output: resolve(options.output), + prometheus: endpoint(options.prometheus, "--prometheus", true), + modelUrl: endpoint(options["model-url"], "--model-url"), model: options.model, + modelRevision: options["model-revision"] ?? "unrecorded", serverRevision: options["server-revision"] ?? "unrecorded", + selected, cache, modelOptions, temperature, + repetitions: integer(options.repetitions, 3, 100, "--repetitions"), + maxTurns: integer(options["max-turns"], 24, 100, "--max-turns"), + maxTokens: integer(options["max-tokens"], 2200, 32768, "--max-tokens"), + timeoutMs: integer(options["timeout-ms"], 180000, 600000, "--timeout-ms"), + }; +} + +export function validateFixture(payload) { + assert.equal(payload.status, "success", "fixture query failed"); + assert.ok(!payload.warnings?.length && payload.isPartial !== true, "partial fixture response"); + assert.equal(payload.data?.resultType, "matrix", "fixture must return original samples"); + const result = payload.data.result; + assert.ok(Array.isArray(result)); + assert.equal(result.length, cases.filter(item => item.count).length, "unexpected fixture series"); + const canonical = []; + for (const scenario of cases) { + const matches = result.filter(series => series.metric?.Hostname === scenario.host); + assert.equal(matches.length, scenario.count ? 1 : 0, scenario.id); + if (!scenario.count) continue; + const series = matches[0]; + assert.deepEqual(series.metric, { __name__: metric, Hostname: scenario.host, gpu: "0" }); + assert.equal(series.values?.length, scenario.count, scenario.id); + assert.ok(!series.histograms?.length, "histograms are not the test fixture"); + for (let i = 0; i < scenario.count; i++) { + assert.deepEqual(series.values[i], [start + i * interval, String(scenario.value)], `${scenario.id} sample ${i}`); + } + assert.equal(series.values.at(-1)[0], scenario.last); + canonical.push({ metric: series.metric, values: series.values }); + } + return { sampleCount: cases.reduce((sum, item) => sum + item.count, 0), sha256: sha(JSON.stringify(canonical)) }; +} + +async function verifyFixture(url, output) { + const query = new URL(`${url}/api/v1/query`); + query.searchParams.set("query", `${metric}[${end - start + 1}s]`); + query.searchParams.set("time", String(end)); + const response = await nativeFetch(query, { redirect: "error", signal: AbortSignal.timeout(15000) }); + const body = await response.text(); + json(join(output, "fixture-preflight.json"), { url: query.toString(), status: response.status, body }); + if (!response.ok) throw new Error(`Fixture HTTP ${response.status}`); + return validateFixture(JSON.parse(body)); +} + +function sourceSnapshot(source, output) { + const files = [...readdirSync(join(source, extension)).filter(name => name.endsWith(".ts")).map(name => `${extension}/${name}`), skillPath].sort(); + const hashes = {}; + for (const path of files) { + const data = readFileSync(join(source, path)); + hashes[path] = sha(data); + const target = join(output, "source", path); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, data); + } + let commit = null; + try { commit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch {} + return { path: source, commit, hashes }; +} + +async function loadTools(source) { + // Only the SDK registration wrapper is stubbed. Real metrics plugin code, + // schemas, history helper and skill are loaded from the requested revision. + const hooks = registerHooks({ resolve(specifier, context, next) { + return specifier === "openclaw/plugin-sdk/plugin-entry" + ? { url: "data:text/javascript,export const definePluginEntry = value => value;", shortCircuit: true } + : next(specifier, context); + } }); + let plugin; + try { ({ default: plugin } = await import(pathToFileURL(join(source, extension, "index.ts")))); } + finally { hooks.deregister(); } + const tools = new Map(); + plugin.register({ pluginConfig: { prometheusUrl: "http://metrics-fixture.invalid", grafanaEnabled: false }, + registerTool: tool => tools.set(tool.name, tool) }); + if (!tools.has("observability_range_query") || [...tools.keys()].some(name => !name.startsWith("observability_"))) throw new Error("Unexpected replay tool set"); + return tools; +} + +export function initialMessages(skill, scenario) { + return [ + { role: "system", content: `You are Mosaic, an operations assistant. Current time: ${iso(end)}. Use the available read-only observability tools to investigate the user's question. This environment scrapes metrics every ${interval} seconds. The observability skill is loaded below. Answer in the user's language.\n\n${skill}` }, + { role: "user", content: userPrompt(scenario) }, + ]; +} + +export async function runCase(options, scenario, repetition, tools, skill) { + const output = join(options.output, `${scenario.id}-${repetition}`); + mkdirSync(output); + const messages = initialMessages(skill, scenario); + const schemas = [...tools.values()].map(tool => ({ type: "function", function: { name: tool.name, description: tool.description, parameters: tool.parameters } })); + const calls = [], usages = [], backend = []; + const started = performance.now(); + let final = null, stop = "turn_budget", modelTurns = 0; + const previousFetch = globalThis.fetch, previousDate = globalThis.Date; + class FrozenDate extends NativeDate { + constructor(...args) { super(...(args.length ? args : [end * 1000])); } + static now() { return end * 1000; } + } + globalThis.Date = FrozenDate; + globalThis.fetch = async (input, init = {}) => { + const url = new URL(input); + if (url.origin !== "http://metrics-fixture.invalid" || (init.method ?? "GET") !== "GET" || + !["/api/v1/query", "/api/v1/query_range", "/api/v1/label/__name__/values"].includes(url.pathname)) { + throw new Error("Tool request outside the read-only fixture"); + } + const forwarded = new URL(options.prometheus + url.pathname + url.search); + if (url.pathname === "/api/v1/query" && !forwarded.searchParams.has("time")) forwarded.searchParams.set("time", String(end)); + try { + const response = await nativeFetch(forwarded, { ...init, redirect: "error", signal: AbortSignal.any([AbortSignal.timeout(15000), ...(init.signal ? [init.signal] : [])]) }); + const body = await response.text(); + const record = { requestedUrl: url.toString(), forwardedUrl: forwarded.toString(), status: response.status, body }; + backend.push(record); + log(join(output, "backend.jsonl"), record); + return new Response(body, { status: response.status, headers: { "content-type": "application/json" } }); + } catch (error) { + const record = { requestedUrl: url.toString(), forwardedUrl: forwarded.toString(), transportError: error.name }; + backend.push(record); + log(join(output, "backend.jsonl"), record); + throw error; + } + }; + try { + if (scenario.followup) { + const params = { query: `${metric}{Hostname="${scenario.host}",gpu="0"}`, rangeMinutes: 120, end: iso(end), stepSeconds: interval }; + const call = { id: "seeded-recent-window", type: "function", function: { name: "observability_range_query", arguments: JSON.stringify(params) } }; + const result = await tools.get(call.function.name).execute(call.id, params); + json(join(output, "seeded-tool.json"), { call, params, result }); + const seed = JSON.parse(textResult(result)); + if (result.isError || seed.resultType !== "matrix" || seed.seriesCount !== 0 || !Array.isArray(seed.result) || seed.result.length !== 0) { + throw new Error("Follow-up seed must be a successful empty recent-window query"); + } + messages.push({ role: "assistant", content: null, tool_calls: [call] }, + { role: "tool", tool_call_id: call.id, content: textResult(result) }, + { role: "assistant", content: "最近两小时,这个主机的 GPU 0 指标没有返回样本。" }, + { role: "user", content: "这个问题是什么时候开始的?请给出时间依据。" }); + } + json(join(output, "initial-messages.json"), messages); + json(join(output, "tool-schemas.json"), schemas); + for (let turn = 0; turn < options.maxTurns; turn++) { + const body = { ...options.modelOptions, model: options.model, messages, tools: schemas, tool_choice: "auto", + temperature: options.temperature, seed: repetition, max_tokens: options.maxTokens, parallel_tool_calls: false }; + if (options.cache === "isolated") body.cache_salt = `metric-history-${randomUUID()}`; + // Auth headers are never recorded. Provider options must not contain secrets. + log(join(output, "model-requests.jsonl"), { turn, body }); + const requestStart = performance.now(); + const response = await nativeFetch(`${options.modelUrl}/chat/completions`, { + method: "POST", redirect: "error", signal: AbortSignal.timeout(options.timeoutMs), + headers: { "content-type": "application/json", ...(process.env.MOSAIC_REPLAY_API_KEY ? { authorization: `Bearer ${process.env.MOSAIC_REPLAY_API_KEY}` } : {}) }, + body: JSON.stringify(body), + }); + const raw = await response.text(); + log(join(output, "model-responses.jsonl"), { turn, status: response.status, elapsedMs: performance.now() - requestStart, body: raw }); + modelTurns++; + if (!response.ok) throw new Error(`Model HTTP ${response.status}`); + const payload = JSON.parse(raw), choice = payload.choices?.[0], answer = choice?.message; + usages.push(payload.usage ?? null); + if (!answer || (answer.content != null && typeof answer.content !== "string")) throw new Error("Invalid model response"); + if (answer.tool_calls !== undefined && !Array.isArray(answer.tool_calls)) throw new Error("Invalid tool_calls"); + messages.push({ role: "assistant", content: answer.content ?? null, ...(answer.tool_calls?.length ? { tool_calls: answer.tool_calls } : {}) }); + if (!answer.tool_calls?.length) { + final = answer.content ?? ""; + stop = /| 16) throw new Error("Too many tool calls in one turn"); + for (const call of answer.tool_calls) { + if (typeof call.id !== "string" || !call.function || typeof call.function.name !== "string") throw new Error("Malformed tool call"); + let params = null, result; + try { + params = JSON.parse(call.function.arguments); + const tool = tools.get(call.function.name); + if (!tool) throw new Error("Tool unavailable in this fixture"); + result = await tool.execute(call.id, params); + } catch (error) { result = { isError: true, content: [{ type: "text", text: error.message }] }; } + const record = { turn, name: call.function.name, params, result }; + calls.push(record); + log(join(output, "tools.jsonl"), record); + messages.push({ role: "tool", tool_call_id: call.id, content: textResult(result) }); + } + json(join(output, "transcript.json"), messages); + } + } catch (error) { + stop = "replay_error"; + json(join(output, "error.json"), { name: error.name, message: error.message }); + } finally { + globalThis.fetch = previousFetch; + globalThis.Date = previousDate; + } + json(join(output, "transcript.json"), messages); + writeFileSync(join(output, "answer.txt"), final ?? "No completed answer.\n"); + const summary = { + case: scenario.id, repetition, stop, modelTurns, toolCalls: calls.length, + toolErrors: calls.filter(call => call.result.isError || call.result.details?.status === "incomplete").length, + backendRequests: backend.length, elapsedMs: performance.now() - started, usages, final, + // These are execution observations, not a grade for natural-language claims. + completed: stop === "stop", investigated: calls.length > 0, + answerReview: "pending", expected: { ...scenario, lastIso: scenario.last === null ? null : iso(scenario.last), missingIso: scenario.missing === null ? null : iso(scenario.missing) }, + }; + json(join(output, "summary.json"), summary); + return summary; +} + +export async function runSuite(options) { + mkdirSync(options.output); // Never mix with or overwrite an earlier experiment. + const fixture = await verifyFixture(options.prometheus, options.output); + const source = sourceSnapshot(options.source, options.output); + const tools = await loadTools(options.source); + const skill = readFileSync(join(options.source, skillPath), "utf8"); + const suiteHash = sha(readFileSync(join(here, "fixtures/metric-history-cases.mjs"))); + const manifest = { + schemaVersion: 1, startedAt: new NativeDate().toISOString(), mode: "isolated-model-tool-replay", + source, runnerSha256: sha(readFileSync(fileURLToPath(import.meta.url))), suiteSha256: suiteHash, + fixture, fixedTime: iso(end), node: process.version, + model: options.model, modelUrl: options.modelUrl, modelRevision: options.modelRevision, serverRevision: options.serverRevision, + settings: { temperature: options.temperature, seedPolicy: "repetition-number", repetitions: options.repetitions, maxTurns: options.maxTurns, + maxTokens: options.maxTokens, timeoutMs: options.timeoutMs, cache: options.cache, modelOptions: options.modelOptions, cases: options.selected }, + scope: "Real metrics plugin and explicitly loaded skill. SDK registration wrapper stubbed. Native OpenClaw routing, skill selection, full prompts and UI are not exercised.", + }; + json(join(options.output, "manifest.json"), manifest); + const summaries = []; + for (const id of options.selected) { + for (let repetition = 1; repetition <= options.repetitions; repetition++) { + const summary = await runCase(options, replayCases.find(item => item.id === id), repetition, tools, skill); + summaries.push(summary); + json(join(options.output, "summaries.json"), summaries); + console.log(JSON.stringify({ case: id, repetition, stop: summary.stop, toolCalls: summary.toolCalls, answerReview: summary.answerReview })); + } + } + return summaries; +} + +export function compareRuns(leftPath, rightPath) { + const read = (dir, name) => JSON.parse(readFileSync(join(dir, name), "utf8")); + const left = read(leftPath, "manifest.json"), right = read(rightPath, "manifest.json"); + for (const key of ["schemaVersion", "mode", "runnerSha256", "suiteSha256", "fixture", "fixedTime", "node", "model", "modelUrl", "modelRevision", "serverRevision", "settings"]) { + assert.deepEqual(left[key], right[key], `Not a matched harness comparison: ${key} differs`); + } + const a = read(leftPath, "summaries.json"), b = read(rightPath, "summaries.json"); + const expected = left.settings.cases.flatMap(id => Array.from({ length: left.settings.repetitions }, (_, i) => `${id}:${i + 1}`)); + const index = list => new Map(list.map(item => [`${item.case}:${item.repetition}`, item])); + const ai = index(a), bi = index(b); + for (const [items, map] of [[a, ai], [b, bi]]) { + assert.equal(items.length, expected.length, "Incomplete experiment"); + assert.equal(map.size, expected.length, "Duplicate case/repetition"); + assert.ok(expected.every(key => map.has(key)), "Missing case/repetition"); + } + return { left: { path: resolve(leftPath), manifest: left }, right: { path: resolve(rightPath), manifest: right }, + interpretation: "Matched execution comparison only. Every answer requires the documented factual review; completion is not correctness. Unrecorded model/server revisions limit reproducibility.", + rows: expected.map(key => ({ key, baseline: ai.get(key), candidate: bi.get(key) })) }; +} + +async function main() { + const [command, ...args] = process.argv.slice(2); + if (command === "compare" && args.length === 2) { + console.log(JSON.stringify(compareRuns(...args), null, 2)); + } else if (command === "run") { + const rows = await runSuite(parseOptions(args)); + // Distinguish unfinished/no-investigation runs from transport success. This + // exit status still does not assert that the final answer is accurate. + if (rows.some(row => !row.completed || !row.investigated)) process.exitCode = 1; + } else { + throw new Error("Usage: node metric-history-replay.mjs run --source REPO --prometheus http://127.0.0.1:19093 --model-url URL/v1 --model MODEL_ID --output NEW_DIR [--case all --repetitions 3 --max-turns 24 --cache normal]; or compare BASELINE_DIR CANDIDATE_DIR"); + } +} +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch(error => { console.error(error.message); process.exitCode = 1; }); +} diff --git a/helm/mosaic-stack/tests/metric-history-replay.test.mjs b/helm/mosaic-stack/tests/metric-history-replay.test.mjs new file mode 100644 index 00000000..58ad75ed --- /dev/null +++ b/helm/mosaic-stack/tests/metric-history-replay.test.mjs @@ -0,0 +1,203 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; +import { compareRuns, endpoint, initialMessages, parseOptions, runSuite, validateFixture } from "./metric-history-replay.mjs"; +import { cases, replayCases, start, end, interval, metric } from "./fixtures/metric-history-cases.mjs"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); +const payload = () => ({ status: "success", data: { resultType: "matrix", result: cases.filter(item => item.count).map(item => ({ + metric: { __name__: metric, Hostname: item.host, gpu: "0" }, + values: Array.from({ length: item.count }, (_, i) => [start + i * interval, String(item.value)]), +})) } }); +const read = path => JSON.parse(readFileSync(path, "utf8")); +const write = (path, value) => writeFileSync(path, JSON.stringify(value)); +function scratch(t) { + const path = mkdtempSync(join(tmpdir(), "mosaic-history-replay-")); + t.after(() => rmSync(path, { recursive: true, force: true })); + return path; +} +async function server(t, handler) { + const requests = []; + const http = createServer(async (req, res) => { + try { + let raw = ""; + for await (const chunk of req) raw += chunk; + const item = { method: req.method, url: new URL(req.url, "http://localhost"), headers: req.headers, body: raw ? JSON.parse(raw) : null }; + requests.push(item); + const result = handler(item, requests.length); + res.writeHead(result.httpStatus ?? 200, { "content-type": "application/json" }); + res.end(JSON.stringify(result.body ?? result)); + } catch (error) { res.writeHead(500); res.end(JSON.stringify({ error: error.message })); } + }); + await new Promise((accept, reject) => { http.once("error", reject); http.listen(0, "127.0.0.1", accept); }); + t.after(() => { http.closeAllConnections(); http.close(); }); + return { url: `http://127.0.0.1:${http.address().port}`, requests }; +} +async function fixture(t) { + return server(t, request => { + assert.equal(request.method, "GET"); + const data = payload(); + if (request.url.pathname.endsWith("query_range")) { data.data.result = []; return data; } + const query = request.url.searchParams.get("query"); + const range = Number(query.match(/\[(\d+)s\]$/)?.[1]); + const stop = Number(request.url.searchParams.get("time")); + const host = query.match(/Hostname="([^"]+)"/)?.[1]; + data.data.result = data.data.result.filter(item => !host || item.metric.Hostname === host) + .map(item => ({ ...item, values: item.values.filter(([ts]) => ts > stop - range && ts <= stop) })) + .filter(item => item.values.length); + return data; + }); +} +function options(output, backend, model, extra = []) { + return parseOptions(["--source", root, "--prometheus", backend, "--model-url", `${model}/v1`, "--model", "fixture-model", + "--output", output, "--case", "three-day-gap-followup", "--repetitions", "1", ...extra]); +} +function response(content, tool_calls) { + return { choices: [{ finish_reason: tool_calls ? "tool_calls" : "stop", message: { role: "assistant", content, ...(tool_calls ? { tool_calls } : {}) } }], + usage: { prompt_tokens: 42, completion_tokens: 12, total_tokens: 54 } }; +} +const historyCall = () => [{ id: "history-call", type: "function", function: { name: "observability_metric_last_seen", arguments: JSON.stringify({ + metric, labels: { Hostname: "fixture-gpu-a", gpu: "0" }, expectedIntervalSeconds: interval, +}) } }]; + +test("fixture validation checks all original timestamps and values, not just endpoints", () => { + assert.equal(validateFixture(payload()).sampleCount, 30121); + for (const mutate of [ + data => { data.data.result[0].values[1234][0]++; }, + data => { data.data.result[0].values[1234][1] = "0"; }, + data => { data.data.result[0].metric.extra = "ambiguous"; }, + data => { data.warnings = ["partial"]; }, + data => { data.data.result.pop(); }, + data => { data.data.result[0].values.pop(); }, + ]) { + const data = payload(); mutate(data); + assert.throws(() => validateFixture(data)); + } +}); + +test("prompts do not disclose ground truth or label the unknown history as a failure", () => { + for (const scenario of replayCases) { + const input = initialMessages("example skill", scenario); + assert.equal(input.length, 2); + assert.ok(input[0].content.includes("60 seconds")); + assert.ok(input[1].content.includes(scenario.host)); + assert.ok(!JSON.stringify(input).includes(scenario.id)); + if (scenario.last && scenario.last !== end) assert.ok(!JSON.stringify(input).includes(new Date(scenario.last * 1000).toISOString())); + assert.ok(!JSON.stringify(input).includes("7200")); + } +}); + +test("configuration rejects credential URLs, production metrics, unknown controls and overwrites of fixed model fields", t => { + for (const url of ["https://user:password@models.example/v1", "https://models.example/v1?api_key=secret", "file:///tmp/model"]) { + assert.throws(() => endpoint(url, "model")); + } + assert.throws(() => endpoint("https://production.example", "fixture", true)); + assert.throws(() => options("/tmp/run", "http://127.0.0.1:19093", "http://models.example", ["--cache", "unknown"])); + assert.throws(() => options("/tmp/run", "http://127.0.0.1:19093", "http://models.example", ["--max-turns", "0"])); + for (const value of ["NaN", "-1", "3"]) assert.throws(() => options("/tmp/run", "http://127.0.0.1:19093", "http://models.example", ["--temperature", value])); + const dir = scratch(t), path = join(dir, "options.json"); + for (const invalid of [{ messages: [] }, { top_p: 0 }, { top_p: "0.95" }, { chat_template_kwargs: [] }, { reasoning_effort: false }]) { + write(path, invalid); + assert.throws(() => options("/tmp/run", "http://127.0.0.1:19093", "http://models.example", ["--model-options", path])); + } +}); + +test("real plugin replay preserves the follow-up, original evidence, all messages, and auth separation", async t => { + const dir = scratch(t), backend = await fixture(t); + const priorKey = process.env.MOSAIC_REPLAY_API_KEY; + process.env.MOSAIC_REPLAY_API_KEY = "synthetic-test-key-never-in-artifacts"; + t.after(() => { if (priorKey === undefined) delete process.env.MOSAIC_REPLAY_API_KEY; else process.env.MOSAIC_REPLAY_API_KEY = priorKey; }); + const model = await server(t, (request, count) => { + assert.equal(request.headers.authorization, "Bearer synthetic-test-key-never-in-artifacts"); + assert.equal(request.url.pathname, "/v1/chat/completions"); + assert.equal(request.body.model, "fixture-model"); + assert.equal(request.body.temperature, 1); + assert.equal(request.body.top_p, 0.95); + assert.deepEqual(request.body.chat_template_kwargs, { enable_thinking: false }); + assert.equal(request.body.cache_salt, undefined); + if (count === 1) { + assert.equal(request.body.messages.length, 6); + assert.equal(request.body.messages.at(-1).content, "这个问题是什么时候开始的?请给出时间依据。"); + return response(null, historyCall()); + } + const evidence = JSON.parse(request.body.messages.at(-1).content); + assert.equal(evidence.lastSample.iso, "2026-11-02T23:59:00.000Z"); + assert.equal(evidence.firstExpectedMissing.iso, "2026-11-03T00:00:00.000Z"); + return response("Last actual source sample: 2026-11-02T23:59:00Z. With a 60-second cadence, the first expected missing sample is 2026-11-03T00:00:00Z. Physical fault time is unproven."); + }); + const output = join(dir, "run"), before = Date; + const config = join(dir, "model-options.json"); + write(config, { top_p: 0.95, chat_template_kwargs: { enable_thinking: false } }); + const result = await runSuite(options(output, backend.url, model.url, ["--temperature", "1", "--model-options", config])); + assert.equal(globalThis.Date, before); + assert.equal(result[0].stop, "stop"); + assert.equal(result[0].modelTurns, 2); + assert.equal(result[0].toolCalls, 1); + assert.equal(result[0].backendRequests, 6); // seeded range + five raw windows + assert.equal(result[0].answerReview, "pending"); + assert.equal(read(join(output, "manifest.json")).fixture.sampleCount, 30121); + assert.equal(read(join(output, "manifest.json")).settings.temperature, 1); + const run = join(output, "three-day-gap-followup-1"); + const messages = read(join(run, "transcript.json")); + assert.equal(messages.length, 9); + assert.equal(readFileSync(join(run, "answer.txt"), "utf8"), result[0].final); + for (const entry of readdirSync(run)) assert.ok(!readFileSync(join(run, entry), "utf8").includes("synthetic-test-key-never-in-artifacts")); + await assert.rejects(runSuite(options(output, backend.url, model.url)), /EEXIST/); +}); + +test("a failed recent-window query cannot become a fabricated empty follow-up context", async t => { + const dir = scratch(t); + const backend = await server(t, request => request.url.pathname.endsWith("query_range") + ? { httpStatus: 503, body: { status: "error", error: "synthetic backend failure" } } : payload()); + const model = await server(t, () => { assert.fail("invalid seed reached the model"); }); + const result = await runSuite(options(join(dir, "run"), backend.url, model.url)); + assert.equal(result[0].stop, "replay_error"); + assert.equal(result[0].modelTurns, 0); + assert.equal(model.requests.length, 0); +}); + +test("completion without investigation and exhausted turns remain visible failures, never correctness passes", async t => { + const dir = scratch(t), backend = await fixture(t); + const noTools = await server(t, () => response("The outage began three days ago.")); + const done = await runSuite(options(join(dir, "no-tools"), backend.url, noTools.url)); + assert.equal(done[0].completed, true); + assert.equal(done[0].investigated, false); + assert.equal(done[0].answerReview, "pending"); + const looping = await server(t, () => response(null, historyCall())); + const budget = await runSuite(options(join(dir, "budget"), backend.url, looping.url, ["--max-turns", "1"])); + assert.equal(budget[0].stop, "turn_budget"); + assert.equal(budget[0].completed, false); + const unavailable = await server(t, () => ({ httpStatus: 503, body: { error: "model unavailable" } })); + const failed = await runSuite(options(join(dir, "error"), backend.url, unavailable.url)); + assert.equal(failed[0].stop, "replay_error"); + assert.equal(failed[0].completed, false); + assert.equal(read(join(dir, "error/three-day-gap-followup-1/error.json")).message, "Model HTTP 503"); +}); + +test("comparisons require identical model, fixture, budgets, runner and complete case/repetition coverage", async t => { + const dir = scratch(t), backend = await fixture(t); + const model = await server(t, () => response("Review this answer separately.")); + const left = join(dir, "left"), right = join(dir, "right"); + await runSuite(options(left, backend.url, model.url)); + await runSuite(options(right, backend.url, model.url)); + const report = compareRuns(left, right); + assert.equal(report.rows.length, 1); + assert.equal(report.rows[0].candidate.answerReview, "pending"); + const manifestPath = join(right, "manifest.json"), original = read(manifestPath); + for (const change of [{ model: "different-model" }, { modelRevision: "different-weights" }, { runnerSha256: "modified" }, + { fixture: { ...original.fixture, sha256: "other-data" } }, { settings: { ...original.settings, cache: "isolated" } }, + { settings: { ...original.settings, temperature: 1 } }]) { + write(manifestPath, { ...original, ...change }); + assert.throws(() => compareRuns(left, right), /Not a matched/); + } + write(manifestPath, original); + write(join(right, "summaries.json"), []); + assert.throws(() => compareRuns(left, right), /Incomplete experiment/); +}); diff --git a/helm/mosaic-stack/tests/metric-history.md b/helm/mosaic-stack/tests/metric-history.md index 33e2ff7d..6a6a57e6 100644 --- a/helm/mosaic-stack/tests/metric-history.md +++ b/helm/mosaic-stack/tests/metric-history.md @@ -82,7 +82,8 @@ No stale markers are inserted, so real Prometheus lookback behavior is exercised | Two-hour gap | fixture-gpu-d | 0 | 23 | 11400 | 2026-11-05T21:59:00Z | 2026-11-05T22:00:00Z | The generator emits every one of the 30121 samples. It can also support -model/tool replay tests. Passing the deterministic tool tests does not prove +the [matched model/tool replay](metric-history-replay.md) for comparing harness +revisions with the same model and fixture. Passing the deterministic tool tests does not prove that an agent selected the skill, invoked the tool, or completed the answer; those require separate native-runtime behavior tests. From 4e6c153297316ba2859d97303e9b2b067489e98d Mon Sep 17 00:00:00 2001 From: JeffPengCoder Date: Tue, 10 Nov 2026 00:49:35 +0800 Subject: [PATCH 3/3] docs: record Nemotron 3 Super metric-history backtest Signed-off-by: JeffPengCoder --- .../tests/metric-history-replay.md | 6 + ...ory-results-nemotron-3-super-20261109.json | 4531 +++++++++++++++++ ...story-results-nemotron-3-super-20261109.md | 97 + 3 files changed, 4634 insertions(+) create mode 100644 helm/mosaic-stack/tests/metric-history-results-nemotron-3-super-20261109.json create mode 100644 helm/mosaic-stack/tests/metric-history-results-nemotron-3-super-20261109.md diff --git a/helm/mosaic-stack/tests/metric-history-replay.md b/helm/mosaic-stack/tests/metric-history-replay.md index b4697aca..a1420e8d 100644 --- a/helm/mosaic-stack/tests/metric-history-replay.md +++ b/helm/mosaic-stack/tests/metric-history-replay.md @@ -13,6 +13,12 @@ wrapper. It does not test native OpenClaw skill selection, complete runtime prompts, session routing, approvals or the UI. Those need a separate deployment test. No score or model behavior is asserted by adding this runner. +An observed [Nemotron 3 Super NVFP4 comparison](metric-history-results-nemotron-3-super-20261109.md) +records all 30 outcomes and their factual review, including failures. Evidence +retrieval improves in that run, but complete-answer reliability and native +OpenClaw acceptance remain unresolved. Its exact configuration and full final +answers are recorded separately from the runner's deterministic tests. + ## Prepare the fixture and revisions Use Node.js 24 or newer and the running local Prometheus fixture documented in diff --git a/helm/mosaic-stack/tests/metric-history-results-nemotron-3-super-20261109.json b/helm/mosaic-stack/tests/metric-history-results-nemotron-3-super-20261109.json new file mode 100644 index 00000000..24e2e278 --- /dev/null +++ b/helm/mosaic-stack/tests/metric-history-results-nemotron-3-super-20261109.json @@ -0,0 +1,4531 @@ +{ + "schema_version": 1, + "observed_date_utc": "2026-11-09", + "scope": "Five synthetic missing-metric cases, three seeds, thinking disabled. Not a broad model benchmark or native OpenClaw acceptance.", + "model": { + "checkpoint": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", + "revision": "ff433f5493e25d631c9f12b5d55c674229923d02", + "architecture": "aarch64", + "gpu": "NVIDIA GB10, 580.173.02", + "quantization": "Unchanged official ModelOpt MIXED_PRECISION checkpoint: 40961 NVFP4 and 139 FP8 layer entries; auto-detected modelopt_mixed", + "versions": { + "torch": "2.13.0+cu130", + "torch_cuda": "13.0", + "vllm": "0.27.1" + }, + "protocol_sha256": { + "config.json": "14a1ad3ab6ec49e09e9c30f8686c8e3ca5b5338ee0eb0c79dff4f285fc29778d", + "hf_quant_config.json": "dbabb3970ad65a6cca51488db15d66d976c2486ff5448c4aee906389a02df316", + "tokenizer.json": "623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7", + "tokenizer_config.json": "10f93eabcb9b1602fbb991d6308e787ce1df28ee9cd7a1c6d1e8c3f338b957bc", + "chat_template.jinja": "575fb74f54ed264df9047d0ecce3c98938aae953fb4f50356675706264cbb68a", + "super_v3_reasoning_parser.py": "f7fc71d1697ed79931787cf7485ee297365a41c3442cc1e05e6ebab43e0a39c1", + "generation_config.json": "27d520b2df55efe6b978d06fbfe8213ae93288d063d462f40b4ed94637c019bf" + }, + "prefix_caching": false, + "inference": { + "temperature": 1, + "top_p": 0.95, + "enable_thinking": false, + "max_tokens": 8192, + "max_turns": 24, + "timeout_ms": 180000, + "seed": "repetition index (1, 2, 3)", + "cache": "normal", + "concurrency": 1 + } + }, + "serving": { + "image": { + "id": "sha256:2c211a1273b48e8929f893b267aeb1509e6b84654cdbde1bad56d79e3964224d", + "architecture": "arm64", + "repo_digests": [ + "vllm/vllm-openai@sha256:1c8e60a0841b333c700488cb029d3664807249da0c071e862191b00fe34b228c" + ] + }, + "command": [ + "--model", + "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", + "--revision", + "ff433f5493e25d631c9f12b5d55c674229923d02", + "--served-model-name", + "nvidia/nemotron-3-super", + "--host", + "0.0.0.0", + "--port", + "8000", + "--async-scheduling", + "--dtype", + "auto", + "--kv-cache-dtype", + "fp8", + "--tensor-parallel-size", + "1", + "--pipeline-parallel-size", + "1", + "--data-parallel-size", + "1", + "--trust-remote-code", + "--gpu-memory-utilization", + "0.75", + "--enable-chunked-prefill", + "--max-num-seqs", + "1", + "--max-model-len", + "262144", + "--moe-backend", + "marlin", + "--mamba-ssm-cache-dtype", + "float16", + "--reasoning-parser-plugin", + "/root/.cache/huggingface/hub/models--nvidia--NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4/snapshots/ff433f5493e25d631c9f12b5d55c674229923d02/super_v3_reasoning_parser.py", + "--reasoning-parser", + "super_v3", + "--enable-auto-tool-choice", + "--tool-call-parser", + "qwen3_coder" + ], + "environment": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "HF_HUB_DISABLE_IMPLICIT_TOKEN": "1", + "VLLM_NVFP4_GEMM_BACKEND": "marlin", + "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm", + "VLLM_USE_FLASHINFER_MOE_FP4": "0" + }, + "effective_runtime_announcements": [ + "(APIServer pid=1) WARNING 11-09 15:46:32 [envs.py:2128] Unknown vLLM environment variable detected: VLLM_NVFP4_GEMM_BACKEND", + "(APIServer pid=1) WARNING 11-09 15:46:32 [envs.py:2128] Unknown vLLM environment variable detected: VLLM_USE_FLASHINFER_MOE_FP4", + "(EngineCore pid=231) INFO 11-09 15:46:52 [__init__.py:665] Selected FlashInferFP8ScaledMMLinearKernel for ModelOptFp8LinearMethod", + "(EngineCore pid=231) INFO 11-09 15:46:52 [__init__.py:1077] Using FlashInferCutlassNvFp4LinearKernel for NVFP4 GEMM", + "(EngineCore pid=231) INFO 11-09 15:46:52 [nvfp4.py:239] Using 'MARLIN' NvFp4 MoE backend out of potential backends: ['FLASHINFER_TRTLLM', 'FLASHINFER_CUTEDSL', 'FLASHINFER_CUTEDSL_BATCHED', 'FLASHINFER_CUTLASS', 'VLLM_CUTLASS', 'MARLIN', 'HUMMING', 'EMULATION'].", + "(EngineCore pid=231) INFO 11-09 15:46:52 [cuda.py:482] Using FLASHINFER attention backend out of potential backends: ['FLASHINFER', 'TRITON_ATTN'].", + "(EngineCore pid=231) INFO 11-09 15:55:38 [gpu_model_runner.py:5405] Model loading took 69.62 GiB memory and 515.902561 seconds", + "(EngineCore pid=231) INFO 11-09 15:56:37 [gpu_worker.py:563] Available KV cache memory: 31.87 GiB", + "(EngineCore pid=231) INFO 11-09 15:56:37 [kv_cache_utils.py:2235] GPU KV cache size: 7,625,306 tokens" + ] + }, + "source_revisions": { + "baseline": { + "commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "file_sha256": { + "helm/mosaic-stack/files/openclaw-seed/extensions/observability/grafana.ts": "4ccd76f10c41e86be37f28c1781075708c71b4c3f3308447949675c16b182236", + "helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts": "34fa25b3e0491c9b946a36d3990b8c1629ad527bc93be11267128aab5309ba1c", + "helm/mosaic-stack/files/openclaw-seed/extensions/observability/loki.ts": "4948c6b18e1dc657b3a0112ae6d4bb0c8620983075be6483d3e3898848ad7b46", + "helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md": "4a53faf9612f8f996e56f3e26b6d5dfa9dd02c203e8e9f6c4336e788514b8220" + } + }, + "candidate": { + "commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "file_sha256": { + "helm/mosaic-stack/files/openclaw-seed/extensions/observability/grafana.ts": "4ccd76f10c41e86be37f28c1781075708c71b4c3f3308447949675c16b182236", + "helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts": "da773298a233edd6792e848dc4680d105065a135fe54bbe4e57a5a65bdfc0fd2", + "helm/mosaic-stack/files/openclaw-seed/extensions/observability/loki.ts": "4948c6b18e1dc657b3a0112ae6d4bb0c8620983075be6483d3e3898848ad7b46", + "helm/mosaic-stack/files/openclaw-seed/extensions/observability/metric-history.ts": "1f7f6a3994653dc76c1243a2ed1c6bea9dfa8d65f971d51b6331477277c65bc5", + "helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md": "089c6169447ed0b123f93b31512198797abe792bfa35b597862660d0cc0c02d2" + } + } + }, + "harness": { + "runnerSha256": "b2d2e3b59a610765f9bd20a1629643e98bbf89cab524015cd3e1f264eb7b0728", + "suiteSha256": "6697c43bec1636f6427900169479ac976f4247e02d09afafc2b559747241b424", + "fixture": { + "sampleCount": 30121, + "sha256": "87c7e049aa340047632a62c0821a12321189de80fc3ac3035fe17bb2c11a882b" + }, + "fixedTime": "2026-11-06T00:00:00.000Z", + "node": "v25.8.1", + "settings": { + "temperature": 1, + "seedPolicy": "repetition-number", + "repetitions": 3, + "maxTurns": 24, + "maxTokens": 8192, + "timeoutMs": 180000, + "cache": "normal", + "modelOptions": { + "top_p": 0.95, + "chat_template_kwargs": { + "enable_thinking": false + } + }, + "cases": [ + "three-day-gap", + "history-unavailable", + "healthy-zero", + "two-hour-gap", + "three-day-gap-followup" + ] + } + }, + "generated_sample_file_sha256": "9a8b53d4d92115c8917f663e21b33caa9e803bbc7f770caa28965495a2f8cba9", + "checkpoint_integrity": { + "revision": "ff433f5493e25d631c9f12b5d55c674229923d02", + "files": [ + { + "file": ".gitattributes", + "expected_size": 1635, + "actual_size": 1635, + "algorithm": "git-blob-sha1", + "expected": "aa7aacd0134a92c3c1943fdecc75cd8b7420cce6", + "actual": "aa7aacd0134a92c3c1943fdecc75cd8b7420cce6", + "pass": true + }, + { + "file": "README.md", + "expected_size": 81626, + "actual_size": 81626, + "algorithm": "git-blob-sha1", + "expected": "9439260819d84611797ff779da798ba1fea09125", + "actual": "9439260819d84611797ff779da798ba1fea09125", + "pass": true + }, + { + "file": "__init__.py", + "expected_size": 0, + "actual_size": 0, + "algorithm": "git-blob-sha1", + "expected": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "actual": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "pass": true + }, + { + "file": "accuracy_chart.png", + "expected_size": 79162, + "actual_size": 79162, + "algorithm": "git-blob-sha1", + "expected": "b02796fcc95a5b948f843088cc3bf8da96b39601", + "actual": "b02796fcc95a5b948f843088cc3bf8da96b39601", + "pass": true + }, + { + "file": "bias.md", + "expected_size": 2626, + "actual_size": 2626, + "algorithm": "git-blob-sha1", + "expected": "6406714ae56570a4a00d16678ae7579117a97f08", + "actual": "6406714ae56570a4a00d16678ae7579117a97f08", + "pass": true + }, + { + "file": "chat_template.jinja", + "expected_size": 10771, + "actual_size": 10771, + "algorithm": "git-blob-sha1", + "expected": "71935c3a25edcdde07a0b7c4bec26f68bbce73a4", + "actual": "71935c3a25edcdde07a0b7c4bec26f68bbce73a4", + "pass": true + }, + { + "file": "config.json", + "expected_size": 7411248, + "actual_size": 7411248, + "algorithm": "git-blob-sha1", + "expected": "25fdfa00f468ad953fc5516aeab78a128332da1a", + "actual": "25fdfa00f468ad953fc5516aeab78a128332da1a", + "pass": true + }, + { + "file": "configuration_nemotron_h.py", + "expected_size": 19822, + "actual_size": 19822, + "algorithm": "git-blob-sha1", + "expected": "b0577e12c52dac6d40529081c2c316b08b3b7ecf", + "actual": "b0577e12c52dac6d40529081c2c316b08b3b7ecf", + "pass": true + }, + { + "file": "explainability.md", + "expected_size": 3155, + "actual_size": 3155, + "algorithm": "git-blob-sha1", + "expected": "24936994e396a15aa18c75dd1cd43d41d36275fd", + "actual": "24936994e396a15aa18c75dd1cd43d41d36275fd", + "pass": true + }, + { + "file": "generation_config.json", + "expected_size": 210, + "actual_size": 210, + "algorithm": "git-blob-sha1", + "expected": "9f6546e3b99dd85a2899453a2487850007815ead", + "actual": "9f6546e3b99dd85a2899453a2487850007815ead", + "pass": true + }, + { + "file": "hf_quant_config.json", + "expected_size": 6146797, + "actual_size": 6146797, + "algorithm": "git-blob-sha1", + "expected": "756004b69ae1a798e8cff5ffeaf0a53e46e6ddc2", + "actual": "756004b69ae1a798e8cff5ffeaf0a53e46e6ddc2", + "pass": true + }, + { + "file": "model-00001-of-00017.safetensors", + "expected_size": 5000122600, + "actual_size": 5000122600, + "algorithm": "sha256", + "expected": "3d31c241fb777f44342ac3c298147bb82ddcaa64214c555494d8fab397218d0e", + "actual": "3d31c241fb777f44342ac3c298147bb82ddcaa64214c555494d8fab397218d0e", + "pass": true + }, + { + "file": "model-00002-of-00017.safetensors", + "expected_size": 4977660672, + "actual_size": 4977660672, + "algorithm": "sha256", + "expected": "78d4a5ad26841e04ea92ae3d6b6b29ab9069b4ea27155b93213a5bfbeef85081", + "actual": "78d4a5ad26841e04ea92ae3d6b6b29ab9069b4ea27155b93213a5bfbeef85081", + "pass": true + }, + { + "file": "model-00003-of-00017.safetensors", + "expected_size": 5001219144, + "actual_size": 5001219144, + "algorithm": "sha256", + "expected": "ecb72debb5d274b4beb74fb993a44e4f8ac27d640ecadaf656ed7de178942429", + "actual": "ecb72debb5d274b4beb74fb993a44e4f8ac27d640ecadaf656ed7de178942429", + "pass": true + }, + { + "file": "model-00004-of-00017.safetensors", + "expected_size": 5000841128, + "actual_size": 5000841128, + "algorithm": "sha256", + "expected": "a7d1c68e827296ee4bfc0ca4fe2c6a545c48b7a7f5148d9ebf4872c6fb123106", + "actual": "a7d1c68e827296ee4bfc0ca4fe2c6a545c48b7a7f5148d9ebf4872c6fb123106", + "pass": true + }, + { + "file": "model-00005-of-00017.safetensors", + "expected_size": 5000389560, + "actual_size": 5000389560, + "algorithm": "sha256", + "expected": "1d432c9c783453c6efa69d01228023b407be77e7bfea2909b2023c03df07df6a", + "actual": "1d432c9c783453c6efa69d01228023b407be77e7bfea2909b2023c03df07df6a", + "pass": true + }, + { + "file": "model-00006-of-00017.safetensors", + "expected_size": 4995800552, + "actual_size": 4995800552, + "algorithm": "sha256", + "expected": "037ea60b87cb2b8ce12beb7164c5382f4d97723c24cdbacc9fc188ae5a28f47b", + "actual": "037ea60b87cb2b8ce12beb7164c5382f4d97723c24cdbacc9fc188ae5a28f47b", + "pass": true + }, + { + "file": "model-00007-of-00017.safetensors", + "expected_size": 5000688960, + "actual_size": 5000688960, + "algorithm": "sha256", + "expected": "3c0527dac96fd91874427f04e92c1df43585503608a487a4c8ffcf9d5f925bd4", + "actual": "3c0527dac96fd91874427f04e92c1df43585503608a487a4c8ffcf9d5f925bd4", + "pass": true + }, + { + "file": "model-00008-of-00017.safetensors", + "expected_size": 5000478392, + "actual_size": 5000478392, + "algorithm": "sha256", + "expected": "ef6e73f64c3d7242a50230a288811eaa56aa2d9c6da9040b90361e6b9245bc26", + "actual": "ef6e73f64c3d7242a50230a288811eaa56aa2d9c6da9040b90361e6b9245bc26", + "pass": true + }, + { + "file": "model-00009-of-00017.safetensors", + "expected_size": 5000199288, + "actual_size": 5000199288, + "algorithm": "sha256", + "expected": "b4f598068350b93d42c486e537239dabc4661545ea2d39c8f0bb7ca23223c4f5", + "actual": "b4f598068350b93d42c486e537239dabc4661545ea2d39c8f0bb7ca23223c4f5", + "pass": true + }, + { + "file": "model-00010-of-00017.safetensors", + "expected_size": 4997005048, + "actual_size": 4997005048, + "algorithm": "sha256", + "expected": "bf8bf1f4e2e01591c3d91f4b51f4f00252e14715132278177cab6ecc76511884", + "actual": "bf8bf1f4e2e01591c3d91f4b51f4f00252e14715132278177cab6ecc76511884", + "pass": true + }, + { + "file": "model-00011-of-00017.safetensors", + "expected_size": 5000780360, + "actual_size": 5000780360, + "algorithm": "sha256", + "expected": "ac436f2e1af8185dc3b093e1527f3297b93fd639ab84ad208376bd82d5e94e94", + "actual": "ac436f2e1af8185dc3b093e1527f3297b93fd639ab84ad208376bd82d5e94e94", + "pass": true + }, + { + "file": "model-00012-of-00017.safetensors", + "expected_size": 5000527232, + "actual_size": 5000527232, + "algorithm": "sha256", + "expected": "7814c9ae0dc394beb4709ebf0c7e976214e2770b322fa2422bc467446dd97023", + "actual": "7814c9ae0dc394beb4709ebf0c7e976214e2770b322fa2422bc467446dd97023", + "pass": true + }, + { + "file": "model-00013-of-00017.safetensors", + "expected_size": 5001126000, + "actual_size": 5001126000, + "algorithm": "sha256", + "expected": "b13aa79c0879539c4a514489497bccd21e23ec597d8112ba91808767fcbd4d92", + "actual": "b13aa79c0879539c4a514489497bccd21e23ec597d8112ba91808767fcbd4d92", + "pass": true + }, + { + "file": "model-00014-of-00017.safetensors", + "expected_size": 5000330504, + "actual_size": 5000330504, + "algorithm": "sha256", + "expected": "1d8781b2643ca6566e7bdae77c275b62785de3502c5337b8cb53558df2477eed", + "actual": "1d8781b2643ca6566e7bdae77c275b62785de3502c5337b8cb53558df2477eed", + "pass": true + }, + { + "file": "model-00015-of-00017.safetensors", + "expected_size": 4996320352, + "actual_size": 4996320352, + "algorithm": "sha256", + "expected": "5f6c3c0f842055418b0420dbeeba6b737d892d9d849725d956603a714f44745e", + "actual": "5f6c3c0f842055418b0420dbeeba6b737d892d9d849725d956603a714f44745e", + "pass": true + }, + { + "file": "model-00016-of-00017.safetensors", + "expected_size": 4998674800, + "actual_size": 4998674800, + "algorithm": "sha256", + "expected": "a5f795056a1ac69546ce8f5bccdcab3b6cb7142426024a81f23c15d3d45aa51e", + "actual": "a5f795056a1ac69546ce8f5bccdcab3b6cb7142426024a81f23c15d3d45aa51e", + "pass": true + }, + { + "file": "model-00017-of-00017.safetensors", + "expected_size": 345784264, + "actual_size": 345784264, + "algorithm": "sha256", + "expected": "218355cf69f1f9d3efa1b5c8b5f531e28e1aa50a3d869ee768ba9835d936e182", + "actual": "218355cf69f1f9d3efa1b5c8b5f531e28e1aa50a3d869ee768ba9835d936e182", + "pass": true + }, + { + "file": "model.safetensors.index.json", + "expected_size": 16634043, + "actual_size": 16634043, + "algorithm": "sha256", + "expected": "e0ca77ceed3a9511db61dc0ae6dbed2f71e6f6ee9f345fe3352f5cfe50cd094f", + "actual": "e0ca77ceed3a9511db61dc0ae6dbed2f71e6f6ee9f345fe3352f5cfe50cd094f", + "pass": true + }, + { + "file": "modeling_nemotron_h.py", + "expected_size": 82338, + "actual_size": 82338, + "algorithm": "git-blob-sha1", + "expected": "de290021078450c4d5042f66dd74b8eecf1d3eb3", + "actual": "de290021078450c4d5042f66dd74b8eecf1d3eb3", + "pass": true + }, + { + "file": "privacy.md", + "expected_size": 2688, + "actual_size": 2688, + "algorithm": "git-blob-sha1", + "expected": "ad6fad984525fd246ec6ba62ca2b1e34f8524d94", + "actual": "ad6fad984525fd246ec6ba62ca2b1e34f8524d94", + "pass": true + }, + { + "file": "safety.md", + "expected_size": 2120, + "actual_size": 2120, + "algorithm": "git-blob-sha1", + "expected": "8d4a9212d934db3437e1b7e9816f679a69740b61", + "actual": "8d4a9212d934db3437e1b7e9816f679a69740b61", + "pass": true + }, + { + "file": "special_tokens_map.json", + "expected_size": 563, + "actual_size": 563, + "algorithm": "git-blob-sha1", + "expected": "0451f37912edb7f8a4bf04d77eef13f6130f0515", + "actual": "0451f37912edb7f8a4bf04d77eef13f6130f0515", + "pass": true + }, + { + "file": "super_v3_reasoning_parser.py", + "expected_size": 1909, + "actual_size": 1909, + "algorithm": "git-blob-sha1", + "expected": "e3ac376058bf821fce8d7482ab1c2523d6e9c801", + "actual": "e3ac376058bf821fce8d7482ab1c2523d6e9c801", + "pass": true + }, + { + "file": "tokenizer.json", + "expected_size": 17077484, + "actual_size": 17077484, + "algorithm": "sha256", + "expected": "623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7", + "actual": "623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7", + "pass": true + }, + { + "file": "tokenizer_config.json", + "expected_size": 177209, + "actual_size": 177209, + "algorithm": "git-blob-sha1", + "expected": "c96e5ad00986bfa0b666d4bc425666f1c51d6d1b", + "actual": "c96e5ad00986bfa0b666d4bc425666f1c51d6d1b", + "pass": true + } + ] + }, + "grading": "Codex reviewed each complete answer and its tool evidence against the pre-run case rubric; this is not a blinded human evaluation. Binary pass requires the required finding and no material unsupported or contradictory claim. A transport failure is not an accuracy zero and is reconciled separately.", + "statistics": { + "baseline": { + "count": 15, + "completed": 15, + "investigated": 15, + "whole_answer_passes": 0, + "whole_answer_failures": 15, + "binary_success_rate": 0.0, + "raw_0_or_1_mean": 0.0, + "raw_definition": "This suite has a binary rubric only; the raw 0/1 mean equals the binary success rate. No graded partial-credit score is claimed.", + "elapsed_ms_distribution_in_case_order": [ + 260086.234375, + 71130.164292, + 121235.46570799999, + 173526.92987500003, + 54876.97199999995, + 143930.376583, + 76637.89241700002, + 37269.4169999999, + 53659.87624999997, + 57634.843125000014, + 38191.36337500007, + 83473.820084, + 14951.630125000025, + 53945.77766600007, + 18542.130292000016 + ], + "elapsed_ms_sum": 1259092.893167, + "elapsed_ms_median": 57634.843125000014, + "tool_calls_distribution_in_case_order": [ + 21, + 3, + 9, + 12, + 3, + 9, + 5, + 2, + 2, + 4, + 2, + 3, + 2, + 3, + 1 + ], + "tool_calls_sum": 81, + "tool_errors_sum": 4, + "model_turns_sum": 96, + "prompt_tokens_sum": 428628, + "completion_tokens_sum": 16480 + }, + "candidate": { + "count": 15, + "completed": 15, + "investigated": 15, + "whole_answer_passes": 3, + "whole_answer_failures": 12, + "binary_success_rate": 0.2, + "raw_0_or_1_mean": 0.2, + "raw_definition": "This suite has a binary rubric only; the raw 0/1 mean equals the binary success rate. No graded partial-credit score is claimed.", + "elapsed_ms_distribution_in_case_order": [ + 60741.11075, + 62153.414958, + 65861.58895800001, + 56609.450709, + 53374.12275000004, + 41847.43212499999, + 56204.36637500004, + 172472.60320800002, + 43697.91341600008, + 61339.434583000024, + 71659.88299999991, + 60770.42316699994, + 24349.793666000012, + 45494.16991699999, + 27443.519417000003 + ], + "elapsed_ms_sum": 904019.2269990001, + "elapsed_ms_median": 56609.450709, + "tool_calls_distribution_in_case_order": [ + 3, + 3, + 4, + 2, + 3, + 3, + 3, + 9, + 2, + 3, + 4, + 3, + 1, + 1, + 1 + ], + "tool_calls_sum": 45, + "tool_errors_sum": 0, + "model_turns_sum": 60, + "prompt_tokens_sum": 381635, + "completion_tokens_sum": 11130 + } + }, + "diagnostic_findings": { + "correct_original_last_sample_in_gap_answers": { + "baseline": 0, + "candidate": 7, + "denominator_per_side": 9 + }, + "zero_is_present_core_distinction": { + "baseline": 2, + "candidate": 3, + "denominator_per_side": 3 + }, + "note": "These diagnostics do not override the complete-answer verdicts. A correct timestamp or zero-versus-missing distinction can coexist with contradictory or unsupported prose." + }, + "reconciliation": { + "expected_unique_ids": 30, + "actual_unique_ids": 30, + "missing": [], + "extra": [], + "duplicates": [], + "formal_attempts": 31, + "canonical_model_outcomes": 30, + "smoke_attempts_outside_formal": 2 + }, + "transport_retry_policy": "Exactly one baseline three-day-gap seed-1 fetch failure was replaced by one same-seed replay. All original records remain retained. No answer-quality failure was rerun or removed.", + "records": [ + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "three-day-gap", + "repetition": 1, + "stop": "stop", + "modelTurns": 22, + "toolCalls": 21, + "toolErrors": 4, + "backendRequests": 19, + "elapsedMs": 260086.234375, + "usages": [ + { + "prompt_tokens": 2472, + "total_tokens": 2572, + "completion_tokens": 100, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2638, + "total_tokens": 2717, + "completion_tokens": 79, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2808, + "total_tokens": 2937, + "completion_tokens": 129, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3069, + "total_tokens": 3186, + "completion_tokens": 117, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3318, + "total_tokens": 3435, + "completion_tokens": 117, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3689, + "total_tokens": 3816, + "completion_tokens": 127, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3948, + "total_tokens": 4150, + "completion_tokens": 202, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4181, + "total_tokens": 4313, + "completion_tokens": 132, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4445, + "total_tokens": 4673, + "completion_tokens": 228, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4805, + "total_tokens": 4938, + "completion_tokens": 133, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 5070, + "total_tokens": 5177, + "completion_tokens": 107, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 5309, + "total_tokens": 5396, + "completion_tokens": 87, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 5528, + "total_tokens": 5647, + "completion_tokens": 119, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 5779, + "total_tokens": 5874, + "completion_tokens": 95, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6006, + "total_tokens": 6123, + "completion_tokens": 117, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6255, + "total_tokens": 6359, + "completion_tokens": 104, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6491, + "total_tokens": 6592, + "completion_tokens": 101, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6846, + "total_tokens": 7086, + "completion_tokens": 240, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 7117, + "total_tokens": 7237, + "completion_tokens": 120, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 7288, + "total_tokens": 7424, + "completion_tokens": 136, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 7475, + "total_tokens": 7623, + "completion_tokens": 148, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 7755, + "total_tokens": 8143, + "completion_tokens": 388, + "prompt_tokens_details": null + } + ], + "final": " perfect!我现在可以明确回答了:\n\n## 结论\n\n根据观测数据分析,fixture-gpu-a 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标数据大约从 **2026-09-03T00:02:00Z** 开始缺失。\n\n### 时间依据:\n\n1. **最后可用数据点**:\n - 时间戳:1793664000 (对应 2026-09-03T00:00:00Z),数值:42\n - 时间戳:1793664120 (对应 2026-09-03T00:02:00Z),数值:42\n\n2. **数据缺失验证**:\n - 在2026-09-03T00:02:00Z之后的时间范围内(即从1793664120开始),查询该时间范围内的所有后续时间点均无数据返回\n - 最近30分钟、2小时、6小时、12小时、24小时、36小时、48小时内均无该指标数据\n\n因此可以确定,GPU 0 的利用率指标数据在 **2026-09-03T00:02:00Z** 之后停止更新,从此时开始已经约 **64天** 没有数据了(基于当前时间2026-11-06T00:00:00Z)。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-transport-retry", + "side": "baseline", + "case": "three-day-gap", + "repetition": 1, + "verdict": "fail", + "core_finding_correct": false, + "claims": [ + "大约从 2026-09-03T00:02:00Z 开始缺失", + "约64天没有数据" + ], + "reason": "After 21 tool calls, still mis-converts November evaluated timestamps into September, treats lookback-reused sample evaluations as original samples, and gives a false 64-day outage duration. This is a valid model failure, separate from the preceding transport-invalid attempt." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "three-day-gap", + "repetition": 2, + "stop": "stop", + "modelTurns": 4, + "toolCalls": 3, + "toolErrors": 0, + "backendRequests": 3, + "elapsedMs": 71130.164292, + "usages": [ + { + "prompt_tokens": 2472, + "total_tokens": 2586, + "completion_tokens": 114, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2677, + "total_tokens": 2815, + "completion_tokens": 138, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2948, + "total_tokens": 3062, + "completion_tokens": 114, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 9730, + "total_tokens": 10369, + "completion_tokens": 639, + "prompt_tokens_details": null + } + ], + "final": "根据查询结果,我可以看到 fixture-gpu-a 的 GPU 0 在 DCGM_FI_DEV_GPU_UTIL 指标上的历史数据:\n\n1. 最近的数据点时间戳是:1793663400(对应UTC时间:2026-11-03T14:30:00Z)\n2. 数据值一直保持在42(表示GPU利用率42%)\n3. 之后再也没有任何数据点\n\n当前时间是2026-11-06T00:00:00.000Z(对应时间戳:1793923200)\n\n因此,数据大约从 **2026-11-03T14:30:00Z** 开始缺失,已经中断约 **2天9小时30分钟**。\n\n时间依据:\n- 最新数据点:时间戳 1793663400 = 2026-11-03T14:30:00Z\n- 当前时间:2026-11-06T00:00:00.000Z\n- 数据缺失时长:约 2天9小时30分钟\n\n过去24小时和7天的范围查询证实了这一点:24小时范围内无数据,而7天范围内只有到2026-11-03T14:30:00Z的历史数据。 fixture-gpu-a 的 GPU 0 在 DCGM_FI_DEV_GPU_UTIL 指标上的数据大约从 **2026-11-03T14:30:00Z** 开始缺失。目前已经没有数据约 **2天9小时30分钟**。\n\n时间依据:\n- 最新可用数据点:时间戳 1793663400 对应 UTC 时间 2026-11-03T14:30:00Z,数值为 42(GPU利用率42%)\n- 当前系统时间:2026-11-06T00:00:00.000Z\n- 查询过去24小时(1440分钟)内无任何该指标数据点\n- 查询过去7天(10080分钟)显示数据仅存在至 2026-11-03T14:30:00Z 之后即中断\n\n因此可以判定,监控数据或指标采集大约在该时间点后停止工作。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "three-day-gap", + "repetition": 2, + "verdict": "fail", + "claims": [ + "1793663400(对应UTC时间:2026-11-03T14:30:00Z)", + "已经中断约 2天9小时30分钟" + ], + "reason": "Wrong UTC conversion and wrong onset. The cited evaluated/downsampled timestamp is not the original last sample 2026-11-02T23:59:00Z; required cadence-qualified first expected absence is 2026-11-03T00:00:00Z." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "three-day-gap", + "repetition": 3, + "stop": "stop", + "modelTurns": 10, + "toolCalls": 9, + "toolErrors": 0, + "backendRequests": 9, + "elapsedMs": 121235.46570799999, + "usages": [ + { + "prompt_tokens": 2472, + "total_tokens": 2581, + "completion_tokens": 109, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2672, + "total_tokens": 2800, + "completion_tokens": 128, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2932, + "total_tokens": 3048, + "completion_tokens": 116, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3180, + "total_tokens": 3280, + "completion_tokens": 100, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3412, + "total_tokens": 3502, + "completion_tokens": 90, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3588, + "total_tokens": 3689, + "completion_tokens": 101, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3756, + "total_tokens": 3839, + "completion_tokens": 83, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3916, + "total_tokens": 4000, + "completion_tokens": 84, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4156, + "total_tokens": 4369, + "completion_tokens": 213, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 23459, + "total_tokens": 23922, + "completion_tokens": 463, + "prompt_tokens_details": null + } + ], + "final": "根据查询结果,我可以清楚地看到fixture-gpu-a的GPU 0的DCGM_FI_DEV_GPU_UTIL指标数据情况:\n\n从返回的数据中可以看到:\n1. fixture-gpu-a的GPU 0在1793318400时间戳(对应2026-06-30T00:00:00Z)开始有持续的数据\n2. 数据显示该GPU的利用率一直稳定在42%\n3. 最后一个数据点是在1793562900时间戳(对应2026-07-02T21:15:00Z)\n4. 之后就再也没有该机器的任何DCGM指标数据\n\n因此,fixture-gpu-a的GPU 0的DCGM_FI_DEV_GPU_UTIL指标数据大约从**2026-07-02T21:15:00Z**开始缺失。\n\n时间依据:\n- 查询显示fixture-gpu-a的DCGM_FI_DEV_GPU_UTIL在1793562900时间戳(2026-07-02T21:15:00Z)后没有更多数据点\n- 而其他机器如fixture-gpu-c和fixture-gpu-d的DCGM指标持续更新到当前时间1793923200(2026-11-06T00:00:00Z)\n- 这表明是fixture-gpu-a这台机器的监控采集在2026年7月2日晚9:15左右中断了\n\n这个时间点与其他机器的数据形成鲜明对比:其他机器的DCGM指标持续报送到当前时间,而fixture-gpu-a则在7月2日停止报送。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "three-day-gap", + "repetition": 3, + "verdict": "fail", + "claims": [ + "2026-07-02T21:15:00Z", + "fixture-gpu-c和fixture-gpu-d的DCGM指标持续更新到当前时间" + ], + "reason": "Dates the November gap to July using an incorrect Unix-time conversion and a compacted range point; does not establish the original last sample or cadence-qualified first absence. Also incorrectly claims the fixture-gpu-d series continues to the current time, despite its two-hour gap." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "history-unavailable", + "repetition": 1, + "stop": "stop", + "modelTurns": 13, + "toolCalls": 12, + "toolErrors": 0, + "backendRequests": 12, + "elapsedMs": 173526.92987500003, + "usages": [ + { + "prompt_tokens": 2472, + "total_tokens": 2562, + "completion_tokens": 90, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2628, + "total_tokens": 2698, + "completion_tokens": 70, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2789, + "total_tokens": 2919, + "completion_tokens": 130, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3051, + "total_tokens": 3227, + "completion_tokens": 176, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3359, + "total_tokens": 3534, + "completion_tokens": 175, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3667, + "total_tokens": 3830, + "completion_tokens": 163, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3916, + "total_tokens": 3986, + "completion_tokens": 70, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4143, + "total_tokens": 4291, + "completion_tokens": 148, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4368, + "total_tokens": 4532, + "completion_tokens": 164, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 9618, + "total_tokens": 9825, + "completion_tokens": 207, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 9954, + "total_tokens": 10176, + "completion_tokens": 222, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 10272, + "total_tokens": 10351, + "completion_tokens": 79, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 10426, + "total_tokens": 10927, + "completion_tokens": 501, + "prompt_tokens_details": null + } + ], + "final": " fixture-gpu-b 主机完全没有任何监控指标数据返回。这表明该主机可能已经:\n\n1. 完全停止向监控系统报告metrics\n2. 监控系统无法从该主机采集metrics\n3. 主机可能已经下线或网络不可达\n\n基于我的检查:\n- fixture-gpu-c 主机的 GPU 0 目前正常报告DCGM_FI_DEV_GPU_UTIL指标(值为0)\n- fixture-gpu-b 主机在最近24小时内没有任何监控指标数据\n- 具体来说,DCGM_FI_DEV_GPU_UTIL{Hostname=\"fixture-gpu-b\", gpu=\"0\"} 在最近24小时内没有任何数据点\n\n由于我没有看到 fixture-gpu-b 主机在任何时间范围内有数据返回(即使是24小时范围),我无法确定确切的数据中断时间点。然而,根据可用的证据:\n\n**时间依据:**\n1. 目前时间是2026-11-06T00:00:00.000Z\n2. fixture-gpu-c 主机的 GPU 0 在最近30分钟内有持续数据流(所有值为0)\n3. fixture-gpu-b 主机在最近24小时内完全没有任何监控指标数据返回\n4. 这表明 fixture-gpu-b 主机的监控数据中断很可能发生在2026-11-05T00:00:00.000Z之前(至少24小时前)\n\n然而,由于即使在24小时范围内也没有数据点,我不能精确的确定数据是什么时候开始缺失的。我只能说数据已经缺失了至少24小时。\n\n根据我的分析,**fixture-gpu-b 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标数据已经缺失了至少24小时**,可能更长时间。数据中断很可能开始于2026-11-05T00:00:00.000Z之前。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "history-unavailable", + "host": "fixture-gpu-b", + "last": null, + "missing": null, + "value": 42, + "count": 0, + "lastIso": null, + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "history-unavailable", + "repetition": 1, + "verdict": "fail", + "claims": [ + "监控数据中断很可能发生在2026-11-05T00:00:00.000Z之前", + "数据已经缺失了至少24小时" + ], + "reason": "Reports unknown exact onset but nevertheless infers an outage predating the 24-hour search and a minimum outage duration from entirely empty history. The fixture provides no evidence the series ever existed." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "history-unavailable", + "repetition": 2, + "stop": "stop", + "modelTurns": 4, + "toolCalls": 3, + "toolErrors": 0, + "backendRequests": 3, + "elapsedMs": 54876.97199999995, + "usages": [ + { + "prompt_tokens": 2472, + "total_tokens": 2586, + "completion_tokens": 114, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2677, + "total_tokens": 2841, + "completion_tokens": 164, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2974, + "total_tokens": 3112, + "completion_tokens": 138, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3244, + "total_tokens": 3619, + "completion_tokens": 375, + "prompt_tokens_details": null + } + ], + "final": "过去 6 小时内仍然没有任何数据点。这表明 fixture-gpu-b 的 GPU 0 在 DCGM_FI_DEV_GPU_UTIL 指标上至少在最近 6 小时内没有上报数据。\n\n由于当前时间是 2026-11-06T00:00:00.000Z,往前推 6 小时就是 2026-11-05T18:00:00.000Z。因此,可以判断该指标的数据大约从 **2026-11-05T18:00:00.000Z 之前开始缺失**,且在截至当前时间的观测窗口内均未恢复。\n\n> 注意:这是基于可用的观测工具和只读查询得出的结论;未更改任何配置。若需要更早时间的历史数据,需要进一步扩大 rangeMinutes 的范围,但根据已有查询,最近 6 小时内均无数据。\n\n**结论**:fixture-gpu-b 的 GPU 0 在 DCGM_FI_DEV_GPU_UTIL 指标上的数据大约从 **2026-11-05T18:00:00.000Z 起开始缺失**,直至当前时间(2026-11-06T00:00:00.000Z)均未恢复。时间依据是最近 6 小时的范围查询均未返回任何样本点。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "history-unavailable", + "host": "fixture-gpu-b", + "last": null, + "missing": null, + "value": 42, + "count": 0, + "lastIso": null, + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "history-unavailable", + "repetition": 2, + "verdict": "fail", + "claims": [ + "数据大约从 2026-11-05T18:00:00.000Z 起开始缺失" + ], + "reason": "Dates onset to the six-hour query boundary and asserts continuing non-recovery from empty history; prior existence and incident onset are unknown." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "history-unavailable", + "repetition": 3, + "stop": "stop", + "modelTurns": 10, + "toolCalls": 9, + "toolErrors": 0, + "backendRequests": 9, + "elapsedMs": 143930.376583, + "usages": [ + { + "prompt_tokens": 2472, + "total_tokens": 2559, + "completion_tokens": 87, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2650, + "total_tokens": 2778, + "completion_tokens": 128, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2910, + "total_tokens": 3038, + "completion_tokens": 128, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3170, + "total_tokens": 3343, + "completion_tokens": 173, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3476, + "total_tokens": 3639, + "completion_tokens": 163, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3725, + "total_tokens": 3843, + "completion_tokens": 118, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3929, + "total_tokens": 4001, + "completion_tokens": 72, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4086, + "total_tokens": 4221, + "completion_tokens": 135, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4388, + "total_tokens": 4578, + "completion_tokens": 190, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4706, + "total_tokens": 5508, + "completion_tokens": 802, + "prompt_tokens_details": null + } + ], + "final": "根据我的检查,fixture-gpu-b 的 GPU 0 状态指标 DCGM_FI_DEV_GPU_UTIL 的数据已经缺失较长时间。具体情况如下:\n\n**查询结果:**\n- 当前查询 fixture-gpu-b 的 DCGM_FI_DEV_GPU_UTIL{Hostname=\"fixture-gpu-b\", gpu=\"0\"} 返回无数据\n- 过去 24 小时范围查询:无数据\n- 过去 7 天范围查询:无数据\n- 过去 30 天范围查询:无数据\n- 过去 60 天范围查询:无数据\n\n**对比验证:**\n- 其他机器(如 fixture-gpu-c)的相同指标仍在正常收集数据\n- 这表明问题不是监控系统全局性故障,而是特定于 fixture-gpu-b 这台机器\n\n**时间判断:**\n由于过去 60 天(约两个月)内均未查询到 fixture-gpu-b 的任何 DCGM 指标数据(包括 GPU 利用率、温度、功耗等),可以得出结论:\n\n**fixture-gpu-b 的 GPU 0 状态指标 DCGM_FI_DEV_GPU_UTIL 的数据大约从 2026 年 9 月 7 日左右开始缺失。**\n\n**时间依据:** 基于当前时间 2026-11-06T00:00:00.000Z 往前推 60 天得到的近似时间点,以及过去 60 天范围查询均未返回任何该机器的 DCGM 指标数据这一事实。其他机器的 DCGM 指标正常收集进一步确认了这是 fixture-gpu-b 特有的问题。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "history-unavailable", + "host": "fixture-gpu-b", + "last": null, + "missing": null, + "value": 42, + "count": 0, + "lastIso": null, + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "history-unavailable", + "repetition": 3, + "verdict": "fail", + "claims": [ + "过去 60 天范围查询:无数据", + "数据大约从 2026 年 9 月 7 日左右开始缺失" + ], + "reason": "Invents onset from a requested 60-day window. Actual returned range-query bounds remained 1793318400 through 1793923200 (7 days) because the tool clamps the range. Empty history establishes neither prior existence nor a dated outage; also overstates checked coverage as 30 and 60 days." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "healthy-zero", + "repetition": 1, + "stop": "stop", + "modelTurns": 6, + "toolCalls": 5, + "toolErrors": 0, + "backendRequests": 5, + "elapsedMs": 76637.89241700002, + "usages": [ + { + "prompt_tokens": 2467, + "total_tokens": 2570, + "completion_tokens": 103, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2636, + "total_tokens": 2718, + "completion_tokens": 82, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2889, + "total_tokens": 3017, + "completion_tokens": 128, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4052, + "total_tokens": 4151, + "completion_tokens": 99, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4317, + "total_tokens": 4400, + "completion_tokens": 83, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4555, + "total_tokens": 5100, + "completion_tokens": 545, + "prompt_tokens_details": null + } + ], + "final": "根据我的调查,我可以得出以下结论:\n\n** fixture-gpu-c 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标确实持续为 0 **:\n\n1. 最新值为 0(时间戳:1793923200,对应于 2026-11-06T00:00:00Z)\n2. 过去 15 分钟内(以 30 秒间隔采样)的所有 31 个数据点均为 0\n3. fixture-gpu-c 只有一个 GPU(gpu=\"0\"),没有其他 GPU 可以进行比较\n4. 更令人担忧的是,整个监控系统中似乎只有这个单一的时间序列:在查询所有 DCGM_FI_DEV_GPU_UTIL 指标时,只返回了这个单一系列(来自 fixture-gpu-c 的 GPU 0)\n\n这种模式强烈表明存在采集问题,因为:\n- 在正常运行的 GPU 系统中,即使是空闲状态,GPU 利用率也应该偶尔显示非零值(即使只是个位数的百分比)\n- 完全没有其他 GPU 指标被报告(无论是来自 fixture-gpu-c 还是其他主机)表明 DCGM 指标的采集可能已经中断或配置存在问题\n- 持续多个采样周期的完全零值(在15分钟内超过30个连续采样点)极不可能是真实的GPU行为\n\n然而,基于只读可观测性工具,我无法区分这是以下哪种情况:\n- fixture-gpu-c 上的 DCGM 代理已停止或无法运行\n- 从 fixture-gpu-c 收集指标的 Prometheus/VictoriaMetrics 抓取已失败\n- 指标端点本身返回了零值或没有数据\n- 更广泛的基础设施问题影响了度量采集\n\n数据明确表明, fixture-gpu-c 的 GPU 0 在所检查的时间窗口内一直报告零利用率,并且整个 DCGM_FI_DEV_GPU_UTIL 指标域似乎只有这一个时间序列,这高度暗示了采集中断。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "healthy-zero", + "host": "fixture-gpu-c", + "last": 1793923200, + "missing": null, + "value": 0, + "count": 11521, + "lastIso": "2026-11-06T00:00:00.000Z", + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "healthy-zero", + "repetition": 1, + "verdict": "fail", + "core_finding_correct": false, + "claims": [ + "持续多个采样周期的完全零值极不可能是真实的GPU行为", + "这高度暗示了采集中断" + ], + "reason": "Confuses a fresh, continuously present zero-valued series with collection failure; invents a requirement for utilization to occasionally be nonzero and infers physical GPU inventory from observed metric series." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "healthy-zero", + "repetition": 2, + "stop": "stop", + "modelTurns": 3, + "toolCalls": 2, + "toolErrors": 0, + "backendRequests": 2, + "elapsedMs": 37269.4169999999, + "usages": [ + { + "prompt_tokens": 2467, + "total_tokens": 2558, + "completion_tokens": 91, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2729, + "total_tokens": 2878, + "completion_tokens": 149, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6344, + "total_tokens": 6596, + "completion_tokens": 252, + "prompt_tokens_details": null + } + ], + "final": "**结论**: fixture-gpu-c 的 GPU 0 在最近一小时内,DCGM_FI_DEV_GPU_UTIL 指标持续为 0(所有 121 个采样点均为 0)。这表明在此期间 GPU 完全未被利用,但**并不等同于采集中断**——若采集真的中断,监控系统通常会返回无数据(即查询结果为空),而这里我们能够成功获取到数值序列且均为 0,说明采集器(如 DCGM Exporter)仍在正常工作,只是该 GPU 当前没有运行任何任务。\n\n因此,**最近的值为 0 并不说明采集已经中断**,而是真实反映了 GPU 0 在观测窗口内的空闲状态。如果需要进一步确认采集器健康状态,可检查其他 GPU 或其它 DCGM 指标(如温度、功率等)是否有更新;但基于当前查询,无证据表明采集中断。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "healthy-zero", + "host": "fixture-gpu-c", + "last": 1793923200, + "missing": null, + "value": 0, + "count": 11521, + "lastIso": "2026-11-06T00:00:00.000Z", + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "healthy-zero", + "repetition": 2, + "verdict": "fail", + "core_finding_correct": true, + "claims": [ + "并不等同于采集中断", + "采集器仍在正常工作,只是该 GPU 当前没有运行任何任务" + ], + "reason": "Correctly distinguishes present zero samples from absence, but overstates collector health and asserts no workload is running from utilization alone. Overall-answer review fails on those unsupported claims; the central zero-versus-missing distinction is correct." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "healthy-zero", + "repetition": 3, + "stop": "stop", + "modelTurns": 3, + "toolCalls": 2, + "toolErrors": 0, + "backendRequests": 2, + "elapsedMs": 53659.87624999997, + "usages": [ + { + "prompt_tokens": 2467, + "total_tokens": 2566, + "completion_tokens": 99, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2737, + "total_tokens": 2868, + "completion_tokens": 131, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6334, + "total_tokens": 6869, + "completion_tokens": 535, + "prompt_tokens_details": null + } + ], + "final": "根据查询结果,我可以明确回答您的问题:\n\n**结论:fixture-gpu-c 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标最近的值确实一直是 0。**\n\n**证据:**\n1. 当前瞬时值(时间戳 1793923200,对应 2026-11-06T00:00:00Z)为 0\n2. 过去 60 分钟内(从 1793919600 到 1793923200),共采集了 121 个数据点(每 30 秒一个点),**所有数据点的值均为 0**\n\n**分析:**\n这种持续性的零值表明 GPU 0 在过去一小时内完全没有被利用(GPU 利用率持续为 0%)。虽然这本身不一定表示采集中断——它可能真实反映了 GPU 处于完全空闲状态——但持续一小时完全为零在实际生产环境中较为少见,除非该GPU被明确关闭或未被调度任何工作负载。\n\n然而,基于可用的监控数据,**无法仅凭该指标的持续零值确定采集是否中断**,因为:\n- 指标本身在持续返回值(全为0),说明采集机制似乎仍在工作\n- 如果采集真正中断,我们通常会看到指标完全消失或返回NaN/错误值,而不是持续返回有效的零值\n\n要更 definitively 判断采集状态,建议:\n1. 检查同一主机上其他GPU的利用率指标,看是否也全为零\n2. 查看DCGM相关的采集健康指标(如DCGM_FI_DEV_GPU_TEMP是否有合理变化)\n3. 检查该主机其他DCGM指标是否也出现异常模式\n\n但根据您提供的信息和查询结果,**该指标的采集似乎仍在正常工作**,只是持续返回零值,表明GPU 0在此期间未被使用。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "healthy-zero", + "host": "fixture-gpu-c", + "last": 1793923200, + "missing": null, + "value": 0, + "count": 11521, + "lastIso": "2026-11-06T00:00:00.000Z", + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "healthy-zero", + "repetition": 3, + "verdict": "fail", + "core_finding_correct": true, + "claims": [ + "采集似乎仍在正常工作", + "表明GPU 0在此期间未被使用" + ], + "reason": "Recognizes fresh zero samples and qualifies collection health, but concludes the GPU was unused from utilization alone. Also describes 121 evaluated 30-second query points as collected samples although the supplied source cadence is 60 seconds. Overall-answer review fails on unsupported workload inference." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "two-hour-gap", + "repetition": 1, + "stop": "stop", + "modelTurns": 5, + "toolCalls": 4, + "toolErrors": 0, + "backendRequests": 4, + "elapsedMs": 57634.843125000014, + "usages": [ + { + "prompt_tokens": 2472, + "total_tokens": 2562, + "completion_tokens": 90, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2628, + "total_tokens": 2698, + "completion_tokens": 70, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2789, + "total_tokens": 2919, + "completion_tokens": 130, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3340, + "total_tokens": 3517, + "completion_tokens": 177, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 7300, + "total_tokens": 7600, + "completion_tokens": 300, + "prompt_tokens_details": null + } + ], + "final": "根据查询结果,我可以确定fixture-gpu-d上GPU 0的DCGM_FI_DEV_GPU_UTIL指标数据缺失的时间点。\n\n最后一个数据点出现在时间戳1793916210,对应的UTC时间是:\n2026-09-05T09:37:00Z\n\n之后(从1793916240开始,即2026-09-05T09:40:00Z)就没有再收到任何数据点。这表明数据大约是在**2026年9月5日09:40左右开始缺失的**。\n\n当前时间是2026-11-06T00:00:00.000Z,所以数据已经缺失大约1个半月左右。\n\n时间依据:通过观察范围查询结果,可以看到从时间戳1793912400到1793916210(每30秒一个点)都有持续的数据值23,但在那之后就没有更多的数据点了,说明监控或数据采集在大约09:40:00Z时中断了。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "two-hour-gap", + "host": "fixture-gpu-d", + "last": 1793915940, + "missing": 1793916000, + "value": 23, + "count": 11400, + "lastIso": "2026-11-05T21:59:00.000Z", + "missingIso": "2026-11-05T22:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "two-hour-gap", + "repetition": 1, + "verdict": "fail", + "claims": [ + "1793916210,对应的UTC时间是:2026-09-05T09:37:00Z", + "2026年9月5日09:40左右开始缺失", + "数据已经缺失大约1个半月左右" + ], + "reason": "Incorrect Unix-to-UTC conversion and duration; treats a lookback-evaluated point as the original last sample. Required original sample is 2026-11-05T21:59:00Z and cadence-qualified first expected absence is 22:00:00Z." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "two-hour-gap", + "repetition": 2, + "stop": "stop", + "modelTurns": 3, + "toolCalls": 2, + "toolErrors": 0, + "backendRequests": 2, + "elapsedMs": 38191.36337500007, + "usages": [ + { + "prompt_tokens": 2472, + "total_tokens": 2590, + "completion_tokens": 118, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2681, + "total_tokens": 2821, + "completion_tokens": 140, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6436, + "total_tokens": 6687, + "completion_tokens": 251, + "prompt_tokens_details": null + } + ], + "final": "从返回的历史数据可以看到,DCGM_FI_DEV_GPU_UTIL 指标在 fixture-gpu-d 的 GPU 0 上,从时间戳 1793901600(对应 UTC 时间 2026-11-05 20:00:00)开始持续上报,数值稳定在 23 左右,直到时间戳 1793916120(对应 UTC 时间 2026-11-06 00:02:00)之后便不再有任何数据点。\n\n换言之,**该指标大约从 2026-11-06 00:02:00 UTC 开始缺失**,这一点由范围查询中最后一个数据点的时间戳直接得出。\n\n(注:当前系统时间为 2026-11-06T00:00:00.000Z,因此缺失大约开始于当前时间约 2 分钟前。)", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "two-hour-gap", + "host": "fixture-gpu-d", + "last": 1793915940, + "missing": 1793916000, + "value": 23, + "count": 11400, + "lastIso": "2026-11-05T21:59:00.000Z", + "missingIso": "2026-11-05T22:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "two-hour-gap", + "repetition": 2, + "verdict": "fail", + "claims": [ + "该指标大约从 2026-11-06 00:02:00 UTC 开始缺失", + "当前时间约 2 分钟前" + ], + "reason": "Mis-converts a lookback-evaluated point into 00:02 UTC, which is after the stated investigation time, then calls it two minutes earlier. Does not establish original last sample 21:59 or first expected absence 22:00 on November 5." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "two-hour-gap", + "repetition": 3, + "stop": "stop", + "modelTurns": 4, + "toolCalls": 3, + "toolErrors": 0, + "backendRequests": 3, + "elapsedMs": 83473.820084, + "usages": [ + { + "prompt_tokens": 2472, + "total_tokens": 2559, + "completion_tokens": 87, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2650, + "total_tokens": 2766, + "completion_tokens": 116, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 9154, + "total_tokens": 9323, + "completion_tokens": 169, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 9436, + "total_tokens": 10218, + "completion_tokens": 782, + "prompt_tokens_details": null + } + ], + "final": "让我使用更直接的方式来理解这个时间戳。当前时间是2026-11-06T00:00:00.000Z,我来计算一下这个时间戳对应的时间。\n\n1793915640 是 Unix 时间戳(秒)。让我计算一下这是从1970-01-01开始的秒数对应的日期。\n\n实际上,我可以通过查看数据点的时间间隔来判断。数据点是每5分钟(300秒)一个点,因为:\n- 1793836800, 1793837160, 1793837520... 每次增加360秒(6分钟)\n\n等等,让我重新看看时间间隔:\n1793837160 - 1793836800 = 360秒(6分钟)\n1793837520 - 1793837160 = 360秒(6分钟)\n\n所以数据点是每6分钟一个。\n\n最后一个数据点是1793915640,对应的值是23。\n当前时间大约是1793923200(基于range查询的end参数)。\n\n让我计算一下时间差:\n1793923200 - 1793915640 = 7560秒 = 126分钟 = 2小时6分钟\n\n这意味着最后一个数据点是在当前时间之前大约2小时6分钟。\n\n不过,让我更精确地计算一下。根据range查询:\n- start: 1793836800 \n- end: 1793923200\n- 这是24小时的范围(1440分钟)\n\n所以end时间1793923200对应的是查询时的当前时间。\n\n最后一个数据点时间戳: 1793915640\n查询的end时间: 1793923200\n时间差: 1793923200 - 1793915640 = 7560秒 = 126分钟 = 2小时6分钟\n\n因此,数据大约在当前时间之前2小时6分钟停止更新。\n\n当前时间是2026-11-06T00:00:00.000Z,所以数据中断大约发生在:\n2026-11-05T21:54:00.000Z\n\n让我验证一下这个时间点:\n1793915640 时间戳 = 2026-11-05T21:54:00Z\n\n所以,fixture-gpu-d 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标数据大约从2026-11-05T21:54:00.000Z开始缺失,即当前时间(2026-11-06T00:00:00.000Z)之前大约2小时6分钟。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "two-hour-gap", + "host": "fixture-gpu-d", + "last": 1793915940, + "missing": 1793916000, + "value": 23, + "count": 11400, + "lastIso": "2026-11-05T21:59:00.000Z", + "missingIso": "2026-11-05T22:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "two-hour-gap", + "repetition": 3, + "verdict": "fail", + "claims": [ + "2026-11-05T21:54:00.000Z开始缺失" + ], + "reason": "Correctly converts the compacted returned point to 21:54 UTC but treats that evaluated/downsampled point as the incident onset. The original last sample is 21:59 UTC and first expected missing sample at known cadence is 22:00 UTC." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "three-day-gap-followup", + "repetition": 1, + "stop": "stop", + "modelTurns": 3, + "toolCalls": 2, + "toolErrors": 0, + "backendRequests": 3, + "elapsedMs": 14951.630125000025, + "usages": [ + { + "prompt_tokens": 2728, + "total_tokens": 2765, + "completion_tokens": 37, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2831, + "total_tokens": 2907, + "completion_tokens": 76, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2996, + "total_tokens": 3045, + "completion_tokens": 49, + "prompt_tokens_details": null + } + ], + "final": " fixture-gpu-a 在最近两小时内(以及更早)没有报告 DCGM_FI_DEV_GPU_UTIL 指标。目前无法查看此 GPU 利用率数据的时间序列。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap-followup", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "followup": true, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "three-day-gap-followup", + "repetition": 1, + "verdict": "fail", + "claims": [ + "在最近两小时内(以及更早)没有报告", + "目前无法查看此 GPU 利用率数据的时间序列" + ], + "reason": "Does not recover accessible older history or establish the last original sample and first expected absence; merely restates the recent lack of data and an unsupported claim about earlier history." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "three-day-gap-followup", + "repetition": 2, + "stop": "stop", + "modelTurns": 4, + "toolCalls": 3, + "toolErrors": 0, + "backendRequests": 4, + "elapsedMs": 53945.77766600007, + "usages": [ + { + "prompt_tokens": 2728, + "total_tokens": 2765, + "completion_tokens": 37, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2831, + "total_tokens": 2866, + "completion_tokens": 35, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3022, + "total_tokens": 3128, + "completion_tokens": 106, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6767, + "total_tokens": 7333, + "completion_tokens": 566, + "prompt_tokens_details": null + } + ], + "final": "**回答**\n\n fixture-gpu-a 的 GPU 0 在最近两小时内没有任何 DCGM_FI_DEV_GPU_UTIL 样本,这意味着监控数据在**约两小时前(2026-11-05 22:00:00 UTC 之前)就已经中止**。\n\n**时间依据**:\n\n1. 查询 fixture-gpu-a、gpu=0 的过去两小时(`start=1793916000` → `end=1793923200`)返回空结果(`result=[]`),说明该时间窗口内根本没有任何样本点被采集到。 \n2. 为了确认该指标本身仍在被其他主机上报,我做了两个检查: \n - `observability_metric_names` 确认 `DCGM_FI_DEV_GPU_UTIL` 仍然是一个有效的监控指标。 \n - 全局瞬时查询 `DCGM_FI_DEV_GPU_UTIL` 只返回 fixture-gpu-c 的一个样本(当前值为 0),说明采集器仍在工作,只是 fixture-gpu-a 不再上报。 \n3. 进一步查询所有匹配 `fixture-gpu-.` 的主机(`observability_range_query`),得到: \n - fixture-gpu-c:在整个两小时窗口内持续返回 0(每分钟一个点,共 121 个点)。 \n - fixture-gpu-d:仅在窗口开始的前 4 分钟返回值 23,之后同样没有更多样本。 \n - fixture-gpu-a:根本没有出现在结果中,说明它在窗口开始时就已经停止上报。\n\n**结论** \n从结果看,fixture-gpu-a 的 GPU 0 监控数据在本次查询窗口开始之前(即 **2026-11-05 22:00:00 UTC**)就已经停止上报;随后两小时内一直没有恢复。若想更精确地定位中止时间,需要查看更早的历史数据(比如查看前 4–6 小时的样本),但基于现有两小时窗口,可以断定问题是在该窗口开始之前就已经存在的。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap-followup", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "followup": true, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "three-day-gap-followup", + "repetition": 2, + "verdict": "fail", + "claims": [ + "约两小时前(2026-11-05 22:00:00 UTC 之前)就已经中止", + "若想更精确地定位中止时间,需要查看更早的历史数据" + ], + "reason": "Stops at the recent empty window and suggests historical queries that are still available rather than executing them; does not recover the three-day original-sample evidence." + } + }, + { + "side": "baseline", + "source_commit": "307afa7e485a334389b4f2c8c68128fbf84ff007", + "summary": { + "case": "three-day-gap-followup", + "repetition": 3, + "stop": "stop", + "modelTurns": 2, + "toolCalls": 1, + "toolErrors": 0, + "backendRequests": 2, + "elapsedMs": 18542.130292000016, + "usages": [ + { + "prompt_tokens": 2728, + "total_tokens": 2837, + "completion_tokens": 109, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2969, + "total_tokens": 3107, + "completion_tokens": 138, + "prompt_tokens_details": null + } + ], + "final": "在过去 24 小时内(截至当前时间 2026-11-06 00:00:00),该系列一直没有样本。这说明问题至少在这段时间内一直存在。若需更精准的起点,可进一步拉长查询窗口或结合主机/GPU 可用性其他指标(如 up、心跳)进行对比。当前基于 DCGM_FI_DEV_GPU_UTIL 的只读数据表明,该 GPU 在最近 24 小时内未产生利用率样本。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap-followup", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "followup": true, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "three-day-gap-followup", + "repetition": 3, + "verdict": "fail", + "claims": [ + "过去 24 小时内该系列一直没有样本", + "可进一步拉长查询窗口" + ], + "reason": "Only widens to 24 hours, then suggests further investigation without recovering accessible original last-sample evidence for the three-day gap." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "three-day-gap", + "repetition": 1, + "stop": "stop", + "modelTurns": 4, + "toolCalls": 3, + "toolErrors": 0, + "backendRequests": 7, + "elapsedMs": 60741.11075, + "usages": [ + { + "prompt_tokens": 4034, + "total_tokens": 4133, + "completion_tokens": 99, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4199, + "total_tokens": 4289, + "completion_tokens": 90, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4380, + "total_tokens": 4507, + "completion_tokens": 127, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6060, + "total_tokens": 6565, + "completion_tokens": 505, + "prompt_tokens_details": null + } + ], + "final": "根据查询结果,我可以得到以下结论:\n\n** fixture-gpu-a 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标数据缺失情况:**\n\n1. **最后观测到的数据时间**:2026年11月2日23:59:00 UTC(ISO格式:2026-11-02T23:59:00.000Z),当时数值为42(代表GPU利用率42%)\n\n2. **数据缺失开始时间**:从2026年11月3日00:00:00 UTC开始(紧接最后一个样本之后)\n\n3. **缺失持续时间**:截至当前时间(2026年11月6日00:00:00 UTC),数据已经缺失了大约3小时1分钟(确切地说是3 days, 0 hours, 1 minutes, 0 seconds)\n\n4. **时间依据**:通过系统atic地搜索了最近79小时的历史数据窗口(从2026年11月2日17:00:00到2026年11月6日00:00:00),发现:\n - 在时间窗口(2026-11-02T17:00:00.000Z, 2026-11-03T17:00:00.000Z]内有419个样本点\n - 在之后的所有时间窗口中(2026-11-03T17:00:00.000Z以后)没有找到任何样本点\n - 最后一个样本的确切时间是2026-11-02T23:59:00.000Z\n\n因此,可以确定 fixture-gpu-a 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标大约从 **2026年11月3日00:00:00 UTC** 开始出现数据缺失。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "three-day-gap", + "repetition": 1, + "verdict": "fail", + "core_finding_correct": true, + "claims": [ + "2026-11-02T23:59:00.000Z", + "从2026年11月3日00:00:00 UTC开始", + "大约3小时1分钟(确切地说是3 days, 0 hours, 1 minutes, 0 seconds)" + ], + "reason": "Recovers the correct original last sample and cadence-qualified first absence, but the full answer contradicts itself on elapsed age: Chinese says 3 hours 1 minute while the evidence and English phrase say 3 days 1 minute. Whole-answer consistency gate fails." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "three-day-gap", + "repetition": 2, + "stop": "stop", + "modelTurns": 4, + "toolCalls": 3, + "toolErrors": 0, + "backendRequests": 7, + "elapsedMs": 62153.414958, + "usages": [ + { + "prompt_tokens": 4034, + "total_tokens": 4125, + "completion_tokens": 91, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4191, + "total_tokens": 4270, + "completion_tokens": 79, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4361, + "total_tokens": 4466, + "completion_tokens": 105, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6019, + "total_tokens": 6584, + "completion_tokens": 565, + "prompt_tokens_details": null + } + ], + "final": "根据 observability_metric_last_seen 的结果,我可以为您提供关于 fixture-gpu-a 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标数据缺失情况的确切信息:\n\n## 数据缺失情况\n\n**最后观测到的数据时间**:2026-11-02T23:59:00.000Z(UTC时间)\n**数值**:42(表示GPU利用率为42%)\n\n**数据开始缺失的时间**:从 2026-11-03T00:00:00.000Z 开始(紧随最后一次观测之后)\n\n**缺失时长**:截至当前时间(2026-11-06T00:00:00.000Z),数据已经缺失了大约 **3天0小时1分钟**\n\n## 时间依据说明\n\n1. 系统通过向前搜索原始样本窗口的方式确定了最后一个真实数据点\n2. 在 2026-11-02T17:00:00.000Z 到 2026-11-03T17:00:00.000Z 这个时间窗口内,发现了419个原始样本\n3. 最后一个样本的时间戳是 2026-11-02T23:59:00.000Z,数值为42\n4. 在所有更晚的时间窗口(从2026-11-03T00:00:00.000Z开始)中,均未发现任何该指标的样本\n5. 因此可以确定数据从最后一个样本时间之后开始缺失\n\n这是基于实际原始样本搜索的确定时间点,而不仅仅是查询窗口的边界。根据系统报告,已经完成了79小时的实际搜索(从2026-11-02T17:00:00.000Z到2026-11-06T00:00:00.000Z),在其中只有最早的时间窗口包含数据,更晚的时间窗口均无数据。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "three-day-gap", + "repetition": 2, + "verdict": "pass", + "core_finding_correct": true, + "claims": [ + "最后观测到的数据时间:2026-11-02T23:59:00.000Z", + "从 2026-11-03T00:00:00.000Z 开始", + "完成了79小时的实际搜索" + ], + "reason": "Correct original last sample, following expected sample time using the prompt-supplied 60-second cadence, and actual 79-hour search coverage. Does not attribute a hardware cause. The optional cadence was omitted from the tool call, so the estimate is supported by the system prompt plus the original timestamp rather than the helper firstExpectedMissing field.", + "precision_note": "Calls approximately 3 days 1 minute the missing duration; that is sample age, one cadence interval longer than the first-expected-absence duration. Accepted as approximate wording here because both boundary timestamps are correct and no physical failure time is asserted." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "three-day-gap", + "repetition": 3, + "stop": "stop", + "modelTurns": 5, + "toolCalls": 4, + "toolErrors": 0, + "backendRequests": 8, + "elapsedMs": 65861.58895800001, + "usages": [ + { + "prompt_tokens": 4034, + "total_tokens": 4110, + "completion_tokens": 76, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4176, + "total_tokens": 4225, + "completion_tokens": 49, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4316, + "total_tokens": 4380, + "completion_tokens": 64, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4699, + "total_tokens": 4764, + "completion_tokens": 65, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6317, + "total_tokens": 6913, + "completion_tokens": 596, + "prompt_tokens_details": null + } + ], + "final": "根据观测数据,fixture-gpu-a 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标最近的数据缺失大约开始于 **2026-11-02T23:59:00.000Z**。\n\n**时间依据:**\n- 通过 `observability_metric_last_seen` 工具在最近 168 小时(7 天)的历史窗口中搜索原始样本,发现最后一个实际数据点的时间戳为 1793663940(即 **2026-11-02T23:59:00.000Z**),此时数值为 42。\n- 在该时间之后(从 2026-11-02T23:59:00.000Z 起至当前时间 2026-11-06T00:00:00.000Z),未检测到任何新的原始样本。\n- 搜索覆盖了从 2026-11-02T17:00:00.000Z 到 2026-11-06T00:00:00.000Z 的时间窗口(共 79 小时),在最早的一段(2026-11-02T17:00:00.000Z 到 2026-11-03T17:00:00.000Z)中发现了 419 个样本,证明采集在此时间窗口内曾正常工作;而之后的时间段(2026-11-03T17:00:00.000Z 起)所有窗口均无样本,确认数据在此之后中断。\n\n因此,可判定该指标的数据大约从 **2026-11-02T23:59:00.000Z** 开始缺失,已持续约 **3 天 0 小时 1 分钟**。此结论基于实际样本的时间戳,而非查询窗口边界,具有时序依据。后续若需更早的历史溯源,可考虑扩展 `lookbackHours` 参数(当前受限于 7 天),但已足够定位最近的中断时间点。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "three-day-gap", + "repetition": 3, + "verdict": "fail", + "core_finding_correct": false, + "claims": [ + "大约从 2026-11-02T23:59:00.000Z 开始缺失", + "lookbackHours 参数(当前受限于 7 天)" + ], + "reason": "Finds the correct original last sample but calls that present-sample timestamp the beginning of missing data and omits the cadence-qualified 00:00 first expected absence. Also misstates the tool maximum as 7 days despite evidence reporting a 720-hour maximum and 168-hour requested budget." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "history-unavailable", + "repetition": 1, + "stop": "stop", + "modelTurns": 3, + "toolCalls": 2, + "toolErrors": 0, + "backendRequests": 10, + "elapsedMs": 56609.450709, + "usages": [ + { + "prompt_tokens": 4034, + "total_tokens": 4142, + "completion_tokens": 108, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4233, + "total_tokens": 4333, + "completion_tokens": 100, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6367, + "total_tokens": 6952, + "completion_tokens": 585, + "prompt_tokens_details": null + } + ], + "final": "根据检查结果,我可以得出以下结论:\n\n fixture-gpu-b 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标在最近的图表上没有数据,这是由于在检查的时间窗口内完全没有找到该指标的任何样本数据。\n\n**时间依据:**\n- 检查时间窗口:2026-10-30T00:00:00.000Z 到 2026-11-06T00:00:00.000Z(共168小时,即7天)\n- 在该完整时间窗口内,所有针对指标 `DCGM_FI_DEV_GPU_UTIL{gpu=\"0\",Hostname=\"fixture-gpu-b\"}` 的范围查询都返回了0个样本\n- 系统明确指出:`status: \"not_found_in_checked_history\"` 和 `conclusion: \"No matching source samples were found in (2026-10-30T00:00:00.000Z, 2026-11-06T00:00:00.000Z].\"`\n\n**重要限制声明:**\n由于在7天的检查窗口内完全没有找到任何该指标的样本数据,我们无法确定数据究竟是从什么时候开始缺失的。系统明确指出:\n- \"It is unknown whether this exact series ever existed.\"(未知此精确序列是否曾经存在)\n- \"The onset is unknown, not established to predate this window.\"( onset 未知,未确定早于此窗口)\n- \"Do not infer an outage duration from empty history.\"(不要从空历史中推断故障持续时间)\n\n因此,基于可用的观测数据,**无法确定该指标数据缺失的具体开始时间**。只能确定的是,在最近7天(从2026-10-30T00:00:00.000Z开始)内,该指标从未出现过任何数据样本。更早的历史数据可能存在,但超出了当前工具的检查范围(默认7天,最大可扩展至30天)。如果需要进一步调查更早时间的情况,需要用户明确指定更早的时间范围进行检查。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "history-unavailable", + "host": "fixture-gpu-b", + "last": null, + "missing": null, + "value": 42, + "count": 0, + "lastIso": null, + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "history-unavailable", + "repetition": 1, + "verdict": "pass", + "core_finding_correct": true, + "claims": [ + "无法确定该指标数据缺失的具体开始时间", + "未知此精确序列是否曾经存在", + "默认7天,最大可扩展至30天" + ], + "reason": "Correctly keeps onset and prior existence unknown, reports the actual complete 168-hour search, and does not date an outage from the query boundary or claim backend retention." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "history-unavailable", + "repetition": 2, + "stop": "stop", + "modelTurns": 4, + "toolCalls": 3, + "toolErrors": 0, + "backendRequests": 11, + "elapsedMs": 53374.12275000004, + "usages": [ + { + "prompt_tokens": 4034, + "total_tokens": 4125, + "completion_tokens": 91, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4191, + "total_tokens": 4277, + "completion_tokens": 86, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4368, + "total_tokens": 4475, + "completion_tokens": 107, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6509, + "total_tokens": 6924, + "completion_tokens": 415, + "prompt_tokens_details": null + } + ], + "final": "根据观测性工具的分析结果,我可以得出以下结论:\n\n**发现:**\n在 fixture-gpu-b 主机上的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标最近没有数据。\n\n**时间依据:**\n系统在检查了从2026年10月30日00:00:00 UTC到2026年11月6日00:00:00 UTC(共168小时,即7天)的历史数据窗口内,均未找到该指标的任何样本数据。所有检查的时间窗口都返回了0个样本。\n\n**结论:**\n根据可用的观测数据,**无法确定数据缺失的确切开始时间**。系统明确指出:\n- 在检查的7天窗口内(2026-10-30T00:00:00.000Z到2026-11-06T00:00:00.000Z)未发现任何匹配的源样本\n- 未知该指标序列是否曾经存在过\n- 无法确定缺失是否早于此检查窗口之前\n- 查询边界(开始或结束时间)都不应被视为故障开始时间\n- 无法从空历史记录推断出故障持续时间\n\n因此,基于目前可观测到的7天历史数据,可以确定的是:**在最近的7天时间内(从2026年10月30日起),该GPU的利用率指标一直没有数据报告**。更早时间的情况无法从现有数据中确定。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "history-unavailable", + "host": "fixture-gpu-b", + "last": null, + "missing": null, + "value": 42, + "count": 0, + "lastIso": null, + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "history-unavailable", + "repetition": 2, + "verdict": "pass", + "core_finding_correct": true, + "claims": [ + "未知该指标序列是否曾经存在过", + "无法从空历史记录推断出故障持续时间", + "共168小时,即7天" + ], + "reason": "Correctly keeps onset and prior existence unknown, reports the actual complete 168-hour search, and does not date an outage from the query boundary or claim backend retention." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "history-unavailable", + "repetition": 3, + "stop": "stop", + "modelTurns": 4, + "toolCalls": 3, + "toolErrors": 0, + "backendRequests": 11, + "elapsedMs": 41847.43212499999, + "usages": [ + { + "prompt_tokens": 4034, + "total_tokens": 4071, + "completion_tokens": 37, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4137, + "total_tokens": 4186, + "completion_tokens": 49, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4277, + "total_tokens": 4342, + "completion_tokens": 65, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6376, + "total_tokens": 6722, + "completion_tokens": 346, + "prompt_tokens_details": null + } + ], + "final": "根据当前可观测数据,**DCGM_FI_DEV_GPU_UTIL{Hostname=\"fixture-gpu-b\",gpu=\"0\"}** 在过去 7 天(即自 2026-10-30T00:00:00.000Z 起)内未出现任何样本。工具在逐步扩大的历史窗口中进行了检查,最终确认在该时间范围内没有任何原始样本被记录。\n\n由于在检查窗口内根本未发现该指标系列的任何样本,**无法确定该指标何时开始缺失**——也不能断定该系列此前曾经存在过。工具明确结论为:“无法判断此精确系列是否曾经存在;缺失 onset 未知,不能确定其早于已检查窗口。”\n\n因此,基于目前可用的 7 天历史数据,**只能说该指标在此窗口之前就已经缺失(或从未存在)**,但无法给出更具体的时间点。若需进一步追溯,需要更长的后端保留时间(当前工具支持最多 30 天回溯),但需用户明确授权扩大 `lookbackHours`(目前未更改配置)。在不更改配置的前提下,只能报告已检查窗口内的观察结果。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "history-unavailable", + "host": "fixture-gpu-b", + "last": null, + "missing": null, + "value": 42, + "count": 0, + "lastIso": null, + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "history-unavailable", + "repetition": 3, + "verdict": "fail", + "core_finding_correct": false, + "claims": [ + "不能确定其早于已检查窗口", + "只能说该指标在此窗口之前就已经缺失(或从未存在)", + "需要更长的后端保留时间" + ], + "reason": "Initially states the correct unknown-onset limitation, then contradicts it by asserting absence predates the checked window (or never existed). Also implies insufficient backend retention without evidence; the actual tool reports retention unknown and supports longer searches." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "healthy-zero", + "repetition": 1, + "stop": "stop", + "modelTurns": 4, + "toolCalls": 3, + "toolErrors": 0, + "backendRequests": 3, + "elapsedMs": 56204.36637500004, + "usages": [ + { + "prompt_tokens": 4029, + "total_tokens": 4124, + "completion_tokens": 95, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4190, + "total_tokens": 4268, + "completion_tokens": 78, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4439, + "total_tokens": 4560, + "completion_tokens": 121, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6724, + "total_tokens": 7170, + "completion_tokens": 446, + "prompt_tokens_details": null + } + ], + "final": "根据我的检查,我可以告诉您:\n\n**结论:fixture-gpu-c 上 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标最近的值一直是 0,但这** **不一定** **表明采集已经中断。**\n\n**证据:**\n\n1. **指标存在且可查询**:DCGM_FI_DEV_GPU_UTIL 指标确实存在于系统中,并且我们能够成功查询到 fixture-gpu-c 上 GPU 0 的数据。\n\n2. **数据持续更新**:在过去60分钟内(从2026-11-05T23:00:00.000Z到2026-11-06T00:00:00.000Z),该指标有61个数据点,每个数据点都成功返回了值(0),没有任何缺失或错误。\n\n3. **时间戳正常**:所有数据点的时间戳都是连续且正常的,表明监控系统正在定期采集并更新这个指标。\n\n4. **值解释**:DCGM_FI_DEV_GPU_UTIL 表示GPU利用率百分比。持续为0意味着GPU在这段时间内没有被使用(空闲状态),而不是监控中断。\n\n**区分采集中断 vs 实际零值:**\n- 如果采集中断,我们会看到数据点缺失、时间戳停滞或查询返回错误\n- 而这里我们看到的是持续的、时间戳正常的零值序列,这更符合GPU确实处于空闲状态的情况\n\n因此,**最近的值一直是0更可能表示GPU 0在这段时间内确实没有被利用(空闲),而不是采集中断。** 监控系统似乎正常工作并持续在采集这个指标。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "healthy-zero", + "host": "fixture-gpu-c", + "last": 1793923200, + "missing": null, + "value": 0, + "count": 11521, + "lastIso": "2026-11-06T00:00:00.000Z", + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "healthy-zero", + "repetition": 1, + "verdict": "fail", + "core_finding_correct": true, + "claims": [ + "过去60分钟内该指标有61个数据点", + "持续为0意味着GPU在这段时间内没有被使用(空闲状态)" + ], + "reason": "Correctly identifies current and historical zero samples as present, but asserts the GPU was unused/idle from utilization alone. This does not establish workload state or complete device health." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "healthy-zero", + "repetition": 2, + "stop": "stop", + "modelTurns": 10, + "toolCalls": 9, + "toolErrors": 0, + "backendRequests": 9, + "elapsedMs": 172472.60320800002, + "usages": [ + { + "prompt_tokens": 4029, + "total_tokens": 4132, + "completion_tokens": 103, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4198, + "total_tokens": 4283, + "completion_tokens": 85, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4454, + "total_tokens": 4573, + "completion_tokens": 119, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6737, + "total_tokens": 6827, + "completion_tokens": 90, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 13826, + "total_tokens": 13920, + "completion_tokens": 94, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 20920, + "total_tokens": 21041, + "completion_tokens": 121, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 21940, + "total_tokens": 22018, + "completion_tokens": 78, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 22184, + "total_tokens": 22275, + "completion_tokens": 91, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 22365, + "total_tokens": 22411, + "completion_tokens": 46, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 22567, + "total_tokens": 23199, + "completion_tokens": 632, + "prompt_tokens_details": null + } + ], + "final": "根据我的调查,我可以得出以下结论:\n\n## 检查结果\n\n**fixture-gpu-c 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标确实一直保持为 0**,根据我的检查:\n\n1. **当前值验证**:最新查询显示该指标值为 0(时间戳:2026-11-06T00:00:00.000Z)\n\n2. **历史数据分析**:\n - 过去 1 小时:所有 60 个采样点都为 0\n - 过去 6 小时:所有采样点都为 0(经下sampling后显示)\n - 过去 24 小时:所有采样点都为 0(经下sampling后显示)\n\n3. **指标存在性确认**:使用 `observability_metric_last_seen` 确认该指标确实存在并被采集,最后一个实际样本是在 2026-11-06T00:00:00.000Z,值为 0\n\n4. **主机范围检查**:fixture-gpu-c 似乎只有 GPU 0,且这是唯一报告此指标的主机\n\n## 是否表明采集中断?\n\n**不,这不一定表明采集已经中断**。根据我的分析:\n\n- **指标仍在被采集**:时间戳更新显示采集器仍在工作(最后样本时间是当前时间)\n- **值为0是有效的监控数据**:GPU 利用率为 0 表示 GPU 当前处于空闲状态,这是一种完全正常的情况\n- **没有中断的迹象**:如果采集真正中断了,我们应该会看到:\n - 指标完全消失(无法查询到)\n - 或者时间戳停止更新(不再有新样本)\n - 或者出现采集错误的迹象\n\n## 结论\n\nfixture-gpu-c 的 GPU 0 目前利用率为 0,这是一个**有效的监控读数**,表明 GPU 当前没有被使用(空闲状态),而不是采集中断。采集器正常工作,只是报告了 GPU 没有工作负载的事实。\n\n如果您怀疑应该有工作负载在运行但看到利用率为 0,这可能表明:\n- 工作负载实际上没有在运行\n- 工作负载没有使用 GPU 资源\n- 工作负载在其他 GPU 上运行(但此主机似乎只有 1 个 GPU)", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "healthy-zero", + "host": "fixture-gpu-c", + "last": 1793923200, + "missing": null, + "value": 0, + "count": 11521, + "lastIso": "2026-11-06T00:00:00.000Z", + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "healthy-zero", + "repetition": 2, + "verdict": "fail", + "core_finding_correct": true, + "claims": [ + "最后一个实际样本是在 2026-11-06T00:00:00.000Z,值为 0", + "采集器正常工作,只是报告了 GPU 没有工作负载的事实" + ], + "reason": "Correctly establishes the current original zero sample and distinguishes presence from absence, but asserts no workload exists and normal collector operation from utilization alone; the queried data cannot establish those facts." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "healthy-zero", + "repetition": 3, + "stop": "stop", + "modelTurns": 3, + "toolCalls": 2, + "toolErrors": 0, + "backendRequests": 2, + "elapsedMs": 43697.91341600008, + "usages": [ + { + "prompt_tokens": 4029, + "total_tokens": 4116, + "completion_tokens": 87, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4287, + "total_tokens": 4420, + "completion_tokens": 133, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6584, + "total_tokens": 6929, + "completion_tokens": 345, + "prompt_tokens_details": null + } + ], + "final": "根据查询结果,我可以得出以下结论:\n\n**fixture-gpu-c 的 GPU 0 的 DCGM_FI_DEV_GPU_UTIL 指标确实存在且持续采集中,但其值一直为 0。**\n\n具体证据:\n1. 该指标在最新查询中存在(timestamp: 1793923200, 对应 2026-11-06T00:00:00.000Z)\n2. 在过去一小时(60分钟)内,该指标持续采集,每分钟一个数据点,共有61个数据点\n3. 所有61个数据点的值都是0,从2026-11-05T23:00:00.000Z开始一直到当前时间\n\n因此,**这并不说明采集已经中断**。相反,监控系统正在正常采集该指标数据,只是该GPU的利用率持续为0(可能表示GPU当前空闲或没有运行任何计算任务)。\n\n如果采集真正中断了,我们应该看到:\n- 指标完全消失(查询返回空结果)\n- 或者数据点停止更新(时间戳不再推进)\n\n而在这里我们看到的是持续的、带时间戳的零值数据点,表明采集器正在工作,只是被监控的GPU当前没有被使用。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "healthy-zero", + "host": "fixture-gpu-c", + "last": 1793923200, + "missing": null, + "value": 0, + "count": 11521, + "lastIso": "2026-11-06T00:00:00.000Z", + "missingIso": null + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "healthy-zero", + "repetition": 3, + "verdict": "fail", + "core_finding_correct": true, + "claims": [ + "共61个数据点", + "可能表示GPU当前空闲", + "只是被监控的GPU当前没有被使用" + ], + "reason": "Correctly establishes fresh zero-valued data. Initially qualifies idle-workload interpretation as possible, but the final sentence asserts the GPU was unused; utilization alone cannot establish that workload fact." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "two-hour-gap", + "repetition": 1, + "stop": "stop", + "modelTurns": 4, + "toolCalls": 3, + "toolErrors": 0, + "backendRequests": 4, + "elapsedMs": 61339.434583000024, + "usages": [ + { + "prompt_tokens": 4034, + "total_tokens": 4127, + "completion_tokens": 93, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4193, + "total_tokens": 4281, + "completion_tokens": 88, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4372, + "total_tokens": 4491, + "completion_tokens": 119, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 5557, + "total_tokens": 6063, + "completion_tokens": 506, + "prompt_tokens_details": null + } + ], + "final": "根据查询结果,我可以为您提供以下信息:\n\n**fixture-gpu-d 的 GPU 0 在 DCGM_FI_DEV_GPU_UTIL 指标上的数据缺失情况:**\n\n1. **最后一次成功观测时间**:2026-11-05T21:59:00.000Z(价值为23)\n2. **数据开始缺失的时间**:大约在2026-11-05T21:59:00.000Z之后\n3. **截至当前时间的缺失时长**:大约2小时1分钟(从最后一次采样到当前时间2026-11-06T00:00:00.000Z)\n\n**时间依据**:\n- 系统在最近的检查窗口(2026-11-05T17:00:00.000Z到2026-11-06T00:00:00.000Z)内进行了分阶段搜索\n- 在2026-11-05T17:00:00.000Z到2026-11-05T23:00:00.000Z之间找到了有效数据(299个样本点)\n- 在2026-11-05T23:00:00.000Z到2026-11-06T00:00:00.000Z这一小时窗口内未找到任何样本\n- 最后一个有效样本的确切时间戳是2026-11-05T21:59:00.000Z\n\n因此,可以判断fixture-gpu-d的GPU 0在DCGM_FI_DEV_GPU_UTIL指标上的数据大约从2026-11-05T21:59:00.000Z开始缺失,截至目前已经缺失了约2小时1分钟。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "two-hour-gap", + "host": "fixture-gpu-d", + "last": 1793915940, + "missing": 1793916000, + "value": 23, + "count": 11400, + "lastIso": "2026-11-05T21:59:00.000Z", + "missingIso": "2026-11-05T22:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "two-hour-gap", + "repetition": 1, + "verdict": "fail", + "core_finding_correct": false, + "claims": [ + "最后一次成功观测时间:2026-11-05T21:59:00.000Z", + "从2026-11-05T21:59:00.000Z开始缺失" + ], + "reason": "Recovers the correct original last sample and sample age, but never states the cadence-qualified 22:00 first expected missing sample. The final sentence instead dates missing data to the still-present 21:59 sample." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "two-hour-gap", + "repetition": 2, + "stop": "stop", + "modelTurns": 5, + "toolCalls": 4, + "toolErrors": 0, + "backendRequests": 5, + "elapsedMs": 71659.88299999991, + "usages": [ + { + "prompt_tokens": 4034, + "total_tokens": 4121, + "completion_tokens": 87, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4187, + "total_tokens": 4257, + "completion_tokens": 70, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4348, + "total_tokens": 4439, + "completion_tokens": 91, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4758, + "total_tokens": 4864, + "completion_tokens": 106, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 5930, + "total_tokens": 6503, + "completion_tokens": 573, + "prompt_tokens_details": null + } + ], + "final": "根据查询结果,我可以为您提供以下信息:\n\n**关于 fixture-gpu-d 的 GPU 0 的 GPU 使用率指标 (DCGM_FI_DEV_GPU_UTIL) 缺失情况的分析:**\n\n1. **最后一次观测到的数据时间**:2026-11-05T21:59:00.000Z(UTC时间)\n2. **该时间点的数值**:23(表示GPU使用率为23%)\n3. **数据缺失开始时间**:从最后一次观测之后开始缺失,即大约从2026-11-05T21:59:00.000Z开始\n4. **缺失时长**:截至当前时间(2026-11-06T00:00:00.000Z),数据已经缺失了大约2小时1分钟\n\n**时间依据**:\n- 通过 `observability_metric_last_seen` 工具在7小时的窗口内(2026-11-05T17:00:00.000Z 到 2026-11-06T00:00:00.000Z)搜索了该指标的原始样本\n- 在时间窗口的早期部分(2026-11-05T17:00:00.000Z 到 2026-11-05T23:00:00.000Z)发现了299个样本,最后一个样本时间戳为2026-11-05T21:59:00.000Z\n- 在最近的时间窗口(2026-11-05T23:00:00.000Z 到 2026-11-06T00:00:00.000Z)内未发现任何样本,表明数据从最后一个样本时间之后停止上报\n\n因此,可以判断fixture-gpu-d的GPU 0的GPU使用率指标大约从**2026-11-05T21:59:00.000Z**开始缺失,截止到目前已经缺失了约2小时1分钟。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "two-hour-gap", + "host": "fixture-gpu-d", + "last": 1793915940, + "missing": 1793916000, + "value": 23, + "count": 11400, + "lastIso": "2026-11-05T21:59:00.000Z", + "missingIso": "2026-11-05T22:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "two-hour-gap", + "repetition": 2, + "verdict": "fail", + "core_finding_correct": false, + "claims": [ + "最后一个实际样本时间:2026-11-05T21:59:00.000Z", + "从2026-11-05T21:59:00.000Z开始缺失" + ], + "reason": "Correct original last sample, sample age, and seven-hour actual search coverage, but omits the known-cadence first expected absence at 22:00 and instead labels the present 21:59 sample as the onset." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "two-hour-gap", + "repetition": 3, + "stop": "stop", + "modelTurns": 4, + "toolCalls": 3, + "toolErrors": 0, + "backendRequests": 4, + "elapsedMs": 60770.42316699994, + "usages": [ + { + "prompt_tokens": 4034, + "total_tokens": 4126, + "completion_tokens": 92, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4192, + "total_tokens": 4278, + "completion_tokens": 86, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 4369, + "total_tokens": 4480, + "completion_tokens": 111, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 5546, + "total_tokens": 6076, + "completion_tokens": 530, + "prompt_tokens_details": null + } + ], + "final": "根据查询结果,我可以得出以下结论:\n\n**关于 fixture-gpu-d 的 GPU 0 在 DCGM_FI_DEV_GPU_UTIL 指标上的数据缺失情况:**\n\n1. **最后一个有效样本时间**:2026-11-05T21:59:00.000Z(值为23)\n2. **数据开始缺失的时间**:大约在2026-11-05T21:59:00.000Z之后\n3. **时间依据**:\n - 在检查窗口(2026-11-05T17:00:00.000Z到2026-11-06T00:00:00.000Z)中,系统发现:\n - 从2026-11-05T17:00:00.000Z到2026-11-05T23:00:00.000Z有数据(299个样本)\n - 从2026-11-05T23:00:00.000Z到2026-11-06T00:00:00.000Z(最近1小时)没有任何样本\n - 最后一个实际的源样本时间戳是2026-11-05T21:59:00.000Z\n - 从该样本到当前时间(2026-11-06T00:00:00.000Z)已过去约2小时1分钟\n\n因此,**fixture-gpu-d 的 GPU 0 在 DCGM_FI_DEV_GPU_UTIL 指标上的数据大约从2026-11-05T21:59:00.000Z开始缺失**,即当前时间的约2小时1分钟前。这个时间依据是通过在指定时间范围内搜索原始样本,找到最后一个有效样本,并在之后的时间窗口中未发现任何新样本来确定的。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "two-hour-gap", + "host": "fixture-gpu-d", + "last": 1793915940, + "missing": 1793916000, + "value": 23, + "count": 11400, + "lastIso": "2026-11-05T21:59:00.000Z", + "missingIso": "2026-11-05T22:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "two-hour-gap", + "repetition": 3, + "verdict": "fail", + "core_finding_correct": false, + "claims": [ + "最后一个实际样本时间:2026-11-05T21:59:00.000Z", + "从2026-11-05T21:59:00.000Z开始缺失" + ], + "reason": "Correct original last sample, sample age, and seven-hour actual search coverage, but omits the known-cadence first expected absence at 22:00 and instead labels the present 21:59 sample as the onset." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "three-day-gap-followup", + "repetition": 1, + "stop": "stop", + "modelTurns": 2, + "toolCalls": 1, + "toolErrors": 0, + "backendRequests": 5, + "elapsedMs": 24349.793666000012, + "usages": [ + { + "prompt_tokens": 4477, + "total_tokens": 4588, + "completion_tokens": 111, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 5813, + "total_tokens": 6006, + "completion_tokens": 193, + "prompt_tokens_details": null + } + ], + "final": " fixture-gpu-a 的 GPU 0 在最近 48 小时(从 2026-11-04T00:00:00Z 到 2026-11-06T00:00:00Z)都没有 DCGM_FI_DEV_GPU_UTIL 样本。检索工具未在该时间窗口内找到任何此指标的原始样本,也无法确定该序列此前是否曾经存在。因此,**无法给出问题开始的确切时间**;目前只能说在这 48 小时内该指标持续缺失。若需要进一步确定更早时间的状况,需要扩大回溯窗口或检查采集器/导出器的健康状态。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap-followup", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "followup": true, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "three-day-gap-followup", + "repetition": 1, + "verdict": "fail", + "core_finding_correct": false, + "claims": [ + "最近 48 小时没有原始样本", + "需要扩大回溯窗口" + ], + "reason": "Stops after a 48-hour empty search and suggests further queries instead of using accessible history to recover the actual three-day last-sample evidence." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "three-day-gap-followup", + "repetition": 2, + "stop": "stop", + "modelTurns": 2, + "toolCalls": 1, + "toolErrors": 0, + "backendRequests": 5, + "elapsedMs": 45494.16991699999, + "usages": [ + { + "prompt_tokens": 4477, + "total_tokens": 4588, + "completion_tokens": 111, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 5813, + "total_tokens": 6364, + "completion_tokens": 551, + "prompt_tokens_details": null + } + ], + "final": "**回答**\n\n在检查的时间窗口内(最近 48 小时),没有发现任何 **DCGM_FI_DEV_GPU_UTIL{gpu=\"0\",Hostname=\"fixture-gpu-a\"}** 的原始样本。因此:\n\n- **无法确定此指标何时停止上报**(onset 未知)。\n- 不能断言此系列在此窗口之前曾经存在;它可能从未被采集过,也可能在此 48 小时之前就停止了,但无法从已有数据中得出确切时间。\n- 最近两小时(2026-11-05T22:00:00Z 至 2026-11-06T00:00:00Z)确实没有样本,这仅说明该窗口内没有数据,**不能作为故障开始时间**。\n\n**时间依据**(来自 `observability_metric_last_seen`):\n\n- 检查范围:`requestedStartIso=\"2026-11-04T00:00:00.000Z\"` 到 `requestedEndIso=\"2026-11-06T00:00:00.000Z\"`(共 48 小时)。\n- 已实际搜索的窗口:`searchedDurationHours=48`,分四段逐步向前查询,所有窗口均返回 `sampleCount=0`、`seriesCount=0`。\n- 结论字段明确指出:“No matching source samples were found … It is unknown whether this exact series ever existed. The onset is unknown, not established to predate this window.”\n\n因此,**基于目前可观测的数据,无法给出更早的确切停止时间**。如果需要进一步回溯,可将 `lookbackHours` 调大(最多可查 720 小时),但同样只能得出“在此延长窗口内仍未发现样本”而不能断言更早的故障时间。若希望确认该系列是否曾经存在,需检查更早的历史或确认采集器配置。当前只能说:**最近两小时无数据,且在最近 48 小时内均未见该指标样本,故障开始时间未知**。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap-followup", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "followup": true, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "three-day-gap-followup", + "repetition": 2, + "verdict": "fail", + "core_finding_correct": false, + "claims": [ + "最近 48 小时内均未见该指标样本,故障开始时间未知", + "调大lookbackHours但同样只能得出延长窗口内仍未发现样本" + ], + "reason": "Accurately reports the checked 48-hour empty window, but stops before reaching accessible older evidence. Also predicts that an unexecuted longer search would remain empty, although the fixture contains older samples." + } + }, + { + "side": "candidate", + "source_commit": "8305d59e44470e10d268a4e60fc36870929b2e27", + "summary": { + "case": "three-day-gap-followup", + "repetition": 3, + "stop": "stop", + "modelTurns": 2, + "toolCalls": 1, + "toolErrors": 0, + "backendRequests": 6, + "elapsedMs": 27443.519417000003, + "usages": [ + { + "prompt_tokens": 4477, + "total_tokens": 4589, + "completion_tokens": 112, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 6142, + "total_tokens": 6378, + "completion_tokens": 236, + "prompt_tokens_details": null + } + ], + "final": "问题开始时间:2026-11-02T23:59:00.000Z\n\n时间依据:\n- 该指标 DCGM_FI_DEV_GPU_UTIL{gpu=\"0\",Hostname=\"fixture-gpu-a\"} 的最后一个观测样本出现在 2026-11-02T23:59:00.000Z,数值为 42。\n- 自该时间起至当前 2026-11-06T00:00:00.000Z,系统已连续 3 天 1 分钟未再采集到任何新样本(在最近两小时窗口以及向前扩展的 79 小时检索中均未发现样本)。\n- 因此可以确定数据中断始于最后一次成功采样之后,即 2026-11-02T23:59:00.000Z。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap-followup", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "followup": true, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "candidate", + "case": "three-day-gap-followup", + "repetition": 3, + "verdict": "fail", + "core_finding_correct": false, + "claims": [ + "问题开始时间:2026-11-02T23:59:00.000Z", + "向前扩展的 79 小时检索中均未发现样本" + ], + "reason": "Finds the correct original last sample but labels that present-sample timestamp as onset and omits the known-cadence 00:00 first expected absence. Incorrectly says the entire 79-hour search had no samples despite finding 419 samples in its oldest window." + } + } + ], + "non_scoring_attempts": [ + { + "stage": "smoke", + "side": "baseline", + "summary": { + "case": "three-day-gap-followup", + "repetition": 1, + "stop": "stop", + "modelTurns": 5, + "toolCalls": 4, + "toolErrors": 0, + "backendRequests": 5, + "elapsedMs": 64337.828041999994, + "usages": [ + { + "prompt_tokens": 2728, + "total_tokens": 2765, + "completion_tokens": 37, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2831, + "total_tokens": 2907, + "completion_tokens": 76, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2996, + "total_tokens": 3072, + "completion_tokens": 76, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3228, + "total_tokens": 3419, + "completion_tokens": 191, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 3486, + "total_tokens": 3974, + "completion_tokens": 488, + "prompt_tokens_details": null + } + ], + "final": "目前可观测系统里只有 `DCGM_FI_DEV_GPU_UTIL` 这一条指标。也就是说我们只能依赖 DCGM 相关的数据来判断 fixture-gpu-a 是否在上报。\n\n刚才我们已经看到:\n- 在当前时间(2026-11-06 00:00:00 UTC)只能查到 fixture-gpu-c 的 GPU 0 有样本(值为 0);\n- fixture-gpu-a 在这个时间点没有任何 DCGM_FI_DEV_GPU_UTIL 样本;\n- 在最近两小时的范围查询里(1793916000–1793923200 Unix 时间戳,即 2026-11-05 12:00:00–2026-11-06 00:00:00 UTC)同样也没有返回任何样本。\n\n因此可以得出结论: fixture-gpu-a 的 GPU 0 在 **整个最近两小时窗口内都没有上报 DCGM_FI_DEV_GPU_UTIL 指标**,即监控系统在此时间段内完全未收到该节点该 GPU 的任何利用率样本。这一断点正是问题开始的时间——即在观测窗口开始之前(2026-11-05 12:00:00 UTC 以前)监控就已丢失该节点的上报。\n\n若需要更精确的“最后一次成功上报”时间点,则需要把查询窗口向过去延伸(例如过去 6 小时、12 小时或 24 小时),直到出现非空样本为止。但在目前所给的「最近两小时」范围内,我们只能说:**自观测窗口起点(2026-11-05 12:00:00 UTC)开始, fixture-gpu-a GPU 0 的 DCGM_FI_DEV_GPU_UTIL 已持续失联**。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap-followup", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "followup": true, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "smoke", + "side": "baseline", + "case": "three-day-gap-followup", + "repetition": 1, + "verdict": "fail", + "claims": [ + "自观测窗口起点(2026-11-05 12:00:00 UTC)开始", + "这一断点正是问题开始的时间" + ], + "reason": "Dates onset from the recent query window, mis-converts that boundary as 12:00 instead of 22:00, and never finds the original last sample at 2026-11-02T23:59:00Z." + }, + "error": null + }, + { + "stage": "smoke", + "side": "candidate", + "summary": { + "case": "three-day-gap-followup", + "repetition": 1, + "stop": "stop", + "modelTurns": 2, + "toolCalls": 1, + "toolErrors": 0, + "backendRequests": 5, + "elapsedMs": 23275.22475, + "usages": [ + { + "prompt_tokens": 4477, + "total_tokens": 4588, + "completion_tokens": 111, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 5813, + "total_tokens": 5982, + "completion_tokens": 169, + "prompt_tokens_details": null + } + ], + "final": " fixture-gpu-a 的 GPU 0 在最近 48 小时(从 2026-11-04T00:00:00Z 到 2026-11-06T00:00:00Z)都没有 DCGM_FI_DEV_GPU_UTIL 样本。检查工具在该窗口内逐段搜索原始样本,未发现任何匹配的时间序列。由于没有更早的历史证据,无法断定该指标何时停止上报,也不能确定该系列曾经存在。因此,只能说明缺失已至少持续 48 小时,但具体中断时间未知。", + "completed": true, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap-followup", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "followup": true, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "smoke", + "side": "candidate", + "case": "three-day-gap-followup", + "repetition": 1, + "verdict": "fail", + "claims": [ + "缺失已至少持续 48 小时", + "具体中断时间未知" + ], + "reason": "Stops after a 48-hour empty-history search even though accessible older samples establish the three-day gap; also treats absence throughout the checked window as a minimum outage duration while admitting prior existence is unproven." + }, + "error": null + }, + { + "stage": "formal-attempt-1", + "side": "baseline", + "summary": { + "case": "three-day-gap", + "repetition": 1, + "stop": "replay_error", + "modelTurns": 2, + "toolCalls": 2, + "toolErrors": 0, + "backendRequests": 2, + "elapsedMs": 34881.631458, + "usages": [ + { + "prompt_tokens": 2472, + "total_tokens": 2580, + "completion_tokens": 108, + "prompt_tokens_details": null + }, + { + "prompt_tokens": 2671, + "total_tokens": 2831, + "completion_tokens": 160, + "prompt_tokens_details": null + } + ], + "final": null, + "completed": false, + "investigated": true, + "answerReview": "pending", + "expected": { + "id": "three-day-gap", + "host": "fixture-gpu-a", + "last": 1793663940, + "missing": 1793664000, + "value": 42, + "count": 7200, + "lastIso": "2026-11-02T23:59:00.000Z", + "missingIso": "2026-11-03T00:00:00.000Z" + } + }, + "review": { + "stage": "formal-attempt-1", + "side": "baseline", + "case": "three-day-gap", + "repetition": 1, + "verdict": "not_assessed", + "claims": [], + "reason": "Transport TypeError: fetch failed after two successful model-selected tool calls; no completed answer. Endpoint remained healthy and the next replay proceeded. Exact socket cause is not recorded. Reconciliation required." + }, + "error": { + "name": "TypeError", + "message": "fetch failed" + } + } + ], + "evidence_retention": { + "canonical_artifact_manifest_sha256": "2121d4a2b5267dc4ab226b468587cae8dec72656ad1224bf8a76a070579f9eb9", + "full_artifact_manifest": { + "files": [ + { + "path": "baseline/fixture-preflight.json", + "bytes": 591444, + "sha256": "843f5be6191f9f56a0fec7b4b31a6104760ff54d4dd572d5898eadc896745090" + }, + { + "path": "baseline/healthy-zero-1/answer.txt", + "bytes": 1618, + "sha256": "2d4b92bc26b95efd07f1811a1e8fbf2969278ce79ee34adad5de849dcfec62ee" + }, + { + "path": "baseline/healthy-zero-1/backend.jsonl", + "bytes": 2804, + "sha256": "7f24d17a6e1da1d9ce33387baeb75700a9c726a8eb219212ec80c9fea5433381" + }, + { + "path": "baseline/healthy-zero-1/initial-messages.json", + "bytes": 7632, + "sha256": "2c6bb0ccb86e37707c1301fd4d8718b6728a3d36c7c97148b347a8164b773eeb" + }, + { + "path": "baseline/healthy-zero-1/model-requests.jsonl", + "bytes": 78656, + "sha256": "bf84b15f81090d3bada15396c80c16a3c2f7174024f6035d11dcde5955e7e7bf" + }, + { + "path": "baseline/healthy-zero-1/model-responses.jsonl", + "bytes": 8591, + "sha256": "d59f8acb24800fd2261ed698c66dae3e331cfe6d9c54a40c0f978ef6798e43f2" + }, + { + "path": "baseline/healthy-zero-1/summary.json", + "bytes": 2975, + "sha256": "0e1bf5d551c673c61a8e0ce0e81a13f4e5eb3461dfc513ac9052b41f04af3188" + }, + { + "path": "baseline/healthy-zero-1/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/healthy-zero-1/tools.jsonl", + "bytes": 7983, + "sha256": "b663820513d49cffc6ee85ba83de9944fc1debb9ba78ce43bbffa20139b25aac" + }, + { + "path": "baseline/healthy-zero-1/transcript.json", + "bytes": 17104, + "sha256": "6083cdca78dbea75d6a9dd564daa42044e03c660d8f6215d3d61564c84961021" + }, + { + "path": "baseline/healthy-zero-2/answer.txt", + "bytes": 806, + "sha256": "c887d7be51ba1bd0429293e27f966120355412761cddef89400b9bc9962648ab" + }, + { + "path": "baseline/healthy-zero-2/backend.jsonl", + "bytes": 3391, + "sha256": "9736f20d2100e18dde46f31d0d40fa7be782e32fe77ef1dcf6c16c377568671d" + }, + { + "path": "baseline/healthy-zero-2/initial-messages.json", + "bytes": 7632, + "sha256": "2c6bb0ccb86e37707c1301fd4d8718b6728a3d36c7c97148b347a8164b773eeb" + }, + { + "path": "baseline/healthy-zero-2/model-requests.jsonl", + "bytes": 42185, + "sha256": "cc434ff0e050cea8c6e6d9763ab42d513d5a67f85555148a1a2e58d190889d1f" + }, + { + "path": "baseline/healthy-zero-2/model-responses.jsonl", + "bytes": 4157, + "sha256": "c996560ecf2d42119d05fb0f52c20ba81df1be9dfe74c8fdb711f671c0b803ad" + }, + { + "path": "baseline/healthy-zero-2/summary.json", + "bytes": 1730, + "sha256": "bfc63f95cf082415fe01e847c5ed0ae0a67232a135fd62947b66c9ad3923097b" + }, + { + "path": "baseline/healthy-zero-2/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/healthy-zero-2/tools.jsonl", + "bytes": 16599, + "sha256": "db717babffa8e1f5f63deb1b8d108b5f9e2cc76be0e120011b2367cf12ca40f7" + }, + { + "path": "baseline/healthy-zero-2/transcript.json", + "bytes": 21095, + "sha256": "cd80745abe800332fa516c3c8db2232f693cc0676b311d88f848659982d90a65" + }, + { + "path": "baseline/healthy-zero-3/answer.txt", + "bytes": 1547, + "sha256": "2e0e4a636446250ff2dac25ec8190f4012de207fd4ef5288c2782aee69492303" + }, + { + "path": "baseline/healthy-zero-3/backend.jsonl", + "bytes": 3391, + "sha256": "9736f20d2100e18dde46f31d0d40fa7be782e32fe77ef1dcf6c16c377568671d" + }, + { + "path": "baseline/healthy-zero-3/initial-messages.json", + "bytes": 7632, + "sha256": "2c6bb0ccb86e37707c1301fd4d8718b6728a3d36c7c97148b347a8164b773eeb" + }, + { + "path": "baseline/healthy-zero-3/model-requests.jsonl", + "bytes": 42230, + "sha256": "b7c47eebcc24e5f689612a5c24f3474071703b6c21a6acebd94efaa543d607c5" + }, + { + "path": "baseline/healthy-zero-3/model-responses.jsonl", + "bytes": 4947, + "sha256": "0b72a27e67b67f0ee4d2e9b01329d9f37ca12450ee5684e010fe0435868b6555" + }, + { + "path": "baseline/healthy-zero-3/summary.json", + "bytes": 2490, + "sha256": "24db31f54799824c9179ec80acddac13f965eaae17b36fa6514657bc8a409d7c" + }, + { + "path": "baseline/healthy-zero-3/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/healthy-zero-3/tools.jsonl", + "bytes": 16599, + "sha256": "db717babffa8e1f5f63deb1b8d108b5f9e2cc76be0e120011b2367cf12ca40f7" + }, + { + "path": "baseline/healthy-zero-3/transcript.json", + "bytes": 21866, + "sha256": "accfaa9e5da27d8e203e5d23a92db69b01955095de1238f342b0b508367e2d51" + }, + { + "path": "baseline/history-unavailable-1/answer.txt", + "bytes": 1482, + "sha256": "ffb82cd36ce6136997e14eb2aac1d88fdca3249a5a487965d7b0597349e14a02" + }, + { + "path": "baseline/history-unavailable-1/backend.jsonl", + "bytes": 8282, + "sha256": "a9a8c06bd5e171846536d2d9d8688b9178219a9ea4c29108854a16b7edc45fe4" + }, + { + "path": "baseline/history-unavailable-1/initial-messages.json", + "bytes": 7652, + "sha256": "b53a1a727553b7e4bfa014be06d6753142320de8a2564eb54766b8798e2016e3" + }, + { + "path": "baseline/history-unavailable-1/model-requests.jsonl", + "bytes": 250731, + "sha256": "eb308807fc5ea51ea5bc7c54f557d9cf44a566a5d7c7abd75706d2260b44e9cf" + }, + { + "path": "baseline/history-unavailable-1/model-responses.jsonl", + "bytes": 18125, + "sha256": "5ceedff383c32a75f487d7fb919c55f23a42677badeb2334e8e6dc677f0f2841" + }, + { + "path": "baseline/history-unavailable-1/summary.json", + "bytes": 3799, + "sha256": "eac53671e72428b7318cf233a5579c1692ce0eadfe32d528d84427667ea62d1a" + }, + { + "path": "baseline/history-unavailable-1/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/history-unavailable-1/tools.jsonl", + "bytes": 29928, + "sha256": "181226fbf1b0d37fe8711239c2a4317793f66a1a0a17c153940a2031352a9d26" + }, + { + "path": "baseline/history-unavailable-1/transcript.json", + "bytes": 36445, + "sha256": "6a345d0d3dd7d2b6967d80bb839f94649f99ea576e10adc381900a2fe20ca000" + }, + { + "path": "baseline/history-unavailable-2/answer.txt", + "bytes": 946, + "sha256": "ef8a3500eb64eafd10052c0bd5a451fb058f08141f65d337ee53a8633e5c6419" + }, + { + "path": "baseline/history-unavailable-2/backend.jsonl", + "bytes": 1322, + "sha256": "4e52af89b3d4dd1263e82629994b4c1b268450a859a75b82a668925b396b5a8a" + }, + { + "path": "baseline/history-unavailable-2/initial-messages.json", + "bytes": 7652, + "sha256": "b53a1a727553b7e4bfa014be06d6753142320de8a2564eb54766b8798e2016e3" + }, + { + "path": "baseline/history-unavailable-2/model-requests.jsonl", + "bytes": 43823, + "sha256": "ff1325f91be8283db7ed312b8e48d5accb4e431808e4291814e56e509ea0e455" + }, + { + "path": "baseline/history-unavailable-2/model-responses.jsonl", + "bytes": 5823, + "sha256": "943e50107c3abf25cb4c6b2c09fb8941ee840fde754c83060e37bbc88f78ce03" + }, + { + "path": "baseline/history-unavailable-2/summary.json", + "bytes": 1997, + "sha256": "175856ba3965bc59a1940b9ddc0f43e31594fe444ca9bf760147f34c836a616c" + }, + { + "path": "baseline/history-unavailable-2/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/history-unavailable-2/tools.jsonl", + "bytes": 2141, + "sha256": "217d0837cc3e589392ef8c9ad1f48cf0741a4ac17ab88118fbf2bf788ecd14f1" + }, + { + "path": "baseline/history-unavailable-2/transcript.json", + "bytes": 11709, + "sha256": "be8a380417aa9c873311168368e4525b6922ecbb1dd43d23cf313ec59409a12f" + }, + { + "path": "baseline/history-unavailable-3/answer.txt", + "bytes": 1231, + "sha256": "e627eef04f517f07e137e70b0fdad2da5b443c900abd4c50ab1b8d5136d62999" + }, + { + "path": "baseline/history-unavailable-3/backend.jsonl", + "bytes": 3786, + "sha256": "71ba073a06ef874f82d9dab8509b64b46a7d20cd050decdab84c0d90db57e67a" + }, + { + "path": "baseline/history-unavailable-3/initial-messages.json", + "bytes": 7652, + "sha256": "b53a1a727553b7e4bfa014be06d6753142320de8a2564eb54766b8798e2016e3" + }, + { + "path": "baseline/history-unavailable-3/model-requests.jsonl", + "bytes": 134845, + "sha256": "9b80c7e247dd460830d2aaf6a436e57999a21e19be6ee3b91e81ed2e86e0a10d" + }, + { + "path": "baseline/history-unavailable-3/model-responses.jsonl", + "bytes": 15234, + "sha256": "fe6c1fa24ad126bcdcdd3be5efa7e5fe31cb6040b9e2a732277fcc4dcec87f2a" + }, + { + "path": "baseline/history-unavailable-3/summary.json", + "bytes": 3121, + "sha256": "66510da581bfe9df352e4c606beceb4ee20b6ce01ad5777097c0f86e9ed67027" + }, + { + "path": "baseline/history-unavailable-3/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/history-unavailable-3/tools.jsonl", + "bytes": 6316, + "sha256": "49824689f6fc128a02a36d26899f789ed6ada56a7a6347f3ffb2047cfb055d2a" + }, + { + "path": "baseline/history-unavailable-3/transcript.json", + "bytes": 18079, + "sha256": "b6be3eba39b84b7d74b9712cf9917f766af7661a9a20c3798398e77d500a0ced" + }, + { + "path": "baseline/manifest.json", + "bytes": 2496, + "sha256": "10a63070412f6dc0427cb0280452c9e1cc44e9d9cd4540b1226d77b27641068f" + }, + { + "path": "baseline/reviews.json", + "bytes": 8835, + "sha256": "67de94d435868834cc7beaa4346b8e15e0d6c9b2ad427f5563a848ff7d36eed0" + }, + { + "path": "baseline/source/helm/mosaic-stack/files/openclaw-seed/extensions/observability/grafana.ts", + "bytes": 20362, + "sha256": "4ccd76f10c41e86be37f28c1781075708c71b4c3f3308447949675c16b182236" + }, + { + "path": "baseline/source/helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts", + "bytes": 9893, + "sha256": "34fa25b3e0491c9b946a36d3990b8c1629ad527bc93be11267128aab5309ba1c" + }, + { + "path": "baseline/source/helm/mosaic-stack/files/openclaw-seed/extensions/observability/loki.ts", + "bytes": 9441, + "sha256": "4948c6b18e1dc657b3a0112ae6d4bb0c8620983075be6483d3e3898848ad7b46" + }, + { + "path": "baseline/source/helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md", + "bytes": 6928, + "sha256": "4a53faf9612f8f996e56f3e26b6d5dfa9dd02c203e8e9f6c4336e788514b8220" + }, + { + "path": "baseline/summaries.json", + "bytes": 39669, + "sha256": "885b3f3fba40ceceb2db3f274129e6d325c85f889c1a562850fb95be12028586" + }, + { + "path": "baseline/three-day-gap-1/answer.txt", + "bytes": 881, + "sha256": "2ea84728f089dc933662519373cb327e141dc45a4acb825df6594137d9e71a42" + }, + { + "path": "baseline/three-day-gap-1/backend.jsonl", + "bytes": 9041, + "sha256": "bfe34c79aa85e8e9cf2b9c3171fe1867e58add2a4d6667455a6a14dc0a96476b" + }, + { + "path": "baseline/three-day-gap-1/initial-messages.json", + "bytes": 7652, + "sha256": "ae39d3ea3eb8d2af471798908e6e468e5dce5229df40dd629fa024bb8eba10ca" + }, + { + "path": "baseline/three-day-gap-1/model-requests.jsonl", + "bytes": 404097, + "sha256": "9ae7c74d2912267f3013db5e4e404002df8b52eebafc2ccdc5734c7709ac4e04" + }, + { + "path": "baseline/three-day-gap-1/model-responses.jsonl", + "bytes": 28569, + "sha256": "b936267f74a9c6056137196266b840b90d1d443f47775ebb8c36d4ff4d001ba7" + }, + { + "path": "baseline/three-day-gap-1/summary.json", + "bytes": 4469, + "sha256": "aa64589474b72fef6f96c046a5b45f2a8b9827f461ef4d10e853047ec50d39f5" + }, + { + "path": "baseline/three-day-gap-1/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/three-day-gap-1/tools.jsonl", + "bytes": 15284, + "sha256": "eec8c4725453f7ecae7a371a0421ee3d0cce55b04c6fc557e82cfd3f740aff30" + }, + { + "path": "baseline/three-day-gap-1/transcript.json", + "bytes": 28527, + "sha256": "f27562ecf1043893b885050c8a1de991fb7d00178cd364bf4ae11065e2bd06d3" + }, + { + "path": "baseline/three-day-gap-2/answer.txt", + "bytes": 1418, + "sha256": "e1516f9fcc5dde646e5b4d416ef071b438573b5d7ec2fcc7f524e1c0230dc95b" + }, + { + "path": "baseline/three-day-gap-2/backend.jsonl", + "bytes": 116712, + "sha256": "19a247732e93994ee69549caa09a86ee03bda962815b860cbbc73febff60af2d" + }, + { + "path": "baseline/three-day-gap-2/initial-messages.json", + "bytes": 7652, + "sha256": "ae39d3ea3eb8d2af471798908e6e468e5dce5229df40dd629fa024bb8eba10ca" + }, + { + "path": "baseline/three-day-gap-2/model-requests.jsonl", + "bytes": 63777, + "sha256": "d865d0adb25c6235f3b0aa1d65839bcae84332dd512ad050d1e2722a3969598d" + }, + { + "path": "baseline/three-day-gap-2/model-responses.jsonl", + "bytes": 6210, + "sha256": "b9c066214b32a50b75ad83a4c9fee09f601a4db3c0b39b17f79abe9c34f26e53" + }, + { + "path": "baseline/three-day-gap-2/summary.json", + "bytes": 2529, + "sha256": "f7edc9e458df994bd9c98e7ed925b70bbb423771b2636f68edfaeae8a0f6f395" + }, + { + "path": "baseline/three-day-gap-2/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/three-day-gap-2/tools.jsonl", + "bytes": 30626, + "sha256": "010163040360269f28c03b7e0e98776ea0b0ee80005ced200c07aaf495b61764" + }, + { + "path": "baseline/three-day-gap-2/transcript.json", + "bytes": 32182, + "sha256": "c60ae177bf0004388d511054b648ba45abb0cd66d395bc8abd0bc8860b015b88" + }, + { + "path": "baseline/three-day-gap-3/answer.txt", + "bytes": 1082, + "sha256": "df1e7a5fe1459374739b4f08426b5e673f9638b7555c9f85b7578e84ccbe8d99" + }, + { + "path": "baseline/three-day-gap-3/backend.jsonl", + "bytes": 105122, + "sha256": "aab0c282243cb3a98e9fbdda55286f3456964bf490e84c67a6f363b2a03b92e8" + }, + { + "path": "baseline/three-day-gap-3/initial-messages.json", + "bytes": 7652, + "sha256": "ae39d3ea3eb8d2af471798908e6e468e5dce5229df40dd629fa024bb8eba10ca" + }, + { + "path": "baseline/three-day-gap-3/model-requests.jsonl", + "bytes": 190314, + "sha256": "70a65abca9753c43473f70c6374350fcd24dbf692ca208a90b8a30fcea1f78ff" + }, + { + "path": "baseline/three-day-gap-3/model-responses.jsonl", + "bytes": 13275, + "sha256": "fbbf0ab397d61d027af24fee5ced34022a2f9a62c22915eae6d0caa9ed5c2f25" + }, + { + "path": "baseline/three-day-gap-3/summary.json", + "bytes": 3018, + "sha256": "bbb085321a057c12e89cee423261aaed669c7dde52e81dacbc40d8c4d34853d0" + }, + { + "path": "baseline/three-day-gap-3/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/three-day-gap-3/tools.jsonl", + "bytes": 89249, + "sha256": "40da6d38618d013ca8d604f86314116aa578b4979cad4635bd0c10b708ff020c" + }, + { + "path": "baseline/three-day-gap-3/transcript.json", + "bytes": 75855, + "sha256": "43e92ee8c2331d49862c3ae311c025752080dbf5490b56e222ce71c2e07dfc25" + }, + { + "path": "baseline/three-day-gap-followup-1/answer.txt", + "bytes": 156, + "sha256": "cbdc95a222c5f187f9549db9f3cb5fbf07afbfc89b9701d2d25fedd750b2fa2a" + }, + { + "path": "baseline/three-day-gap-followup-1/backend.jsonl", + "bytes": 1105, + "sha256": "cf76f2d7333837f728b68b92bbd662c2967f12981b88dc3064dfc16935c178b6" + }, + { + "path": "baseline/three-day-gap-followup-1/initial-messages.json", + "bytes": 8601, + "sha256": "b3c1acc83407f4527a94b4b2640a27fbf3494c020a6792cc50e3ea38653da110" + }, + { + "path": "baseline/three-day-gap-followup-1/model-requests.jsonl", + "bytes": 32760, + "sha256": "76f1d3a59aa956f67ebdca7d57aa168ceecd398b8c6fb7c890f36c4344aaecca" + }, + { + "path": "baseline/three-day-gap-followup-1/model-responses.jsonl", + "bytes": 3078, + "sha256": "1dfa9e0216a98b566dfa36f95aa8cc201a877085f3a9d3b4bc489b171d5385c1" + }, + { + "path": "baseline/three-day-gap-followup-1/seeded-tool.json", + "bytes": 1246, + "sha256": "242cd05c2a5d2a4f326caa7805415bdd65be0dadbbe5c3419b872ed46f35faaf" + }, + { + "path": "baseline/three-day-gap-followup-1/summary.json", + "bytes": 1148, + "sha256": "6ba4d3ffef76ac01e4a9b3ec5677e816e5cf964715b67c09c91abc410fe75d1b" + }, + { + "path": "baseline/three-day-gap-followup-1/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/three-day-gap-followup-1/tools.jsonl", + "bytes": 995, + "sha256": "bd9f438849a1bf057004dd8ba4df79e12d6bfe9ff20e7e68015e2acd9cea905c" + }, + { + "path": "baseline/three-day-gap-followup-1/transcript.json", + "bytes": 10078, + "sha256": "5ecbe177c436561cfeafcde2c2b4c349bc993b3db6c4d208ce4b55db13a9f673" + }, + { + "path": "baseline/three-day-gap-followup-2/answer.txt", + "bytes": 1633, + "sha256": "974dae96cc905597584209be5c50baf7c59e05da8a13576f9fc3948d03b08513" + }, + { + "path": "baseline/three-day-gap-followup-2/backend.jsonl", + "bytes": 4143, + "sha256": "ee62091ee28225c4ab38d52a8657cdd25aea715d6466235815ff28673a00eaa9" + }, + { + "path": "baseline/three-day-gap-followup-2/initial-messages.json", + "bytes": 8601, + "sha256": "b3c1acc83407f4527a94b4b2640a27fbf3494c020a6792cc50e3ea38653da110" + }, + { + "path": "baseline/three-day-gap-followup-2/model-requests.jsonl", + "bytes": 56266, + "sha256": "88cea2ae03f51f265384cac0011f8e07242c99c0f5546fea5dbb7dc54d7c7aa3" + }, + { + "path": "baseline/three-day-gap-followup-2/model-responses.jsonl", + "bytes": 5651, + "sha256": "4e6f9e6a384e0dd2d2059d91ede9df5649c635da3b616a7bf3d4b2c9e50c386c" + }, + { + "path": "baseline/three-day-gap-followup-2/seeded-tool.json", + "bytes": 1246, + "sha256": "242cd05c2a5d2a4f326caa7805415bdd65be0dadbbe5c3419b872ed46f35faaf" + }, + { + "path": "baseline/three-day-gap-followup-2/summary.json", + "bytes": 2779, + "sha256": "ad321373d40940244fee09e0da19ab89405e3d9acb6e0474044d040ee450d410" + }, + { + "path": "baseline/three-day-gap-followup-2/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/three-day-gap-followup-2/tools.jsonl", + "bytes": 17680, + "sha256": "e23ca45c367f8f85f6ed88847b5edd09a6e207bfedaef609b27afa1b98ad9c4c" + }, + { + "path": "baseline/three-day-gap-followup-2/transcript.json", + "bytes": 23613, + "sha256": "f606f892aa375267b63068e7da80f42c3a911cd8769dc8625e684d0df3839048" + }, + { + "path": "baseline/three-day-gap-followup-3/answer.txt", + "bytes": 402, + "sha256": "fd6700be6e22f2b2dc8efa5b0269e83cf7b8e05566a047b77ed4b6cd23cb9c8e" + }, + { + "path": "baseline/three-day-gap-followup-3/backend.jsonl", + "bytes": 928, + "sha256": "8ad254c896059530793cde687d4f5caedce7983adbb7b8ae72fe3cd779af883c" + }, + { + "path": "baseline/three-day-gap-followup-3/initial-messages.json", + "bytes": 8601, + "sha256": "b3c1acc83407f4527a94b4b2640a27fbf3494c020a6792cc50e3ea38653da110" + }, + { + "path": "baseline/three-day-gap-followup-3/model-requests.jsonl", + "bytes": 21608, + "sha256": "2824a1c6a94536615a1e46575b47845a0f0a0816a5005035409274a886d8833b" + }, + { + "path": "baseline/three-day-gap-followup-3/model-responses.jsonl", + "bytes": 2390, + "sha256": "42a61b880d9ee642b6e93511406d5b43b5b85b264851ce275a556f4363156c0c" + }, + { + "path": "baseline/three-day-gap-followup-3/seeded-tool.json", + "bytes": 1246, + "sha256": "242cd05c2a5d2a4f326caa7805415bdd65be0dadbbe5c3419b872ed46f35faaf" + }, + { + "path": "baseline/three-day-gap-followup-3/summary.json", + "bytes": 1259, + "sha256": "7933a1dafb0ca74100ccb6336f2d807da68235faeb1b01fc328769e19aef378e" + }, + { + "path": "baseline/three-day-gap-followup-3/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/three-day-gap-followup-3/tools.jsonl", + "bytes": 801, + "sha256": "aeaa25106f5bca5229032274bba16a6a8a3e27261428ae9c23178b9ca8dd9cad" + }, + { + "path": "baseline/three-day-gap-followup-3/transcript.json", + "bytes": 9913, + "sha256": "5b5a83632d669791101fd7188727819807bdf9bac0dd96f0a992cd43f9422f31" + }, + { + "path": "baseline/two-hour-gap-1/answer.txt", + "bytes": 732, + "sha256": "b296046cf3526fed326190c8c2b91c8db540cedc33940b783c317051076544ef" + }, + { + "path": "baseline/two-hour-gap-1/backend.jsonl", + "bytes": 4493, + "sha256": "e74837afc6f8057cc5888ab7144065ce8e9a4ea409d4c926078d7c8efc81594a" + }, + { + "path": "baseline/two-hour-gap-1/initial-messages.json", + "bytes": 7652, + "sha256": "daa66419650577fd8e6d9136135e1c547ee98c3c24075736f1903d15859d4a8a" + }, + { + "path": "baseline/two-hour-gap-1/model-requests.jsonl", + "bytes": 68382, + "sha256": "1f24857ff1337ff288f966e7bda63fd4d5e92767ef2b5f448adc01ded43d33a7" + }, + { + "path": "baseline/two-hour-gap-1/model-responses.jsonl", + "bytes": 6659, + "sha256": "e715c34a37f5aca5f39d58fcef6dd24e7991feda95c84e4de92517c3b7a0f75d" + }, + { + "path": "baseline/two-hour-gap-1/summary.json", + "bytes": 1969, + "sha256": "05dc9e08839a5f8d1b06ee8bc10b4a4de14e8c190a5ee3d1b586be6a0194b69b" + }, + { + "path": "baseline/two-hour-gap-1/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/two-hour-gap-1/tools.jsonl", + "bytes": 19785, + "sha256": "3f2011cbbdec5c08fac6d5d89455dd3375dd083b373acefe02262e03f1f226fc" + }, + { + "path": "baseline/two-hour-gap-1/transcript.json", + "bytes": 24213, + "sha256": "6fdcd0ece4bf8602fcead7364187600b1f229cea80eb9ed84c1fcf462b6813f5" + }, + { + "path": "baseline/two-hour-gap-2/answer.txt", + "bytes": 591, + "sha256": "41d0ccb33d37e558200ddad212158229c019944d97608681177dd6cf4716b5c7" + }, + { + "path": "baseline/two-hour-gap-2/backend.jsonl", + "bytes": 5846, + "sha256": "66f328fa86a1ce4fe113848a95f1e67f261623836568dfa7a2db8a9db4bb0de4" + }, + { + "path": "baseline/two-hour-gap-2/initial-messages.json", + "bytes": 7652, + "sha256": "daa66419650577fd8e6d9136135e1c547ee98c3c24075736f1903d15859d4a8a" + }, + { + "path": "baseline/two-hour-gap-2/model-requests.jsonl", + "bytes": 42191, + "sha256": "bb9b3db33f984054a1067546586dfca794789b3f657135a836b1079f7966a153" + }, + { + "path": "baseline/two-hour-gap-2/model-responses.jsonl", + "bytes": 4078, + "sha256": "658dca5c1c92575899c0a8414c50d2d8905cebc2bd98e3e81b40aae6bebe437a" + }, + { + "path": "baseline/two-hour-gap-2/summary.json", + "bytes": 1548, + "sha256": "ee3107e424e65dee2dce10d5820a614036fc4e8fd5bedbd2b9aa975c414e6d3d" + }, + { + "path": "baseline/two-hour-gap-2/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/two-hour-gap-2/tools.jsonl", + "bytes": 16587, + "sha256": "acabad4e6aef271503f09ff8736986da4848e9256f1a79d3a7b4c292d5497d27" + }, + { + "path": "baseline/two-hour-gap-2/transcript.json", + "bytes": 20987, + "sha256": "f11249279ca14339c2c76061a94515188696bce47d455f6710ad34c7804e342e" + }, + { + "path": "baseline/two-hour-gap-3/answer.txt", + "bytes": 1756, + "sha256": "f2f34767c6398491b8d7cd779573fe8206b34f8fc16c1d4ffd1864874bee8764" + }, + { + "path": "baseline/two-hour-gap-3/backend.jsonl", + "bytes": 27735, + "sha256": "b9203bc5ea22715acb9b958c63e794c1bf8bfa59d21741db15e0ea659f43a55d" + }, + { + "path": "baseline/two-hour-gap-3/initial-messages.json", + "bytes": 7652, + "sha256": "daa66419650577fd8e6d9136135e1c547ee98c3c24075736f1903d15859d4a8a" + }, + { + "path": "baseline/two-hour-gap-3/model-requests.jsonl", + "bytes": 81820, + "sha256": "a9d3b2ab797976f9f237e6a0e6551d5b89f7fd5937fdc7bbe3d309421f0b0982" + }, + { + "path": "baseline/two-hour-gap-3/model-responses.jsonl", + "bytes": 6558, + "sha256": "dd659dc4f957a7e43267f1353c2970a4a86990e020cf20e97da16e371ab0d7ea" + }, + { + "path": "baseline/two-hour-gap-3/summary.json", + "bytes": 2882, + "sha256": "9929b5e8ef77a303ca590f48e29a67c900193b906fbf8d9fd91239567386751d" + }, + { + "path": "baseline/two-hour-gap-3/tool-schemas.json", + "bytes": 2767, + "sha256": "f34273e2f44478702449d98b49a79be82707abee500537b0747821713052d7c3" + }, + { + "path": "baseline/two-hour-gap-3/tools.jsonl", + "bytes": 29283, + "sha256": "6dbf2e26afa8e8fd51c32c91d344c0ceb3f4fa6cf914ab69cb95a797ccdc67de" + }, + { + "path": "baseline/two-hour-gap-3/transcript.json", + "bytes": 31655, + "sha256": "ad40500e2880eb0b24a499bc55bbc7f99525dca28249ccaca8b9cc326c113689" + }, + { + "path": "candidate/fixture-preflight.json", + "bytes": 591444, + "sha256": "843f5be6191f9f56a0fec7b4b31a6104760ff54d4dd572d5898eadc896745090" + }, + { + "path": "candidate/healthy-zero-1/answer.txt", + "bytes": 1354, + "sha256": "d60408e8528bf44f71d61279a1db85f3df52bda01316ad6954ae65d330ea3eb4" + }, + { + "path": "candidate/healthy-zero-1/backend.jsonl", + "bytes": 2482, + "sha256": "59f01fae1ac94ad67c5d242c285b2f772302b601a7388d2f9f91c4be59c0e96e" + }, + { + "path": "candidate/healthy-zero-1/initial-messages.json", + "bytes": 11863, + "sha256": "1190f15455d1582dd5662a83a5891e76094356a8afa3e564d67eb7f865c52b32" + }, + { + "path": "candidate/healthy-zero-1/model-requests.jsonl", + "bytes": 75906, + "sha256": "b712c97c17d6671b325bb69c245597d9a4574624aa3f091379d0460085ad3f71" + }, + { + "path": "candidate/healthy-zero-1/model-responses.jsonl", + "bytes": 5855, + "sha256": "70d9eacfe07d0629e8aa7b2b22d97c33010db37c4c864599573a7c86b365e551" + }, + { + "path": "candidate/healthy-zero-1/summary.json", + "bytes": 2432, + "sha256": "734ed793fef0dd9c1e12668ee686d3fadfb51954506a514c2523cbe10af8a219" + }, + { + "path": "candidate/healthy-zero-1/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/healthy-zero-1/tools.jsonl", + "bytes": 11657, + "sha256": "a8263ccac8a1d72137bc494157b6c69f93b15a5ff07bfc9329fb8bb95c9b08a7" + }, + { + "path": "candidate/healthy-zero-1/transcript.json", + "bytes": 22471, + "sha256": "44d89563a8ca8bbbc474a1af1cfbbd69ec72d0501501aa9722ea32fb24112a07" + }, + { + "path": "candidate/healthy-zero-2/answer.txt", + "bytes": 1870, + "sha256": "06c388d61291230ac03689631a8c94bf80169fd60b5672ff31762dbf90107278" + }, + { + "path": "candidate/healthy-zero-2/backend.jsonl", + "bytes": 40816, + "sha256": "9a97b1dceb78a13da2a94268fe725e8168f5c9e7403553249e274106c6f66825" + }, + { + "path": "candidate/healthy-zero-2/initial-messages.json", + "bytes": 11863, + "sha256": "1190f15455d1582dd5662a83a5891e76094356a8afa3e564d67eb7f865c52b32" + }, + { + "path": "candidate/healthy-zero-2/model-requests.jsonl", + "bytes": 488836, + "sha256": "4d534fc882a69e0135c0be395677bb1b481831e320a0292a6cbc542de872f889" + }, + { + "path": "candidate/healthy-zero-2/model-responses.jsonl", + "bytes": 13629, + "sha256": "92b980c042de958a1a4977b3eed8485234eb43baf3f2ae5634069a2a30e33c40" + }, + { + "path": "candidate/healthy-zero-2/summary.json", + "bytes": 3803, + "sha256": "3ab5f107013eb48141e3d0be7a765cd13b497edb150009c2fc40930727169d92" + }, + { + "path": "candidate/healthy-zero-2/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/healthy-zero-2/tools.jsonl", + "bytes": 81671, + "sha256": "f184dd17b1981b0acaab5e0729f82bbb7c929d6cf4be771d4518f7a3cee83691" + }, + { + "path": "candidate/healthy-zero-2/transcript.json", + "bytes": 73730, + "sha256": "6b43775e07f3c916c317be9c421e26f995cc0dc8cd46e1a6aa712d5df7fc3f44" + }, + { + "path": "candidate/healthy-zero-3/answer.txt", + "bytes": 1010, + "sha256": "095f02cfb56ea737e09dc204887aea0657a9b2c2d419bed0d40c5f62b99c6879" + }, + { + "path": "candidate/healthy-zero-3/backend.jsonl", + "bytes": 2251, + "sha256": "45322f1b8aa09985a677444458d6504b5706c88485ca778a35a109b01bdbdfd0" + }, + { + "path": "candidate/healthy-zero-3/initial-messages.json", + "bytes": 11863, + "sha256": "1190f15455d1582dd5662a83a5891e76094356a8afa3e564d67eb7f865c52b32" + }, + { + "path": "candidate/healthy-zero-3/model-requests.jsonl", + "bytes": 57901, + "sha256": "ec0441c0c74e9a21de4f9d58bc09eca04f04fab80c92df841e3f5d7c99876655" + }, + { + "path": "candidate/healthy-zero-3/model-responses.jsonl", + "bytes": 4357, + "sha256": "22d8721d93a21577e4eb98521c03aee0865db58a4ac9b65f12a4d5a7b1e7b25b" + }, + { + "path": "candidate/healthy-zero-3/summary.json", + "bytes": 1948, + "sha256": "49b0e9187bc721c0d6d93f98af12e2c31971ca26e7236d1dce96b9043a6c2386" + }, + { + "path": "candidate/healthy-zero-3/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/healthy-zero-3/tools.jsonl", + "bytes": 11265, + "sha256": "bb891a2b55c83c7a57ea804675ddd1e2714255a4220335dd879fe840881d3fb4" + }, + { + "path": "candidate/healthy-zero-3/transcript.json", + "bytes": 21429, + "sha256": "fa03f664bda84d94969ef0210c47fd13cd4fe0822ba5e4ae8763a38d411a4368" + }, + { + "path": "candidate/history-unavailable-1/answer.txt", + "bytes": 1662, + "sha256": "dce95c920084618e2933f9c1961885d85b125c2f7914b14ba7b9f50038cb10a3" + }, + { + "path": "candidate/history-unavailable-1/backend.jsonl", + "bytes": 4240, + "sha256": "8c3fdffb09132251136649253d818ddb283618ccb4cca36de4263bf67a000c71" + }, + { + "path": "candidate/history-unavailable-1/initial-messages.json", + "bytes": 11883, + "sha256": "1b56dc098181825746ef091aa968e5a850edffce1b0d90d74e91c03e095c4d66" + }, + { + "path": "candidate/history-unavailable-1/model-requests.jsonl", + "bytes": 55530, + "sha256": "da993e7959e645577dce25c63ed77ade107b7eade79b6521af5a7acf3ea138a0" + }, + { + "path": "candidate/history-unavailable-1/model-responses.jsonl", + "bytes": 5047, + "sha256": "83266dd6aa8be621cdc3eafc0bce0af6854d7481dbb5933eab21e6f9a85817c5" + }, + { + "path": "candidate/history-unavailable-1/summary.json", + "bytes": 2594, + "sha256": "7097c061be2abacfb3d5b5c058056772117cba5ec8081f3c136af4db278381ed" + }, + { + "path": "candidate/history-unavailable-1/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/history-unavailable-1/tools.jsonl", + "bytes": 9027, + "sha256": "6c9a33d80b44379b88264a4916689d9ee9bcdb3c5720b6966e317065cae4d23c" + }, + { + "path": "candidate/history-unavailable-1/transcript.json", + "bytes": 19856, + "sha256": "439286e0380ad3d3b65b67b02d796a9a1e83d5f88408c86d117755b2da78750e" + }, + { + "path": "candidate/history-unavailable-2/answer.txt", + "bytes": 1129, + "sha256": "0bbbaccb8ecd51e9bc06720e38f3f61858bf4159da5c5b7ece3c4bc4cb90244f" + }, + { + "path": "candidate/history-unavailable-2/backend.jsonl", + "bytes": 4471, + "sha256": "db430707127781e028f215c73db0431ece05fe0fdf5a883e8a15f3b46e075586" + }, + { + "path": "candidate/history-unavailable-2/initial-messages.json", + "bytes": 11883, + "sha256": "1b56dc098181825746ef091aa968e5a850edffce1b0d90d74e91c03e095c4d66" + }, + { + "path": "candidate/history-unavailable-2/model-requests.jsonl", + "bytes": 73465, + "sha256": "346fcba1cb1b59b867860369399475bf3393643255e57fb6ac728416ce7f495c" + }, + { + "path": "candidate/history-unavailable-2/model-responses.jsonl", + "bytes": 5605, + "sha256": "44558591f696bb5809396935e3c15eb2c77dd5164a8a8e141c7c2f17270d6da1" + }, + { + "path": "candidate/history-unavailable-2/summary.json", + "bytes": 2189, + "sha256": "3deaca164a258b90c926990d8298114543f559726c4d81575a70547366184e5a" + }, + { + "path": "candidate/history-unavailable-2/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/history-unavailable-2/tools.jsonl", + "bytes": 9419, + "sha256": "975da20592c52832a4a9a54fb3267596d0111a9fc1890c0f158bec89ff32fc1e" + }, + { + "path": "candidate/history-unavailable-2/transcript.json", + "bytes": 19982, + "sha256": "eb7f72404bb44720a66e2929c1a6ebd09b4e6d90d39220ee60f301996122ac95" + }, + { + "path": "candidate/history-unavailable-3/answer.txt", + "bytes": 1039, + "sha256": "bf0de58b4cab9d2dce597de8dd41da8ae8a7d6311d40384423d4e1baef1c5d90" + }, + { + "path": "candidate/history-unavailable-3/backend.jsonl", + "bytes": 4469, + "sha256": "f775a8145dcde32a37fd60896969c3f8fa5f293b289b64615f13168e4923fbac" + }, + { + "path": "candidate/history-unavailable-3/initial-messages.json", + "bytes": 11883, + "sha256": "1b56dc098181825746ef091aa968e5a850edffce1b0d90d74e91c03e095c4d66" + }, + { + "path": "candidate/history-unavailable-3/model-requests.jsonl", + "bytes": 72581, + "sha256": "1a941856601cf72253f3b8ba2b15fe3efe6d2f9136e023d9f596ec1d63c5f0c9" + }, + { + "path": "candidate/history-unavailable-3/model-responses.jsonl", + "bytes": 5062, + "sha256": "a025ca4385008c4a6297021171ae4c3d352bd2555b76df9d95711c87da105733" + }, + { + "path": "candidate/history-unavailable-3/summary.json", + "bytes": 2090, + "sha256": "52d69346fe22ca18b4f1a00c1c79c59415ca6a9e26f861b18205551c964e8e22" + }, + { + "path": "candidate/history-unavailable-3/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/history-unavailable-3/tools.jsonl", + "bytes": 9416, + "sha256": "f7e0d1b3e5869d1b44d7159f08a5e330f5d6812a31871871a5b035439415de76" + }, + { + "path": "candidate/history-unavailable-3/transcript.json", + "bytes": 19451, + "sha256": "e49ebc674ff264e1fe3ad25f2e1c549e6c7afeff21f7744a4940094e9ffb6930" + }, + { + "path": "candidate/manifest.json", + "bytes": 2653, + "sha256": "89432df57a776bc6a3c3f60793322e1abdccb44c1afc583b30fdbadf52381669" + }, + { + "path": "candidate/reviews.json", + "bytes": 9957, + "sha256": "81214fdca2f12828c7ebce8654627f35f590f7de270ccdc53217455df533a7c0" + }, + { + "path": "candidate/source/helm/mosaic-stack/files/openclaw-seed/extensions/observability/grafana.ts", + "bytes": 20362, + "sha256": "4ccd76f10c41e86be37f28c1781075708c71b4c3f3308447949675c16b182236" + }, + { + "path": "candidate/source/helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts", + "bytes": 13426, + "sha256": "da773298a233edd6792e848dc4680d105065a135fe54bbe4e57a5a65bdfc0fd2" + }, + { + "path": "candidate/source/helm/mosaic-stack/files/openclaw-seed/extensions/observability/loki.ts", + "bytes": 9441, + "sha256": "4948c6b18e1dc657b3a0112ae6d4bb0c8620983075be6483d3e3898848ad7b46" + }, + { + "path": "candidate/source/helm/mosaic-stack/files/openclaw-seed/extensions/observability/metric-history.ts", + "bytes": 12812, + "sha256": "1f7f6a3994653dc76c1243a2ed1c6bea9dfa8d65f971d51b6331477277c65bc5" + }, + { + "path": "candidate/source/helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md", + "bytes": 11095, + "sha256": "089c6169447ed0b123f93b31512198797abe792bfa35b597862660d0cc0c02d2" + }, + { + "path": "candidate/summaries.json", + "bytes": 35899, + "sha256": "8004225eb8d08afb480f1c9fe36232e621a8d2c9e2207641a7781c3ab774b533" + }, + { + "path": "candidate/three-day-gap-1/answer.txt", + "bytes": 1156, + "sha256": "9dfa49849643cd961b4fd24adc33b9d87cf41d821c63dc676a3e1a2ed9883b2c" + }, + { + "path": "candidate/three-day-gap-1/backend.jsonl", + "bytes": 11249, + "sha256": "95c08989254237ebf7fd4a1407707765a1fea6a888398171befc00e3ddeda3c8" + }, + { + "path": "candidate/three-day-gap-1/initial-messages.json", + "bytes": 11883, + "sha256": "fa554f53c5a054290c54f98f5f73558a8048a570d2acfdf7fa65417266e43eb3" + }, + { + "path": "candidate/three-day-gap-1/model-requests.jsonl", + "bytes": 72548, + "sha256": "d618f691a6d33298c86b7602ca72c691c1e9b0c67eab028b19a1f8dcb76fffcb" + }, + { + "path": "candidate/three-day-gap-1/model-responses.jsonl", + "bytes": 5747, + "sha256": "2b77ddd4024fa3b0a10780944f71c218c7a031400f8bd542880cad6e70950ad3" + }, + { + "path": "candidate/three-day-gap-1/summary.json", + "bytes": 2255, + "sha256": "47a2acb8b9f14f54295aae523139270ecd863d4ff3f4d1bc3532a51effdabf4b" + }, + { + "path": "candidate/three-day-gap-1/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/three-day-gap-1/tools.jsonl", + "bytes": 7506, + "sha256": "854dd288b7ace043b580b75edd63ed83011a64784bfee33bfa00f084984ccec0" + }, + { + "path": "candidate/three-day-gap-1/transcript.json", + "bytes": 19010, + "sha256": "e8bc70a238576c5b23dad3c17ebdd31a73a2a81b5cdac7e166b3f4d47e850b5e" + }, + { + "path": "candidate/three-day-gap-2/answer.txt", + "bytes": 1344, + "sha256": "bd19789dd443f49613a9159b088b62ba4bf53bf3b25be998a76b8d21c9e2bb9d" + }, + { + "path": "candidate/three-day-gap-2/backend.jsonl", + "bytes": 11249, + "sha256": "95c08989254237ebf7fd4a1407707765a1fea6a888398171befc00e3ddeda3c8" + }, + { + "path": "candidate/three-day-gap-2/initial-messages.json", + "bytes": 11883, + "sha256": "fa554f53c5a054290c54f98f5f73558a8048a570d2acfdf7fa65417266e43eb3" + }, + { + "path": "candidate/three-day-gap-2/model-requests.jsonl", + "bytes": 72279, + "sha256": "0d73ecf402c370c9a22275465fe60ed861ebdcb8759d63761673c8922c1720b2" + }, + { + "path": "candidate/three-day-gap-2/model-responses.jsonl", + "bytes": 5789, + "sha256": "55acf89038545d09ff91abb9f24848babd2ea4f446eb8027ebf2785d5c9213bf" + }, + { + "path": "candidate/three-day-gap-2/summary.json", + "bytes": 2448, + "sha256": "30ac8e9095e6f8b19ac423b12600bab1343ab13ce35dc2d39cc122121dbf1828" + }, + { + "path": "candidate/three-day-gap-2/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/three-day-gap-2/tools.jsonl", + "bytes": 7506, + "sha256": "854dd288b7ace043b580b75edd63ed83011a64784bfee33bfa00f084984ccec0" + }, + { + "path": "candidate/three-day-gap-2/transcript.json", + "bytes": 19044, + "sha256": "df09abb82620bf125afe5fefde8ce50dfad742f24162eb72c585e76be49cabc6" + }, + { + "path": "candidate/three-day-gap-3/answer.txt", + "bytes": 1305, + "sha256": "889fb74dc9f979ccbc34376394b22a33b01f2ce46f62e9051cf53b3f5a957100" + }, + { + "path": "candidate/three-day-gap-3/backend.jsonl", + "bytes": 11711, + "sha256": "d35af28ff8beab601a88acd3bd99ce43db3f2cab6229d34ed44775861ec2dc33" + }, + { + "path": "candidate/three-day-gap-3/initial-messages.json", + "bytes": 11883, + "sha256": "fa554f53c5a054290c54f98f5f73558a8048a570d2acfdf7fa65417266e43eb3" + }, + { + "path": "candidate/three-day-gap-3/model-requests.jsonl", + "bytes": 91869, + "sha256": "fc876d2896c3407320b4d264a0a5beefe4b79fb0056ff46ec1a657b3da0e7ba1" + }, + { + "path": "candidate/three-day-gap-3/model-responses.jsonl", + "bytes": 6541, + "sha256": "27c82587342a6a24f7d21239b34ba7c4b6e22a01628c75f6c0996397e70cce61" + }, + { + "path": "candidate/three-day-gap-3/summary.json", + "bytes": 2538, + "sha256": "11011e48a3f5b939ffb19f0d2c4b9879fabb33890389ac33e364ca17ac25196c" + }, + { + "path": "candidate/three-day-gap-3/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/three-day-gap-3/tools.jsonl", + "bytes": 9543, + "sha256": "d147789d0d027933b1bf6b2750a74802f00434c3ebe7149ade5b3292c0dcf38a" + }, + { + "path": "candidate/three-day-gap-3/transcript.json", + "bytes": 20180, + "sha256": "e5f4fdd62dcce55cbe868b7f2d5dcf103b8e7b6cc432775ae6a28cbcab437ebb" + }, + { + "path": "candidate/three-day-gap-followup-1/answer.txt", + "bytes": 496, + "sha256": "ee8f2c6e8720f0c7efa7a2778841daf9fa4d103ef288078641ea4bbe837c4416" + }, + { + "path": "candidate/three-day-gap-followup-1/backend.jsonl", + "bytes": 2174, + "sha256": "64733a8d3f2e9fd62bfca821486aefe6bd59f16512b6a1af149e974bc6671430" + }, + { + "path": "candidate/three-day-gap-followup-1/initial-messages.json", + "bytes": 13512, + "sha256": "69f56da1771cfa2a22dea9c4cdc8593cbb8022767e54acce489681f4a8ecd1b4" + }, + { + "path": "candidate/three-day-gap-followup-1/model-requests.jsonl", + "bytes": 38821, + "sha256": "ed00f7e4a27b13e39b49b0c9f3a71d2cedd06c23e940cdc67faa94c98bf91436" + }, + { + "path": "candidate/three-day-gap-followup-1/model-responses.jsonl", + "bytes": 2485, + "sha256": "f5f426840e8f0a9d3f30505e930a20f293207295d287600e869d5721867dd5d8" + }, + { + "path": "candidate/three-day-gap-followup-1/seeded-tool.json", + "bytes": 2606, + "sha256": "a19c6ad9ea5fe18b1700a7ba42221ea5c55b04471103d97da02389d2a43fc29b" + }, + { + "path": "candidate/three-day-gap-followup-1/summary.json", + "bytes": 1353, + "sha256": "fd4673a65b5f4a61bf159989f164447a872ebca40f42804cf0ccfe4bde69e8a5" + }, + { + "path": "candidate/three-day-gap-followup-1/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/three-day-gap-followup-1/tools.jsonl", + "bytes": 5480, + "sha256": "4af604d166c63261f767e272a9a06f80720b1ca505029886d0b89e7e3437fac9" + }, + { + "path": "candidate/three-day-gap-followup-1/transcript.json", + "bytes": 17547, + "sha256": "3c8a385879e6a2d4eda2981a13e416aba1f64a911eae9dab278dded77056f6aa" + }, + { + "path": "candidate/three-day-gap-followup-2/answer.txt", + "bytes": 1636, + "sha256": "1e553ec1df1245c90e54d09e78aaae7e0f1ed3b0db27de49738eedf6d9bfa504" + }, + { + "path": "candidate/three-day-gap-followup-2/backend.jsonl", + "bytes": 2174, + "sha256": "64733a8d3f2e9fd62bfca821486aefe6bd59f16512b6a1af149e974bc6671430" + }, + { + "path": "candidate/three-day-gap-followup-2/initial-messages.json", + "bytes": 13512, + "sha256": "69f56da1771cfa2a22dea9c4cdc8593cbb8022767e54acce489681f4a8ecd1b4" + }, + { + "path": "candidate/three-day-gap-followup-2/model-requests.jsonl", + "bytes": 38821, + "sha256": "e182276afd606bc29261541dc2df7207e54155c942401a4ba4be1e5663dfdea7" + }, + { + "path": "candidate/three-day-gap-followup-2/model-responses.jsonl", + "bytes": 3675, + "sha256": "e64fcc6a96613bdce09dec055078972c87c090233999e75538f1d837f8540917" + }, + { + "path": "candidate/three-day-gap-followup-2/seeded-tool.json", + "bytes": 2606, + "sha256": "a19c6ad9ea5fe18b1700a7ba42221ea5c55b04471103d97da02389d2a43fc29b" + }, + { + "path": "candidate/three-day-gap-followup-2/summary.json", + "bytes": 2514, + "sha256": "3aa558d947dfed59a61b81dd36059c21329a817a3ea35997a97f729d01dac37b" + }, + { + "path": "candidate/three-day-gap-followup-2/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/three-day-gap-followup-2/tools.jsonl", + "bytes": 5480, + "sha256": "4af604d166c63261f767e272a9a06f80720b1ca505029886d0b89e7e3437fac9" + }, + { + "path": "candidate/three-day-gap-followup-2/transcript.json", + "bytes": 18709, + "sha256": "4bcff9c3eefc182568b70f358eba3d922daeaf19979d0884d137c8c8933142ea" + }, + { + "path": "candidate/three-day-gap-followup-3/answer.txt", + "bytes": 518, + "sha256": "e536fc7018477568599ac5b741bca8d5972e0e91ac3d830d22b65c0a90a5386d" + }, + { + "path": "candidate/three-day-gap-followup-3/backend.jsonl", + "bytes": 11092, + "sha256": "74f3962bf1c1d6a5c00f671e82e335bd0f0804518a4785b551f7c949c661cc07" + }, + { + "path": "candidate/three-day-gap-followup-3/initial-messages.json", + "bytes": 13512, + "sha256": "69f56da1771cfa2a22dea9c4cdc8593cbb8022767e54acce489681f4a8ecd1b4" + }, + { + "path": "candidate/three-day-gap-followup-3/model-requests.jsonl", + "bytes": 39483, + "sha256": "6687d190544d5f38c95b98d4248ad80f64ebef684bbba00781ed670ff8be02e9" + }, + { + "path": "candidate/three-day-gap-followup-3/model-responses.jsonl", + "bytes": 2530, + "sha256": "b7d1b8be6936b7f7aeb312b53a6ec3d5f815af342be559f0643067714525a736" + }, + { + "path": "candidate/three-day-gap-followup-3/seeded-tool.json", + "bytes": 2606, + "sha256": "a19c6ad9ea5fe18b1700a7ba42221ea5c55b04471103d97da02389d2a43fc29b" + }, + { + "path": "candidate/three-day-gap-followup-3/summary.json", + "bytes": 1384, + "sha256": "2538c332f08f0e6891580348d3cb3b27bc1726646863e0ccaba18da6083ec14c" + }, + { + "path": "candidate/three-day-gap-followup-3/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/three-day-gap-followup-3/tools.jsonl", + "bytes": 6566, + "sha256": "df4f5f5e0be091ed52f70661fff7140066686108a8a8c7fd14288ca9036d1bdb" + }, + { + "path": "candidate/three-day-gap-followup-3/transcript.json", + "bytes": 18240, + "sha256": "2d4c7ee89fc7c47cfb3ac1ddc0b611d8455114820c8a1e4c665b0c91bb2567b2" + }, + { + "path": "candidate/two-hour-gap-1/answer.txt", + "bytes": 1029, + "sha256": "3597b2c55c1431bcd2fc61ce4346fdb9a4b332766480714878d7ceb75d0aa87e" + }, + { + "path": "candidate/two-hour-gap-1/backend.jsonl", + "bytes": 7565, + "sha256": "0cc7be36ca900ff4c0309210a4ac2ed3c448e58a75b6a44c27f9d9273ba59abe" + }, + { + "path": "candidate/two-hour-gap-1/initial-messages.json", + "bytes": 11883, + "sha256": "b23b28346c618fb2fe92ebee7b95ff9a9ca8480ad4c7f550827b9d9f1cc0d01b" + }, + { + "path": "candidate/two-hour-gap-1/model-requests.jsonl", + "bytes": 71373, + "sha256": "48b0c2cc80e14b4bf6f5a113d49d52d774c2fb42ce8d368f0c0f33aaa8a8c79d" + }, + { + "path": "candidate/two-hour-gap-1/model-responses.jsonl", + "bytes": 5582, + "sha256": "12252f9e8b8fe1c78ef38b7547eec7a38c54448335522231102c70cd9a4f117d" + }, + { + "path": "candidate/two-hour-gap-1/summary.json", + "bytes": 2133, + "sha256": "21f99a3cdc4be2d5b4c3d7df998f1242ed3f68196e66f56aed17246081f12831" + }, + { + "path": "candidate/two-hour-gap-1/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/two-hour-gap-1/tools.jsonl", + "bytes": 5704, + "sha256": "7fb1b106a31d3402961fb9186943247f4a683ecdd90670a6ae46476af171ac6c" + }, + { + "path": "candidate/two-hour-gap-1/transcript.json", + "bytes": 17755, + "sha256": "f4e9198f67ca59fed28a316ae2f108fa57f8c0a6354ba2a91e35b166c399649e" + }, + { + "path": "candidate/two-hour-gap-2/answer.txt", + "bytes": 1241, + "sha256": "ce2b02052ebb3c0030b20c08ea01e911c1588f39313743136855c2c58d9c7f2e" + }, + { + "path": "candidate/two-hour-gap-2/backend.jsonl", + "bytes": 8031, + "sha256": "171daa6f52aa9dad145ef16cb46a2454231d67d59f9cebf1bef990bacbaee04d" + }, + { + "path": "candidate/two-hour-gap-2/initial-messages.json", + "bytes": 11883, + "sha256": "b23b28346c618fb2fe92ebee7b95ff9a9ca8480ad4c7f550827b9d9f1cc0d01b" + }, + { + "path": "candidate/two-hour-gap-2/model-requests.jsonl", + "bytes": 91527, + "sha256": "f606808a2b4f4a3de7a9a259302b028f19b62093d40d82ba5d358a32bb0af3b6" + }, + { + "path": "candidate/two-hour-gap-2/model-responses.jsonl", + "bytes": 6863, + "sha256": "82ef62d782b0b556448398073ee195578a039d23a865e3f59131c50eb9a79014" + }, + { + "path": "candidate/two-hour-gap-2/summary.json", + "bytes": 2481, + "sha256": "1b2974affc6d1fcac7c3fa9129d2b2a943f913707f90b5e689400aa2eef08bfb" + }, + { + "path": "candidate/two-hour-gap-2/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/two-hour-gap-2/tools.jsonl", + "bytes": 7745, + "sha256": "7dd43721a31a0c991e94c52a8c1b84b584ac9ce76b5e9141e568f5d1d597862b" + }, + { + "path": "candidate/two-hour-gap-2/transcript.json", + "bytes": 19412, + "sha256": "99601cc66894d2bdbb1307f4000a16fb32c336dad4d0278af94e86ac9deff4ae" + }, + { + "path": "candidate/two-hour-gap-3/answer.txt", + "bytes": 1111, + "sha256": "1834f09ae2e211d64005e1bc5655330c03f2216aea5754d834c25ffaa552560b" + }, + { + "path": "candidate/two-hour-gap-3/backend.jsonl", + "bytes": 7565, + "sha256": "0cc7be36ca900ff4c0309210a4ac2ed3c448e58a75b6a44c27f9d9273ba59abe" + }, + { + "path": "candidate/two-hour-gap-3/initial-messages.json", + "bytes": 11883, + "sha256": "b23b28346c618fb2fe92ebee7b95ff9a9ca8480ad4c7f550827b9d9f1cc0d01b" + }, + { + "path": "candidate/two-hour-gap-3/model-requests.jsonl", + "bytes": 71338, + "sha256": "f53abd4d280b52bc8ee623cf33b70c407cb98b5598612ebcfd6e1a850fce5f65" + }, + { + "path": "candidate/two-hour-gap-3/model-responses.jsonl", + "bytes": 5625, + "sha256": "a9ab2b589bac0d7ca23755e9b00e76c4f729f8f589a92bd83fcd53fef68f8a6f" + }, + { + "path": "candidate/two-hour-gap-3/summary.json", + "bytes": 2213, + "sha256": "c323981f9bbbe29b1bb6bf924ded12861d44aa44f7ed727bb4edc24e35ea70f5" + }, + { + "path": "candidate/two-hour-gap-3/tool-schemas.json", + "bytes": 5850, + "sha256": "b898787e4883f81874db544e916e4039435d50494e17d312b4e64d266011fddf" + }, + { + "path": "candidate/two-hour-gap-3/tools.jsonl", + "bytes": 5704, + "sha256": "7fb1b106a31d3402961fb9186943247f4a683ecdd90670a6ae46476af171ac6c" + }, + { + "path": "candidate/two-hour-gap-3/transcript.json", + "bytes": 17801, + "sha256": "d90e48b4fe51c882c146f2c3b83d46cb85b758a40ddf813f225b0e0cc36fcba8" + }, + { + "path": "comparison.json", + "bytes": 89631, + "sha256": "8f79cd5a8e374c5af3f40f73b4090d64913df9179ed62e0a05d44e422fd0a0a8" + }, + { + "path": "reconciliation.json", + "bytes": 5122, + "sha256": "08820c61cb058681fa3ba00df6132d5ea28090f098131758615eec5959461099" + }, + { + "path": "result-summary.json", + "bytes": 3198, + "sha256": "ffeae049b67c59c696dd5d850df85852f9c29313be12a0323c2f85ed8101e7ab" + }, + { + "path": "server-record.json", + "bytes": 5784, + "sha256": "a7ab5582799c9b633c0b4924eeec847c474a7f208a91735cd902f5383178b8a6" + } + ] + }, + "included": "All 30 canonical summaries, complete final answers, every per-turn token usage record, review reasons, both smoke outcomes, and the transport-invalid attempt.", + "retained_locally": "Full model requests/responses, transcripts, real tool/backend results, source snapshots, fixture and serving logs, and original deployment recovery evidence.", + "not_published": "Deployment-specific host addresses, local workspace paths, original production-container identity and recovery records. No credentials were included in these benchmark outputs." + } +} diff --git a/helm/mosaic-stack/tests/metric-history-results-nemotron-3-super-20261109.md b/helm/mosaic-stack/tests/metric-history-results-nemotron-3-super-20261109.md new file mode 100644 index 00000000..e1aa8983 --- /dev/null +++ b/helm/mosaic-stack/tests/metric-history-results-nemotron-3-super-20261109.md @@ -0,0 +1,97 @@ +# Nemotron 3 Super NVFP4 missing-metric backtest + +Observed on 2026-11-09 UTC. This is the five-case isolated model/tool replay for PR #104, not a general model benchmark or native OpenClaw acceptance test. + +The candidate improves evidence retrieval, but the complete answers remain unreliable. Under the pre-run rubric, baseline passes **0/15** and candidate **3/15**. Correct original last-sample times appear in **0/9** baseline gap answers and **7/9** candidate gap answers. These are different measurements: finding a correct timestamp does not establish a correct final conclusion. + +The source-sample helper is useful; this result does **not** justify closing #103. Follow-up backtracking, known-cadence handling, and factual output consistency still need native-runtime work. + +## Exact experiment + +| Field | Value | +| --- | --- | +| Model | `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4` | +| Weights revision | `ff433f5493e25d631c9f12b5d55c674229923d02` | +| Precision | Unchanged ModelOpt mixed policy: 40,961 NVFP4 and 139 FP8 entries; FP8 KV cache | +| Hardware / serving | DGX Spark GB10; driver 580.173.02; Linux ARM64; vLLM 0.27.1; Torch 2.13.0+cu130 | +| ARM64 image | `vllm/vllm-openai@sha256:1c8e60a0841b333c700488cb029d3664807249da0c071e862191b00fe34b228c` | +| Baseline | `307afa7e485a334389b4f2c8c68128fbf84ff007` | +| Candidate | `8305d59e44470e10d268a4e60fc36870929b2e27` | +| Protocol | Checkpoint-pinned tokenizer/template and `super_v3` reasoning parser; `qwen3_coder` tool parser | +| Generation | Temperature 1.0; top_p 0.95; thinking explicitly disabled; seeds 1, 2, 3 | +| Budgets | 24 model turns; 8,192 generated tokens per turn; 180,000 ms per request | +| Capacity | One serial replay and one serving sequence; 262,144-token context; TP=PP=DP=1 | +| Cache | Normal request behavior; no request salt. This server reports prefix caching disabled | +| Data | Real Prometheus 3.16.0; all 30,121 synthetic original samples validated; fixed investigation time 2026-11-06T00:00:00Z | +| Coverage | Five cases × three seeds × two revisions = 30 canonical model outcomes | + +The model is **Nemotron 3 Super**, not the different 3.5 Super VL checkpoint in the current AFOA Super profile. Thinking-disabled settings mirror that profile choice; they do not measure thinking-enabled performance. The customer's actual model was not established by this experiment. + +The two legacy environment variables `VLLM_NVFP4_GEMM_BACKEND` and `VLLM_USE_FLASHINFER_MOE_FP4` were reported as unknown by this image. Effective kernels were FlashInfer/Cutlass NVFP4 linear operations and Marlin MoE, as recorded in the full serving record. Quantization was automatically detected as `modelopt_mixed`; weights were not converted. + +## Review method and limitations + +Codex checked each complete answer and its tool evidence against the [pre-run rubric](metric-history-replay.md#review-every-answer-against-its-evidence). This is not a blinded human evaluation. Pass requires the required case finding without a material contradiction or unsupported conclusion elsewhere in the answer. All answers and reasons are retained in the accompanying JSON for independent review. + +For the zero-valued series, 2/3 baseline and 3/3 candidate answers recognized presence. Their complete-answer verdicts still fail because they asserted idle/no-workload state or collector health from utilization alone; one baseline answer actually inferred collection failure from zeros. These distinct findings must not be collapsed into a single claim that the model cannot recognize zero. + +Candidate three-day-gap seed 2 is accepted with a documented precision note: both boundary timestamps and actual search coverage are correct, but its approximate duration uses sample age (three days one minute), one cadence interval longer than time since the first expected absence. The helper call omitted the optional cadence; the one-minute estimate is instead supported by the cadence supplied in the system prompt. Other gap failures either omit that distinction or contain materially wrong dates, durations, coverage, or limits. + +Each seed is one observation, not a statistical guarantee. Matching seeds do not guarantee bitwise identical outputs. The suite explicitly loads the skill and real plugin with only SDK registration stubbed; it does not exercise automatic skill selection, full native prompts, session routing, UI, or final-output enforcement. No serving/cache change is proposed by these results. + +## Every canonical outcome + +Durations are milliseconds as recorded by the runner, not rounded estimates. B = baseline, C = candidate. Token columns sum all model turns; the JSON retains each individual usage object. + +| Side | Case | Seed | Stop | Turns | Tool calls | Tool errors | Backend requests | Elapsed ms | Prompt tokens | Completion tokens | Whole-answer review | +| --- | --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| baseline | three-day-gap | 1 | stop | 22 | 21 | 4 | 19 | 260086.234375 | 112292 | 3126 | fail | +| baseline | three-day-gap | 2 | stop | 4 | 3 | 0 | 3 | 71130.164292 | 17827 | 1005 | fail | +| baseline | three-day-gap | 3 | stop | 10 | 9 | 0 | 9 | 121235.46570799999 | 53543 | 1487 | fail | +| baseline | history-unavailable | 1 | stop | 13 | 12 | 0 | 12 | 173526.92987500003 | 70663 | 2195 | fail | +| baseline | history-unavailable | 2 | stop | 4 | 3 | 0 | 3 | 54876.97199999995 | 11367 | 791 | fail | +| baseline | history-unavailable | 3 | stop | 10 | 9 | 0 | 9 | 143930.376583 | 35512 | 1996 | fail | +| baseline | healthy-zero | 1 | stop | 6 | 5 | 0 | 5 | 76637.89241700002 | 20916 | 1040 | fail | +| baseline | healthy-zero | 2 | stop | 3 | 2 | 0 | 2 | 37269.4169999999 | 11540 | 492 | fail | +| baseline | healthy-zero | 3 | stop | 3 | 2 | 0 | 2 | 53659.87624999997 | 11538 | 765 | fail | +| baseline | two-hour-gap | 1 | stop | 5 | 4 | 0 | 4 | 57634.843125000014 | 18529 | 767 | fail | +| baseline | two-hour-gap | 2 | stop | 3 | 2 | 0 | 2 | 38191.36337500007 | 11589 | 509 | fail | +| baseline | two-hour-gap | 3 | stop | 4 | 3 | 0 | 3 | 83473.820084 | 23712 | 1154 | fail | +| baseline | three-day-gap-followup | 1 | stop | 3 | 2 | 0 | 3 | 14951.630125000025 | 8555 | 162 | fail | +| baseline | three-day-gap-followup | 2 | stop | 4 | 3 | 0 | 4 | 53945.77766600007 | 15348 | 744 | fail | +| baseline | three-day-gap-followup | 3 | stop | 2 | 1 | 0 | 2 | 18542.130292000016 | 5697 | 247 | fail | +| candidate | three-day-gap | 1 | stop | 4 | 3 | 0 | 7 | 60741.11075 | 18673 | 821 | fail | +| candidate | three-day-gap | 2 | stop | 4 | 3 | 0 | 7 | 62153.414958 | 18605 | 840 | pass | +| candidate | three-day-gap | 3 | stop | 5 | 4 | 0 | 8 | 65861.58895800001 | 23542 | 850 | fail | +| candidate | history-unavailable | 1 | stop | 3 | 2 | 0 | 10 | 56609.450709 | 14634 | 793 | pass | +| candidate | history-unavailable | 2 | stop | 4 | 3 | 0 | 11 | 53374.12275000004 | 19102 | 699 | pass | +| candidate | history-unavailable | 3 | stop | 4 | 3 | 0 | 11 | 41847.43212499999 | 18824 | 497 | fail | +| candidate | healthy-zero | 1 | stop | 4 | 3 | 0 | 3 | 56204.36637500004 | 19382 | 740 | fail | +| candidate | healthy-zero | 2 | stop | 10 | 9 | 0 | 9 | 172472.60320800002 | 143220 | 1459 | fail | +| candidate | healthy-zero | 3 | stop | 3 | 2 | 0 | 2 | 43697.91341600008 | 14900 | 565 | fail | +| candidate | two-hour-gap | 1 | stop | 4 | 3 | 0 | 4 | 61339.434583000024 | 18156 | 806 | fail | +| candidate | two-hour-gap | 2 | stop | 5 | 4 | 0 | 5 | 71659.88299999991 | 23257 | 927 | fail | +| candidate | two-hour-gap | 3 | stop | 4 | 3 | 0 | 4 | 60770.42316699994 | 18141 | 819 | fail | +| candidate | three-day-gap-followup | 1 | stop | 2 | 1 | 0 | 5 | 24349.793666000012 | 10290 | 304 | fail | +| candidate | three-day-gap-followup | 2 | stop | 2 | 1 | 0 | 5 | 45494.16991699999 | 10290 | 662 | fail | +| candidate | three-day-gap-followup | 3 | stop | 2 | 1 | 0 | 6 | 27443.519417000003 | 10619 | 348 | fail | + +## Transport reconciliation and smoke outcomes + +The original formal baseline three-day-gap seed-1 attempt ended with `TypeError: fetch failed` after two successful tool calls. It has no final answer and is not an accuracy zero. The endpoint remained healthy; the remaining runs proceeded. One same-seed retry completed with 21 tool calls and four tool errors, then gave an incorrect September onset. That valid model failure is the canonical result. The original failed attempt remains in the JSON. No low-quality answer was retried to improve the score. + +Both smoke replays completed and invoked tools, but both failed factual review. They are outside the formal denominator. + +| Stage | Side | Case | Seed | Stop | Turns | Tool calls | Tool errors | Backend requests | Elapsed ms | Review | +| --- | --- | --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | --- | +| smoke | baseline | three-day-gap-followup | 1 | stop | 5 | 4 | 0 | 5 | 64337.828041999994 | fail | +| smoke | candidate | three-day-gap-followup | 1 | stop | 2 | 1 | 0 | 5 | 23275.22475 | fail | +| formal-attempt-1 | baseline | three-day-gap | 1 | replay_error | 2 | 2 | 0 | 2 | 34881.631458 | not_assessed | + +## Complete review record + +[metric-history-results-nemotron-3-super-20261109.json](metric-history-results-nemotron-3-super-20261109.json) contains all 30 full final answers and reviews, all per-turn token usage, both smoke answers, the failed transport attempt, both source file-hash inventories, every checkpoint-file integrity result, the exact serving command/settings, full timing/tool-call distributions and the retained-artifact hash manifest. + +It is a compact run record, not the full request/response rollout. Full requests, backend/tool results and source snapshots remain retained externally; deployment-specific addresses, workspace paths and recovery details are not published. + +Reproduce with [metric-history-replay.md](metric-history-replay.md), the exact pins/settings above, and the generated fixture. The source revision with the recorded results adds documentation only; the tested candidate code remains the pinned commit listed above.