From eb00ad75f8ae663850f888bd558decb825bbba31 Mon Sep 17 00:00:00 2001 From: Vjeran Grozdanic Date: Fri, 9 Oct 2026 14:27:04 +0200 Subject: [PATCH 1/2] feat(ai): Build LLM cache shadow reports --- .../issues/llm_cache_detection/query.py | 59 ++++++++++++++ .../issues/llm_cache_detection/reporting.py | 80 +++++++++++++++++++ .../issues/llm_cache_detection/test_query.py | 49 ++++++++++++ .../llm_cache_detection/test_reporting.py | 73 +++++++++++++++++ 4 files changed, 261 insertions(+) create mode 100644 src/sentry/ai_monitoring/issues/llm_cache_detection/reporting.py create mode 100644 tests/sentry/ai_monitoring/issues/llm_cache_detection/test_reporting.py diff --git a/src/sentry/ai_monitoring/issues/llm_cache_detection/query.py b/src/sentry/ai_monitoring/issues/llm_cache_detection/query.py index 77ce6d11ad70..6f99d8bb5eb9 100644 --- a/src/sentry/ai_monitoring/issues/llm_cache_detection/query.py +++ b/src/sentry/ai_monitoring/issues/llm_cache_detection/query.py @@ -58,6 +58,7 @@ # Order by input tokens so the highest-volume groups fit when the query reaches # its limit. Agent operation groups may be folded after this limit is applied. CALL_SITE_GROUPS_LIMIT = 300 +SAMPLE_CALLS_LIMIT = 3 class DroppedRowReason(StrEnum): @@ -75,6 +76,18 @@ class CallSiteQueryResult: truncated: bool +@dataclass(frozen=True) +class SampleCall: + """Trace metadata and token counts for one example call.""" + + trace_id: str + span_id: str + timestamp: str + input_tokens: float + cache_read_tokens: float + cache_creation_tokens: float + + def _build_group_filter(stats: CallSiteStats) -> str | None: """Build an exact-match filter for a call site, if its values are expressible.""" agent_attribute = ( @@ -276,3 +289,49 @@ def count_spans_with_cache_attributes( ) data = result.get("data", []) return int(data[0].get(COUNT) or 0) if data else 0 + + +def fetch_sample_calls( + project: Project, stats: CallSiteStats, window: DetectionWindow +) -> list[SampleCall] | None: + """Return the largest call from up to three traces, without prompt content.""" + group_filter = _build_group_filter(stats) + if group_filter is None: + return None + result = _run_spans_query( + project, + window, + query_string=group_filter, + selected_columns=[ + "trace", + "id", + "timestamp", + INPUT_TOKENS, + *CACHE_TOKEN_ATTRIBUTES, + ], + orderby=[f"-{INPUT_TOKENS}"], + limit=SAMPLE_CALLS_LIMIT * 3, + ) + + samples: list[SampleCall] = [] + seen_trace_ids: set[str] = set() + for row in result.get("data", []): + trace_id = row.get("trace") + span_id = row.get("id") + timestamp = row.get("timestamp") + if not trace_id or not span_id or not timestamp or trace_id in seen_trace_ids: + continue + seen_trace_ids.add(trace_id) + samples.append( + SampleCall( + trace_id=trace_id, + span_id=span_id, + timestamp=timestamp, + input_tokens=_token_count(row, INPUT_TOKENS), + cache_read_tokens=_token_count(row, CACHE_READ_TOKENS), + cache_creation_tokens=_token_count(row, CACHE_CREATION_TOKENS), + ) + ) + if len(samples) == SAMPLE_CALLS_LIMIT: + break + return samples diff --git a/src/sentry/ai_monitoring/issues/llm_cache_detection/reporting.py b/src/sentry/ai_monitoring/issues/llm_cache_detection/reporting.py new file mode 100644 index 000000000000..5f710b46e286 --- /dev/null +++ b/src/sentry/ai_monitoring/issues/llm_cache_detection/reporting.py @@ -0,0 +1,80 @@ +"""Structured shadow reports for LLM prompt-cache detection.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from enum import StrEnum +from typing import Any + +from sentry.ai_monitoring.issues.llm_cache_detection.detection import ( + CacheFinding, + CallSiteWarmth, + Classification, + ProbeGap, + min_cacheable_prefix_tokens, +) +from sentry.ai_monitoring.issues.llm_cache_detection.pricing import PricingGap, SavingsEstimate +from sentry.ai_monitoring.issues.llm_cache_detection.query import SampleCall + + +class Disposition(StrEnum): + """What issue creation would do with a finding.""" + + WOULD_CREATE = "would_create" + OVER_CAP = "over_cap" + + +@dataclass(frozen=True) +class CandidateReport: + """A provisional candidate and its final shadow result.""" + + finding: CacheFinding + initial: Classification + rank: int + disposition: Disposition | None = None + pricing: SavingsEstimate | PricingGap | None = None + sample_calls: list[SampleCall] | ProbeGap | None = None + + +def _candidate_fields(report: CandidateReport) -> dict[str, Any]: + finding = report.finding + stats = finding.stats + fields: dict[str, Any] = { + "rank": report.rank, + "severity": finding.severity, + "initial_outcome": report.initial.outcome.value, + "initial_reason": report.initial.reason.value, + "outcome": finding.outcome.value, + "reason": finding.classification.reason.value, + "disposition": report.disposition.value if report.disposition else None, + **asdict(stats), + "hit_rate": stats.hit_rate, + "write_read_ratio": stats.write_read_ratio, + "uncached_tokens": stats.uncached_tokens, + "unrecouped_cache_write_tokens": stats.unrecouped_cache_write_tokens, + "cache_exceeds_input": stats.cache_exceeds_input, + "min_cacheable_prefix_tokens": min_cacheable_prefix_tokens(stats.model), + } + + for name, measurement in ( + ("warmth", finding.warmth), + ("cache_presence", finding.spans_with_cache_attributes), + ("pricing", report.pricing), + ("sample_calls", report.sample_calls), + ): + if isinstance(measurement, StrEnum): + fields[f"{name}_gap"] = measurement.value + + if isinstance(finding.warmth, CallSiteWarmth): + fields.update( + asdict(finding.warmth), + cacheable_share=finding.warmth.cacheable_share, + long_ttl_cacheable_share=finding.warmth.long_ttl_cacheable_share, + ) + if isinstance(finding.spans_with_cache_attributes, int): + fields["spans_with_cache_attributes"] = finding.spans_with_cache_attributes + if isinstance(report.pricing, SavingsEstimate): + fields.update(asdict(report.pricing)) + if isinstance(report.sample_calls, list): + fields["sample_calls"] = [asdict(sample) for sample in report.sample_calls] + return fields diff --git a/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_query.py b/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_query.py index 0a0a0575dfe9..ffb09a5ad960 100644 --- a/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_query.py +++ b/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_query.py @@ -19,8 +19,10 @@ CALL_SITE_GROUPS_LIMIT, COUNT, COUNT_SAMPLE, + INPUT_TOKENS, MODEL, OPERATION_NAME, + SAMPLE_CALLS_LIMIT, SPAN_NAME, SUM_CACHE_CREATION_TOKENS, SUM_CACHE_READ_TOKENS, @@ -31,6 +33,7 @@ count_spans_with_cache_attributes, fetch_call_site_stats, fetch_call_site_warmth, + fetch_sample_calls, ) from sentry.models.project import Project from sentry.search.events.types import SnubaRow @@ -215,6 +218,52 @@ def test_count_spans_with_cache_attributes() -> None: assert f"(has:{CACHE_READ_TOKENS} OR has:{CACHE_CREATION_TOKENS})" in kwargs["query_string"] +def test_fetch_sample_calls_returns_one_call_per_trace() -> None: + project = mock.Mock(spec=Project) + window = DetectionWindow( + start=datetime(2026, 1, 1, tzinfo=UTC), + end=datetime(2026, 1, 8, tzinfo=UTC), + ) + stats = _to_call_sites([make_row()])[0][0] + rows = [ + { + "trace": "trace-a", + "id": "span-1", + "timestamp": "2026-01-07T00:00:00Z", + INPUT_TOKENS: 4_000, + CACHE_READ_TOKENS: 100, + }, + {"trace": "trace-a", "id": "span-2", "timestamp": "2026-01-06T00:00:00Z"}, + {"trace": "trace-b", "id": "span-3", "timestamp": "2026-01-05T00:00:00Z"}, + {"trace": "trace-c", "id": "span-4"}, + ] + + with mock.patch.object( + query_module, "_run_spans_query", return_value={"data": rows} + ) as run_query: + samples = fetch_sample_calls(project, stats, window) + + assert samples is not None + assert [(sample.trace_id, sample.span_id) for sample in samples] == [ + ("trace-a", "span-1"), + ("trace-b", "span-3"), + ] + assert samples[0].input_tokens == 4_000 + assert samples[0].cache_read_tokens == 100 + assert samples[0].cache_creation_tokens == 0 + kwargs = run_query.call_args.kwargs + assert kwargs["selected_columns"] == [ + "trace", + "id", + "timestamp", + INPUT_TOKENS, + CACHE_READ_TOKENS, + CACHE_CREATION_TOKENS, + ] + assert kwargs["orderby"] == [f"-{INPUT_TOKENS}"] + assert kwargs["limit"] == SAMPLE_CALLS_LIMIT * 3 + + def test_to_call_sites_treats_missing_token_aggregates_as_zero() -> None: row = make_row( **{ diff --git a/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_reporting.py b/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_reporting.py new file mode 100644 index 000000000000..d6dcee1275c1 --- /dev/null +++ b/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_reporting.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from sentry.ai_monitoring.issues.llm_cache_detection.detection import ( + CacheFinding, + CacheOutcome, + CallSiteWarmth, + Classification, + OutcomeReason, + ProbeGap, +) +from sentry.ai_monitoring.issues.llm_cache_detection.pricing import PricingGap, SavingsEstimate +from sentry.ai_monitoring.issues.llm_cache_detection.query import SampleCall +from sentry.ai_monitoring.issues.llm_cache_detection.reporting import ( + CandidateReport, + Disposition, + _candidate_fields, +) +from tests.sentry.ai_monitoring.issues.llm_cache_detection.test_utils import make_stats + +INITIAL = Classification(CacheOutcome.NOT_CACHING, OutcomeReason.CACHE_ACTIVITY) +FINAL = Classification(CacheOutcome.THRASH, OutcomeReason.CACHE_ACTIVITY) + + +def test_candidate_fields_include_evidence_and_enrichment() -> None: + finding = CacheFinding( + classification=FINAL, + stats=make_stats(model="claude-sonnet-4", hit_rate=0.1, write_read_ratio=2), + warmth=CallSiteWarmth(1_000, 500, 800, 900), + spans_with_cache_attributes=400, + ) + report = CandidateReport( + finding=finding, + initial=INITIAL, + rank=2, + disposition=Disposition.WOULD_CREATE, + pricing=SavingsEstimate(12.5, 0.000003, 0.0000003, 0.00000375), + sample_calls=[SampleCall("trace-id", "span-id", "2026-01-01T00:00:00Z", 4_000, 100, 200)], + ) + + fields = _candidate_fields(report) + + assert fields["rank"] == 2 + assert fields["initial_outcome"] == "not_caching" + assert fields["outcome"] == "thrash" + assert fields["disposition"] == "would_create" + assert fields["cacheable_share"] == 0.8 + assert fields["spans_with_cache_attributes"] == 400 + assert fields["estimated_savings_usd"] == 12.5 + assert fields["sample_calls"][0]["trace_id"] == "trace-id" + assert fields["min_cacheable_prefix_tokens"] == 1_024 + + +def test_candidate_fields_explain_missing_measurements() -> None: + finding = CacheFinding( + classification=FINAL, + stats=make_stats(), + warmth=ProbeGap.OUT_OF_TIME, + spans_with_cache_attributes=ProbeGap.UNQUERYABLE, + ) + report = CandidateReport( + finding=finding, + initial=INITIAL, + rank=1, + pricing=PricingGap.NO_METADATA, + sample_calls=ProbeGap.FAILED, + ) + + fields = _candidate_fields(report) + + assert fields["warmth_gap"] == "out_of_time" + assert fields["cache_presence_gap"] == "unqueryable_call_site" + assert fields["pricing_gap"] == "no_metadata" + assert fields["sample_calls_gap"] == "probe_failed" From a0b447b99e4653cffcdfcfbe724221ded55b0ae7 Mon Sep 17 00:00:00 2001 From: Vjeran Grozdanic Date: Mon, 12 Oct 2026 14:43:03 +0200 Subject: [PATCH 2/2] refactor(ai): Report fixed cache threshold --- .../ai_monitoring/issues/llm_cache_detection/reporting.py | 4 ++-- .../issues/llm_cache_detection/test_reporting.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sentry/ai_monitoring/issues/llm_cache_detection/reporting.py b/src/sentry/ai_monitoring/issues/llm_cache_detection/reporting.py index 5f710b46e286..da3f213033f2 100644 --- a/src/sentry/ai_monitoring/issues/llm_cache_detection/reporting.py +++ b/src/sentry/ai_monitoring/issues/llm_cache_detection/reporting.py @@ -7,11 +7,11 @@ from typing import Any from sentry.ai_monitoring.issues.llm_cache_detection.detection import ( + MIN_CACHEABLE_PREFIX_TOKENS, CacheFinding, CallSiteWarmth, Classification, ProbeGap, - min_cacheable_prefix_tokens, ) from sentry.ai_monitoring.issues.llm_cache_detection.pricing import PricingGap, SavingsEstimate from sentry.ai_monitoring.issues.llm_cache_detection.query import SampleCall @@ -53,7 +53,7 @@ def _candidate_fields(report: CandidateReport) -> dict[str, Any]: "uncached_tokens": stats.uncached_tokens, "unrecouped_cache_write_tokens": stats.unrecouped_cache_write_tokens, "cache_exceeds_input": stats.cache_exceeds_input, - "min_cacheable_prefix_tokens": min_cacheable_prefix_tokens(stats.model), + "min_cacheable_prefix_tokens": MIN_CACHEABLE_PREFIX_TOKENS, } for name, measurement in ( diff --git a/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_reporting.py b/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_reporting.py index d6dcee1275c1..28b343699b71 100644 --- a/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_reporting.py +++ b/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_reporting.py @@ -47,7 +47,7 @@ def test_candidate_fields_include_evidence_and_enrichment() -> None: assert fields["spans_with_cache_attributes"] == 400 assert fields["estimated_savings_usd"] == 12.5 assert fields["sample_calls"][0]["trace_id"] == "trace-id" - assert fields["min_cacheable_prefix_tokens"] == 1_024 + assert fields["min_cacheable_prefix_tokens"] == 4_096 def test_candidate_fields_explain_missing_measurements() -> None: