Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions src/sentry/ai_monitoring/issues/llm_cache_detection/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
# Order by input tokens so the highest-volume groups fit when the query reaches
# its limit. Agent operation groups may be folded after this limit is applied.
CALL_SITE_GROUPS_LIMIT = 300
SAMPLE_CALLS_LIMIT = 3


class DroppedRowReason(StrEnum):
Expand All @@ -75,6 +76,18 @@ class CallSiteQueryResult:
truncated: bool


@dataclass(frozen=True)
class SampleCall:
"""Trace metadata and token counts for one example call."""

trace_id: str
span_id: str
timestamp: str
input_tokens: float
cache_read_tokens: float
cache_creation_tokens: float


def _build_group_filter(stats: CallSiteStats) -> str | None:
"""Build an exact-match filter for a call site, if its values are expressible."""
agent_attribute = (
Expand Down Expand Up @@ -276,3 +289,49 @@ def count_spans_with_cache_attributes(
)
data = result.get("data", [])
return int(data[0].get(COUNT) or 0) if data else 0


def fetch_sample_calls(
project: Project, stats: CallSiteStats, window: DetectionWindow
) -> list[SampleCall] | None:
"""Return the largest call from up to three traces, without prompt content."""
group_filter = _build_group_filter(stats)
if group_filter is None:
return None
result = _run_spans_query(
project,
window,
query_string=group_filter,
selected_columns=[
"trace",
"id",
"timestamp",
INPUT_TOKENS,
*CACHE_TOKEN_ATTRIBUTES,
],
orderby=[f"-{INPUT_TOKENS}"],
limit=SAMPLE_CALLS_LIMIT * 3,
)

samples: list[SampleCall] = []
seen_trace_ids: set[str] = set()
for row in result.get("data", []):
trace_id = row.get("trace")
span_id = row.get("id")
timestamp = row.get("timestamp")
if not trace_id or not span_id or not timestamp or trace_id in seen_trace_ids:
continue
seen_trace_ids.add(trace_id)
samples.append(
SampleCall(
trace_id=trace_id,
span_id=span_id,
timestamp=timestamp,
input_tokens=_token_count(row, INPUT_TOKENS),
cache_read_tokens=_token_count(row, CACHE_READ_TOKENS),
cache_creation_tokens=_token_count(row, CACHE_CREATION_TOKENS),
)
)
if len(samples) == SAMPLE_CALLS_LIMIT:
break
return samples
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
"""Structured shadow reports for LLM prompt-cache detection."""

from __future__ import annotations

from dataclasses import asdict, dataclass
from enum import StrEnum
from typing import Any

from sentry.ai_monitoring.issues.llm_cache_detection.detection import (
MIN_CACHEABLE_PREFIX_TOKENS,
CacheFinding,
CallSiteWarmth,
Classification,
ProbeGap,
)
from sentry.ai_monitoring.issues.llm_cache_detection.pricing import PricingGap, SavingsEstimate
from sentry.ai_monitoring.issues.llm_cache_detection.query import SampleCall


class Disposition(StrEnum):
"""What issue creation would do with a finding."""

WOULD_CREATE = "would_create"
OVER_CAP = "over_cap"


@dataclass(frozen=True)
class CandidateReport:
"""A provisional candidate and its final shadow result."""

finding: CacheFinding
initial: Classification
rank: int
disposition: Disposition | None = None
pricing: SavingsEstimate | PricingGap | None = None
sample_calls: list[SampleCall] | ProbeGap | None = None


def _candidate_fields(report: CandidateReport) -> dict[str, Any]:
finding = report.finding
stats = finding.stats
fields: dict[str, Any] = {
"rank": report.rank,
"severity": finding.severity,
"initial_outcome": report.initial.outcome.value,
"initial_reason": report.initial.reason.value,
"outcome": finding.outcome.value,
"reason": finding.classification.reason.value,
"disposition": report.disposition.value if report.disposition else None,
**asdict(stats),
"hit_rate": stats.hit_rate,
"write_read_ratio": stats.write_read_ratio,
"uncached_tokens": stats.uncached_tokens,
"unrecouped_cache_write_tokens": stats.unrecouped_cache_write_tokens,
"cache_exceeds_input": stats.cache_exceeds_input,
"min_cacheable_prefix_tokens": MIN_CACHEABLE_PREFIX_TOKENS,
}

for name, measurement in (
("warmth", finding.warmth),
("cache_presence", finding.spans_with_cache_attributes),
("pricing", report.pricing),
("sample_calls", report.sample_calls),
):
if isinstance(measurement, StrEnum):
fields[f"{name}_gap"] = measurement.value

if isinstance(finding.warmth, CallSiteWarmth):
fields.update(
asdict(finding.warmth),
cacheable_share=finding.warmth.cacheable_share,
long_ttl_cacheable_share=finding.warmth.long_ttl_cacheable_share,
)
if isinstance(finding.spans_with_cache_attributes, int):
fields["spans_with_cache_attributes"] = finding.spans_with_cache_attributes
if isinstance(report.pricing, SavingsEstimate):
fields.update(asdict(report.pricing))
if isinstance(report.sample_calls, list):
fields["sample_calls"] = [asdict(sample) for sample in report.sample_calls]
return fields
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@
CALL_SITE_GROUPS_LIMIT,
COUNT,
COUNT_SAMPLE,
INPUT_TOKENS,
MODEL,
OPERATION_NAME,
SAMPLE_CALLS_LIMIT,
SPAN_NAME,
SUM_CACHE_CREATION_TOKENS,
SUM_CACHE_READ_TOKENS,
Expand All @@ -31,6 +33,7 @@
count_spans_with_cache_attributes,
fetch_call_site_stats,
fetch_call_site_warmth,
fetch_sample_calls,
)
from sentry.models.project import Project
from sentry.search.events.types import SnubaRow
Expand Down Expand Up @@ -215,6 +218,52 @@ def test_count_spans_with_cache_attributes() -> None:
assert f"(has:{CACHE_READ_TOKENS} OR has:{CACHE_CREATION_TOKENS})" in kwargs["query_string"]


def test_fetch_sample_calls_returns_one_call_per_trace() -> None:
project = mock.Mock(spec=Project)
window = DetectionWindow(
start=datetime(2026, 1, 1, tzinfo=UTC),
end=datetime(2026, 1, 8, tzinfo=UTC),
)
stats = _to_call_sites([make_row()])[0][0]
rows = [
{
"trace": "trace-a",
"id": "span-1",
"timestamp": "2026-01-07T00:00:00Z",
INPUT_TOKENS: 4_000,
CACHE_READ_TOKENS: 100,
},
{"trace": "trace-a", "id": "span-2", "timestamp": "2026-01-06T00:00:00Z"},
{"trace": "trace-b", "id": "span-3", "timestamp": "2026-01-05T00:00:00Z"},
{"trace": "trace-c", "id": "span-4"},
]

with mock.patch.object(
query_module, "_run_spans_query", return_value={"data": rows}
) as run_query:
samples = fetch_sample_calls(project, stats, window)

assert samples is not None
assert [(sample.trace_id, sample.span_id) for sample in samples] == [
("trace-a", "span-1"),
("trace-b", "span-3"),
]
assert samples[0].input_tokens == 4_000
assert samples[0].cache_read_tokens == 100
assert samples[0].cache_creation_tokens == 0
kwargs = run_query.call_args.kwargs
assert kwargs["selected_columns"] == [
"trace",
"id",
"timestamp",
INPUT_TOKENS,
CACHE_READ_TOKENS,
CACHE_CREATION_TOKENS,
]
assert kwargs["orderby"] == [f"-{INPUT_TOKENS}"]
assert kwargs["limit"] == SAMPLE_CALLS_LIMIT * 3


def test_to_call_sites_treats_missing_token_aggregates_as_zero() -> None:
row = make_row(
**{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
from __future__ import annotations

from sentry.ai_monitoring.issues.llm_cache_detection.detection import (
CacheFinding,
CacheOutcome,
CallSiteWarmth,
Classification,
OutcomeReason,
ProbeGap,
)
from sentry.ai_monitoring.issues.llm_cache_detection.pricing import PricingGap, SavingsEstimate
from sentry.ai_monitoring.issues.llm_cache_detection.query import SampleCall
from sentry.ai_monitoring.issues.llm_cache_detection.reporting import (
CandidateReport,
Disposition,
_candidate_fields,
)
from tests.sentry.ai_monitoring.issues.llm_cache_detection.test_utils import make_stats

INITIAL = Classification(CacheOutcome.NOT_CACHING, OutcomeReason.CACHE_ACTIVITY)
FINAL = Classification(CacheOutcome.THRASH, OutcomeReason.CACHE_ACTIVITY)


def test_candidate_fields_include_evidence_and_enrichment() -> None:
finding = CacheFinding(
classification=FINAL,
stats=make_stats(model="claude-sonnet-4", hit_rate=0.1, write_read_ratio=2),
warmth=CallSiteWarmth(1_000, 500, 800, 900),
spans_with_cache_attributes=400,
)
report = CandidateReport(
finding=finding,
initial=INITIAL,
rank=2,
disposition=Disposition.WOULD_CREATE,
pricing=SavingsEstimate(12.5, 0.000003, 0.0000003, 0.00000375),
sample_calls=[SampleCall("trace-id", "span-id", "2026-01-01T00:00:00Z", 4_000, 100, 200)],
)

fields = _candidate_fields(report)

assert fields["rank"] == 2
assert fields["initial_outcome"] == "not_caching"
assert fields["outcome"] == "thrash"
assert fields["disposition"] == "would_create"
assert fields["cacheable_share"] == 0.8
assert fields["spans_with_cache_attributes"] == 400
assert fields["estimated_savings_usd"] == 12.5
assert fields["sample_calls"][0]["trace_id"] == "trace-id"
assert fields["min_cacheable_prefix_tokens"] == 4_096


def test_candidate_fields_explain_missing_measurements() -> None:
finding = CacheFinding(
classification=FINAL,
stats=make_stats(),
warmth=ProbeGap.OUT_OF_TIME,
spans_with_cache_attributes=ProbeGap.UNQUERYABLE,
)
report = CandidateReport(
finding=finding,
initial=INITIAL,
rank=1,
pricing=PricingGap.NO_METADATA,
sample_calls=ProbeGap.FAILED,
)

fields = _candidate_fields(report)

assert fields["warmth_gap"] == "out_of_time"
assert fields["cache_presence_gap"] == "unqueryable_call_site"
assert fields["pricing_gap"] == "no_metadata"
assert fields["sample_calls_gap"] == "probe_failed"
Loading