Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,25 @@
from __future__ import annotations

import math
import re
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from enum import StrEnum

DETECTION_WINDOW_DAYS = 7

# Smaller prompts do not produce material savings, even when a provider can cache them.
MIN_CACHEABLE_PREFIX_TOKENS = 4_096
MIN_CALLS_FOR_CONFIDENCE = 200
MIN_SAMPLED_CALLS = 50
NOT_CACHING_MAX_HIT_RATE = 0.05
THRASH_MAX_HIT_RATE = 0.30
THRASH_MIN_CREATION_INPUT_FRACTION = 0.3

# These integrations omit cache-token attributes when their values are zero.
POSITIVE_ONLY_CACHE_REPORTING_MODEL_MARKERS = ("gemini", "gpt")
POSITIVE_ONLY_CACHE_REPORTING_MODEL_PATTERN = re.compile(r"(?:^|[/:])o\d")


@dataclass(frozen=True)
class DetectionWindow:
Expand All @@ -30,6 +43,31 @@ class AgentLabelSource(StrEnum):
OPERATION_NAME = "gen_ai.operation.name"


class CacheOutcome(StrEnum):
HEALTHY = "healthy"
NOT_CACHING = "not_caching"
THRASH = "thrash"
INELIGIBLE = "ineligible"


FLAGGED_OUTCOMES = frozenset({CacheOutcome.NOT_CACHING, CacheOutcome.THRASH})


class OutcomeReason(StrEnum):
SMALL_PROMPTS = "small_prompts"
LOW_VOLUME = "low_volume"
FEW_STORED_SPANS = "few_stored_spans"
CACHE_ACTIVITY = "cache_activity"
ZERO_CACHE_TOKENS = "zero_cache_tokens"
POSITIVE_ONLY_REPORTER = "positive_only_reporter"


@dataclass(frozen=True)
class Classification:
outcome: CacheOutcome
reason: OutcomeReason


@dataclass(frozen=True)
class CallSiteStats:
"""Aggregates for one agent label, span name, and model.
Expand Down Expand Up @@ -88,3 +126,53 @@ def unrecouped_cache_write_tokens(self) -> float:
@property
def has_cache_activity(self) -> bool:
return self.sum_cache_read_tokens > 0 or self.sum_cache_creation_tokens > 0


@dataclass(frozen=True)
class CacheFinding:
classification: Classification
stats: CallSiteStats

@property
def outcome(self) -> CacheOutcome:
return self.classification.outcome

@property
def severity(self) -> float:
return self.stats.uncached_tokens + self.stats.unrecouped_cache_write_tokens


def classify_call_site(stats: CallSiteStats) -> Classification:
"""Classify token sums before traffic and instrumentation probes."""
if stats.avg_input_tokens < MIN_CACHEABLE_PREFIX_TOKENS:
return Classification(CacheOutcome.INELIGIBLE, OutcomeReason.SMALL_PROMPTS)
if stats.call_count < MIN_CALLS_FOR_CONFIDENCE:
return Classification(CacheOutcome.INELIGIBLE, OutcomeReason.LOW_VOLUME)
if stats.sampled_call_count < MIN_SAMPLED_CALLS:
return Classification(CacheOutcome.INELIGIBLE, OutcomeReason.FEW_STORED_SPANS)

if (
stats.sum_cache_creation_tokens > 0
and stats.hit_rate < THRASH_MAX_HIT_RATE
and stats.sum_cache_creation_tokens
>= THRASH_MIN_CREATION_INPUT_FRACTION * stats.sum_input_tokens
):
return Classification(CacheOutcome.THRASH, OutcomeReason.CACHE_ACTIVITY)

if stats.hit_rate < NOT_CACHING_MAX_HIT_RATE:
if stats.has_cache_activity:
reason = OutcomeReason.CACHE_ACTIVITY
elif reports_only_positive_cache_values(stats.model):
reason = OutcomeReason.POSITIVE_ONLY_REPORTER
else:
reason = OutcomeReason.ZERO_CACHE_TOKENS
return Classification(CacheOutcome.NOT_CACHING, reason)

return Classification(CacheOutcome.HEALTHY, OutcomeReason.CACHE_ACTIVITY)


def reports_only_positive_cache_values(model: str) -> bool:
normalized = model.lower()
if any(marker in normalized for marker in POSITIVE_ONLY_CACHE_REPORTING_MODEL_MARKERS):
return True
return POSITIVE_ONLY_CACHE_REPORTING_MODEL_PATTERN.search(normalized) is not None
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,21 @@
from dataclasses import replace
from datetime import timedelta

import pytest

from sentry.ai_monitoring.issues.llm_cache_detection.detection import (
DETECTION_WINDOW_DAYS,
MIN_CACHEABLE_PREFIX_TOKENS,
MIN_CALLS_FOR_CONFIDENCE,
MIN_SAMPLED_CALLS,
AgentLabelSource,
CacheFinding,
CacheOutcome,
CallSiteStats,
Classification,
DetectionWindow,
OutcomeReason,
classify_call_site,
)


Expand All @@ -22,7 +32,7 @@ def make_stats() -> CallSiteStats:
sum_input_tokens=1_000_000,
sum_cache_read_tokens=100_000,
sum_cache_creation_tokens=150_000,
avg_input_tokens=1_000,
avg_input_tokens=5_000,
)


Expand Down Expand Up @@ -71,3 +81,74 @@ def test_token_derived_properties() -> None:
assert empty.hit_rate == 0
assert empty.write_read_ratio is None
assert not empty.has_cache_activity


@pytest.mark.parametrize(
("stats", "expected"),
[
pytest.param(
make_stats(),
Classification(CacheOutcome.HEALTHY, OutcomeReason.CACHE_ACTIVITY),
id="healthy",
),
pytest.param(
replace(make_stats(), sum_cache_read_tokens=1_000, sum_cache_creation_tokens=0),
Classification(CacheOutcome.NOT_CACHING, OutcomeReason.CACHE_ACTIVITY),
id="not-caching",
),
pytest.param(
replace(make_stats(), sum_cache_read_tokens=0, sum_cache_creation_tokens=0),
Classification(CacheOutcome.NOT_CACHING, OutcomeReason.ZERO_CACHE_TOKENS),
id="ambiguous-zero",
),
pytest.param(
replace(
make_stats(),
model="gemini-2.5-pro",
sum_cache_read_tokens=0,
sum_cache_creation_tokens=0,
),
Classification(CacheOutcome.NOT_CACHING, OutcomeReason.POSITIVE_ONLY_REPORTER),
id="positive-only-reporter",
),
pytest.param(
replace(
make_stats(),
sum_cache_read_tokens=80_000,
sum_cache_creation_tokens=400_000,
),
Classification(CacheOutcome.THRASH, OutcomeReason.CACHE_ACTIVITY),
id="thrash",
),
pytest.param(
replace(make_stats(), avg_input_tokens=MIN_CACHEABLE_PREFIX_TOKENS - 1),
Classification(CacheOutcome.INELIGIBLE, OutcomeReason.SMALL_PROMPTS),
id="small-prompts",
),
pytest.param(
replace(make_stats(), call_count=MIN_CALLS_FOR_CONFIDENCE - 1),
Classification(CacheOutcome.INELIGIBLE, OutcomeReason.LOW_VOLUME),
id="low-volume",
),
pytest.param(
replace(make_stats(), sampled_call_count=MIN_SAMPLED_CALLS - 1),
Classification(CacheOutcome.INELIGIBLE, OutcomeReason.FEW_STORED_SPANS),
id="few-stored-spans",
),
],
)
def test_classify_call_site(stats: CallSiteStats, expected: Classification) -> None:
assert classify_call_site(stats) == expected


def test_severity_counts_uncached_input_and_unrecouped_writes() -> None:
finding = CacheFinding(
classification=Classification(CacheOutcome.THRASH, OutcomeReason.CACHE_ACTIVITY),
stats=replace(
make_stats(),
sum_cache_read_tokens=80_000,
sum_cache_creation_tokens=400_000,
),
)

assert finding.severity == 840_000
Loading