diff --git a/src/sentry/ai_monitoring/issues/llm_cache_detection/pricing.py b/src/sentry/ai_monitoring/issues/llm_cache_detection/pricing.py new file mode 100644 index 000000000000..89085b269cc5 --- /dev/null +++ b/src/sentry/ai_monitoring/issues/llm_cache_detection/pricing.py @@ -0,0 +1,70 @@ +"""Estimate savings for LLM prompt-cache findings.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + +from sentry.ai_monitoring.issues.llm_cache_detection.detection import CacheFinding, CacheOutcome +from sentry.ai_monitoring.utils import model_costs +from sentry.relay.config.ai_model_costs import AIModelMetadataConfig + + +@dataclass(frozen=True) +class SavingsEstimate: + """Estimated savings and the prices used to calculate them.""" + + estimated_savings_usd: float + price_per_input_token: float + price_per_cached_input_token: float + price_per_cache_write_token: float + + +class PricingGap(StrEnum): + """Why a finding could not be priced.""" + + NO_METADATA = "no_metadata" + UNKNOWN_MODEL = "unknown_model" + NO_INPUT_PRICE = "no_input_price" + NO_CACHED_PRICE = "no_cached_price" + NO_WRITE_PREMIUM = "no_write_premium" + NOTHING_TO_RECOVER = "nothing_to_recover" + + +def estimate_savings( + finding: CacheFinding, config: AIModelMetadataConfig | None +) -> SavingsEstimate | PricingGap: + if config is None: + return PricingGap.NO_METADATA + costs = model_costs(finding.stats.model, config) + if costs is None: + return PricingGap.UNKNOWN_MODEL + + input_price = costs["inputPerToken"] + cached_input_price = costs["inputCachedPerToken"] + cache_write_price = costs["inputCacheWritePerToken"] + # Zero means no known price in the metadata feed, not free usage. + if input_price <= 0: + return PricingGap.NO_INPUT_PRICE + if cached_input_price <= 0: + return PricingGap.NO_CACHED_PRICE + + stats = finding.stats + if finding.outcome == CacheOutcome.THRASH: + if cache_write_price <= cached_input_price: + return PricingGap.NO_WRITE_PREMIUM + # A stable prefix would turn cache writes into cache reads. + savings = stats.sum_cache_creation_tokens * (cache_write_price - cached_input_price) + else: + # This is an upper bound because the shared prefix length is unknown. + savings = stats.uncached_tokens * (input_price - cached_input_price) + + if savings <= 0: + return PricingGap.NOTHING_TO_RECOVER + + return SavingsEstimate( + estimated_savings_usd=savings, + price_per_input_token=input_price, + price_per_cached_input_token=cached_input_price, + price_per_cache_write_token=cache_write_price, + ) diff --git a/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_pricing.py b/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_pricing.py new file mode 100644 index 000000000000..46828c536ebb --- /dev/null +++ b/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_pricing.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from dataclasses import replace + +import pytest + +from sentry.ai_monitoring.issues.llm_cache_detection.detection import ( + CacheFinding, + CacheOutcome, + Classification, + OutcomeReason, +) +from sentry.ai_monitoring.issues.llm_cache_detection.pricing import ( + PricingGap, + SavingsEstimate, + estimate_savings, +) +from sentry.relay.config.ai_model_costs import AIModelCost, AIModelMetadataConfig +from tests.sentry.ai_monitoring.issues.llm_cache_detection.test_utils import make_stats + +INPUT_PRICE = 0.000003 +CACHED_PRICE = 0.0000003 +WRITE_PRICE = 0.00000375 +MODEL = "claude-sonnet-4" + + +def metadata( + *, + input_price: float = INPUT_PRICE, + cached_price: float = CACHED_PRICE, + write_price: float = WRITE_PRICE, +) -> AIModelMetadataConfig: + costs: AIModelCost = { + "inputPerToken": input_price, + "outputPerToken": 0.000015, + "outputReasoningPerToken": 0.000015, + "inputCachedPerToken": cached_price, + "inputCacheWritePerToken": write_price, + } + return {"version": 1, "models": {MODEL: {"costs": costs}}} + + +def finding( + outcome: CacheOutcome, + *, + model: str = MODEL, + read_tokens: float = 0, + write_tokens: float = 0, +) -> CacheFinding: + stats = replace( + make_stats(model=model), + sum_input_tokens=10_000_000, + sum_cache_read_tokens=read_tokens, + sum_cache_creation_tokens=write_tokens, + ) + return CacheFinding( + classification=Classification(outcome, OutcomeReason.CACHE_ACTIVITY), + stats=stats, + ) + + +@pytest.mark.parametrize( + ("cache_finding", "expected"), + [ + ( + finding(CacheOutcome.NOT_CACHING), + 10_000_000 * (INPUT_PRICE - CACHED_PRICE), + ), + ( + finding(CacheOutcome.THRASH, read_tokens=200_000, write_tokens=8_000_000), + 8_000_000 * (WRITE_PRICE - CACHED_PRICE), + ), + ], +) +def test_estimates_savings(cache_finding: CacheFinding, expected: float) -> None: + estimate = estimate_savings(cache_finding, metadata()) + + assert isinstance(estimate, SavingsEstimate) + assert estimate.estimated_savings_usd == pytest.approx(expected) + assert estimate.price_per_input_token == INPUT_PRICE + assert estimate.price_per_cached_input_token == CACHED_PRICE + assert estimate.price_per_cache_write_token == WRITE_PRICE + + +@pytest.mark.parametrize( + ("cache_finding", "config", "expected"), + [ + (finding(CacheOutcome.NOT_CACHING), None, PricingGap.NO_METADATA), + (finding(CacheOutcome.NOT_CACHING, model="unknown"), metadata(), PricingGap.UNKNOWN_MODEL), + ( + finding(CacheOutcome.NOT_CACHING), + metadata(input_price=0), + PricingGap.NO_INPUT_PRICE, + ), + ( + finding(CacheOutcome.NOT_CACHING), + metadata(cached_price=0), + PricingGap.NO_CACHED_PRICE, + ), + ( + finding(CacheOutcome.THRASH, write_tokens=8_000_000), + metadata(write_price=0), + PricingGap.NO_WRITE_PREMIUM, + ), + ( + finding(CacheOutcome.NOT_CACHING), + metadata(cached_price=INPUT_PRICE), + PricingGap.NOTHING_TO_RECOVER, + ), + ], +) +def test_reports_pricing_gaps( + cache_finding: CacheFinding, + config: AIModelMetadataConfig | None, + expected: PricingGap, +) -> None: + assert estimate_savings(cache_finding, config) == expected diff --git a/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_utils.py b/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_utils.py new file mode 100644 index 000000000000..96f1af843120 --- /dev/null +++ b/tests/sentry/ai_monitoring/issues/llm_cache_detection/test_utils.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from sentry.ai_monitoring.issues.llm_cache_detection.detection import ( + AgentLabelSource, + CallSiteStats, +) + + +def make_stats( + *, + agent_label: str = "Some Agent", + span_name: str = "generate_content generate_structured", + model: str = "model-x", + call_count: int = 10_000, + avg_input_tokens: float = 5_000, + hit_rate: float = 0.0, + write_read_ratio: float = 0.0, + sampled_call_count: int | None = None, +) -> CallSiteStats: + """Build stats from cache ratios instead of raw token sums.""" + sum_input = call_count * avg_input_tokens + sum_read = hit_rate * sum_input + sum_creation = write_read_ratio * sum_read + return CallSiteStats( + agent_label=agent_label, + agent_label_source=AgentLabelSource.AGENT_NAME, + span_name=span_name, + model=model, + call_count=call_count, + sampled_call_count=call_count if sampled_call_count is None else sampled_call_count, + sum_input_tokens=sum_input, + sum_cache_read_tokens=sum_read, + sum_cache_creation_tokens=sum_creation, + avg_input_tokens=avg_input_tokens, + )