Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions src/sentry/ai_monitoring/issues/llm_cache_detection/pricing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
"""Estimate savings for LLM prompt-cache findings."""

from __future__ import annotations

from dataclasses import dataclass
from enum import StrEnum

from sentry.ai_monitoring.issues.llm_cache_detection.detection import CacheFinding, CacheOutcome
from sentry.ai_monitoring.utils import model_costs
from sentry.relay.config.ai_model_costs import AIModelMetadataConfig


@dataclass(frozen=True)
class SavingsEstimate:
"""Estimated savings and the prices used to calculate them."""

estimated_savings_usd: float
price_per_input_token: float
price_per_cached_input_token: float
price_per_cache_write_token: float


class PricingGap(StrEnum):
"""Why a finding could not be priced."""

NO_METADATA = "no_metadata"
UNKNOWN_MODEL = "unknown_model"
NO_INPUT_PRICE = "no_input_price"
NO_CACHED_PRICE = "no_cached_price"
NO_WRITE_PREMIUM = "no_write_premium"
NOTHING_TO_RECOVER = "nothing_to_recover"


def estimate_savings(
finding: CacheFinding, config: AIModelMetadataConfig | None
) -> SavingsEstimate | PricingGap:
if config is None:
return PricingGap.NO_METADATA
costs = model_costs(finding.stats.model, config)
if costs is None:
return PricingGap.UNKNOWN_MODEL

input_price = costs["inputPerToken"]
cached_input_price = costs["inputCachedPerToken"]
cache_write_price = costs["inputCacheWritePerToken"]
# Zero means no known price in the metadata feed, not free usage.
if input_price <= 0:
return PricingGap.NO_INPUT_PRICE
if cached_input_price <= 0:
return PricingGap.NO_CACHED_PRICE

stats = finding.stats
if finding.outcome == CacheOutcome.THRASH:
if cache_write_price <= cached_input_price:
return PricingGap.NO_WRITE_PREMIUM
# A stable prefix would turn cache writes into cache reads.
savings = stats.sum_cache_creation_tokens * (cache_write_price - cached_input_price)
else:
# This is an upper bound because the shared prefix length is unknown.
savings = stats.uncached_tokens * (input_price - cached_input_price)

if savings <= 0:
return PricingGap.NOTHING_TO_RECOVER

return SavingsEstimate(
estimated_savings_usd=savings,
price_per_input_token=input_price,
price_per_cached_input_token=cached_input_price,
price_per_cache_write_token=cache_write_price,
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
from __future__ import annotations

from dataclasses import replace

import pytest

from sentry.ai_monitoring.issues.llm_cache_detection.detection import (
CacheFinding,
CacheOutcome,
Classification,
OutcomeReason,
)
from sentry.ai_monitoring.issues.llm_cache_detection.pricing import (
PricingGap,
SavingsEstimate,
estimate_savings,
)
from sentry.relay.config.ai_model_costs import AIModelCost, AIModelMetadataConfig
from tests.sentry.ai_monitoring.issues.llm_cache_detection.test_utils import make_stats

INPUT_PRICE = 0.000003
CACHED_PRICE = 0.0000003
WRITE_PRICE = 0.00000375
MODEL = "claude-sonnet-4"


def metadata(
*,
input_price: float = INPUT_PRICE,
cached_price: float = CACHED_PRICE,
write_price: float = WRITE_PRICE,
) -> AIModelMetadataConfig:
costs: AIModelCost = {
"inputPerToken": input_price,
"outputPerToken": 0.000015,
"outputReasoningPerToken": 0.000015,
"inputCachedPerToken": cached_price,
"inputCacheWritePerToken": write_price,
}
return {"version": 1, "models": {MODEL: {"costs": costs}}}


def finding(
outcome: CacheOutcome,
*,
model: str = MODEL,
read_tokens: float = 0,
write_tokens: float = 0,
) -> CacheFinding:
stats = replace(
make_stats(model=model),
sum_input_tokens=10_000_000,
sum_cache_read_tokens=read_tokens,
sum_cache_creation_tokens=write_tokens,
)
return CacheFinding(
classification=Classification(outcome, OutcomeReason.CACHE_ACTIVITY),
stats=stats,
)


@pytest.mark.parametrize(
("cache_finding", "expected"),
[
(
finding(CacheOutcome.NOT_CACHING),
10_000_000 * (INPUT_PRICE - CACHED_PRICE),
),
(
finding(CacheOutcome.THRASH, read_tokens=200_000, write_tokens=8_000_000),
8_000_000 * (WRITE_PRICE - CACHED_PRICE),
),
],
)
def test_estimates_savings(cache_finding: CacheFinding, expected: float) -> None:
estimate = estimate_savings(cache_finding, metadata())

assert isinstance(estimate, SavingsEstimate)
assert estimate.estimated_savings_usd == pytest.approx(expected)
assert estimate.price_per_input_token == INPUT_PRICE
assert estimate.price_per_cached_input_token == CACHED_PRICE
assert estimate.price_per_cache_write_token == WRITE_PRICE


@pytest.mark.parametrize(
("cache_finding", "config", "expected"),
[
(finding(CacheOutcome.NOT_CACHING), None, PricingGap.NO_METADATA),
(finding(CacheOutcome.NOT_CACHING, model="unknown"), metadata(), PricingGap.UNKNOWN_MODEL),
(
finding(CacheOutcome.NOT_CACHING),
metadata(input_price=0),
PricingGap.NO_INPUT_PRICE,
),
(
finding(CacheOutcome.NOT_CACHING),
metadata(cached_price=0),
PricingGap.NO_CACHED_PRICE,
),
(
finding(CacheOutcome.THRASH, write_tokens=8_000_000),
metadata(write_price=0),
PricingGap.NO_WRITE_PREMIUM,
),
(
finding(CacheOutcome.NOT_CACHING),
metadata(cached_price=INPUT_PRICE),
PricingGap.NOTHING_TO_RECOVER,
),
],
)
def test_reports_pricing_gaps(
cache_finding: CacheFinding,
config: AIModelMetadataConfig | None,
expected: PricingGap,
) -> None:
assert estimate_savings(cache_finding, config) == expected
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
from __future__ import annotations

from sentry.ai_monitoring.issues.llm_cache_detection.detection import (
AgentLabelSource,
CallSiteStats,
)


def make_stats(
*,
agent_label: str = "Some Agent",
span_name: str = "generate_content generate_structured",
model: str = "model-x",
call_count: int = 10_000,
avg_input_tokens: float = 5_000,
hit_rate: float = 0.0,
write_read_ratio: float = 0.0,
sampled_call_count: int | None = None,
) -> CallSiteStats:
"""Build stats from cache ratios instead of raw token sums."""
sum_input = call_count * avg_input_tokens
sum_read = hit_rate * sum_input
sum_creation = write_read_ratio * sum_read
return CallSiteStats(
agent_label=agent_label,
agent_label_source=AgentLabelSource.AGENT_NAME,
span_name=span_name,
model=model,
call_count=call_count,
sampled_call_count=call_count if sampled_call_count is None else sampled_call_count,
sum_input_tokens=sum_input,
sum_cache_read_tokens=sum_read,
sum_cache_creation_tokens=sum_creation,
avg_input_tokens=avg_input_tokens,
)
Loading