From 432f2a3bc3a03bc0e16690a1ced0154d39b2b910 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 21:29:27 +0000 Subject: [PATCH] perf(artifact-bundles): Bound the URL lookup by an index row budget The URL lookup in `get_artifact_bundles_containing_url` checks every `ArtifactBundleIndex` row of every bundle in the release with a substring match. It cannot stop early: a file is usually in a single bundle, so the `LIMIT 5` is never reached, and hits cost as much as misses. Releases whose name never changes accumulate millions of rows. Behind `sourcemaps.artifact-bundles.url-lookup.max-index-rows` (0, the default, keeps the current behavior), the lookup only scans bundles up to that many index rows, counted by `artifact_count`: - first the active bundles, uploaded or renewed (`date_added`) within the renewal threshold plus `sourcemaps.artifact-bundles.url-lookup.active-margin-days`, newest first, - then the others, newest first, - always at least one bundle. A release with fewer indexed files than the budget is still scanned completely. A bundle in use is renewed when a lookup returns it, so it stays active; lookups for files in bundles beyond the budget miss. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Lz5HnKSszfgNef7wkQGiPR --- src/sentry/debug_files/artifact_bundles.py | 109 ++++++++++++++++-- src/sentry/options/defaults.py | 20 ++++ .../debug_files/test_artifact_bundles.py | 100 ++++++++++++++++ 3 files changed, 217 insertions(+), 12 deletions(-) diff --git a/src/sentry/debug_files/artifact_bundles.py b/src/sentry/debug_files/artifact_bundles.py index 4a9774bfd69a..aeceb1f530ba 100644 --- a/src/sentry/debug_files/artifact_bundles.py +++ b/src/sentry/debug_files/artifact_bundles.py @@ -32,6 +32,11 @@ # A value of 3 means that the third upload will trigger indexing and backfill. INDEXING_THRESHOLD = 3 +# The maximum number of active and of idle bundles that the URL lookup considers when +# `sourcemaps.artifact-bundles.url-lookup.max-index-rows` is set. This bounds the queries that +# pick the bundles to scan; the row budget usually stops earlier. +URL_LOOKUP_MAX_CANDIDATE_BUNDLES = 1000 + # We want to keep the bundle as being indexed for 600 seconds = 10 minutes. We might need to revise this number and # optimize it based on the time taken to perform the indexing (on average). @@ -386,21 +391,98 @@ def get_artifact_bundles_containing_debug_id( ) +def get_url_lookup_candidates( + project: Project, release_name: str, dist_name: str +) -> list[int] | None: + """ + Returns the ids of the bundles whose files the URL lookup should scan, or `None` to scan + every bundle in the release. + + The URL lookup checks every `ArtifactBundleIndex` row of the bundles it scans, whether + the file is found or not, and some releases have millions of them. When + `sourcemaps.artifact-bundles.url-lookup.max-index-rows` is set, that budget is spent on + the active bundles first, those uploaded or renewed (`date_added`) within the renewal + threshold plus a margin, newest first, and then on the remaining bundles, newest first. + A release with fewer indexed files than the budget, and fewer active and fewer idle + bundles than `URL_LOOKUP_MAX_CANDIDATE_BUNDLES`, is still scanned completely. + + A bundle that is in use is renewed when a lookup returns it, so it stays active. Lookups + of bundles beyond the budget can no longer find them by URL. + """ + max_index_rows = options.get("sourcemaps.artifact-bundles.url-lookup.max-index-rows") + if max_index_rows <= 0: + return None + + active_days = options.get("system.debug-files-renewal-age-threshold-days") + options.get( + "sourcemaps.artifact-bundles.url-lookup.active-margin-days" + ) + active_since = timezone.now() - timedelta(days=active_days) + + release_bundles = ArtifactBundle.objects.filter( + organization_id=project.organization_id, + releaseartifactbundle__organization_id=project.organization_id, + releaseartifactbundle__release_name=release_name, + releaseartifactbundle__dist_name=dist_name, + projectartifactbundle__project_id=project.id, + ) + + # Using a dict to keep the order and drop bundles repeated by duplicate link rows. + candidates: dict[int, None] = {} + index_rows = 0 + truncated = False + for bundles in ( + release_bundles.filter(date_added__gte=active_since), + release_bundles.filter(date_added__lt=active_since), + ): + rows = list( + bundles.values_list("id", "artifact_count").order_by("-id")[ + :URL_LOOKUP_MAX_CANDIDATE_BUNDLES + ] + ) + for bundle_id, artifact_count in rows: + if bundle_id in candidates: + continue + # We always scan at least one bundle, even if it alone exceeds the budget. + if candidates and index_rows + artifact_count > max_index_rows: + truncated = True + break + candidates[bundle_id] = None + index_rows += artifact_count + if truncated or len(rows) == URL_LOOKUP_MAX_CANDIDATE_BUNDLES: + # Either the budget is spent, or this group has more bundles than we looked at, + # and those come before any bundle of the next group. + truncated = True + break + + metrics.distribution("artifact_bundle_url_lookup.index_rows", index_rows) + metrics.incr( + "artifact_bundle_url_lookup.candidates", + tags={"truncated": "true" if truncated else "false"}, + ) + return list(candidates) + + def get_artifact_bundles_containing_url( project: Project, release_name: str, dist_name: str, url: str ) -> set[tuple[int, datetime]]: """ - Returns the most recently uploaded bundle containing a file matching the `release`, `dist` and `url`. + Returns the most recently uploaded bundles containing a file matching the `release`, `dist` and `url`. """ - return set( - ArtifactBundle.objects.filter( - Exists( - ArtifactBundleIndex.objects.filter( - artifact_bundle_id=OuterRef("pk"), - organization_id=project.organization.id, - url__icontains=url, - ) - ), + contains_url = Exists( + ArtifactBundleIndex.objects.filter( + artifact_bundle_id=OuterRef("pk"), + organization_id=project.organization.id, + url__icontains=url, + ) + ) + + candidates = get_url_lookup_candidates(project, release_name, dist_name) + if candidates is not None: + # The candidates already belong to the project, release and dist. + bundles = ArtifactBundle.objects.filter(contains_url, id__in=candidates) + else: + bundles = ArtifactBundle.objects.filter( + contains_url, Exists( ProjectArtifactBundle.objects.filter( artifact_bundle_id=OuterRef("pk"), @@ -416,8 +498,11 @@ def get_artifact_bundles_containing_url( ) ), ) - .values_list("id", "date_added") - .order_by("-date_last_modified", "-id")[:MAX_BUNDLES_QUERY] + + return set( + bundles.values_list("id", "date_added").order_by("-date_last_modified", "-id")[ + :MAX_BUNDLES_QUERY + ] ) diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index 458f2d04a2cf..4c34bbc330ee 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -1117,6 +1117,26 @@ flags=FLAG_AUTOMATOR_MODIFIABLE, ) +# Upper bound on the number of `ArtifactBundleIndex` rows a single URL lookup in the +# artifact-lookup endpoint scans. Releases that have more indexed files than this scan +# their active bundles (uploaded or renewed recently) first, then the others, newest +# first, until the budget is spent. 0 disables the limit and scans every bundle. +register( + "sourcemaps.artifact-bundles.url-lookup.max-index-rows", + type=Int, + default=0, + flags=FLAG_AUTOMATOR_MODIFIABLE, +) +# Days added to `system.debug-files-renewal-age-threshold-days` to decide which bundles +# count as active for the URL lookup. A bundle in use is renewed once it is older than the +# threshold, so the margin must cover the time between lookups of a bundle in use. +register( + "sourcemaps.artifact-bundles.url-lookup.active-margin-days", + type=Int, + default=7, + flags=FLAG_AUTOMATOR_MODIFIABLE, +) + # Killswitch to stop storing any reprocessing payloads. register("store.reprocessing-force-disable", default=False, flags=FLAG_AUTOMATOR_MODIFIABLE) diff --git a/tests/sentry/debug_files/test_artifact_bundles.py b/tests/sentry/debug_files/test_artifact_bundles.py index 2625299cb1c5..599f21561cbc 100644 --- a/tests/sentry/debug_files/test_artifact_bundles.py +++ b/tests/sentry/debug_files/test_artifact_bundles.py @@ -20,6 +20,7 @@ from sentry.models.files.fileblob import FileBlob from sentry.tasks.assemble import assemble_artifacts from sentry.testutils.cases import TestCase +from sentry.testutils.helpers.options import override_options from sentry.utils import json @@ -455,3 +456,102 @@ def test_multiple_bundles_with_different_urls(self) -> None: ), {bundle3.id}, ) + + def index_url(self, bundle: ArtifactBundle, url: str) -> None: + ArtifactBundleIndex.objects.create( + organization_id=self.organization.id, + artifact_bundle=bundle, + url=url, + date_added=bundle.date_added, + ) + + def lookup(self, url: str) -> set[int]: + return { + bundle_id + for bundle_id, _ in get_artifact_bundles_containing_url( + self.project, self.release_name, self.dist_name, url + ) + } + + @override_options({"sourcemaps.artifact-bundles.url-lookup.max-index-rows": 1000}) + def test_url_lookup_budget_scans_small_release_completely(self) -> None: + old = self.create_bundle(date_added=timezone.now() - timedelta(days=90)) + new = self.create_bundle() + self.index_url(old, "~/path/to/old.js") + self.index_url(new, "~/path/to/new.js") + + assert self.lookup("/path/to/old") == {old.id} + assert self.lookup("/path/to/new") == {new.id} + + @override_options( + { + # Each bundle has 5 files, so the budget covers two bundles. + "sourcemaps.artifact-bundles.url-lookup.max-index-rows": 10, + "system.debug-files-renewal-age-threshold-days": 7, + "sourcemaps.artifact-bundles.url-lookup.active-margin-days": 7, + } + ) + def test_url_lookup_budget_prefers_active_bundles(self) -> None: + now = timezone.now() + renewed = self.create_bundle( + date_added=now - timedelta(days=3), date_last_modified=now - timedelta(days=60) + ) + idle = self.create_bundle(date_added=now - timedelta(days=50)) + newest = self.create_bundle() + self.index_url(renewed, "~/path/to/renewed.js") + self.index_url(idle, "~/path/to/idle.js") + self.index_url(newest, "~/path/to/newest.js") + + # `idle` was uploaded after `renewed`, but only the active bundles fit in the budget. + assert self.lookup("/path/to/renewed") == {renewed.id} + assert self.lookup("/path/to/newest") == {newest.id} + assert self.lookup("/path/to/idle") == set() + + @override_options( + { + "sourcemaps.artifact-bundles.url-lookup.max-index-rows": 10, + "system.debug-files-renewal-age-threshold-days": 7, + "sourcemaps.artifact-bundles.url-lookup.active-margin-days": 7, + } + ) + def test_url_lookup_budget_uses_newest_idle_bundles(self) -> None: + now = timezone.now() + oldest = self.create_bundle(date_added=now - timedelta(days=90)) + older = self.create_bundle(date_added=now - timedelta(days=80)) + newer = self.create_bundle(date_added=now - timedelta(days=70)) + for bundle, name in ((oldest, "oldest"), (older, "older"), (newer, "newer")): + self.index_url(bundle, f"~/path/to/{name}.js") + + # Without active bundles, the budget goes to the most recently uploaded bundles. + assert self.lookup("/path/to/newer") == {newer.id} + assert self.lookup("/path/to/older") == {older.id} + assert self.lookup("/path/to/oldest") == set() + + @override_options({"sourcemaps.artifact-bundles.url-lookup.max-index-rows": 1}) + def test_url_lookup_budget_scans_at_least_one_bundle(self) -> None: + older = self.create_bundle() + newer = self.create_bundle() + self.index_url(older, "~/path/to/app.js") + self.index_url(newer, "~/path/to/app.js") + + assert self.lookup("/path/to/app") == {newer.id} + + @override_options({"sourcemaps.artifact-bundles.url-lookup.max-index-rows": 1000}) + def test_url_lookup_budget_respects_project_and_release(self) -> None: + bundle = self.create_bundle() + self.index_url(bundle, "~/path/to/app.js") + + other_project = self.create_project(organization=self.organization) + assert ( + get_artifact_bundles_containing_url( + other_project, self.release_name, self.dist_name, "/path/to/app" + ) + == set() + ) + assert ( + get_artifact_bundles_containing_url( + self.project, "2.0.0", self.dist_name, "/path/to/app" + ) + == set() + ) + assert self.lookup("/path/to/app") == {bundle.id}