Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 97 additions & 12 deletions src/sentry/debug_files/artifact_bundles.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@
# A value of 3 means that the third upload will trigger indexing and backfill.
INDEXING_THRESHOLD = 3

# The maximum number of active and of idle bundles that the URL lookup considers when
# `sourcemaps.artifact-bundles.url-lookup.max-index-rows` is set. This bounds the queries that
# pick the bundles to scan; the row budget usually stops earlier.
URL_LOOKUP_MAX_CANDIDATE_BUNDLES = 1000


# We want to keep the bundle as being indexed for 600 seconds = 10 minutes. We might need to revise this number and
# optimize it based on the time taken to perform the indexing (on average).
Expand Down Expand Up @@ -412,21 +417,98 @@ def get_artifact_bundles_containing_debug_id(
)


def get_url_lookup_candidates(
project: Project, release_name: str, dist_name: str
) -> list[int] | None:
"""
Returns the ids of the bundles whose files the URL lookup should scan, or `None` to scan
every bundle in the release.

The URL lookup checks every `ArtifactBundleIndex` row of the bundles it scans, whether
the file is found or not, and some releases have millions of them. When
`sourcemaps.artifact-bundles.url-lookup.max-index-rows` is set, that budget is spent on
the active bundles first, those uploaded or renewed (`date_added`) within the renewal
threshold plus a margin, newest first, and then on the remaining bundles, newest first.
A release with fewer indexed files than the budget, and fewer active and fewer idle
bundles than `URL_LOOKUP_MAX_CANDIDATE_BUNDLES`, is still scanned completely.

A bundle that is in use is renewed when a lookup returns it, so it stays active. Lookups
of bundles beyond the budget can no longer find them by URL.
"""
max_index_rows = options.get("sourcemaps.artifact-bundles.url-lookup.max-index-rows")
if max_index_rows <= 0:
return None

active_days = options.get("system.debug-files-renewal-age-threshold-days") + options.get(
"sourcemaps.artifact-bundles.url-lookup.active-margin-days"
)
active_since = timezone.now() - timedelta(days=active_days)

release_bundles = ArtifactBundle.objects.filter(
organization_id=project.organization_id,
releaseartifactbundle__organization_id=project.organization_id,
releaseartifactbundle__release_name=release_name,
releaseartifactbundle__dist_name=dist_name,
projectartifactbundle__project_id=project.id,
)

# Using a dict to keep the order and drop bundles repeated by duplicate link rows.
candidates: dict[int, None] = {}
index_rows = 0
truncated = False
for bundles in (
release_bundles.filter(date_added__gte=active_since),
release_bundles.filter(date_added__lt=active_since),
):
rows = list(
bundles.values_list("id", "artifact_count").order_by("-id")[
:URL_LOOKUP_MAX_CANDIDATE_BUNDLES
]
Comment thread
mwarkentin marked this conversation as resolved.
Resolved
)
for bundle_id, artifact_count in rows:
if bundle_id in candidates:
continue
# We always scan at least one bundle, even if it alone exceeds the budget.
if candidates and index_rows + artifact_count > max_index_rows:
truncated = True
break
candidates[bundle_id] = None
index_rows += artifact_count
if truncated or len(rows) == URL_LOOKUP_MAX_CANDIDATE_BUNDLES:
# Either the budget is spent, or this group has more bundles than we looked at,
# and those come before any bundle of the next group.
truncated = True
break

metrics.distribution("artifact_bundle_url_lookup.index_rows", index_rows)
metrics.incr(
"artifact_bundle_url_lookup.candidates",
tags={"truncated": "true" if truncated else "false"},
)
return list(candidates)


def get_artifact_bundles_containing_url(
project: Project, release_name: str, dist_name: str, url: str
) -> set[tuple[int, datetime]]:
"""
Returns the most recently uploaded bundle containing a file matching the `release`, `dist` and `url`.
Returns the most recently uploaded bundles containing a file matching the `release`, `dist` and `url`.
"""
return set(
ArtifactBundle.objects.filter(
Exists(
ArtifactBundleIndex.objects.filter(
artifact_bundle_id=OuterRef("pk"),
organization_id=project.organization.id,
url__icontains=url,
)
),
contains_url = Exists(
ArtifactBundleIndex.objects.filter(
artifact_bundle_id=OuterRef("pk"),
organization_id=project.organization.id,
url__icontains=url,
)
)

candidates = get_url_lookup_candidates(project, release_name, dist_name)
if candidates is not None:
# The candidates already belong to the project, release and dist.
bundles = ArtifactBundle.objects.filter(contains_url, id__in=candidates)
else:
bundles = ArtifactBundle.objects.filter(
contains_url,
Exists(
ProjectArtifactBundle.objects.filter(
artifact_bundle_id=OuterRef("pk"),
Expand All @@ -442,8 +524,11 @@ def get_artifact_bundles_containing_url(
)
),
)
.values_list("id", "date_added")
.order_by("-date_last_modified", "-id")[:MAX_BUNDLES_QUERY]

return set(
bundles.values_list("id", "date_added").order_by("-date_last_modified", "-id")[
:MAX_BUNDLES_QUERY
]
)


Expand Down
20 changes: 20 additions & 0 deletions src/sentry/options/defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -1131,6 +1131,26 @@
flags=FLAG_AUTOMATOR_MODIFIABLE,
)

# Upper bound on the number of `ArtifactBundleIndex` rows a single URL lookup in the
# artifact-lookup endpoint scans. Releases that have more indexed files than this scan
# their active bundles (uploaded or renewed recently) first, then the others, newest
# first, until the budget is spent. 0 disables the limit and scans every bundle.
register(
"sourcemaps.artifact-bundles.url-lookup.max-index-rows",
type=Int,
default=0,
flags=FLAG_AUTOMATOR_MODIFIABLE,
)
# Days added to `system.debug-files-renewal-age-threshold-days` to decide which bundles
# count as active for the URL lookup. A bundle in use is renewed once it is older than the
# threshold, so the margin must cover the time between lookups of a bundle in use.
register(
"sourcemaps.artifact-bundles.url-lookup.active-margin-days",
type=Int,
default=7,
flags=FLAG_AUTOMATOR_MODIFIABLE,
)

# Do not add `ArtifactBundleIndex` rows for files stored under a name built from their own
# debug ID (`~/<debug-id>-<n>.js`), which lookups find by debug ID rather than by URL.
register(
Expand Down
99 changes: 99 additions & 0 deletions tests/sentry/debug_files/test_artifact_bundles.py
Original file line number Diff line number Diff line change
Expand Up @@ -525,3 +525,102 @@ def test_multiple_bundles_with_different_urls(self) -> None:
),
{bundle3.id},
)

def index_url(self, bundle: ArtifactBundle, url: str) -> None:
ArtifactBundleIndex.objects.create(
organization_id=self.organization.id,
artifact_bundle=bundle,
url=url,
date_added=bundle.date_added,
)

def lookup(self, url: str) -> set[int]:
return {
bundle_id
for bundle_id, _ in get_artifact_bundles_containing_url(
self.project, self.release_name, self.dist_name, url
)
}

@override_options({"sourcemaps.artifact-bundles.url-lookup.max-index-rows": 1000})
def test_url_lookup_budget_scans_small_release_completely(self) -> None:
old = self.create_bundle(date_added=timezone.now() - timedelta(days=90))
new = self.create_bundle()
self.index_url(old, "~/path/to/old.js")
self.index_url(new, "~/path/to/new.js")

assert self.lookup("/path/to/old") == {old.id}
assert self.lookup("/path/to/new") == {new.id}

@override_options(
{
# Each bundle has 5 files, so the budget covers two bundles.
"sourcemaps.artifact-bundles.url-lookup.max-index-rows": 10,
"system.debug-files-renewal-age-threshold-days": 7,
"sourcemaps.artifact-bundles.url-lookup.active-margin-days": 7,
}
)
def test_url_lookup_budget_prefers_active_bundles(self) -> None:
now = timezone.now()
renewed = self.create_bundle(
date_added=now - timedelta(days=3), date_last_modified=now - timedelta(days=60)
)
idle = self.create_bundle(date_added=now - timedelta(days=50))
newest = self.create_bundle()
self.index_url(renewed, "~/path/to/renewed.js")
self.index_url(idle, "~/path/to/idle.js")
self.index_url(newest, "~/path/to/newest.js")

# `idle` was uploaded after `renewed`, but only the active bundles fit in the budget.
assert self.lookup("/path/to/renewed") == {renewed.id}
assert self.lookup("/path/to/newest") == {newest.id}
assert self.lookup("/path/to/idle") == set()

@override_options(
{
"sourcemaps.artifact-bundles.url-lookup.max-index-rows": 10,
"system.debug-files-renewal-age-threshold-days": 7,
"sourcemaps.artifact-bundles.url-lookup.active-margin-days": 7,
}
)
def test_url_lookup_budget_uses_newest_idle_bundles(self) -> None:
now = timezone.now()
oldest = self.create_bundle(date_added=now - timedelta(days=90))
older = self.create_bundle(date_added=now - timedelta(days=80))
newer = self.create_bundle(date_added=now - timedelta(days=70))
for bundle, name in ((oldest, "oldest"), (older, "older"), (newer, "newer")):
self.index_url(bundle, f"~/path/to/{name}.js")

# Without active bundles, the budget goes to the most recently uploaded bundles.
assert self.lookup("/path/to/newer") == {newer.id}
assert self.lookup("/path/to/older") == {older.id}
assert self.lookup("/path/to/oldest") == set()

@override_options({"sourcemaps.artifact-bundles.url-lookup.max-index-rows": 1})
def test_url_lookup_budget_scans_at_least_one_bundle(self) -> None:
older = self.create_bundle()
newer = self.create_bundle()
self.index_url(older, "~/path/to/app.js")
self.index_url(newer, "~/path/to/app.js")

assert self.lookup("/path/to/app") == {newer.id}

@override_options({"sourcemaps.artifact-bundles.url-lookup.max-index-rows": 1000})
def test_url_lookup_budget_respects_project_and_release(self) -> None:
bundle = self.create_bundle()
self.index_url(bundle, "~/path/to/app.js")

other_project = self.create_project(organization=self.organization)
assert (
get_artifact_bundles_containing_url(
other_project, self.release_name, self.dist_name, "/path/to/app"
)
== set()
)
assert (
get_artifact_bundles_containing_url(
self.project, "2.0.0", self.dist_name, "/path/to/app"
)
== set()
)
assert self.lookup("/path/to/app") == {bundle.id}
Loading