diff --git a/src/sentry/issues/derived/check.py b/src/sentry/issues/derived/check.py index cce9ce816219..9b8cf7d19ead 100644 --- a/src/sentry/issues/derived/check.py +++ b/src/sentry/issues/derived/check.py @@ -6,6 +6,8 @@ from typing import Any, NamedTuple, Protocol, TypedDict from uuid import uuid4 +from django.utils import timezone + from sentry.issues.derived.features import NO_CHANGE_RECONCILE_IDS, STATUS, IssueStatus from sentry.issues.derived.framework import Feature, State from sentry.issues.derived.gate import derived_should_be_correct @@ -20,6 +22,8 @@ logger = logging.getLogger(__name__) +_RECENT_LOG_ENTRY_WINDOW = timedelta(minutes=10) + class _GroupActionPipeline(PipelineFeatures, Protocol): @property @@ -140,6 +144,13 @@ class CheckInvalidated: pass +@dataclass(frozen=True) +class CheckRecentBackfill: + """A check skipped because its replay includes a recently backfilled log entry.""" + + pass + + class FeatureDifference(TypedDict): """JSON-safe expected and actual feature values for diagnostics.""" @@ -157,7 +168,7 @@ class CheckFailure: differences: dict[Feature[Any], FeatureDifference] -type CheckResult = CheckPassed | CheckFailure | CheckInvalidated +type CheckResult = CheckPassed | CheckFailure | CheckInvalidated | CheckRecentBackfill def compare_derived_data( @@ -285,6 +296,7 @@ def check_derived_data( check_id = CheckId.new_for_derived_data(target) target_state = GroupDerivedDataStore.load(pipeline, target) + recent_entry_cutoff = timezone.now() - _RECENT_LOG_ENTRY_WINDOW replayed_derived = _check_cache.get(check_id) if replayed_derived is None: @@ -298,6 +310,12 @@ def check_derived_data( deadline = time.monotonic() + timeout.total_seconds() if timeout is not None else None while entries := _entries_through_target_cursor(replayed_derived, check_id, batch_size): + if any( + entry.date_updated >= recent_entry_cutoff and entry.date_added < recent_entry_cutoff + for entry in entries + ): + _check_cache.delete(check_id) + return CheckRecentBackfill() replayed = pipeline.run( entries, state=GroupDerivedDataStore.load(pipeline, replayed_derived) ) diff --git a/src/sentry/issues/derived/tasks_util.py b/src/sentry/issues/derived/tasks_util.py index 295c973c8099..3d851f31db6a 100644 --- a/src/sentry/issues/derived/tasks_util.py +++ b/src/sentry/issues/derived/tasks_util.py @@ -12,7 +12,13 @@ from django.db.models import Max, Min from django.db.utils import OperationalError -from sentry.issues.derived.check import CheckFailure, CheckId, CheckInvalidated, CheckResult +from sentry.issues.derived.check import ( + CheckFailure, + CheckId, + CheckInvalidated, + CheckRecentBackfill, + CheckResult, +) from sentry.issues.models.groupderiveddata import GroupDerivedData from sentry.taskworker.selfchain_idempotency import already_spawned, mark_spawned from sentry.utils import metrics @@ -65,6 +71,8 @@ def mark_spawned(self) -> None: def _record_check_result(result: CheckResult) -> None: outcome = "no_result" if isinstance(result, CheckInvalidated) else "success" + if isinstance(result, CheckRecentBackfill): + outcome = "recent_backfill" if isinstance(result, CheckFailure): outcome = "mismatch" logger.warning( diff --git a/tests/sentry/issues/derived/test_processing.py b/tests/sentry/issues/derived/test_processing.py index 91b6db021a6d..ebe32e8afa78 100644 --- a/tests/sentry/issues/derived/test_processing.py +++ b/tests/sentry/issues/derived/test_processing.py @@ -31,6 +31,7 @@ CheckFailure, CheckInvalidated, CheckPassed, + CheckRecentBackfill, CheckTimeout, FeatureDifference, check_derived_data, @@ -155,6 +156,12 @@ def _publish(*, group: Group, action: GroupAction, actor: GroupActionActor = SYS ) +def _age_action_log_entries(group_id: int) -> None: + GroupActionLogEntry.objects.filter(group_id=group_id).update( + date_updated=django_timezone.now() - timedelta(minutes=11) + ) + + @with_feature("projects:issue-action-log-write-to-db") class ProcessGroupLogTest(TestCase): def setUp(self) -> None: @@ -250,11 +257,22 @@ def test_check_derived_data_matches_replayed_state(self) -> None: assert check_derived_data(derived, PIPELINE) == CheckPassed() + def test_check_derived_data_skips_recently_backfilled_log_entry(self) -> None: + group = self.create_group() + _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) + derived = process_group_log(group.id) + GroupActionLogEntry.objects.filter(group_id=group.id).update( + date_added=django_timezone.now() - timedelta(days=30) + ) + + assert check_derived_data(derived, PIPELINE) == CheckRecentBackfill() + def test_check_derived_data_reports_different_features(self) -> None: group = self.create_group() _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) derived.view_count = 0 + _age_action_log_entries(group.id) assert check_derived_data(derived, PIPELINE) == CheckFailure( group_id=group.id, @@ -277,6 +295,7 @@ def test_check_derived_data_can_resume_after_timeout(self) -> None: _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) + _age_action_log_entries(group.id) with pytest.raises(CheckTimeout) as exc_info: check_derived_data(derived, PIPELINE, timeout=timedelta(0), batch_size=1) @@ -297,6 +316,7 @@ def test_check_derived_data_uses_invocation_scoped_checkpoints(self) -> None: _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) + _age_action_log_entries(group.id) with pytest.raises(CheckTimeout) as first_timeout: check_derived_data(derived, PIPELINE, timeout=timedelta(0), batch_size=1) @@ -325,6 +345,9 @@ def test_check_derived_data_stops_after_partial_batch(self) -> None: _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) entries = list(GroupActionLogEntry.objects.filter(group_id=group.id)) + _age_action_log_entries(group.id) + for entry in entries: + entry.date_updated = django_timezone.now() - timedelta(minutes=11) with patch( "sentry.issues.derived.check._entries_through_target_cursor", side_effect=[entries] @@ -338,6 +361,7 @@ def test_check_derived_data_does_not_resume_across_generations(self) -> None: _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) + _age_action_log_entries(group.id) with pytest.raises(CheckTimeout) as exc_info: check_derived_data(derived, PIPELINE, timeout=timedelta(0), batch_size=1) @@ -362,6 +386,7 @@ def test_check_derived_data_skips_row_invalidated_during_replay(self) -> None: group = self.create_group() _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) + _age_action_log_entries(group.id) original_run = PIPELINE.run def run_and_invalidate( diff --git a/tests/sentry/issues/derived/test_tasks.py b/tests/sentry/issues/derived/test_tasks.py index 1a40210f86bf..87b3ca3eb830 100644 --- a/tests/sentry/issues/derived/test_tasks.py +++ b/tests/sentry/issues/derived/test_tasks.py @@ -6,6 +6,7 @@ import pytest from django.db.utils import OperationalError +from django.utils import timezone as django_timezone from sentry.issues.action_log.publish import publish_action from sentry.issues.action_log.types import ActionSource, GroupActionActor, ViewAction @@ -36,6 +37,7 @@ _pick_random_fresh_group_ranges, group_id_ranges_for_hash, ) +from sentry.issues.models.groupactionlogentry import GroupActionLogEntry from sentry.issues.models.groupderiveddata import GroupDerivedData from sentry.models.group import Group, GroupStatus from sentry.taskworker.selfchain_idempotency import already_spawned, mark_spawned @@ -1164,6 +1166,28 @@ def test_checks_only_fresh_rows_inline(self) -> None: call("issues.derived.check_group", sample_rate=1.0, tags={"result": "success"}), ] + def test_records_recent_backfill(self) -> None: + group = self.create_unprocessed_groups(1)[0] + process_group_log(group.id) + GroupActionLogEntry.objects.filter(group_id=group.id).update( + date_added=django_timezone.now() - timedelta(days=30) + ) + + with patch("sentry.issues.derived.tasks_util.metrics.incr") as mock_incr: + check_fresh_derived_data_batch( + group_id_start=group.id, + group_id_end=group.id + 1, + ) + + assert ( + call( + "issues.derived.check_group", + sample_rate=1.0, + tags={"result": "recent_backfill"}, + ) + in mock_incr.call_args_list + ) + def test_reschedules_timed_out_group_with_check_id(self) -> None: group = self.create_unprocessed_groups(1)[0] derived = process_group_log(group.id)