From 16f23bd90cf7e8546ddb15e3bcfcb1f4f66e602e Mon Sep 17 00:00:00 2001 From: Kyle Consalus Date: Wed, 14 Oct 2026 10:25:40 -0700 Subject: [PATCH 1/2] fix(derived data): Filter out very recent logs in consistency check --- src/sentry/issues/derived/check.py | 17 +++++++++++- src/sentry/issues/derived/tasks_util.py | 10 ++++++- .../sentry/issues/derived/test_processing.py | 26 +++++++++++++++++++ tests/sentry/issues/derived/test_tasks.py | 24 +++++++++++++++++ 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/src/sentry/issues/derived/check.py b/src/sentry/issues/derived/check.py index cce9ce816219..a094b255917e 100644 --- a/src/sentry/issues/derived/check.py +++ b/src/sentry/issues/derived/check.py @@ -6,6 +6,8 @@ from typing import Any, NamedTuple, Protocol, TypedDict from uuid import uuid4 +from django.utils import timezone + from sentry.issues.derived.features import NO_CHANGE_RECONCILE_IDS, STATUS, IssueStatus from sentry.issues.derived.framework import Feature, State from sentry.issues.derived.gate import derived_should_be_correct @@ -20,6 +22,8 @@ logger = logging.getLogger(__name__) +_RECENT_LOG_ENTRY_WINDOW = timedelta(minutes=10) + class _GroupActionPipeline(PipelineFeatures, Protocol): @property @@ -140,6 +144,13 @@ class CheckInvalidated: pass +@dataclass(frozen=True) +class CheckRecentLogEntry: + """A check skipped because its replay includes a recently written log entry.""" + + pass + + class FeatureDifference(TypedDict): """JSON-safe expected and actual feature values for diagnostics.""" @@ -157,7 +168,7 @@ class CheckFailure: differences: dict[Feature[Any], FeatureDifference] -type CheckResult = CheckPassed | CheckFailure | CheckInvalidated +type CheckResult = CheckPassed | CheckFailure | CheckInvalidated | CheckRecentLogEntry def compare_derived_data( @@ -285,6 +296,7 @@ def check_derived_data( check_id = CheckId.new_for_derived_data(target) target_state = GroupDerivedDataStore.load(pipeline, target) + recent_entry_cutoff = timezone.now() - _RECENT_LOG_ENTRY_WINDOW replayed_derived = _check_cache.get(check_id) if replayed_derived is None: @@ -298,6 +310,9 @@ def check_derived_data( deadline = time.monotonic() + timeout.total_seconds() if timeout is not None else None while entries := _entries_through_target_cursor(replayed_derived, check_id, batch_size): + if any(entry.date_updated >= recent_entry_cutoff for entry in entries): + _check_cache.delete(check_id) + return CheckRecentLogEntry() replayed = pipeline.run( entries, state=GroupDerivedDataStore.load(pipeline, replayed_derived) ) diff --git a/src/sentry/issues/derived/tasks_util.py b/src/sentry/issues/derived/tasks_util.py index 295c973c8099..ddb5d65f9873 100644 --- a/src/sentry/issues/derived/tasks_util.py +++ b/src/sentry/issues/derived/tasks_util.py @@ -12,7 +12,13 @@ from django.db.models import Max, Min from django.db.utils import OperationalError -from sentry.issues.derived.check import CheckFailure, CheckId, CheckInvalidated, CheckResult +from sentry.issues.derived.check import ( + CheckFailure, + CheckId, + CheckInvalidated, + CheckRecentLogEntry, + CheckResult, +) from sentry.issues.models.groupderiveddata import GroupDerivedData from sentry.taskworker.selfchain_idempotency import already_spawned, mark_spawned from sentry.utils import metrics @@ -65,6 +71,8 @@ def mark_spawned(self) -> None: def _record_check_result(result: CheckResult) -> None: outcome = "no_result" if isinstance(result, CheckInvalidated) else "success" + if isinstance(result, CheckRecentLogEntry): + outcome = "recent_log_entry" if isinstance(result, CheckFailure): outcome = "mismatch" logger.warning( diff --git a/tests/sentry/issues/derived/test_processing.py b/tests/sentry/issues/derived/test_processing.py index 91b6db021a6d..b248ee54631c 100644 --- a/tests/sentry/issues/derived/test_processing.py +++ b/tests/sentry/issues/derived/test_processing.py @@ -31,6 +31,7 @@ CheckFailure, CheckInvalidated, CheckPassed, + CheckRecentLogEntry, CheckTimeout, FeatureDifference, check_derived_data, @@ -155,6 +156,12 @@ def _publish(*, group: Group, action: GroupAction, actor: GroupActionActor = SYS ) +def _age_action_log_entries(group_id: int) -> None: + GroupActionLogEntry.objects.filter(group_id=group_id).update( + date_updated=django_timezone.now() - timedelta(minutes=11) + ) + + @with_feature("projects:issue-action-log-write-to-db") class ProcessGroupLogTest(TestCase): def setUp(self) -> None: @@ -247,14 +254,26 @@ def test_check_derived_data_matches_replayed_state(self) -> None: group = self.create_group() _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) + _age_action_log_entries(group.id) assert check_derived_data(derived, PIPELINE) == CheckPassed() + def test_check_derived_data_skips_recently_written_log_entry(self) -> None: + group = self.create_group() + _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) + derived = process_group_log(group.id) + GroupActionLogEntry.objects.filter(group_id=group.id).update( + date_added=django_timezone.now() - timedelta(days=30) + ) + + assert check_derived_data(derived, PIPELINE) == CheckRecentLogEntry() + def test_check_derived_data_reports_different_features(self) -> None: group = self.create_group() _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) derived.view_count = 0 + _age_action_log_entries(group.id) assert check_derived_data(derived, PIPELINE) == CheckFailure( group_id=group.id, @@ -277,6 +296,7 @@ def test_check_derived_data_can_resume_after_timeout(self) -> None: _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) + _age_action_log_entries(group.id) with pytest.raises(CheckTimeout) as exc_info: check_derived_data(derived, PIPELINE, timeout=timedelta(0), batch_size=1) @@ -297,6 +317,7 @@ def test_check_derived_data_uses_invocation_scoped_checkpoints(self) -> None: _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) + _age_action_log_entries(group.id) with pytest.raises(CheckTimeout) as first_timeout: check_derived_data(derived, PIPELINE, timeout=timedelta(0), batch_size=1) @@ -325,6 +346,9 @@ def test_check_derived_data_stops_after_partial_batch(self) -> None: _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) entries = list(GroupActionLogEntry.objects.filter(group_id=group.id)) + _age_action_log_entries(group.id) + for entry in entries: + entry.date_updated = django_timezone.now() - timedelta(minutes=11) with patch( "sentry.issues.derived.check._entries_through_target_cursor", side_effect=[entries] @@ -338,6 +362,7 @@ def test_check_derived_data_does_not_resume_across_generations(self) -> None: _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) + _age_action_log_entries(group.id) with pytest.raises(CheckTimeout) as exc_info: check_derived_data(derived, PIPELINE, timeout=timedelta(0), batch_size=1) @@ -362,6 +387,7 @@ def test_check_derived_data_skips_row_invalidated_during_replay(self) -> None: group = self.create_group() _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) + _age_action_log_entries(group.id) original_run = PIPELINE.run def run_and_invalidate( diff --git a/tests/sentry/issues/derived/test_tasks.py b/tests/sentry/issues/derived/test_tasks.py index 1a40210f86bf..5a4eedb71099 100644 --- a/tests/sentry/issues/derived/test_tasks.py +++ b/tests/sentry/issues/derived/test_tasks.py @@ -6,6 +6,7 @@ import pytest from django.db.utils import OperationalError +from django.utils import timezone as django_timezone from sentry.issues.action_log.publish import publish_action from sentry.issues.action_log.types import ActionSource, GroupActionActor, ViewAction @@ -36,6 +37,7 @@ _pick_random_fresh_group_ranges, group_id_ranges_for_hash, ) +from sentry.issues.models.groupactionlogentry import GroupActionLogEntry from sentry.issues.models.groupderiveddata import GroupDerivedData from sentry.models.group import Group, GroupStatus from sentry.taskworker.selfchain_idempotency import already_spawned, mark_spawned @@ -1141,6 +1143,9 @@ def test_checks_only_fresh_rows_inline(self) -> None: group_ids = sorted(group.id for group in groups) for group_id in group_ids: process_group_log(group_id) + GroupActionLogEntry.objects.filter(group_id__in=group_ids).update( + date_updated=django_timezone.now() - timedelta(minutes=11) + ) GroupDerivedData.objects.filter(group_id=group_ids[1]).update(pipeline_hash="stale") with patch("sentry.issues.derived.tasks_util.metrics.incr") as mock_incr: @@ -1164,6 +1169,25 @@ def test_checks_only_fresh_rows_inline(self) -> None: call("issues.derived.check_group", sample_rate=1.0, tags={"result": "success"}), ] + def test_records_recent_log_entry(self) -> None: + group = self.create_unprocessed_groups(1)[0] + process_group_log(group.id) + + with patch("sentry.issues.derived.tasks_util.metrics.incr") as mock_incr: + check_fresh_derived_data_batch( + group_id_start=group.id, + group_id_end=group.id + 1, + ) + + assert ( + call( + "issues.derived.check_group", + sample_rate=1.0, + tags={"result": "recent_log_entry"}, + ) + in mock_incr.call_args_list + ) + def test_reschedules_timed_out_group_with_check_id(self) -> None: group = self.create_unprocessed_groups(1)[0] derived = process_group_log(group.id) From c86156e68ff5d6986a3e1e4106e39b90764cb2e1 Mon Sep 17 00:00:00 2001 From: Kyle Consalus Date: Wed, 14 Oct 2026 10:35:46 -0700 Subject: [PATCH 2/2] more precise --- src/sentry/issues/derived/check.py | 13 ++++++++----- src/sentry/issues/derived/tasks_util.py | 6 +++--- tests/sentry/issues/derived/test_processing.py | 7 +++---- tests/sentry/issues/derived/test_tasks.py | 10 +++++----- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/sentry/issues/derived/check.py b/src/sentry/issues/derived/check.py index a094b255917e..9b8cf7d19ead 100644 --- a/src/sentry/issues/derived/check.py +++ b/src/sentry/issues/derived/check.py @@ -145,8 +145,8 @@ class CheckInvalidated: @dataclass(frozen=True) -class CheckRecentLogEntry: - """A check skipped because its replay includes a recently written log entry.""" +class CheckRecentBackfill: + """A check skipped because its replay includes a recently backfilled log entry.""" pass @@ -168,7 +168,7 @@ class CheckFailure: differences: dict[Feature[Any], FeatureDifference] -type CheckResult = CheckPassed | CheckFailure | CheckInvalidated | CheckRecentLogEntry +type CheckResult = CheckPassed | CheckFailure | CheckInvalidated | CheckRecentBackfill def compare_derived_data( @@ -310,9 +310,12 @@ def check_derived_data( deadline = time.monotonic() + timeout.total_seconds() if timeout is not None else None while entries := _entries_through_target_cursor(replayed_derived, check_id, batch_size): - if any(entry.date_updated >= recent_entry_cutoff for entry in entries): + if any( + entry.date_updated >= recent_entry_cutoff and entry.date_added < recent_entry_cutoff + for entry in entries + ): _check_cache.delete(check_id) - return CheckRecentLogEntry() + return CheckRecentBackfill() replayed = pipeline.run( entries, state=GroupDerivedDataStore.load(pipeline, replayed_derived) ) diff --git a/src/sentry/issues/derived/tasks_util.py b/src/sentry/issues/derived/tasks_util.py index ddb5d65f9873..3d851f31db6a 100644 --- a/src/sentry/issues/derived/tasks_util.py +++ b/src/sentry/issues/derived/tasks_util.py @@ -16,7 +16,7 @@ CheckFailure, CheckId, CheckInvalidated, - CheckRecentLogEntry, + CheckRecentBackfill, CheckResult, ) from sentry.issues.models.groupderiveddata import GroupDerivedData @@ -71,8 +71,8 @@ def mark_spawned(self) -> None: def _record_check_result(result: CheckResult) -> None: outcome = "no_result" if isinstance(result, CheckInvalidated) else "success" - if isinstance(result, CheckRecentLogEntry): - outcome = "recent_log_entry" + if isinstance(result, CheckRecentBackfill): + outcome = "recent_backfill" if isinstance(result, CheckFailure): outcome = "mismatch" logger.warning( diff --git a/tests/sentry/issues/derived/test_processing.py b/tests/sentry/issues/derived/test_processing.py index b248ee54631c..ebe32e8afa78 100644 --- a/tests/sentry/issues/derived/test_processing.py +++ b/tests/sentry/issues/derived/test_processing.py @@ -31,7 +31,7 @@ CheckFailure, CheckInvalidated, CheckPassed, - CheckRecentLogEntry, + CheckRecentBackfill, CheckTimeout, FeatureDifference, check_derived_data, @@ -254,11 +254,10 @@ def test_check_derived_data_matches_replayed_state(self) -> None: group = self.create_group() _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) - _age_action_log_entries(group.id) assert check_derived_data(derived, PIPELINE) == CheckPassed() - def test_check_derived_data_skips_recently_written_log_entry(self) -> None: + def test_check_derived_data_skips_recently_backfilled_log_entry(self) -> None: group = self.create_group() _publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id)) derived = process_group_log(group.id) @@ -266,7 +265,7 @@ def test_check_derived_data_skips_recently_written_log_entry(self) -> None: date_added=django_timezone.now() - timedelta(days=30) ) - assert check_derived_data(derived, PIPELINE) == CheckRecentLogEntry() + assert check_derived_data(derived, PIPELINE) == CheckRecentBackfill() def test_check_derived_data_reports_different_features(self) -> None: group = self.create_group() diff --git a/tests/sentry/issues/derived/test_tasks.py b/tests/sentry/issues/derived/test_tasks.py index 5a4eedb71099..87b3ca3eb830 100644 --- a/tests/sentry/issues/derived/test_tasks.py +++ b/tests/sentry/issues/derived/test_tasks.py @@ -1143,9 +1143,6 @@ def test_checks_only_fresh_rows_inline(self) -> None: group_ids = sorted(group.id for group in groups) for group_id in group_ids: process_group_log(group_id) - GroupActionLogEntry.objects.filter(group_id__in=group_ids).update( - date_updated=django_timezone.now() - timedelta(minutes=11) - ) GroupDerivedData.objects.filter(group_id=group_ids[1]).update(pipeline_hash="stale") with patch("sentry.issues.derived.tasks_util.metrics.incr") as mock_incr: @@ -1169,9 +1166,12 @@ def test_checks_only_fresh_rows_inline(self) -> None: call("issues.derived.check_group", sample_rate=1.0, tags={"result": "success"}), ] - def test_records_recent_log_entry(self) -> None: + def test_records_recent_backfill(self) -> None: group = self.create_unprocessed_groups(1)[0] process_group_log(group.id) + GroupActionLogEntry.objects.filter(group_id=group.id).update( + date_added=django_timezone.now() - timedelta(days=30) + ) with patch("sentry.issues.derived.tasks_util.metrics.incr") as mock_incr: check_fresh_derived_data_batch( @@ -1183,7 +1183,7 @@ def test_records_recent_log_entry(self) -> None: call( "issues.derived.check_group", sample_rate=1.0, - tags={"result": "recent_log_entry"}, + tags={"result": "recent_backfill"}, ) in mock_incr.call_args_list )