Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion src/sentry/issues/derived/check.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
from typing import Any, NamedTuple, Protocol, TypedDict
from uuid import uuid4

from django.utils import timezone

from sentry.issues.derived.features import NO_CHANGE_RECONCILE_IDS, STATUS, IssueStatus
from sentry.issues.derived.framework import Feature, State
from sentry.issues.derived.gate import derived_should_be_correct
Expand All @@ -20,6 +22,8 @@

logger = logging.getLogger(__name__)

_RECENT_LOG_ENTRY_WINDOW = timedelta(minutes=10)


class _GroupActionPipeline(PipelineFeatures, Protocol):
@property
Expand Down Expand Up @@ -140,6 +144,13 @@ class CheckInvalidated:
pass


@dataclass(frozen=True)
class CheckRecentBackfill:
"""A check skipped because its replay includes a recently backfilled log entry."""

pass


class FeatureDifference(TypedDict):
"""JSON-safe expected and actual feature values for diagnostics."""

Expand All @@ -157,7 +168,7 @@ class CheckFailure:
differences: dict[Feature[Any], FeatureDifference]


type CheckResult = CheckPassed | CheckFailure | CheckInvalidated
type CheckResult = CheckPassed | CheckFailure | CheckInvalidated | CheckRecentBackfill


def compare_derived_data(
Expand Down Expand Up @@ -285,6 +296,7 @@ def check_derived_data(
check_id = CheckId.new_for_derived_data(target)

target_state = GroupDerivedDataStore.load(pipeline, target)
recent_entry_cutoff = timezone.now() - _RECENT_LOG_ENTRY_WINDOW

replayed_derived = _check_cache.get(check_id)
if replayed_derived is None:
Expand All @@ -298,6 +310,12 @@ def check_derived_data(

deadline = time.monotonic() + timeout.total_seconds() if timeout is not None else None
while entries := _entries_through_target_cursor(replayed_derived, check_id, batch_size):
if any(
entry.date_updated >= recent_entry_cutoff and entry.date_added < recent_entry_cutoff
for entry in entries
):
_check_cache.delete(check_id)
return CheckRecentBackfill()
replayed = pipeline.run(
entries, state=GroupDerivedDataStore.load(pipeline, replayed_derived)
)
Expand Down
10 changes: 9 additions & 1 deletion src/sentry/issues/derived/tasks_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,13 @@
from django.db.models import Max, Min
from django.db.utils import OperationalError

from sentry.issues.derived.check import CheckFailure, CheckId, CheckInvalidated, CheckResult
from sentry.issues.derived.check import (
CheckFailure,
CheckId,
CheckInvalidated,
CheckRecentBackfill,
CheckResult,
)
from sentry.issues.models.groupderiveddata import GroupDerivedData
from sentry.taskworker.selfchain_idempotency import already_spawned, mark_spawned
from sentry.utils import metrics
Expand Down Expand Up @@ -65,6 +71,8 @@ def mark_spawned(self) -> None:

def _record_check_result(result: CheckResult) -> None:
outcome = "no_result" if isinstance(result, CheckInvalidated) else "success"
if isinstance(result, CheckRecentBackfill):
outcome = "recent_backfill"
if isinstance(result, CheckFailure):
outcome = "mismatch"
logger.warning(
Expand Down
25 changes: 25 additions & 0 deletions tests/sentry/issues/derived/test_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
CheckFailure,
CheckInvalidated,
CheckPassed,
CheckRecentBackfill,
CheckTimeout,
FeatureDifference,
check_derived_data,
Expand Down Expand Up @@ -155,6 +156,12 @@ def _publish(*, group: Group, action: GroupAction, actor: GroupActionActor = SYS
)


def _age_action_log_entries(group_id: int) -> None:
GroupActionLogEntry.objects.filter(group_id=group_id).update(
date_updated=django_timezone.now() - timedelta(minutes=11)
)


@with_feature("projects:issue-action-log-write-to-db")
class ProcessGroupLogTest(TestCase):
def setUp(self) -> None:
Expand Down Expand Up @@ -250,11 +257,22 @@ def test_check_derived_data_matches_replayed_state(self) -> None:

assert check_derived_data(derived, PIPELINE) == CheckPassed()

def test_check_derived_data_skips_recently_backfilled_log_entry(self) -> None:
group = self.create_group()
_publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id))
derived = process_group_log(group.id)
GroupActionLogEntry.objects.filter(group_id=group.id).update(
date_added=django_timezone.now() - timedelta(days=30)
)

assert check_derived_data(derived, PIPELINE) == CheckRecentBackfill()

def test_check_derived_data_reports_different_features(self) -> None:
group = self.create_group()
_publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id))
derived = process_group_log(group.id)
derived.view_count = 0
_age_action_log_entries(group.id)

assert check_derived_data(derived, PIPELINE) == CheckFailure(
group_id=group.id,
Expand All @@ -277,6 +295,7 @@ def test_check_derived_data_can_resume_after_timeout(self) -> None:
_publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id))
_publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id))
derived = process_group_log(group.id)
_age_action_log_entries(group.id)

with pytest.raises(CheckTimeout) as exc_info:
check_derived_data(derived, PIPELINE, timeout=timedelta(0), batch_size=1)
Expand All @@ -297,6 +316,7 @@ def test_check_derived_data_uses_invocation_scoped_checkpoints(self) -> None:
_publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id))
_publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id))
derived = process_group_log(group.id)
_age_action_log_entries(group.id)

with pytest.raises(CheckTimeout) as first_timeout:
check_derived_data(derived, PIPELINE, timeout=timedelta(0), batch_size=1)
Expand Down Expand Up @@ -325,6 +345,9 @@ def test_check_derived_data_stops_after_partial_batch(self) -> None:
_publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id))
derived = process_group_log(group.id)
entries = list(GroupActionLogEntry.objects.filter(group_id=group.id))
_age_action_log_entries(group.id)
for entry in entries:
entry.date_updated = django_timezone.now() - timedelta(minutes=11)

with patch(
"sentry.issues.derived.check._entries_through_target_cursor", side_effect=[entries]
Expand All @@ -338,6 +361,7 @@ def test_check_derived_data_does_not_resume_across_generations(self) -> None:
_publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id))
_publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id))
derived = process_group_log(group.id)
_age_action_log_entries(group.id)

with pytest.raises(CheckTimeout) as exc_info:
check_derived_data(derived, PIPELINE, timeout=timedelta(0), batch_size=1)
Expand All @@ -362,6 +386,7 @@ def test_check_derived_data_skips_row_invalidated_during_replay(self) -> None:
group = self.create_group()
_publish(group=group, action=ViewAction(), actor=GroupActionActor.user(self.user.id))
derived = process_group_log(group.id)
_age_action_log_entries(group.id)
original_run = PIPELINE.run

def run_and_invalidate(
Expand Down
24 changes: 24 additions & 0 deletions tests/sentry/issues/derived/test_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import pytest
from django.db.utils import OperationalError
from django.utils import timezone as django_timezone

from sentry.issues.action_log.publish import publish_action
from sentry.issues.action_log.types import ActionSource, GroupActionActor, ViewAction
Expand Down Expand Up @@ -36,6 +37,7 @@
_pick_random_fresh_group_ranges,
group_id_ranges_for_hash,
)
from sentry.issues.models.groupactionlogentry import GroupActionLogEntry
from sentry.issues.models.groupderiveddata import GroupDerivedData
from sentry.models.group import Group, GroupStatus
from sentry.taskworker.selfchain_idempotency import already_spawned, mark_spawned
Expand Down Expand Up @@ -1164,6 +1166,28 @@ def test_checks_only_fresh_rows_inline(self) -> None:
call("issues.derived.check_group", sample_rate=1.0, tags={"result": "success"}),
]

def test_records_recent_backfill(self) -> None:
group = self.create_unprocessed_groups(1)[0]
process_group_log(group.id)
GroupActionLogEntry.objects.filter(group_id=group.id).update(
date_added=django_timezone.now() - timedelta(days=30)
)

with patch("sentry.issues.derived.tasks_util.metrics.incr") as mock_incr:
check_fresh_derived_data_batch(
group_id_start=group.id,
group_id_end=group.id + 1,
)

assert (
call(
"issues.derived.check_group",
sample_rate=1.0,
tags={"result": "recent_backfill"},
)
in mock_incr.call_args_list
)

def test_reschedules_timed_out_group_with_check_id(self) -> None:
group = self.create_unprocessed_groups(1)[0]
derived = process_group_log(group.id)
Expand Down
Loading