diff --git a/src/sentry/event_manager.py b/src/sentry/event_manager.py index 1c7452f6d2e2..b54c7db4530d 100644 --- a/src/sentry/event_manager.py +++ b/src/sentry/event_manager.py @@ -456,6 +456,7 @@ def save( cache_key: str | None = None, skip_send_first_transaction: bool = False, attachments: list[CachedAttachment] | None = None, + unprocessed: MutableMapping[str, Any] | None = None, ) -> Event: """ After normalizing and processing an event, save adjacent models such as @@ -491,6 +492,7 @@ def save( "project_id": project.id, "raw": raw, "start_time": start_time, + "unprocessed": unprocessed, } # After calling _pull_out_data we get some keys in the job like the platform @@ -1137,12 +1139,23 @@ def _nodestore_save_many(jobs: Sequence[Job], app_feature: str) -> None: event = job["event"] # We only care about `unprocessed` for error events if event.get_event_type() not in ("transaction", "generic") and job["groups"]: - unprocessed = event_processing_store.get( - cache_key_for_event({"project": event.project_id, "event_id": event.event_id}), - unprocessed=True, - ) + unprocessed = job.get("unprocessed") + unprocessed_source = "inline" + if unprocessed is None: + # Clean up after `store.enable-inline-payloads` is 1.0 and + # `store.disable-processing-store` is `True` + unprocessed_source = "processing_store" + unprocessed = event_processing_store.get( + cache_key_for_event({"project": event.project_id, "event_id": event.event_id}), + unprocessed=True, + ) if unprocessed is not None: subkeys["unprocessed"] = unprocessed + metrics.incr( + "events.unprocessed_copy.promoted", + tags={"source": unprocessed_source}, + sample_rate=1.0, + ) if app_feature: event_size = 0 diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index 92affa54e68e..e72d7e8ef546 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -1796,7 +1796,7 @@ ) # Suppresses working-cache keys only for events entering in the inline cohort. # Keyless events remain inline after rollout changes; keyed events keep writing. -# Unprocessed backups and cleanup remain in Redis. +# Unprocessed backups are also passed inline instead of being written to Redis. register( "store.disable-processing-store", type=Bool, diff --git a/src/sentry/tasks/store.py b/src/sentry/tasks/store.py index f3c2947116d4..dd63ee236803 100644 --- a/src/sentry/tasks/store.py +++ b/src/sentry/tasks/store.py @@ -18,7 +18,11 @@ from sentry.feedback.usecases.ingest.save_event_feedback import ( save_event_feedback as save_event_feedback_impl, ) -from sentry.ingest.event_payload import load_event_payload, prepare_submit +from sentry.ingest.event_payload import ( + get_event_payload_transport, + load_event_payload, + prepare_submit, +) from sentry.ingest.types import ConsumerType from sentry.killswitches import killswitch_matches_context from sentry.lang.native.symbolicator import SymbolicatorTaskKind @@ -71,6 +75,7 @@ def submit_process( from_symbolicate: bool = False, has_attachments: bool = False, data: MutableMapping[str, Any] | None = None, + unprocessed: MutableMapping[str, Any] | None = None, ) -> None: data, cache_key = prepare_submit(data, cache_key, event_id) if from_reprocessing: @@ -85,6 +90,7 @@ def submit_process( from_symbolicate=from_symbolicate, has_attachments=has_attachments, data=data, + unprocessed=unprocessed, ) @@ -102,6 +108,7 @@ def submit_save_event( start_time: float | None, data: MutableMapping[str, Any] | None, inline: bool = False, + unprocessed: MutableMapping[str, Any] | None = None, ) -> None: data, cache_key = prepare_submit(data, cache_key, event_id) @@ -117,6 +124,7 @@ def submit_save_event( "start_time": start_time, "event_id": event_id, "project_id": project_id, + "unprocessed": unprocessed, } if inline: @@ -235,7 +243,12 @@ def _do_preprocess_event( "symbolication_function": symbolication_function_name, }, ): - reprocessing2.backup_unprocessed_event(data=original_data) + unprocessed = None + transport = get_event_payload_transport(event_id) + if transport.cache: + reprocessing2.backup_unprocessed_event(data=original_data) + if transport.inline and not options.get("store.reprocessing-force-disable"): + unprocessed = original_data submit_symbolicate( SymbolicatorTaskKind( @@ -248,6 +261,7 @@ def _do_preprocess_event( has_attachments=has_attachments, symbolicate_functions=symbolicate_functions, data=data, + unprocessed=unprocessed, ) return # else: go directly to process, do not go through the symbolicate queue, do not collect 200 @@ -358,6 +372,7 @@ def do_process_event( data_has_changed: bool = False, from_symbolicate: bool = False, has_attachments: bool = False, + unprocessed: MutableMapping[str, Any] | None = None, ) -> None: data = load_event_payload(data, cache_key, processing.event_processing_store) @@ -390,6 +405,7 @@ def _continue_to_save_event() -> None: event_id=data_event_id, start_time=start_time, data=data, + unprocessed=unprocessed, ) if is_process_disabled(project_id, data_event_id, data.get("platform") or "null"): @@ -481,6 +497,7 @@ def process_event( from_symbolicate: bool = False, has_attachments: bool = False, data: MutableMapping[str, Any] | None = None, + unprocessed: MutableMapping[str, Any] | None = None, **kwargs: Any, ) -> None: """ @@ -502,6 +519,7 @@ def process_event( from_symbolicate=from_symbolicate, has_attachments=has_attachments, data=data, + unprocessed=unprocessed, ) @@ -520,6 +538,7 @@ def process_event_from_reprocessing( from_symbolicate: bool = False, has_attachments: bool = False, data: MutableMapping[str, Any] | None = None, + unprocessed: MutableMapping[str, Any] | None = None, **kwargs: Any, ) -> None: return do_process_event( @@ -531,6 +550,7 @@ def process_event_from_reprocessing( from_symbolicate=from_symbolicate, has_attachments=has_attachments, data=data, + unprocessed=unprocessed, ) @@ -541,6 +561,7 @@ def _do_save_event( project_id: int | None = None, has_attachments: bool = False, consumer_type: str | None = None, + unprocessed: MutableMapping[str, Any] | None = None, **kwargs: Any, ) -> None: """ @@ -628,6 +649,7 @@ def _do_save_event( start_time=start_time, cache_key=cache_key, attachments=attachments, + unprocessed=unprocessed, ) except HashDiscarded: # Mark all the attachments as `rate_limited`, so they are being properly cleaned up in the `finally` block: diff --git a/src/sentry/tasks/symbolication.py b/src/sentry/tasks/symbolication.py index 09b8e1e2ccb7..f43b753021a3 100644 --- a/src/sentry/tasks/symbolication.py +++ b/src/sentry/tasks/symbolication.py @@ -1,5 +1,6 @@ +import copy import logging -from collections.abc import Mapping +from collections.abc import Mapping, MutableMapping from time import time from typing import Any @@ -68,6 +69,7 @@ def _do_symbolicate_event( data: Event | None = None, has_attachments: bool = False, symbolicate_functions: list[SymbolicatorFunction] | None = None, + unprocessed: MutableMapping[str, Any] | None = None, ) -> None: data = load_event_payload(data, cache_key, processing.event_processing_store) @@ -86,6 +88,11 @@ def _do_symbolicate_event( sentry_sdk.set_tag("event_id", event_id) sentry_sdk.set_attribute("event_id", event_id) + # Eager task execution doesn't serialize task arguments, so copy before symbolication + # modifies the payload in place. + if unprocessed is data: + unprocessed = copy.deepcopy(data) + def _continue_to_process_event(was_killswitched: bool = False) -> None: # Go through the remaining symbolication platforms/functions # and submit the next one. @@ -100,6 +107,7 @@ def _continue_to_process_event(was_killswitched: bool = False) -> None: has_attachments=has_attachments, symbolicate_functions=symbolicate_functions, data=data, + unprocessed=unprocessed, ) return # else: @@ -112,6 +120,7 @@ def _continue_to_process_event(was_killswitched: bool = False) -> None: from_symbolicate=True, has_attachments=has_attachments, data=data, + unprocessed=unprocessed, ) symbolication_function = task_kind.function @@ -231,6 +240,7 @@ def submit_symbolicate( has_attachments: bool = False, symbolicate_functions: list[SymbolicatorFunction] | None = None, data: Event | None = None, + unprocessed: MutableMapping[str, Any] | None = None, ) -> None: data, cache_key = prepare_submit(data, cache_key, event_id) @@ -253,6 +263,7 @@ def submit_symbolicate( has_attachments=has_attachments, symbolicate_functions=symbolicate_function_names, data=data, + unprocessed=unprocessed, ) @@ -281,6 +292,7 @@ def symbolication_fn( data: Event | None = None, has_attachments: bool = False, symbolicate_functions: list[str] | None = None, + unprocessed: MutableMapping[str, Any] | None = None, **kwargs: Any, ) -> None: """ @@ -308,6 +320,7 @@ def symbolication_fn( data=data, has_attachments=has_attachments, symbolicate_functions=symbolicate_function_values, + unprocessed=unprocessed, ) fn_name = name.split(".")[-1] diff --git a/tests/sentry/tasks/test_store.py b/tests/sentry/tasks/test_store.py index 7a39d280b12d..61d277bc1dcc 100644 --- a/tests/sentry/tasks/test_store.py +++ b/tests/sentry/tasks/test_store.py @@ -145,6 +145,7 @@ def test_move_to_process_event_inline_save_event_still_submits_process_event( from_symbolicate=False, has_attachments=False, data=None, + unprocessed=None, ) assert mock_save_event.call_count == 0 assert mock_save_event.delay.call_count == 0 @@ -191,6 +192,7 @@ def test_move_to_save_event_inline( start_time=None, event_id=EVENT_ID, project_id=default_project.id, + unprocessed=None, ) assert mock_save_event.delay.call_count == 0 @@ -234,6 +236,7 @@ def test_process_event_mutate_and_save( start_time=1, event_id=EVENT_ID, project_id=default_project.id, + unprocessed=None, ) @@ -264,7 +267,12 @@ def test_process_event_no_mutate_and_save( mock_event_processing_store.store.assert_called_once_with(data) mock_save_event.delay.assert_called_once_with( - cache_key="e:1", data=None, start_time=1, event_id=EVENT_ID, project_id=default_project.id + cache_key="e:1", + data=None, + start_time=1, + event_id=EVENT_ID, + project_id=default_project.id, + unprocessed=None, ) @@ -326,7 +334,12 @@ def test_process_event_unprocessed( assert event["unprocessed"] is True mock_save_event.delay.assert_called_once_with( - cache_key="e:1", data=None, start_time=1, event_id=EVENT_ID, project_id=default_project.id + cache_key="e:1", + data=None, + start_time=1, + event_id=EVENT_ID, + project_id=default_project.id, + unprocessed=None, ) @@ -598,7 +611,12 @@ def more_extra(data): assert event["extra"] == {"ooo": "[Filtered]", "ooo2": "event preprocessor"} mock_save_event.delay.assert_called_once_with( - cache_key="e:1", data=None, start_time=1, event_id=EVENT_ID, project_id=default_project.id + cache_key="e:1", + data=None, + start_time=1, + event_id=EVENT_ID, + project_id=default_project.id, + unprocessed=None, ) @@ -711,6 +729,7 @@ def test_store_consumer_type( start_time=1, event_id=EVENT_ID, project_id=default_project.id, + unprocessed=None, ) transaction_data = { diff --git a/tests/sentry/tasks/test_symbolication.py b/tests/sentry/tasks/test_symbolication.py index ce6882daddf8..8f990cd2ab8f 100644 --- a/tests/sentry/tasks/test_symbolication.py +++ b/tests/sentry/tasks/test_symbolication.py @@ -45,7 +45,10 @@ def mock_event_processing_store(): @django_db_all -@pytest.mark.parametrize("inline", (False, True)) +@pytest.mark.parametrize( + "inline,disable_store,redis_backups", + [(False, False, 1), (True, False, 1), (True, True, 0)], +) def test_move_to_symbolicate_event( default_project, mock_process_event, @@ -53,24 +56,27 @@ def test_move_to_symbolicate_event( mock_symbolicate_event, mock_event_processing_store, inline, + disable_store, + redis_backups, ): data = {"platform": "native", "project": default_project.id, "event_id": EVENT_ID} - cache_key = None if inline else "e:1" + cache_key = None if disable_store else "e:1" mock_event_processing_store.store.return_value = "e:1" with ( override_options( { "store.enable-inline-payloads": float(inline), - "store.disable-processing-store": inline, + "store.disable-processing-store": disable_store, } ), mock.patch("sentry.tasks.store.reprocessing2.backup_unprocessed_event") as backup, ): preprocess_event(cache_key=cache_key, data=data) - backup.assert_called_once_with(data=data) + assert backup.call_args_list == [mock.call(data=data)] * redis_backups assert mock_symbolicate_event.delay.call_count == 1 kwargs = mock_symbolicate_event.delay.call_args.kwargs + assert (kwargs["unprocessed"] == data) is inline assert kwargs["data"] == (data if inline else None) assert kwargs["cache_key"] == cache_key assert mock_process_event.delay.call_count == 0