Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions src/sentry/event_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,7 @@ def save(
cache_key: str | None = None,
skip_send_first_transaction: bool = False,
attachments: list[CachedAttachment] | None = None,
unprocessed: MutableMapping[str, Any] | None = None,
) -> Event:
"""
After normalizing and processing an event, save adjacent models such as
Expand Down Expand Up @@ -491,6 +492,7 @@ def save(
"project_id": project.id,
"raw": raw,
"start_time": start_time,
"unprocessed": unprocessed,
}

# After calling _pull_out_data we get some keys in the job like the platform
Expand Down Expand Up @@ -1137,12 +1139,23 @@ def _nodestore_save_many(jobs: Sequence[Job], app_feature: str) -> None:
event = job["event"]
# We only care about `unprocessed` for error events
if event.get_event_type() not in ("transaction", "generic") and job["groups"]:
unprocessed = event_processing_store.get(
cache_key_for_event({"project": event.project_id, "event_id": event.event_id}),
unprocessed=True,
)
unprocessed = job.get("unprocessed")
unprocessed_source = "inline"
if unprocessed is None:
# Clean up after `store.enable-inline-payloads` is 1.0 and
# `store.disable-processing-store` is `True`
unprocessed_source = "processing_store"
unprocessed = event_processing_store.get(
cache_key_for_event({"project": event.project_id, "event_id": event.event_id}),
unprocessed=True,
)
if unprocessed is not None:
subkeys["unprocessed"] = unprocessed
metrics.incr(
"events.unprocessed_copy.promoted",
tags={"source": unprocessed_source},
sample_rate=1.0,
)

if app_feature:
event_size = 0
Expand Down
2 changes: 1 addition & 1 deletion src/sentry/options/defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -1796,7 +1796,7 @@
)
# Suppresses working-cache keys only for events entering in the inline cohort.
# Keyless events remain inline after rollout changes; keyed events keep writing.
# Unprocessed backups and cleanup remain in Redis.
# Unprocessed backups are also passed inline instead of being written to Redis.
register(
"store.disable-processing-store",
type=Bool,
Expand Down
26 changes: 24 additions & 2 deletions src/sentry/tasks/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@
from sentry.feedback.usecases.ingest.save_event_feedback import (
save_event_feedback as save_event_feedback_impl,
)
from sentry.ingest.event_payload import load_event_payload, prepare_submit
from sentry.ingest.event_payload import (
get_event_payload_transport,
load_event_payload,
prepare_submit,
)
from sentry.ingest.types import ConsumerType
from sentry.killswitches import killswitch_matches_context
from sentry.lang.native.symbolicator import SymbolicatorTaskKind
Expand Down Expand Up @@ -71,6 +75,7 @@ def submit_process(
from_symbolicate: bool = False,
has_attachments: bool = False,
data: MutableMapping[str, Any] | None = None,
unprocessed: MutableMapping[str, Any] | None = None,
) -> None:
data, cache_key = prepare_submit(data, cache_key, event_id)
if from_reprocessing:
Expand All @@ -85,6 +90,7 @@ def submit_process(
from_symbolicate=from_symbolicate,
has_attachments=has_attachments,
data=data,
unprocessed=unprocessed,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is all the data in this dict JSON encodable? Tasks will fail if there are any non-json encodable values in the object.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this is encodable. It contains the original data.

)


Expand All @@ -102,6 +108,7 @@ def submit_save_event(
start_time: float | None,
data: MutableMapping[str, Any] | None,
inline: bool = False,
unprocessed: MutableMapping[str, Any] | None = None,
) -> None:
data, cache_key = prepare_submit(data, cache_key, event_id)

Expand All @@ -117,6 +124,7 @@ def submit_save_event(
"start_time": start_time,
"event_id": event_id,
"project_id": project_id,
"unprocessed": unprocessed,
}

if inline:
Expand Down Expand Up @@ -235,7 +243,12 @@ def _do_preprocess_event(
"symbolication_function": symbolication_function_name,
},
):
reprocessing2.backup_unprocessed_event(data=original_data)
unprocessed = None
transport = get_event_payload_transport(event_id)
if transport.cache:
reprocessing2.backup_unprocessed_event(data=original_data)
if transport.inline and not options.get("store.reprocessing-force-disable"):
unprocessed = original_data

submit_symbolicate(
SymbolicatorTaskKind(
Expand All @@ -248,6 +261,7 @@ def _do_preprocess_event(
has_attachments=has_attachments,
symbolicate_functions=symbolicate_functions,
data=data,
unprocessed=unprocessed,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should unprocessed and cache_key be mutually exclusive? I would think that the end state is that we don't have both being used as the cache key is redundant when we also have the unprocessed + data parameters present.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the end state they will be mutually exclusive and eventually cache_key will be removed from the tasks. During rollout, we also double-write for some time so we can safely roll back at any point.

)
return
# else: go directly to process, do not go through the symbolicate queue, do not collect 200
Expand Down Expand Up @@ -358,6 +372,7 @@ def do_process_event(
data_has_changed: bool = False,
from_symbolicate: bool = False,
has_attachments: bool = False,
unprocessed: MutableMapping[str, Any] | None = None,
) -> None:
data = load_event_payload(data, cache_key, processing.event_processing_store)

Expand Down Expand Up @@ -390,6 +405,7 @@ def _continue_to_save_event() -> None:
event_id=data_event_id,
start_time=start_time,
data=data,
unprocessed=unprocessed,
)

if is_process_disabled(project_id, data_event_id, data.get("platform") or "null"):
Expand Down Expand Up @@ -481,6 +497,7 @@ def process_event(
from_symbolicate: bool = False,
has_attachments: bool = False,
data: MutableMapping[str, Any] | None = None,
unprocessed: MutableMapping[str, Any] | None = None,
**kwargs: Any,
) -> None:
"""
Expand All @@ -502,6 +519,7 @@ def process_event(
from_symbolicate=from_symbolicate,
has_attachments=has_attachments,
data=data,
unprocessed=unprocessed,
)


Expand All @@ -520,6 +538,7 @@ def process_event_from_reprocessing(
from_symbolicate: bool = False,
has_attachments: bool = False,
data: MutableMapping[str, Any] | None = None,
unprocessed: MutableMapping[str, Any] | None = None,
**kwargs: Any,
) -> None:
return do_process_event(
Expand All @@ -531,6 +550,7 @@ def process_event_from_reprocessing(
from_symbolicate=from_symbolicate,
has_attachments=has_attachments,
data=data,
unprocessed=unprocessed,
)


Expand All @@ -541,6 +561,7 @@ def _do_save_event(
project_id: int | None = None,
has_attachments: bool = False,
consumer_type: str | None = None,
unprocessed: MutableMapping[str, Any] | None = None,
**kwargs: Any,
) -> None:
"""
Expand Down Expand Up @@ -628,6 +649,7 @@ def _do_save_event(
start_time=start_time,
cache_key=cache_key,
attachments=attachments,
unprocessed=unprocessed,
)
except HashDiscarded:
# Mark all the attachments as `rate_limited`, so they are being properly cleaned up in the `finally` block:
Expand Down
15 changes: 14 additions & 1 deletion src/sentry/tasks/symbolication.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import copy
import logging
from collections.abc import Mapping
from collections.abc import Mapping, MutableMapping
from time import time
from typing import Any

Expand Down Expand Up @@ -68,6 +69,7 @@ def _do_symbolicate_event(
data: Event | None = None,
has_attachments: bool = False,
symbolicate_functions: list[SymbolicatorFunction] | None = None,
unprocessed: MutableMapping[str, Any] | None = None,
) -> None:
data = load_event_payload(data, cache_key, processing.event_processing_store)

Expand All @@ -86,6 +88,11 @@ def _do_symbolicate_event(
sentry_sdk.set_tag("event_id", event_id)
sentry_sdk.set_attribute("event_id", event_id)

# Eager task execution doesn't serialize task arguments, so copy before symbolication
# modifies the payload in place.
if unprocessed is data:
unprocessed = copy.deepcopy(data)

def _continue_to_process_event(was_killswitched: bool = False) -> None:
# Go through the remaining symbolication platforms/functions
# and submit the next one.
Expand All @@ -100,6 +107,7 @@ def _continue_to_process_event(was_killswitched: bool = False) -> None:
has_attachments=has_attachments,
symbolicate_functions=symbolicate_functions,
data=data,
unprocessed=unprocessed,
)
return
# else:
Expand All @@ -112,6 +120,7 @@ def _continue_to_process_event(was_killswitched: bool = False) -> None:
from_symbolicate=True,
has_attachments=has_attachments,
data=data,
unprocessed=unprocessed,
)

symbolication_function = task_kind.function
Expand Down Expand Up @@ -231,6 +240,7 @@ def submit_symbolicate(
has_attachments: bool = False,
symbolicate_functions: list[SymbolicatorFunction] | None = None,
data: Event | None = None,
unprocessed: MutableMapping[str, Any] | None = None,
) -> None:
data, cache_key = prepare_submit(data, cache_key, event_id)

Expand All @@ -253,6 +263,7 @@ def submit_symbolicate(
has_attachments=has_attachments,
symbolicate_functions=symbolicate_function_names,
data=data,
unprocessed=unprocessed,
)


Expand Down Expand Up @@ -281,6 +292,7 @@ def symbolication_fn(
data: Event | None = None,
has_attachments: bool = False,
symbolicate_functions: list[str] | None = None,
unprocessed: MutableMapping[str, Any] | None = None,
**kwargs: Any,
) -> None:
"""
Expand Down Expand Up @@ -308,6 +320,7 @@ def symbolication_fn(
data=data,
has_attachments=has_attachments,
symbolicate_functions=symbolicate_function_values,
unprocessed=unprocessed,
)

fn_name = name.split(".")[-1]
Expand Down
25 changes: 22 additions & 3 deletions tests/sentry/tasks/test_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ def test_move_to_process_event_inline_save_event_still_submits_process_event(
from_symbolicate=False,
has_attachments=False,
data=None,
unprocessed=None,
)
assert mock_save_event.call_count == 0
assert mock_save_event.delay.call_count == 0
Expand Down Expand Up @@ -191,6 +192,7 @@ def test_move_to_save_event_inline(
start_time=None,
event_id=EVENT_ID,
project_id=default_project.id,
unprocessed=None,
)
assert mock_save_event.delay.call_count == 0

Expand Down Expand Up @@ -234,6 +236,7 @@ def test_process_event_mutate_and_save(
start_time=1,
event_id=EVENT_ID,
project_id=default_project.id,
unprocessed=None,
)


Expand Down Expand Up @@ -264,7 +267,12 @@ def test_process_event_no_mutate_and_save(
mock_event_processing_store.store.assert_called_once_with(data)

mock_save_event.delay.assert_called_once_with(
cache_key="e:1", data=None, start_time=1, event_id=EVENT_ID, project_id=default_project.id
cache_key="e:1",
data=None,
start_time=1,
event_id=EVENT_ID,
project_id=default_project.id,
unprocessed=None,
)


Expand Down Expand Up @@ -326,7 +334,12 @@ def test_process_event_unprocessed(
assert event["unprocessed"] is True

mock_save_event.delay.assert_called_once_with(
cache_key="e:1", data=None, start_time=1, event_id=EVENT_ID, project_id=default_project.id
cache_key="e:1",
data=None,
start_time=1,
event_id=EVENT_ID,
project_id=default_project.id,
unprocessed=None,
)


Expand Down Expand Up @@ -598,7 +611,12 @@ def more_extra(data):
assert event["extra"] == {"ooo": "[Filtered]", "ooo2": "event preprocessor"}

mock_save_event.delay.assert_called_once_with(
cache_key="e:1", data=None, start_time=1, event_id=EVENT_ID, project_id=default_project.id
cache_key="e:1",
data=None,
start_time=1,
event_id=EVENT_ID,
project_id=default_project.id,
unprocessed=None,
)


Expand Down Expand Up @@ -711,6 +729,7 @@ def test_store_consumer_type(
start_time=1,
event_id=EVENT_ID,
project_id=default_project.id,
unprocessed=None,
)

transaction_data = {
Expand Down
14 changes: 10 additions & 4 deletions tests/sentry/tasks/test_symbolication.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,32 +45,38 @@ def mock_event_processing_store():


@django_db_all
@pytest.mark.parametrize("inline", (False, True))
@pytest.mark.parametrize(
"inline,disable_store,redis_backups",
[(False, False, 1), (True, False, 1), (True, True, 0)],
)
def test_move_to_symbolicate_event(
default_project,
mock_process_event,
mock_save_event,
mock_symbolicate_event,
mock_event_processing_store,
inline,
disable_store,
redis_backups,
):
data = {"platform": "native", "project": default_project.id, "event_id": EVENT_ID}
cache_key = None if inline else "e:1"
cache_key = None if disable_store else "e:1"
mock_event_processing_store.store.return_value = "e:1"
with (
override_options(
{
"store.enable-inline-payloads": float(inline),
"store.disable-processing-store": inline,
"store.disable-processing-store": disable_store,
}
),
mock.patch("sentry.tasks.store.reprocessing2.backup_unprocessed_event") as backup,
):
preprocess_event(cache_key=cache_key, data=data)

backup.assert_called_once_with(data=data)
assert backup.call_args_list == [mock.call(data=data)] * redis_backups
assert mock_symbolicate_event.delay.call_count == 1
kwargs = mock_symbolicate_event.delay.call_args.kwargs
assert (kwargs["unprocessed"] == data) is inline
assert kwargs["data"] == (data if inline else None)
assert kwargs["cache_key"] == cache_key
assert mock_process_event.delay.call_count == 0
Expand Down
Loading