Repository navigation
ref(reprocessing): Optionally pass unprocessed copy inline instead of via redis #126810
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,7 +18,11 @@ | |
| from sentry.feedback.usecases.ingest.save_event_feedback import ( | ||
| save_event_feedback as save_event_feedback_impl, | ||
| ) | ||
| from sentry.ingest.event_payload import load_event_payload, prepare_submit | ||
| from sentry.ingest.event_payload import ( | ||
| get_event_payload_transport, | ||
| load_event_payload, | ||
| prepare_submit, | ||
| ) | ||
| from sentry.ingest.types import ConsumerType | ||
| from sentry.killswitches import killswitch_matches_context | ||
| from sentry.lang.native.symbolicator import SymbolicatorTaskKind | ||
|
|
@@ -71,6 +75,7 @@ def submit_process( | |
| from_symbolicate: bool = False, | ||
| has_attachments: bool = False, | ||
| data: MutableMapping[str, Any] | None = None, | ||
| unprocessed: MutableMapping[str, Any] | None = None, | ||
| ) -> None: | ||
| data, cache_key = prepare_submit(data, cache_key, event_id) | ||
| if from_reprocessing: | ||
|
|
@@ -85,6 +90,7 @@ def submit_process( | |
| from_symbolicate=from_symbolicate, | ||
| has_attachments=has_attachments, | ||
| data=data, | ||
| unprocessed=unprocessed, | ||
| ) | ||
|
|
||
|
|
||
|
|
@@ -102,6 +108,7 @@ def submit_save_event( | |
| start_time: float | None, | ||
| data: MutableMapping[str, Any] | None, | ||
| inline: bool = False, | ||
| unprocessed: MutableMapping[str, Any] | None = None, | ||
| ) -> None: | ||
| data, cache_key = prepare_submit(data, cache_key, event_id) | ||
|
|
||
|
|
@@ -117,6 +124,7 @@ def submit_save_event( | |
| "start_time": start_time, | ||
| "event_id": event_id, | ||
| "project_id": project_id, | ||
| "unprocessed": unprocessed, | ||
| } | ||
|
|
||
| if inline: | ||
|
|
@@ -235,7 +243,12 @@ def _do_preprocess_event( | |
| "symbolication_function": symbolication_function_name, | ||
| }, | ||
| ): | ||
| reprocessing2.backup_unprocessed_event(data=original_data) | ||
| unprocessed = None | ||
| transport = get_event_payload_transport(event_id) | ||
| if transport.cache: | ||
| reprocessing2.backup_unprocessed_event(data=original_data) | ||
| if transport.inline and not options.get("store.reprocessing-force-disable"): | ||
| unprocessed = original_data | ||
|
|
||
| submit_symbolicate( | ||
| SymbolicatorTaskKind( | ||
|
|
@@ -248,6 +261,7 @@ def _do_preprocess_event( | |
| has_attachments=has_attachments, | ||
| symbolicate_functions=symbolicate_functions, | ||
| data=data, | ||
| unprocessed=unprocessed, | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In the end state they will be mutually exclusive and eventually cache_key will be removed from the tasks. During rollout, we also double-write for some time so we can safely roll back at any point. |
||
| ) | ||
| return | ||
| # else: go directly to process, do not go through the symbolicate queue, do not collect 200 | ||
|
|
@@ -358,6 +372,7 @@ def do_process_event( | |
| data_has_changed: bool = False, | ||
| from_symbolicate: bool = False, | ||
| has_attachments: bool = False, | ||
| unprocessed: MutableMapping[str, Any] | None = None, | ||
| ) -> None: | ||
| data = load_event_payload(data, cache_key, processing.event_processing_store) | ||
|
|
||
|
|
@@ -390,6 +405,7 @@ def _continue_to_save_event() -> None: | |
| event_id=data_event_id, | ||
| start_time=start_time, | ||
| data=data, | ||
| unprocessed=unprocessed, | ||
| ) | ||
|
|
||
| if is_process_disabled(project_id, data_event_id, data.get("platform") or "null"): | ||
|
|
@@ -481,6 +497,7 @@ def process_event( | |
| from_symbolicate: bool = False, | ||
| has_attachments: bool = False, | ||
| data: MutableMapping[str, Any] | None = None, | ||
| unprocessed: MutableMapping[str, Any] | None = None, | ||
| **kwargs: Any, | ||
| ) -> None: | ||
| """ | ||
|
|
@@ -502,6 +519,7 @@ def process_event( | |
| from_symbolicate=from_symbolicate, | ||
| has_attachments=has_attachments, | ||
| data=data, | ||
| unprocessed=unprocessed, | ||
| ) | ||
|
|
||
|
|
||
|
|
@@ -520,6 +538,7 @@ def process_event_from_reprocessing( | |
| from_symbolicate: bool = False, | ||
| has_attachments: bool = False, | ||
| data: MutableMapping[str, Any] | None = None, | ||
| unprocessed: MutableMapping[str, Any] | None = None, | ||
| **kwargs: Any, | ||
| ) -> None: | ||
| return do_process_event( | ||
|
|
@@ -531,6 +550,7 @@ def process_event_from_reprocessing( | |
| from_symbolicate=from_symbolicate, | ||
| has_attachments=has_attachments, | ||
| data=data, | ||
| unprocessed=unprocessed, | ||
| ) | ||
|
|
||
|
|
||
|
|
@@ -541,6 +561,7 @@ def _do_save_event( | |
| project_id: int | None = None, | ||
| has_attachments: bool = False, | ||
| consumer_type: str | None = None, | ||
| unprocessed: MutableMapping[str, Any] | None = None, | ||
| **kwargs: Any, | ||
| ) -> None: | ||
| """ | ||
|
|
@@ -628,6 +649,7 @@ def _do_save_event( | |
| start_time=start_time, | ||
| cache_key=cache_key, | ||
| attachments=attachments, | ||
| unprocessed=unprocessed, | ||
| ) | ||
| except HashDiscarded: | ||
| # Mark all the attachments as `rate_limited`, so they are being properly cleaned up in the `finally` block: | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is all the data in this dict JSON encodable? Tasks will fail if there are any non-json encodable values in the object.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, this is encodable. It contains the original
data.