From 8c2a7a30b77151be80abab0b8bd7a6cd2e18a9c5 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Mon, 25 Nov 2019 13:39:35 +0100 Subject: [PATCH 1/5] feat(native): Add option to limit crash reports per group --- .../api/endpoints/organization_details.py | 7 +- src/sentry/api/endpoints/project_details.py | 6 +- .../api/serializers/models/organization.py | 6 +- src/sentry/api/serializers/models/project.py | 5 +- src/sentry/coreapi.py | 5 +- src/sentry/deletions/defaults/group.py | 1 + src/sentry/ingest/ingest_consumer.py | 2 +- src/sentry/lang/native/utils.py | 31 ++++- .../forms/organizationGeneralSettings.jsx | 8 +- .../app/data/forms/projectGeneralSettings.jsx | 10 +- .../static/sentry/app/utils/crashReports.tsx | 20 +++ src/sentry/tasks/merge.py | 2 + src/sentry/tasks/store.py | 126 +++++++++++++++--- src/sentry/tasks/unmerge.py | 4 + src/sentry/utils/cache.py | 4 + .../endpoints/test_organization_details.py | 5 +- .../api/endpoints/test_project_details.py | 10 +- tests/sentry/models/test_organization.py | 6 +- tests/symbolicator/test_minidump_full.py | 9 +- tests/symbolicator/test_unreal_full.py | 3 +- 20 files changed, 220 insertions(+), 50 deletions(-) create mode 100644 src/sentry/static/sentry/app/utils/crashReports.tsx diff --git a/src/sentry/api/endpoints/organization_details.py b/src/sentry/api/endpoints/organization_details.py index 6092cdc57ad5..f73355f6cd03 100644 --- a/src/sentry/api/endpoints/organization_details.py +++ b/src/sentry/api/endpoints/organization_details.py @@ -16,6 +16,7 @@ from sentry.api.serializers.models import organization as org_serializers from sentry.api.serializers.rest_framework import ListField from sentry.constants import LEGACY_RATE_LIMIT_OPTIONS, RESERVED_ORGANIZATION_SLUGS +from sentry.lang.native.utils import STORE_CRASH_REPORTS_DEFAULT, convert_crashreport_count from sentry.models import ( AuditLogEntryEvent, Authenticator, @@ -66,8 +67,8 @@ ( "storeCrashReports", "sentry:store_crash_reports", - bool, - org_serializers.STORE_CRASH_REPORTS_DEFAULT, + convert_crashreport_count, + STORE_CRASH_REPORTS_DEFAULT, ), ( "attachmentsRole", @@ -130,7 +131,7 @@ class OrganizationSerializer(serializers.Serializer): dataScrubberDefaults = serializers.BooleanField(required=False) sensitiveFields = ListField(child=serializers.CharField(), required=False) safeFields = ListField(child=serializers.CharField(), required=False) - storeCrashReports = serializers.BooleanField(required=False) + storeCrashReports = serializers.IntegerField(min_value=-1, max_value=20, required=False) attachmentsRole = serializers.CharField(required=True) scrubIPAddresses = serializers.BooleanField(required=False) scrapeJavaScript = serializers.BooleanField(required=False) diff --git a/src/sentry/api/endpoints/project_details.py b/src/sentry/api/endpoints/project_details.py index 986e740bea7f..524b82be6846 100644 --- a/src/sentry/api/endpoints/project_details.py +++ b/src/sentry/api/endpoints/project_details.py @@ -24,6 +24,7 @@ from sentry.api.serializers.rest_framework.origin import OriginField from sentry.constants import RESERVED_PROJECT_SLUGS from sentry.lang.native.symbolicator import parse_sources, InvalidSourcesError +from sentry.lang.native.utils import convert_crashreport_count from sentry.models import ( AuditLogEntryEvent, Group, @@ -109,7 +110,7 @@ class ProjectAdminSerializer(ProjectMemberSerializer): dataScrubberDefaults = serializers.BooleanField(required=False) sensitiveFields = ListField(child=serializers.CharField(), required=False) safeFields = ListField(child=serializers.CharField(), required=False) - storeCrashReports = serializers.BooleanField(required=False) + storeCrashReports = serializers.IntegerField(min_value=-1, max_value=20, required=False) relayPiiConfig = serializers.CharField(required=False, allow_blank=True, allow_null=True) builtinSymbolSources = ListField(child=serializers.CharField(), required=False) symbolSources = serializers.CharField(required=False, allow_blank=True, allow_null=True) @@ -532,7 +533,8 @@ def put(self, request, project): ) if "sentry:store_crash_reports" in options: project.update_option( - "sentry:store_crash_reports", bool(options["sentry:store_crash_reports"]) + "sentry:store_crash_reports", + convert_crashreport_count(options["sentry:store_crash_reports"]), ) if "sentry:relay_pii_config" in options: project.update_option( diff --git a/src/sentry/api/serializers/models/organization.py b/src/sentry/api/serializers/models/organization.py index cfeaca15a41d..fcd98068babc 100644 --- a/src/sentry/api/serializers/models/organization.py +++ b/src/sentry/api/serializers/models/organization.py @@ -8,6 +8,7 @@ from sentry.app import quotas from sentry.api.serializers import Serializer, register, serialize from sentry.constants import LEGACY_RATE_LIMIT_OPTIONS +from sentry.lang.native.utils import convert_crashreport_count from sentry.models import ( ApiKey, Organization, @@ -29,7 +30,6 @@ REQUIRE_SCRUB_DEFAULTS_DEFAULT = False SENSITIVE_FIELDS_DEFAULT = None SAFE_FIELDS_DEFAULT = None -STORE_CRASH_REPORTS_DEFAULT = False ATTACHMENTS_ROLE_DEFAULT = settings.SENTRY_DEFAULT_ROLE REQUIRE_SCRUB_IP_ADDRESS_DEFAULT = False SCRAPE_JAVASCRIPT_DEFAULT = True @@ -174,8 +174,8 @@ def serialize(self, obj, attrs, user, access): ) or [], "safeFields": obj.get_option("sentry:safe_fields", SAFE_FIELDS_DEFAULT) or [], - "storeCrashReports": bool( - obj.get_option("sentry:store_crash_reports", STORE_CRASH_REPORTS_DEFAULT) + "storeCrashReports": convert_crashreport_count( + obj.get_option("sentry:store_crash_reports") ), "attachmentsRole": six.text_type( obj.get_option("sentry:attachments_role", ATTACHMENTS_ROLE_DEFAULT) diff --git a/src/sentry/api/serializers/models/project.py b/src/sentry/api/serializers/models/project.py index fd7a7e35385d..2b883dfef348 100644 --- a/src/sentry/api/serializers/models/project.py +++ b/src/sentry/api/serializers/models/project.py @@ -17,6 +17,7 @@ from sentry.auth.superuser import is_active_superuser from sentry.constants import StatsPeriod from sentry.digests import backend as digests +from sentry.lang.native.utils import convert_crashreport_count from sentry.models import ( EnvironmentProject, Project, @@ -551,8 +552,8 @@ def get_value_with_default(key): "dataScrubber": bool(attrs["options"].get("sentry:scrub_data", True)), "dataScrubberDefaults": bool(attrs["options"].get("sentry:scrub_defaults", True)), "safeFields": attrs["options"].get("sentry:safe_fields", []), - "storeCrashReports": bool( - attrs["options"].get("sentry:store_crash_reports", False) + "storeCrashReports": convert_crashreport_count( + attrs["options"].get("sentry:store_crash_reports") ), "sensitiveFields": attrs["options"].get("sentry:sensitive_fields", []), "subjectTemplate": attrs["options"].get("mail:subject_template") diff --git a/src/sentry/coreapi.py b/src/sentry/coreapi.py index f9741135ec20..86bc89816c76 100644 --- a/src/sentry/coreapi.py +++ b/src/sentry/coreapi.py @@ -22,6 +22,7 @@ from sentry.tasks.store import preprocess_event, preprocess_event_from_reprocessing from sentry.utils import json from sentry.utils.auth import parse_auth_header +from sentry.utils.cache import cache_key_for_event from sentry.utils.http import origin_from_request from sentry.utils.strings import decompress from sentry.utils.sdk import configure_scope @@ -253,10 +254,6 @@ def auth_from_request(cls, request): return auth -def cache_key_for_event(data): - return u"e:{1}:{0}".format(data["project"], data["event_id"]) - - def decompress_deflate(encoded_data): try: return zlib.decompress(encoded_data).decode("utf-8") diff --git a/src/sentry/deletions/defaults/group.py b/src/sentry/deletions/defaults/group.py index 389fa76c9a69..0edcd7f1c17d 100644 --- a/src/sentry/deletions/defaults/group.py +++ b/src/sentry/deletions/defaults/group.py @@ -85,6 +85,7 @@ def get_child_relations(self, instance): models.GroupEmailThread, models.GroupSubscription, models.UserReport, + models.EventAttachment, IncidentGroup, # Event is last as its the most time consuming models.Event, diff --git a/src/sentry/ingest/ingest_consumer.py b/src/sentry/ingest/ingest_consumer.py index 5dfd0cddb4d0..4d2cc6d9a36e 100644 --- a/src/sentry/ingest/ingest_consumer.py +++ b/src/sentry/ingest/ingest_consumer.py @@ -8,12 +8,12 @@ from django.conf import settings from django.core.cache import cache -from sentry.coreapi import cache_key_for_event from sentry.cache import default_cache from sentry.models import Project from sentry.signals import event_accepted from sentry.tasks.store import preprocess_event from sentry.utils import json +from sentry.utils.cache import cache_key_for_event from sentry.utils.kafka import create_batching_kafka_consumer logger = logging.getLogger(__name__) diff --git a/src/sentry/lang/native/utils.py b/src/sentry/lang/native/utils.py index 08b3b68de5f4..89617f758e88 100644 --- a/src/sentry/lang/native/utils.py +++ b/src/sentry/lang/native/utils.py @@ -5,8 +5,8 @@ import logging from sentry.attachments import attachment_cache -from sentry.coreapi import cache_key_for_event from sentry.stacktraces.processing import find_stacktraces_in_data +from sentry.utils.cache import cache_key_for_event from sentry.utils.safe import get_path logger = logging.getLogger(__name__) @@ -29,6 +29,11 @@ "pe", # Windows ) +# Default disables storing crash reports. +STORE_CRASH_REPORTS_DEFAULT = 0 +# Do not limit crash report attachments per group. +STORE_CRASH_REPORTS_ALL = -1 + def is_native_platform(platform): return platform in NATIVE_PLATFORMS @@ -108,3 +113,27 @@ def get_event_attachment(data, attachment_type): cache_key = cache_key_for_event(data) attachments = attachment_cache.get(cache_key) or [] return next((a for a in attachments if a.type == attachment_type), None) + + +def get_crashreport_key(group_id): + return u"cr:%s" % (group_id,) + + +def convert_crashreport_count(value): + """ + Shim to read both legacy and new `sentry:store_crash_reports` project and + organization options. + + The legacy format stored `True` for an unlimited number of crash reports, + and `False` for no crash reports. + + The new format stores `-1` for unbounded storage, `0` for no crash reports, + and a positive number for a bounded number per group. + + Defaults to `0` (no storage). + """ + if value is True: + return STORE_CRASH_REPORTS_ALL + if value is None: + return STORE_CRASH_REPORTS_DEFAULT + return int(value) diff --git a/src/sentry/static/sentry/app/data/forms/organizationGeneralSettings.jsx b/src/sentry/static/sentry/app/data/forms/organizationGeneralSettings.jsx index 46e2937d3ca5..a388052563cb 100644 --- a/src/sentry/static/sentry/app/data/forms/organizationGeneralSettings.jsx +++ b/src/sentry/static/sentry/app/data/forms/organizationGeneralSettings.jsx @@ -1,6 +1,10 @@ import {extractMultilineFields} from 'app/utils'; import {t} from 'app/locale'; import slugify from 'app/utils/slugify'; +import { + STORE_CRASH_REPORTS_VALUES, + formatStoreCrashReports, +} from 'app/utils/crashReports'; // Export route to make these forms searchable by label/help export const route = '/settings/:orgId/'; @@ -194,12 +198,14 @@ const formGroups = [ }, { name: 'storeCrashReports', - type: 'boolean', + type: 'range', label: t('Store Native Crash Reports'), help: t( 'Store native crash reports such as Minidumps for improved processing and download in issue details' ), visible: ({features}) => features.has('event-attachments'), + formatLabel: formatStoreCrashReports, + allowedValues: STORE_CRASH_REPORTS_VALUES, }, { name: 'attachmentsRole', diff --git a/src/sentry/static/sentry/app/data/forms/projectGeneralSettings.jsx b/src/sentry/static/sentry/app/data/forms/projectGeneralSettings.jsx index 4a67fdf01bfe..5294234a0665 100644 --- a/src/sentry/static/sentry/app/data/forms/projectGeneralSettings.jsx +++ b/src/sentry/static/sentry/app/data/forms/projectGeneralSettings.jsx @@ -9,6 +9,10 @@ import getDynamicText from 'app/utils/getDynamicText'; import marked from 'app/utils/marked'; import platforms from 'app/data/platforms'; import slugify from 'app/utils/slugify'; +import { + STORE_CRASH_REPORTS_VALUES, + formatStoreCrashReports, +} from 'app/utils/crashReports'; import space from 'app/styles/space'; import {GroupingConfigItem} from 'app/components/events/groupingInfo'; @@ -330,12 +334,14 @@ export const fields = { }, storeCrashReports: { name: 'storeCrashReports', - type: 'boolean', + type: 'range', label: t('Store Native Crash Reports'), help: t( - 'Store native crash reports such as Minidumps for improved processing and download in issue details' + 'Store native crash reports such as Minidumps for improved processing and download in issue details. Overrides organization settings when enabled.' ), visible: ({features}) => features.has('event-attachments'), + formatLabel: formatStoreCrashReports, + allowedValues: STORE_CRASH_REPORTS_VALUES, }, relayPiiConfig: { name: 'relayPiiConfig', diff --git a/src/sentry/static/sentry/app/utils/crashReports.tsx b/src/sentry/static/sentry/app/utils/crashReports.tsx new file mode 100644 index 000000000000..15d86b9fb744 --- /dev/null +++ b/src/sentry/static/sentry/app/utils/crashReports.tsx @@ -0,0 +1,20 @@ +import {t, tct} from 'app/locale'; + +export function formatStoreCrashReports(value: number) { + if (value === -1) { + return t('Unlimited'); + } else if (value === 0) { + return t('Disabled'); + } else { + return tct('[value] per issue', {value}); + } +} + +function getStoreCrashReportsValues() { + // generate a range from 0 (disabled) to 20 inclusive + const values = Array.from(new Array(21), (_, i) => i); + values.push(-1); // special "Unlimited" at the end + return values; +} + +export const STORE_CRASH_REPORTS_VALUES = getStoreCrashReportsValues(); diff --git a/src/sentry/tasks/merge.py b/src/sentry/tasks/merge.py index fbf8265cbbff..46b5f8c5c4d2 100644 --- a/src/sentry/tasks/merge.py +++ b/src/sentry/tasks/merge.py @@ -42,6 +42,7 @@ def merge_groups( GroupSubscription, Environment, Event, + EventAttachment, UserReport, GroupRedirect, GroupMeta, @@ -98,6 +99,7 @@ def merge_groups( GroupRuleStatus, GroupSubscription, Event, + EventAttachment, UserReport, GroupRedirect, GroupMeta, diff --git a/src/sentry/tasks/store.py b/src/sentry/tasks/store.py index 418d117f3582..10ad13e3dbdc 100644 --- a/src/sentry/tasks/store.py +++ b/src/sentry/tasks/store.py @@ -13,6 +13,11 @@ from sentry.constants import DEFAULT_STORE_NORMALIZER_ARGS from sentry.attachments import attachment_cache from sentry.cache import default_cache +from sentry.lang.native.utils import ( + STORE_CRASH_REPORTS_ALL, + convert_crashreport_count, + get_crashreport_key, +) from sentry.tasks.base import instrumented_task from sentry.utils import metrics from sentry.utils.safe import safe_execute @@ -32,6 +37,9 @@ # Attachment file types that are considered a crash report (PII relevant) CRASH_REPORT_TYPES = ("event.minidump",) +# Timeout for cached group crash report counts +CRASH_REPORT_TIMEOUT = 24 * 3600 # one day + class RetryProcessing(Exception): pass @@ -417,16 +425,104 @@ def save_attachment(event, attachment): if not organization.get_option("sentry:store_crash_reports"): return - file = File.objects.create( - name=attachment.name, - type=attachment.type, - headers={"Content-Type": attachment.content_type}, - ) - file.putfile(six.BytesIO(attachment.data)) - EventAttachment.objects.create( - event_id=event.event_id, project_id=event.project_id, name=attachment.name, file=file - ) +def get_max_crashreports(model): + value = model.get_option("sentry:store_crash_reports") + return convert_crashreport_count(value) + + +def crashreports_exceeded(current_count, max_count): + if max_count == STORE_CRASH_REPORTS_ALL: + return False + return current_count >= max_count + + +def get_stored_crashreports(cache_key, event, max_crashreports): + # There are two common cases: Storing crash reports is disabled, or is + # unbounded. In both cases, there is no need in caching values or querying + # the database. + if max_crashreports in (0, STORE_CRASH_REPORTS_ALL): + return max_crashreports + + cached_reports = default_cache.get(cache_key, None) + if cached_reports >= max_crashreports: + return cached_reports + + # Fall-through if max_crashreports was bumped to get a more accurate number. + return EventAttachment.objects.filter( + group_id=event.group_id, file__type__in=CRASH_REPORT_TYPES + ).count() + + +def save_attachments(cache_key, event): + """ + Persists cached event attachments into the file store. + + This method checks whether event attachments are available and sends them to + the blob store. There is special handling for crash reports which may + contain unstripped PII. If the project or organization is configured to + limit the amount of crash reports per group, the number of stored crashes is + limited. + + :param cache_key: The cache key at which the event payload is stored in the + cache. This is used to retrieve attachments. + :param event: The event model instance. + """ + if not features.has("organizations:event-attachments", event.project.organization, actor=None): + return + + attachments = attachment_cache.get(cache_key) or [] + if not attachments: + return + + # The setting is both an organization and project setting. The project + # setting strictly overrides the organization setting, unless set to the + # default. + max_crashreports = get_max_crashreports(event.project) + if not max_crashreports: + max_crashreports = get_max_crashreports(event.project.organization) + + # The number of crash reports is cached per group + crashreports_key = get_crashreport_key(event.group_id) + + # Only fetch the number of stored crash reports if there is a crash report + # in the list of attachments. Otherwise, we won't require this number. + if any(attachment.type in CRASH_REPORT_TYPES for attachment in attachments): + cached_reports = get_stored_crashreports(crashreports_key, event, max_crashreports) + else: + cached_reports = 0 + stored_reports = cached_reports + + for attachment in attachments: + # If the attachment is a crash report (e.g. minidump), we need to honor + # the store_crash_reports setting. Otherwise, we assume that the client + # has already verified PII and just store the attachment. + if attachment.type in CRASH_REPORT_TYPES: + if crashreports_exceeded(stored_reports, max_crashreports): + continue + stored_reports += 1 + + file = File.objects.create( + name=attachment.name, + type=attachment.type, + headers={"Content-Type": attachment.content_type}, + ) + file.putfile(six.BytesIO(attachment.data)) + + EventAttachment.objects.create( + event_id=event.event_id, + project_id=event.project_id, + group_id=event.group_id, + name=attachment.name, + file=file, + ) + + # Check if we have exceeded the stored crash reports count. If so, we + # persist the current maximum (not the actual number!) into the cache. Next + # time when loading from the cache, we will validate that this number has + # not changed, or otherwise re-fetch from the database. + if crashreports_exceeded(stored_reports, max_crashreports) and stored_reports > cached_reports: + default_cache.set(crashreports_key, max_crashreports, CRASH_REPORT_TIMEOUT) def _do_save_event( @@ -491,13 +587,6 @@ def _do_save_event( # event.project.organization is populated after this statement. event = manager.save(project_id, assume_normalized=True) - # Always load attachments from the cache so we can later prune them. - # Only save them if the event-attachments feature is active, though. - if features.has("organizations:event-attachments", event.project.organization, actor=None): - attachments = attachment_cache.get(cache_key) or [] - for attachment in attachments: - save_attachment(event, attachment) - # This is where we can finally say that we have accepted the event. track_outcome( event.project.organization_id, @@ -536,6 +625,11 @@ def _do_save_event( event_id, ) + else: + if cache_key: + # Note that event is now a model, and no longer the data + save_attachments(cache_key, event) + finally: if cache_key: default_cache.delete(cache_key) diff --git a/src/sentry/tasks/unmerge.py b/src/sentry/tasks/unmerge.py index c24e54f61a46..3279ade5ebbb 100644 --- a/src/sentry/tasks/unmerge.py +++ b/src/sentry/tasks/unmerge.py @@ -20,6 +20,7 @@ Project, Release, UserReport, + EventAttachment, ) from sentry.similarity import features from sentry.snuba.events import Columns @@ -224,6 +225,9 @@ def migrate_events( UserReport.objects.filter(project_id=project.id, event_id__in=event_id_set).update( group=destination_id ) + EventAttachment.objects.filter(project_id=project.id, event_id__in=event_id_set).update( + group=destination_id + ) return (destination.id, eventstream_state) diff --git a/src/sentry/utils/cache.py b/src/sentry/utils/cache.py index 7691f9212da5..39c1d82d3972 100644 --- a/src/sentry/utils/cache.py +++ b/src/sentry/utils/cache.py @@ -72,3 +72,7 @@ def __call__(self, *args, **kwargs): def __get__(self, obj, type=None): return functools.partial(self.__call__, obj) + + +def cache_key_for_event(data): + return u"e:{1}:{0}".format(data["project"], data["event_id"]) diff --git a/tests/sentry/api/endpoints/test_organization_details.py b/tests/sentry/api/endpoints/test_organization_details.py index c38d5e0cf444..7b7aaeeb145c 100644 --- a/tests/sentry/api/endpoints/test_organization_details.py +++ b/tests/sentry/api/endpoints/test_organization_details.py @@ -199,7 +199,7 @@ def test_various_options(self): "dataScrubberDefaults": True, "sensitiveFields": [u"password"], "safeFields": [u"email"], - "storeCrashReports": True, + "storeCrashReports": 10, "scrubIPAddresses": True, "scrapeJavaScript": False, "defaultRole": "owner", @@ -231,7 +231,7 @@ def test_various_options(self): assert options.get("sentry:require_scrub_ip_address") assert options.get("sentry:sensitive_fields") == ["password"] assert options.get("sentry:safe_fields") == ["email"] - assert options.get("sentry:store_crash_reports") is True + assert options.get("sentry:store_crash_reports") == 10 assert options.get("sentry:scrape_javascript") is False assert options.get("sentry:join_requests") is False @@ -250,6 +250,7 @@ def test_various_options(self): assert u"to {}".format(data["dataScrubberDefaults"]) in log.data["dataScrubberDefaults"] assert u"to {}".format(data["sensitiveFields"]) in log.data["sensitiveFields"] assert u"to {}".format(data["safeFields"]) in log.data["safeFields"] + assert u"to {}".format(data["storeCrashReports"]) in log.data["storeCrashReports"] assert u"to {}".format(data["scrubIPAddresses"]) in log.data["scrubIPAddresses"] assert u"to {}".format(data["scrapeJavaScript"]) in log.data["scrapeJavaScript"] assert u"to {}".format(data["allowJoinRequests"]) in log.data["allowJoinRequests"] diff --git a/tests/sentry/api/endpoints/test_project_details.py b/tests/sentry/api/endpoints/test_project_details.py index ae62dcfd9d46..04119969c1e0 100644 --- a/tests/sentry/api/endpoints/test_project_details.py +++ b/tests/sentry/api/endpoints/test_project_details.py @@ -205,7 +205,7 @@ def test_options(self): "sentry:scrub_defaults": False, "sentry:sensitive_fields": ["foo", "bar"], "sentry:safe_fields": ["token"], - "sentry:store_crash_reports": False, + "sentry:store_crash_reports": 0, "sentry:relay_pii_config": '{"applications": {"freeform": []}}', "sentry:csp_ignored_sources_defaults": False, "sentry:csp_ignored_sources": "foo\nbar", @@ -243,7 +243,7 @@ def test_options(self): ).exists() assert project.get_option("sentry:safe_fields", []) == options["sentry:safe_fields"] assert ( - project.get_option("sentry:store_crash_reports", False) + project.get_option("sentry:store_crash_reports") == options["sentry:store_crash_reports"] ) @@ -425,10 +425,10 @@ def test_safe_fields(self): assert resp.data["safeFields"] == ["foobar.com", "https://example.com"] def test_store_crash_reports(self): - resp = self.client.put(self.path, data={"storeCrashReports": True}) + resp = self.client.put(self.path, data={"storeCrashReports": 10}) assert resp.status_code == 200, resp.content - assert self.project.get_option("sentry:store_crash_reports") is True - assert resp.data["storeCrashReports"] is True + assert self.project.get_option("sentry:store_crash_reports") == 10 + assert resp.data["storeCrashReports"] == 10 def test_relay_pii_config(self): with self.feature("organizations:relay"): diff --git a/tests/sentry/models/test_organization.py b/tests/sentry/models/test_organization.py index ad2450993e8f..35e085942e49 100644 --- a/tests/sentry/models/test_organization.py +++ b/tests/sentry/models/test_organization.py @@ -193,11 +193,11 @@ def test_has_changed(self): assert f.has_changed("value") is False OrganizationOption.objects.create( - organization=org, key="sentry:store_crash_reports", value=False + organization=org, key="sentry:store_crash_reports", value=0 ) p = OrganizationOption.objects.get(organization=org, key="sentry:store_crash_reports") - p.value = True - assert p.has_changed("value") is True + p.value = 10 + assert p.has_changed("value") == 10 class Require2fa(TestCase): diff --git a/tests/symbolicator/test_minidump_full.py b/tests/symbolicator/test_minidump_full.py index f1340d79bc69..119a712fa53d 100644 --- a/tests/symbolicator/test_minidump_full.py +++ b/tests/symbolicator/test_minidump_full.py @@ -12,6 +12,7 @@ from sentry import eventstore from sentry.testutils import TransactionTestCase from sentry.models import EventAttachment +from sentry.lang.native.utils import STORE_CRASH_REPORTS_ALL from tests.symbolicator import get_fixture_path, insta_snapshot_stacktrace_data @@ -61,7 +62,7 @@ def upload_symbols(self): assert len(response.data) == 1 def test_full_minidump(self): - self.project.update_option("sentry:store_crash_reports", True) + self.project.update_option("sentry:store_crash_reports", STORE_CRASH_REPORTS_ALL) self.upload_symbols() with self.feature("organizations:event-attachments"): @@ -93,7 +94,7 @@ def test_full_minidump(self): assert minidump.file.checksum == "74bb01c850e8d65d3ffbc5bad5cabc4668fce247" def test_full_minidump_json_extra(self): - self.project.update_option("sentry:store_crash_reports", True) + self.project.update_option("sentry:store_crash_reports", STORE_CRASH_REPORTS_ALL) self.upload_symbols() with self.feature("organizations:event-attachments"): @@ -110,7 +111,7 @@ def test_full_minidump_json_extra(self): # Other assertions are performed by `test_full_minidump` def test_full_minidump_invalid_extra(self): - self.project.update_option("sentry:store_crash_reports", True) + self.project.update_option("sentry:store_crash_reports", STORE_CRASH_REPORTS_ALL) self.upload_symbols() with self.feature("organizations:event-attachments"): @@ -127,7 +128,7 @@ def test_full_minidump_invalid_extra(self): # Other assertions are performed by `test_full_minidump` def test_raw_minidump(self): - self.project.update_option("sentry:store_crash_reports", True) + self.project.update_option("sentry:store_crash_reports", STORE_CRASH_REPORTS_ALL) self.upload_symbols() with self.feature("organizations:event-attachments"): diff --git a/tests/symbolicator/test_unreal_full.py b/tests/symbolicator/test_unreal_full.py index 01443de9246a..21f48b75b27b 100644 --- a/tests/symbolicator/test_unreal_full.py +++ b/tests/symbolicator/test_unreal_full.py @@ -12,6 +12,7 @@ from sentry.testutils import TransactionTestCase from sentry.models import EventAttachment from sentry import eventstore +from sentry.lang.native.utils import STORE_CRASH_REPORTS_ALL from tests.symbolicator import get_fixture_path @@ -69,7 +70,7 @@ def upload_symbols(self): assert len(response.data) == 1 def unreal_crash_test_impl(self, filename): - self.project.update_option("sentry:store_crash_reports", True) + self.project.update_option("sentry:store_crash_reports", STORE_CRASH_REPORTS_ALL) self.upload_symbols() # attachments feature has to be on for the files extract stick around From 3c80aa9a227c2991b6c4f2da28fb7435ced46ede Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Mon, 25 Nov 2019 15:11:00 +0100 Subject: [PATCH 2/5] test: Fix invalid test --- tests/sentry/models/test_organization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sentry/models/test_organization.py b/tests/sentry/models/test_organization.py index 35e085942e49..f10f6179cd42 100644 --- a/tests/sentry/models/test_organization.py +++ b/tests/sentry/models/test_organization.py @@ -197,7 +197,7 @@ def test_has_changed(self): ) p = OrganizationOption.objects.get(organization=org, key="sentry:store_crash_reports") p.value = 10 - assert p.has_changed("value") == 10 + assert p.has_changed("value") is True class Require2fa(TestCase): From 1e6e9a958dca47226429b60904b435a20da9f24d Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Mon, 25 Nov 2019 15:43:36 +0100 Subject: [PATCH 3/5] fix: Use group_id in unmerge --- src/sentry/tasks/unmerge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/tasks/unmerge.py b/src/sentry/tasks/unmerge.py index 3279ade5ebbb..0d6797f0f475 100644 --- a/src/sentry/tasks/unmerge.py +++ b/src/sentry/tasks/unmerge.py @@ -226,7 +226,7 @@ def migrate_events( group=destination_id ) EventAttachment.objects.filter(project_id=project.id, event_id__in=event_id_set).update( - group=destination_id + group_id=destination_id ) return (destination.id, eventstream_state) From a7eb10c67ff817a3f0c7bc59cb52a48d103eee7e Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Mon, 25 Nov 2019 15:59:16 +0100 Subject: [PATCH 4/5] ref: Remove dead code --- src/sentry/tasks/store.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/sentry/tasks/store.py b/src/sentry/tasks/store.py index 10ad13e3dbdc..962be368ce38 100644 --- a/src/sentry/tasks/store.py +++ b/src/sentry/tasks/store.py @@ -26,7 +26,7 @@ from sentry.utils.canonical import CanonicalKeyDict, CANONICAL_TYPES from sentry.utils.dates import to_datetime from sentry.utils.sdk import configure_scope -from sentry.models import EventAttachment, File, ProjectOption, Activity, Organization, Project +from sentry.models import EventAttachment, File, ProjectOption, Activity, Project error_logger = logging.getLogger("sentry.errors.events") info_logger = logging.getLogger("sentry.store") @@ -410,22 +410,6 @@ def create_failed_event( return True -def save_attachment(event, attachment): - """ - Saves an event attachment to blob storage. - """ - - # If the attachment is a crash report (e.g. minidump), we need to honor the - # store_crash_reports setting. Otherwise, we assume that the client has - # already verified PII and just store the attachment. - if attachment.type in CRASH_REPORT_TYPES: - project = Project.objects.get_from_cache(id=event.project_id) - if not project.get_option("sentry:store_crash_reports"): - organization = Organization.objects.get_from_cache(id=project.organization_id) - if not organization.get_option("sentry:store_crash_reports"): - return - - def get_max_crashreports(model): value = model.get_option("sentry:store_crash_reports") return convert_crashreport_count(value) From e73d549b4d7767c104673a16686a2c8313053f4f Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Mon, 25 Nov 2019 18:59:23 +0100 Subject: [PATCH 5/5] ref: Switch to Django cache --- src/sentry/tasks/store.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sentry/tasks/store.py b/src/sentry/tasks/store.py index 962be368ce38..aff6e5083ff4 100644 --- a/src/sentry/tasks/store.py +++ b/src/sentry/tasks/store.py @@ -5,6 +5,7 @@ import six from time import time +from django.core.cache import cache from django.utils import timezone from semaphore.processing import StoreNormalizer @@ -428,7 +429,7 @@ def get_stored_crashreports(cache_key, event, max_crashreports): if max_crashreports in (0, STORE_CRASH_REPORTS_ALL): return max_crashreports - cached_reports = default_cache.get(cache_key, None) + cached_reports = cache.get(cache_key, None) if cached_reports >= max_crashreports: return cached_reports @@ -506,7 +507,7 @@ def save_attachments(cache_key, event): # time when loading from the cache, we will validate that this number has # not changed, or otherwise re-fetch from the database. if crashreports_exceeded(stored_reports, max_crashreports) and stored_reports > cached_reports: - default_cache.set(crashreports_key, max_crashreports, CRASH_REPORT_TIMEOUT) + cache.set(crashreports_key, max_crashreports, CRASH_REPORT_TIMEOUT) def _do_save_event(