Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 41 additions & 20 deletions src/sentry/monitors/tasks/detect_broken_monitor_envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,33 +5,34 @@
from collections.abc import Iterable
from datetime import timedelta
from typing import Any
from urllib.parse import urlencode, urlparse, urlunparse
from urllib.parse import urlencode

from django.db import router, transaction
from django.urls import reverse
from django.utils import timezone as django_timezone

from sentry.constants import ObjectStatus
from sentry.models.organization import Organization
from sentry.models.organizationmember import OrganizationMember
from sentry.models.project import Project
from sentry.models.team import Team
from sentry.monitors.grouptype import MonitorIncidentType
from sentry.monitors.models import (
CheckInStatus,
Monitor,
MonitorCheckIn,
MonitorEnvBrokenDetection,
MonitorIncident,
)
from sentry.monitors.types import DATA_SOURCE_CRON_MONITOR
from sentry.notifications.services import notifications_service
from sentry.notifications.types import NotificationSettingEnum
from sentry.tasks.base import instrumented_task
from sentry.taskworker.namespaces import crons_tasks
from sentry.types.actor import Actor
from sentry.utils.email import MessageBuilder
from sentry.utils.email.manager import get_email_addresses
from sentry.utils.http import absolute_uri
from sentry.utils.query import RangeQuerySetWrapper
from sentry.workflow_engine.models import DataSourceDetector

logger = logging.getLogger("sentry")

Expand All @@ -49,21 +50,16 @@


def generate_monitor_overview_url(organization: Organization):
return absolute_uri(reverse("sentry-organization-crons", args=[organization.slug]))
return organization.absolute_url(f"/organizations/{organization.slug}/monitors/crons/")


def generate_monitor_detail_url(
organization: Organization, project_slug: str, monitor_slug: str, environments: list[str]
organization: Organization, detector_id: int, environments: list[str]
):
url = absolute_uri(
reverse(
"sentry-organization-cron-monitor-details",
args=[organization.slug, project_slug, monitor_slug],
)
return organization.absolute_url(
f"/organizations/{organization.slug}/monitors/{detector_id}/",
query=urlencode({"environment": environments}, doseq=True),
)
url_parts = list(urlparse(url))
url_parts[4] = urlencode({"environment": environments}, doseq=True)
return urlunparse(url_parts)


def update_user_monitor_dictionary(
Expand All @@ -72,6 +68,7 @@ def update_user_monitor_dictionary(
open_incident: MonitorIncident,
project: Project,
environment_name: str,
detector_id: int | None,
) -> None:
user_monitor_entry = user_monitor_entries[user_email][open_incident.monitor.id]
user_monitor_entry.update(
Expand All @@ -82,6 +79,7 @@ def update_user_monitor_dictionary(
),
"project_slug": project.slug,
"slug": open_incident.monitor.slug,
"detector_id": detector_id,
}
)
if len(user_monitor_entry["environment_names"]) < MAX_ENVIRONMENTS_IN_MONITOR_LINK:
Expand Down Expand Up @@ -133,11 +131,14 @@ def generate_monitor_email_context(
(
monitor_entry["slug"],
monitor_entry["project_slug"],
generate_monitor_detail_url(
organization,
monitor_entry["project_slug"],
monitor_entry["slug"],
monitor_entry["environment_names"],
(
generate_monitor_detail_url(
organization,
monitor_entry["detector_id"],
monitor_entry["environment_names"],
)
if monitor_entry["detector_id"] is not None
else generate_monitor_overview_url(organization)
),
monitor_entry["earliest_start"],
)
Expand Down Expand Up @@ -200,6 +201,16 @@ def detect_broken_monitor_envs_for_org(org_id: int):
.select_related("monitor_environment")
.filter(monitor__organization_id=org_id)
)
monitor_ids = set(orgs_open_incidents.values_list("monitor_id", flat=True))
detector_ids_by_monitor_id = {
int(monitor_id): detector_id
for monitor_id, detector_id in DataSourceDetector.objects.filter(
data_source__type=DATA_SOURCE_CRON_MONITOR,
data_source__organization_id=organization.id,
data_source__source_id__in=monitor_ids,
detector__type=MonitorIncidentType.slug,
).values_list("data_source__source_id", "detector_id")
}
# Query for all the broken incidents within the current org we are processing
for open_incident in RangeQuerySetWrapper(
orgs_open_incidents,
Expand Down Expand Up @@ -229,7 +240,12 @@ def detect_broken_monitor_envs_for_org(org_id: int):
continue

update_user_monitor_dictionary(
user_broken_envs, email, open_incident, project, environment_name
user_broken_envs,
email,
open_incident,
project,
environment_name,
detector_ids_by_monitor_id.get(open_incident.monitor_id),
)
elif (
not detection.env_muted_timestamp
Expand All @@ -248,7 +264,12 @@ def detect_broken_monitor_envs_for_org(org_id: int):
continue

update_user_monitor_dictionary(
user_muted_envs, email, open_incident, project, environment_name
user_muted_envs,
email,
open_incident,
project,
environment_name,
detector_ids_by_monitor_id.get(open_incident.monitor_id),
)

# After accumulating all users within the org and which monitors to email them, send the emails
Expand Down
111 changes: 99 additions & 12 deletions tests/sentry/monitors/tasks/test_detect_broken_monitor_envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,15 @@
from datetime import timedelta
from unittest.mock import MagicMock, Mock, call, patch

from django.urls import reverse
from django.db import connection
from django.test.utils import CaptureQueriesContext
from django.utils import timezone

from sentry.constants import ObjectStatus
from sentry.grouping.utils import hash_from_values
from sentry.models.organization import Organization
from sentry.models.project import Project
from sentry.monitors.grouptype import MonitorIncidentType
from sentry.monitors.models import (
CheckInStatus,
Monitor,
Expand All @@ -18,18 +22,22 @@
ScheduleType,
)
from sentry.monitors.tasks.detect_broken_monitor_envs import detect_broken_monitor_envs
from sentry.monitors.types import DATA_SOURCE_CRON_MONITOR
from sentry.notifications.models.notificationsettingoption import NotificationSettingOption
from sentry.silo.base import SiloMode
from sentry.testutils.cases import TestCase
from sentry.testutils.helpers.datetime import before_now
from sentry.testutils.helpers.features import with_feature
from sentry.testutils.silo import assume_test_silo_mode
from sentry.users.models.user_option import UserOption
from sentry.users.models.useremail import UserEmail
from sentry.workflow_engine.models import DataSourceDetector


class MonitorDetectBrokenMonitorEnvTaskTest(TestCase):
def setUp(self) -> None:
super().setUp()
self.detector_ids: dict[str, int] = {}
self._run_tasks = self.tasks()
self._run_tasks.__enter__()

Expand All @@ -45,9 +53,9 @@ def create_monitor_env(self, monitor, environment_id):
)

def generate_cron_monitor_url(self, org_slug: str, project_slug: str, monitor_slug: str) -> str:
return "http://testserver" + reverse(
"sentry-organization-cron-monitor-details",
args=[org_slug, project_slug, monitor_slug],
return (
f"http://testserver/organizations/{org_slug}/monitors/"
f"{self.detector_ids[monitor_slug]}/"
)

def create_monitor_and_env(
Expand All @@ -72,6 +80,17 @@ def create_monitor_and_env(
"checkin_margin": None,
},
)
data_source = self.create_data_source(
organization=Organization.objects.get(id=monitor.organization_id),
source_id=str(monitor.id),
type=DATA_SOURCE_CRON_MONITOR,
)
detector = self.create_detector(
project=Project.objects.get(id=monitor.project_id),
type=MonitorIncidentType.slug,
)
self.create_data_source_detector(data_source=data_source, detector=detector)
self.detector_ids[monitor.slug] = detector.id
return (monitor, self.create_monitor_env(monitor=monitor, environment_id=environment_id))

def create_incident_for_monitor_env(self, monitor, monitor_environment):
Expand Down Expand Up @@ -119,6 +138,74 @@ def test_creates_broken_detection_no_duplicates(
assert len(MonitorEnvBrokenDetection.objects.filter(monitor_incident=incident)) == 1
assert builder.call_count == 1

@patch("sentry.monitors.tasks.detect_broken_monitor_envs.MessageBuilder")
def test_preserves_environment_query_parameters(self, builder: MagicMock) -> None:
first_environment = self.create_environment(self.project, name="production & staging")
second_environment = self.create_environment(self.project, name="canary/us")
monitor, first_monitor_environment = self.create_monitor_and_env(
environment_id=first_environment.id
)
second_monitor_environment = self.create_monitor_env(monitor, second_environment.id)
self.create_incident_for_monitor_env(monitor, first_monitor_environment)
self.create_incident_for_monitor_env(monitor, second_monitor_environment)

detect_broken_monitor_envs()

monitor_url = builder.call_args.kwargs["context"]["broken_monitors"][0][2]
assert monitor_url == (
f"http://testserver/organizations/{self.organization.slug}/monitors/"
f"{self.detector_ids[monitor.slug]}/"
"?environment=production+%26+staging&environment=canary%2Fus"
)

@with_feature("system:multi-region")
@patch("sentry.monitors.tasks.detect_broken_monitor_envs.MessageBuilder")
def test_uses_customer_domain_urls(self, builder: MagicMock) -> None:
monitor, monitor_environment = self.create_monitor_and_env()
self.create_incident_for_monitor_env(monitor, monitor_environment)

detect_broken_monitor_envs()

context = builder.call_args.kwargs["context"]
assert context["broken_monitors"][0][2] == (
f"http://{self.organization.slug}.testserver/monitors/"
f"{self.detector_ids[monitor.slug]}/?environment={self.environment.name}"
)
assert context["view_monitors_link"] == (
f"http://{self.organization.slug}.testserver/monitors/crons/"
)

@patch("sentry.monitors.tasks.detect_broken_monitor_envs.MessageBuilder")
def test_missing_detector_falls_back_to_monitor_list(self, builder: MagicMock) -> None:
monitor, monitor_environment = self.create_monitor_and_env()
DataSourceDetector.objects.filter(detector_id=self.detector_ids[monitor.slug]).delete()
self.create_incident_for_monitor_env(monitor, monitor_environment)

detect_broken_monitor_envs()

monitor_url = builder.call_args.kwargs["context"]["broken_monitors"][0][2]
assert monitor_url == (
f"http://testserver/organizations/{self.organization.slug}/monitors/crons/"
)

@patch("sentry.monitors.tasks.detect_broken_monitor_envs.MessageBuilder")
def test_queries_detectors_once_for_multiple_monitors(self, builder: MagicMock) -> None:
first_monitor, first_monitor_environment = self.create_monitor_and_env()
second_monitor, second_monitor_environment = self.create_monitor_and_env(name="second")
self.create_incident_for_monitor_env(first_monitor, first_monitor_environment)
self.create_incident_for_monitor_env(second_monitor, second_monitor_environment)

with CaptureQueriesContext(connection) as queries:
detect_broken_monitor_envs()

detector_queries = [
query
for query in queries.captured_queries
if "workflow_engine_datasourcedetector" in query["sql"]
]
assert len(detector_queries) == 1
assert len(builder.call_args.kwargs["context"]["broken_monitors"]) == 2

def test_does_not_create_broken_detection_insufficient_duration(self) -> None:
monitor, monitor_environment = self.create_monitor_and_env()

Expand Down Expand Up @@ -239,7 +326,7 @@ def test_sends_emails_to_all_users_across_orgs(
timezone.now() - timedelta(days=14),
)
],
"view_monitors_link": f"http://testserver/organizations/{self.organization.slug}/insights/crons/",
"view_monitors_link": f"http://testserver/organizations/{self.organization.slug}/monitors/crons/",
},
{
"broken_monitors": [
Expand All @@ -256,7 +343,7 @@ def test_sends_emails_to_all_users_across_orgs(
timezone.now() - timedelta(days=14),
),
],
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/insights/crons/",
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/monitors/crons/",
},
{
"broken_monitors": [
Expand All @@ -273,7 +360,7 @@ def test_sends_emails_to_all_users_across_orgs(
timezone.now() - timedelta(days=14),
),
],
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/insights/crons/",
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/monitors/crons/",
},
]
expected_subjects = [
Expand Down Expand Up @@ -380,7 +467,7 @@ def test_disables_environments_and_sends_email(
timezone.now() - timedelta(days=14),
)
],
"view_monitors_link": f"http://testserver/organizations/{self.organization.slug}/insights/crons/",
"view_monitors_link": f"http://testserver/organizations/{self.organization.slug}/monitors/crons/",
},
{
"muted_monitors": [
Expand All @@ -397,7 +484,7 @@ def test_disables_environments_and_sends_email(
timezone.now() - timedelta(days=14),
),
],
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/insights/crons/",
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/monitors/crons/",
},
{
"muted_monitors": [
Expand All @@ -414,7 +501,7 @@ def test_disables_environments_and_sends_email(
timezone.now() - timedelta(days=14),
),
],
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/insights/crons/",
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/monitors/crons/",
},
]
expected_subjects = [
Expand Down Expand Up @@ -503,7 +590,7 @@ def test_disables_corrects_environments_and_sends_email(
timezone.now() - timedelta(days=14),
)
],
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/insights/crons/",
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/monitors/crons/",
},
{
"muted_monitors": [
Expand All @@ -514,7 +601,7 @@ def test_disables_corrects_environments_and_sends_email(
timezone.now() - timedelta(days=14),
)
],
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/insights/crons/",
"view_monitors_link": f"http://testserver/organizations/{second_org.slug}/monitors/crons/",
},
]

Expand Down
Loading