Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion .github/workflows/migrations-drift.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
name: migrations-drift
on:
pull_request:
paths: ['src/sentry/**/migrations/**.py', '.github/workflows/migrations-drift.yml']
paths:
- 'src/sentry/**/migrations/**.py'
- 'tools/migrations/drop_pending_deletions.py'
- '.github/workflows/migrations-drift.yml'

# Cancel in progress workflows on pull_requests.
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
Expand Down Expand Up @@ -50,6 +53,10 @@ jobs:
SENTRY_NO_CREATE_DEFAULT_PROJECT: 1
run: make apply-migrations

# The squashed state has no tables or columns that are pending deletion.
- name: drop pending deletions
run: python3 -um tools.migrations.drop_pending_deletions

- name: capture database schema before
run: docker exec postgres-postgres-1 bash -c 'pg_dumpall -U postgres -s --restrict-key=driftcheck' > schema-before

Expand Down
111 changes: 111 additions & 0 deletions tests/sentry/new_migrations/test_drop_pending_deletions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
from typing import Any

from django.db import connection, models
from django.db.migrations.state import ModelState

from sentry.new_migrations.monkey.fields import SafeRemoveField
from sentry.new_migrations.monkey.models import SafeDeleteModel
from sentry.new_migrations.monkey.state import DeletionAction, SentryProjectState
from sentry.testutils.cases import TestCase
from tools.migrations.drop_pending_deletions import pending_deletion_sql

APP = "drift_test"


def _model(name: str, *fields: tuple[str, models.Field[Any, Any]]) -> ModelState:
return ModelState(APP, name, [("id", models.BigAutoField(primary_key=True)), *fields])


class PendingDeletionSqlTest(TestCase):
def setUp(self) -> None:
super().setUp()
self.state = SentryProjectState()
for model_state in (
_model("Tag"),
_model(
"Thing",
("kept", models.IntegerField(null=True)),
("old", models.IntegerField(null=True)),
("tags", models.ManyToManyField(f"{APP}.Tag")),
(
"members",
models.ManyToManyField(
f"{APP}.Tag",
through=f"{APP}.Membership",
related_name="+",
),
),
),
_model(
"Membership",
("thing", models.ForeignKey(f"{APP}.Thing", models.CASCADE, db_constraint=False)),
("tag", models.ForeignKey(f"{APP}.Tag", models.CASCADE, db_constraint=False)),
),
_model("Gone", ("tags", models.ManyToManyField(f"{APP}.Tag", related_name="+"))),
_model("Unrelated", ("value", models.IntegerField())),
):
self.state.add_model(model_state)

with connection.schema_editor() as editor:
for model in self.state.apps.get_models():
editor.create_model(model)

for operation in (
SafeRemoveField("thing", "old", deletion_action=DeletionAction.MOVE_TO_PENDING),
SafeRemoveField("thing", "tags", deletion_action=DeletionAction.MOVE_TO_PENDING),
SafeRemoveField("thing", "members", deletion_action=DeletionAction.MOVE_TO_PENDING),
SafeDeleteModel("Gone", deletion_action=DeletionAction.MOVE_TO_PENDING),
):
operation.state_forwards(APP, self.state)

def _drop(self) -> list[str]:
statements = pending_deletion_sql(self.state, connection)
with connection.cursor() as cursor:
for statement in statements:
cursor.execute(statement)
return statements

def _tables(self) -> set[str]:
with connection.cursor() as cursor:
return {t for t in connection.introspection.table_names(cursor) if t.startswith(APP)}

def _columns(self, table: str) -> set[str]:
with connection.cursor() as cursor:
return {c.name for c in connection.introspection.get_table_description(cursor, table)}

def test_sql(self) -> None:
assert self._drop() == [
'ALTER TABLE "drift_test_thing" DROP COLUMN "old"',
'DROP TABLE "drift_test_thing_tags"',
'DROP TABLE "drift_test_gone_tags"',
'DROP TABLE "drift_test_gone"',
]

def test_scalar_field(self) -> None:
assert "old" in self._columns("drift_test_thing")
self._drop()
assert self._columns("drift_test_thing") == {"id", "kept"}

def test_m2m_field(self) -> None:
self._drop()
tables = self._tables()
# The auto-created through table goes with its field...
assert "drift_test_thing_tags" not in tables
# ...but an explicit through model isn't the field's to drop.
assert "drift_test_membership" in tables
assert self._columns("drift_test_membership") == {"id", "thing_id", "tag_id"}

def test_model(self) -> None:
assert {"drift_test_gone", "drift_test_gone_tags"} <= self._tables()
self._drop()
assert not {"drift_test_gone", "drift_test_gone_tags"} & self._tables()

def test_unrelated_schema_is_kept(self) -> None:
self._drop()
assert self._tables() == {
"drift_test_tag",
"drift_test_thing",
"drift_test_membership",
"drift_test_unrelated",
}
assert self._columns("drift_test_unrelated") == {"id", "value"}
97 changes: 97 additions & 0 deletions tools/migrations/drop_pending_deletions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
"""Drop tables and columns that migrations have moved to pending deletion.

The first step of a safe deletion (`DeletionAction.MOVE_TO_PENDING`) removes a
model or field from the migration state but leaves it in the database until a
later migration drops it. A schema built from that state never has them, so the
migrations-drift check drops them from the migrated database first.
"""

from __future__ import annotations

from typing import TYPE_CHECKING, Any, cast

if TYPE_CHECKING:
from django.db.backends.base.base import BaseDatabaseWrapper
from django.db.models import Field, ManyToManyField, Model

from sentry.new_migrations.monkey.state import SentryProjectState


def _auto_through_table(field: ManyToManyField[Any, Any]) -> str | None:
through = field.remote_field.through
assert through is not None
return through._meta.db_table if through._meta.auto_created else None


def pending_deletion_sql(state: SentryProjectState, connection: BaseDatabaseWrapper) -> list[str]:
"""SQL for the `DeletionAction.DELETE` half of every pending deletion.

Mirrors `SafeRemoveField` / `SafeDeleteModel`: a many-to-many field or model
takes its auto-created through table with it, while explicit `through`
models are only dropped when they are pending deletion themselves. There's
no `CASCADE`, so anything else depending on these fails loudly instead of
being dropped along with them.
"""
from django.db import router
from django.db.models import ManyToManyField

def allow_migrate(model: type[Model]) -> bool:
return model._meta.can_migrate(connection) and router.allow_migrate_model(
connection.alias, model
)

q = connection.ops.quote_name
columns: list[tuple[str, str]] = []
through_tables: list[str | None] = []
tables: list[str] = []

# The state annotates these as `type[Field]`, but they're field instances.
fields = cast("list[Field[Any, Any]]", list(state.pending_deletion_fields.values()))
for field in fields:
if not allow_migrate(field.model):
continue
if isinstance(field, ManyToManyField):
through_tables.append(_auto_through_table(field))
elif field.db_parameters(connection=connection)["type"] is not None:
columns.append((field.model._meta.db_table, field.column))

for model in state.pending_deletion_models.values():
if not allow_migrate(model):
continue
through_tables += [_auto_through_table(f) for f in model._meta.local_many_to_many]
tables.append(model._meta.db_table)

# Through tables reference the tables they join, so they go first.
return [f"ALTER TABLE {q(table)} DROP COLUMN {q(column)}" for table, column in columns] + [
f"DROP TABLE {q(table)}" for table in [*through_tables, *tables] if table is not None
]


def main() -> int:
from sentry.runner import configure

configure()

from django.conf import settings
from django.db import connections
from django.db.migrations.loader import MigrationLoader

from sentry.new_migrations.monkey.state import SentryProjectState

state = MigrationLoader(connections["default"]).project_state()
assert isinstance(state, SentryProjectState), type(state)

for alias, db in settings.DATABASES.items():
if db.get("REPLICA_OF"):
continue
connection = connections[alias]
with connection.cursor() as cursor:
for statement in pending_deletion_sql(state, connection):
print(f"{alias}: {statement}")
cursor.execute(statement)

return 0


if __name__ == "__main__":
raise SystemExit(main())
Loading