Skip to content

Commit

Permalink
feat: Squash errors migrations (#4854)
Browse files Browse the repository at this point in the history 
We rebuilt the errors table from scratch many years ago in the 0011_rebuild_errors migration. Let's squash everything before that now.

It should save time and complexity for dev/ci/new installations.
  • Loading branch information
@lynnagara
lynnagara authored Oct 19, 2023
1 parent 2d3fc18 commit 73cd686
Show file tree
Hide file tree
Showing 16 changed files with 66 additions and 1,136 deletions.
18 changes: 18 additions & 0 deletions snuba/migrations/migration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ class Migration(ABC):
A Migration should implement the forwards and backwards methods. Migrations should
not use this class directly, rather they should extend either the ClickHouseNodeMigration
(for SQL migrations to be run on ClickHouse) or CodeMigration (for Python migrations).
SquashedMigration can be used for a noop migration.
Migrations that cannot be completed immediately, such as those that contain
a data migration, must be marked with blocking = True.
Expand Down Expand Up @@ -49,6 +50,23 @@ def backwards(self, context: Context, dry_run: bool) -> None:
raise NotImplementedError


class SquashedMigration(Migration):
"""
This is a migration that used to exist. We keep the migration ID in the DB
sequence but it no longer does anything as it is safe to no longer run.
"""

blocking = False

def forwards(self, context: Context, dry_run: bool) -> None:
_migration_id, _logger, update_status = context
update_status(Status.COMPLETED)

def backwards(self, context: Context, dry_run: bool) -> None:
_migration_id, _logger, update_status = context
update_status(Status.NOT_STARTED)


class CodeMigration(Migration, ABC):
"""
Consists of one or more Python functions executed in sequence.
Expand Down
175 changes: 3 additions & 172 deletions snuba/snuba_migrations/events/0001_events_initial.py
View file
Original file line number Diff line number Diff line change
@@ -1,174 +1,5 @@
from typing import List, Sequence
from snuba.migrations import migration

from snuba.clickhouse.columns import (
Array,
Column,
DateTime,
FixedString,
Float,
Nested,
String,
UInt,
)
from snuba.clusters.storage_sets import StorageSetKey
from snuba.migrations import migration, operations, table_engines
from snuba.migrations.columns import MigrationModifiers as Modifiers

columns: List[Column[Modifiers]] = [
Column("event_id", FixedString(32)),
Column("project_id", UInt(64)),
Column("group_id", UInt(64)),
Column("timestamp", DateTime()),
Column("deleted", UInt(8)),
Column("retention_days", UInt(16)),
Column("platform", String(Modifiers(nullable=True))),
Column("message", String(Modifiers(nullable=True))),
Column("primary_hash", FixedString(32, Modifiers(nullable=True))),
Column("received", DateTime(Modifiers(nullable=True))),
Column("search_message", String(Modifiers(nullable=True))),
Column("title", String(Modifiers(nullable=True))),
Column("location", String(Modifiers(nullable=True))),
Column("user_id", String(Modifiers(nullable=True))),
Column("username", String(Modifiers(nullable=True))),
Column("email", String(Modifiers(nullable=True))),
Column("ip_address", String(Modifiers(nullable=True))),
Column("geo_country_code", String(Modifiers(nullable=True))),
Column("geo_region", String(Modifiers(nullable=True))),
Column("geo_city", String(Modifiers(nullable=True))),
Column("sdk_name", String(Modifiers(nullable=True))),
Column("sdk_version", String(Modifiers(nullable=True))),
Column("type", String(Modifiers(nullable=True))),
Column("version", String(Modifiers(nullable=True))),
Column("offset", UInt(64, Modifiers(nullable=True))),
Column("partition", UInt(16, Modifiers(nullable=True))),
Column("message_timestamp", DateTime()),
Column("os_build", String(Modifiers(nullable=True))),
Column("os_kernel_version", String(Modifiers(nullable=True))),
Column("device_name", String(Modifiers(nullable=True))),
Column("device_brand", String(Modifiers(nullable=True))),
Column("device_locale", String(Modifiers(nullable=True))),
Column("device_uuid", String(Modifiers(nullable=True))),
Column("device_model_id", String(Modifiers(nullable=True))),
Column("device_arch", String(Modifiers(nullable=True))),
Column("device_battery_level", Float(32, Modifiers(nullable=True))),
Column("device_orientation", String(Modifiers(nullable=True))),
Column("device_simulator", UInt(8, Modifiers(nullable=True))),
Column("device_online", UInt(8, Modifiers(nullable=True))),
Column("device_charging", UInt(8, Modifiers(nullable=True))),
Column("level", String(Modifiers(nullable=True))),
Column("logger", String(Modifiers(nullable=True))),
Column("server_name", String(Modifiers(nullable=True))),
Column("transaction", String(Modifiers(nullable=True))),
Column("environment", String(Modifiers(nullable=True))),
Column("sentry:release", String(Modifiers(nullable=True))),
Column("sentry:dist", String(Modifiers(nullable=True))),
Column("sentry:user", String(Modifiers(nullable=True))),
Column("site", String(Modifiers(nullable=True))),
Column("url", String(Modifiers(nullable=True))),
Column("app_device", String(Modifiers(nullable=True))),
Column("device", String(Modifiers(nullable=True))),
Column("device_family", String(Modifiers(nullable=True))),
Column("runtime", String(Modifiers(nullable=True))),
Column("runtime_name", String(Modifiers(nullable=True))),
Column("browser", String(Modifiers(nullable=True))),
Column("browser_name", String(Modifiers(nullable=True))),
Column("os", String(Modifiers(nullable=True))),
Column("os_name", String(Modifiers(nullable=True))),
Column("os_rooted", UInt(8, Modifiers(nullable=True))),
Column("tags", Nested([("key", String()), ("value", String())])),
Column("_tags_flattened", String()),
Column("contexts", Nested([("key", String()), ("value", String())])),
Column("http_method", String(Modifiers(nullable=True))),
Column("http_referer", String(Modifiers(nullable=True))),
Column(
"exception_stacks",
Nested(
[
("type", String(Modifiers(nullable=True))),
("value", String(Modifiers(nullable=True))),
("mechanism_type", String(Modifiers(nullable=True))),
("mechanism_handled", UInt(8, Modifiers(nullable=True))),
]
),
),
Column(
"exception_frames",
Nested(
[
("abs_path", String(Modifiers(nullable=True))),
("filename", String(Modifiers(nullable=True))),
("package", String(Modifiers(nullable=True))),
("module", String(Modifiers(nullable=True))),
("function", String(Modifiers(nullable=True))),
("in_app", UInt(8, Modifiers(nullable=True))),
("colno", UInt(32, Modifiers(nullable=True))),
("lineno", UInt(32, Modifiers(nullable=True))),
("stack_level", UInt(16)),
]
),
),
Column("culprit", String(Modifiers(nullable=True))),
Column("sdk_integrations", Array(String())),
Column("modules", Nested([("name", String()), ("version", String())])),
]


class Migration(migration.ClickhouseNodeMigrationLegacy):
blocking = False

def forwards_local(self) -> Sequence[operations.SqlOperation]:
sample_expr = "cityHash64(toString(event_id))"

return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="deleted",
order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr,
partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))",
sample_by=sample_expr,
),
),
]

def backwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.DropTable(
storage_set=StorageSetKey.EVENTS, table_name="sentry_local"
)
]

def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="sentry_local",
sharding_key="cityHash64(toString(event_id))",
),
),
operations.CreateTable(
storage_set=StorageSetKey.EVENTS_RO,
table_name="sentry_dist_ro",
columns=columns,
engine=table_engines.Distributed(
local_table_name="sentry_local",
sharding_key="cityHash64(toString(event_id))",
),
),
]

def backwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.DropTable(
storage_set=StorageSetKey.EVENTS, table_name="sentry_dist"
),
operations.DropTable(
storage_set=StorageSetKey.EVENTS_RO, table_name="sentry_dist_ro"
),
]
class Migration(migration.SquashedMigration):
pass
91 changes: 3 additions & 88 deletions snuba/snuba_migrations/events/0002_events_onpremise_compatibility.py
View file
Original file line number Diff line number Diff line change
@@ -1,90 +1,5 @@
from typing import Sequence
from snuba.migrations import migration

from snuba.clickhouse.columns import Array, Column, DateTime, Nested, String, UInt
from snuba.clusters.storage_sets import StorageSetKey
from snuba.migrations import migration, operations
from snuba.migrations.columns import MigrationModifiers as Modifiers


class Migration(migration.ClickhouseNodeMigrationLegacy):
"""
This is a one-off migration to support on premise users who are upgrading from
any older version of Snuba that used the old migration system. Since their sentry_local
table might be previously created with slightly different columns, this migration
should bring them back in sync by adding and removing the relevant columns that
have changed over time. It should be a no-op if the table is already up to date.
"""

blocking = False

def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column("group_id", UInt(64)),
after="project_id",
),
operations.DropColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column_name="device_model",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column("sdk_integrations", Array(String())),
after="exception_frames",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column("modules.name", Nested([("name", String())])),
after="sdk_integrations",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column("culprit", String(Modifiers(nullable=True))),
after="sdk_integrations",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column("search_message", String(Modifiers(nullable=True))),
after="received",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column("title", String(Modifiers(nullable=True))),
after="search_message",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column("location", String(Modifiers(nullable=True))),
after="title",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column("_tags_flattened", String()),
after="tags",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column("message_timestamp", DateTime()),
after="partition",
),
]

def backwards_local(self) -> Sequence[operations.SqlOperation]:
return []

def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return []

def backwards_dist(self) -> Sequence[operations.SqlOperation]:
return []
class Migration(migration.SquashedMigration):
pass
Loading

0 comments on commit 73cd686

Please sign in to comment.