From 53ac2bcc12fe3f197f9fe920a65b50e39db59de2 Mon Sep 17 00:00:00 2001 From: Joey Orlando Date: Wed, 6 Nov 2024 06:02:21 -0500 Subject: [PATCH] fix: improve performance of recent `SlackChannel` related migrations (#5233) # What this PR does After deploying [`r439-v1.12.0`](https://github.com/grafana/oncall-private/releases/tag/r439-v1.12.0) to staging, I noticed that the migrations were taking a long time, and caused some wonkiness (see https://raintank-corp.slack.com/archives/C08063QES5N). ```bash Apply all migrations: [redacted secret grafana-admin-creds:admin-user], alerts, auth, auth_token, base, contenttypes, email, exotel, fcm_django, google, heartbeat, labels, mobile_app, oss_installation, phone_notifications, schedules, sessions, slack, social_django, telegram, twilioapp, user_management, webhooks, zvonok Running migrations: source=engine:app google_trace_id=none logger=apps.alerts.migrations.0063_migrate_channelfilter_slack_channel_id Starting migration to populate slack_channel field. source=engine:app google_trace_id=none logger=apps.alerts.migrations.0063_migrate_channelfilter_slack_channel_id Bulk updated 1 ChannelFilters with their Slack channel. source=engine:app google_trace_id=none logger=apps.alerts.migrations.0063_migrate_channelfilter_slack_channel_id Finished migration to populate slack_channel field. Applying alerts.0063_migrate_channelfilter_slack_channel_id... OK source=engine:app google_trace_id=none logger=apps.alerts.migrations.0064_migrate_resolutionnoteslackmessage_slack_channel_id Starting migration to populate slack_channel field. source=engine:app google_trace_id=none logger=apps.alerts.migrations.0064_migrate_resolutionnoteslackmessage_slack_channel_id Bulk updated 1 ResolutionNoteSlackMessage records with their Slack channel. source=engine:app google_trace_id=none logger=apps.alerts.migrations.0064_migrate_resolutionnoteslackmessage_slack_channel_id Finished migration to populate slack_channel field. Applying alerts.0064_migrate_resolutionnoteslackmessage_slack_channel_id... OK source=engine:app google_trace_id=none logger=apps.schedules.migrations.0019_auto_20241021_1735 Starting migration to populate slack_channel field. source=engine:app google_trace_id=none logger=apps.schedules.migrations.0019_auto_20241021_1735 Bulk updated 6 OnCallSchedules with their Slack channel. source=engine:app google_trace_id=none logger=apps.schedules.migrations.0019_auto_20241021_1735 Finished migration to populate slack_channel field. Applying schedules.0019_auto_20241021_1735... OK source=engine:app google_trace_id=none logger=apps.user_management.migrations.0026_auto_20241017_1919 Starting migration to populate default_slack_channel field. source=engine:app google_trace_id=none logger=apps.user_management.migrations.0026_auto_20241017_1919 Bulk updated 1 organizations with their default Slack channel. source=engine:app google_trace_id=none logger=apps.user_management.migrations.0026_auto_20241017_1919 Finished migration to populate default_slack_channel field. Applying user_management.0026_auto_20241017_1919... OK ``` **NOTE**: wrt these migrations already being run for certain OSS stacks; it shouldn't have much of an impact on OSS deployments, as it's really only an issue for _very large_ versions of these tables (particularly the `ResolutionNoteSlackMessage` table, which by its nature, has a tendency to generate a lot of data). ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] Added the relevant release notes label (see labels prefixed w/ `release:`). These labels dictate how your PR will show up in the autogenerated release notes. --- ..._migrate_channelfilter_slack_channel_id.py | 55 ++++++----------- ...lutionnoteslackmessage_slack_channel_id.py | 61 ++++++------------- .../migrations/0019_auto_20241021_1735.py | 51 +++++----------- .../migrations/0026_auto_20241017_1919.py | 48 ++++----------- 4 files changed, 64 insertions(+), 151 deletions(-) diff --git a/engine/apps/alerts/migrations/0063_migrate_channelfilter_slack_channel_id.py b/engine/apps/alerts/migrations/0063_migrate_channelfilter_slack_channel_id.py index d7a46b95..dab5a459 100644 --- a/engine/apps/alerts/migrations/0063_migrate_channelfilter_slack_channel_id.py +++ b/engine/apps/alerts/migrations/0063_migrate_channelfilter_slack_channel_id.py @@ -9,49 +9,28 @@ logger = logging.getLogger(__name__) def populate_slack_channel(apps, schema_editor): ChannelFilter = apps.get_model("alerts", "ChannelFilter") SlackChannel = apps.get_model("slack", "SlackChannel") + AlertReceiveChannel = apps.get_model("alerts", "AlertReceiveChannel") + Organization = apps.get_model("user_management", "Organization") logger.info("Starting migration to populate slack_channel field.") - queryset = ChannelFilter.objects.filter( - _slack_channel_id__isnull=False, - alert_receive_channel__organization__slack_team_identity__isnull=False, - ) - total_channel_filters = queryset.count() - updated_channel_filters = 0 - missing_channel_filters = 0 - channel_filters_to_update = [] + sql = f""" + UPDATE {ChannelFilter._meta.db_table} AS cf + JOIN {AlertReceiveChannel._meta.db_table} AS arc ON arc.id = cf.alert_receive_channel_id + JOIN {Organization._meta.db_table} AS org ON org.id = arc.organization_id + JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = cf._slack_channel_id + AND sc.slack_team_identity_id = org.slack_team_identity_id + SET cf.slack_channel_id = sc.id + WHERE cf._slack_channel_id IS NOT NULL + AND org.slack_team_identity_id IS NOT NULL; + """ - logger.info(f"Total channel filters to process: {total_channel_filters}") + with schema_editor.connection.cursor() as cursor: + cursor.execute(sql) + updated_rows = cursor.rowcount # Number of rows updated - for channel_filter in queryset: - slack_id = channel_filter._slack_channel_id - slack_team_identity = channel_filter.alert_receive_channel.organization.slack_team_identity - - try: - slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity) - channel_filter.slack_channel = slack_channel - channel_filters_to_update.append(channel_filter) - - updated_channel_filters += 1 - logger.info( - f"ChannelFilter {channel_filter.id} updated with SlackChannel {slack_channel.id} " - f"(slack_id: {slack_id})." - ) - except SlackChannel.DoesNotExist: - missing_channel_filters += 1 - logger.warning( - f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} " - f"does not exist for ChannelFilter {channel_filter.id}." - ) - - if channel_filters_to_update: - ChannelFilter.objects.bulk_update(channel_filters_to_update, ["slack_channel"]) - logger.info(f"Bulk updated {len(channel_filters_to_update)} ChannelFilters with their Slack channel.") - - logger.info( - f"Finished migration. Total channel filters processed: {total_channel_filters}. " - f"Channel filters updated: {updated_channel_filters}. Missing SlackChannels: {missing_channel_filters}." - ) + logger.info(f"Bulk updated {updated_rows} ChannelFilters with their Slack channel.") + logger.info("Finished migration to populate slack_channel field.") class Migration(migrations.Migration): diff --git a/engine/apps/alerts/migrations/0064_migrate_resolutionnoteslackmessage_slack_channel_id.py b/engine/apps/alerts/migrations/0064_migrate_resolutionnoteslackmessage_slack_channel_id.py index ccb304ed..4f492e31 100644 --- a/engine/apps/alerts/migrations/0064_migrate_resolutionnoteslackmessage_slack_channel_id.py +++ b/engine/apps/alerts/migrations/0064_migrate_resolutionnoteslackmessage_slack_channel_id.py @@ -10,53 +10,30 @@ logger = logging.getLogger(__name__) def populate_slack_channel(apps, schema_editor): ResolutionNoteSlackMessage = apps.get_model("alerts", "ResolutionNoteSlackMessage") SlackChannel = apps.get_model("slack", "SlackChannel") + AlertGroup = apps.get_model("alerts", "AlertGroup") + AlertReceiveChannel = apps.get_model("alerts", "AlertReceiveChannel") + Organization = apps.get_model("user_management", "Organization") logger.info("Starting migration to populate slack_channel field.") - queryset = ResolutionNoteSlackMessage.objects.filter( - _slack_channel_id__isnull=False, - alert_group__channel__organization__slack_team_identity__isnull=False, - ) - total_resolution_notes = queryset.count() - updated_resolution_notes = 0 - missing_resolution_notes = 0 - resolution_notes_to_update = [] + sql = f""" + UPDATE {ResolutionNoteSlackMessage._meta.db_table} AS rsm + JOIN {AlertGroup._meta.db_table} AS ag ON ag.id = rsm.alert_group_id + JOIN {AlertReceiveChannel._meta.db_table} AS arc ON arc.id = ag.channel_id + JOIN {Organization._meta.db_table} AS org ON org.id = arc.organization_id + JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = rsm._slack_channel_id + AND sc.slack_team_identity_id = org.slack_team_identity_id + SET rsm.slack_channel_id = sc.id + WHERE rsm._slack_channel_id IS NOT NULL + AND org.slack_team_identity_id IS NOT NULL; + """ - logger.info(f"Total resolution note slack messages to process: {total_resolution_notes}") - - for resolution_note in queryset: - slack_id = resolution_note._slack_channel_id - slack_team_identity = resolution_note.alert_group.channel.organization.slack_team_identity - - try: - slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity) - resolution_note.slack_channel = slack_channel - resolution_notes_to_update.append(resolution_note) - - updated_resolution_notes += 1 - logger.info( - f"ResolutionNoteSlackMessage {resolution_note.id} updated with SlackChannel {slack_channel.id} " - f"(slack_id: {slack_id})." - ) - except SlackChannel.DoesNotExist: - missing_resolution_notes += 1 - logger.warning( - f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} " - f"does not exist for ResolutionNoteSlackMessage {resolution_note.id}." - ) - - if resolution_notes_to_update: - ResolutionNoteSlackMessage.objects.bulk_update(resolution_notes_to_update, ["slack_channel"]) - logger.info( - f"Bulk updated {len(resolution_notes_to_update)} ResolutionNoteSlackMessage with their Slack channel." - ) - - logger.info( - f"Finished migration. Total resolution note slack messages processed: {total_resolution_notes}. " - f"Resolution note slack messages updated: {updated_resolution_notes}. " - f"Missing SlackChannels: {missing_resolution_notes}." - ) + with schema_editor.connection.cursor() as cursor: + cursor.execute(sql) + updated_rows = cursor.rowcount # Number of rows updated + logger.info(f"Bulk updated {updated_rows} ResolutionNoteSlackMessage records with their Slack channel.") + logger.info("Finished migration to populate slack_channel field.") class Migration(migrations.Migration): diff --git a/engine/apps/schedules/migrations/0019_auto_20241021_1735.py b/engine/apps/schedules/migrations/0019_auto_20241021_1735.py index 8fe349dd..edc89366 100644 --- a/engine/apps/schedules/migrations/0019_auto_20241021_1735.py +++ b/engine/apps/schedules/migrations/0019_auto_20241021_1735.py @@ -9,47 +9,26 @@ logger = logging.getLogger(__name__) def populate_slack_channel(apps, schema_editor): OnCallSchedule = apps.get_model("schedules", "OnCallSchedule") SlackChannel = apps.get_model("slack", "SlackChannel") + Organization = apps.get_model("user_management", "Organization") logger.info("Starting migration to populate slack_channel field.") - queryset = OnCallSchedule.objects.filter(channel__isnull=False, organization__slack_team_identity__isnull=False) - total_schedules = queryset.count() - updated_schedules = 0 - missing_channels = 0 - schedules_to_update = [] + sql = f""" + UPDATE {OnCallSchedule._meta.db_table} AS ocs + JOIN {Organization._meta.db_table} AS org ON org.id = ocs.organization_id + JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = ocs.channel + AND sc.slack_team_identity_id = org.slack_team_identity_id + SET ocs.slack_channel_id = sc.id + WHERE ocs.channel IS NOT NULL + AND org.slack_team_identity_id IS NOT NULL; + """ - logger.info(f"Total schedules to process: {total_schedules}") - - for schedule in queryset: - slack_id = schedule.channel - slack_team_identity = schedule.organization.slack_team_identity - - try: - slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity) - - schedule.slack_channel = slack_channel - schedules_to_update.append(schedule) - - updated_schedules += 1 - logger.info( - f"Schedule {schedule.id} updated with SlackChannel {slack_channel.id} (slack_id: {slack_id})." - ) - except SlackChannel.DoesNotExist: - missing_channels += 1 - logger.warning( - f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} " - f"does not exist for Schedule {schedule.id}." - ) - - if schedules_to_update: - OnCallSchedule.objects.bulk_update(schedules_to_update, ["slack_channel"]) - logger.info(f"Bulk updated {len(schedules_to_update)} OnCallSchedules with their Slack channel.") - - logger.info( - f"Finished migration. Total schedules processed: {total_schedules}. " - f"Schedules updated: {updated_schedules}. Missing SlackChannels: {missing_channels}." - ) + with schema_editor.connection.cursor() as cursor: + cursor.execute(sql) + updated_rows = cursor.rowcount # Number of rows updated + logger.info(f"Bulk updated {updated_rows} OnCallSchedules with their Slack channel.") + logger.info("Finished migration to populate slack_channel field.") class Migration(migrations.Migration): diff --git a/engine/apps/user_management/migrations/0026_auto_20241017_1919.py b/engine/apps/user_management/migrations/0026_auto_20241017_1919.py index c0869217..df28b026 100644 --- a/engine/apps/user_management/migrations/0026_auto_20241017_1919.py +++ b/engine/apps/user_management/migrations/0026_auto_20241017_1919.py @@ -13,43 +13,21 @@ def populate_default_slack_channel(apps, schema_editor): logger.info("Starting migration to populate default_slack_channel field.") - queryset = Organization.objects.filter(general_log_channel_id__isnull=False, slack_team_identity__isnull=False) - total_orgs = queryset.count() - updated_orgs = 0 - missing_channels = 0 - organizations_to_update = [] + sql = f""" + UPDATE {Organization._meta.db_table} AS org + JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = org.general_log_channel_id + AND sc.slack_team_identity_id = org.slack_team_identity_id + SET org.default_slack_channel_id = sc.id + WHERE org.general_log_channel_id IS NOT NULL + AND org.slack_team_identity_id IS NOT NULL; + """ - logger.info(f"Total organizations to process: {total_orgs}") + with schema_editor.connection.cursor() as cursor: + cursor.execute(sql) + updated_rows = cursor.rowcount # Number of rows updated - for org in queryset: - slack_id = org.general_log_channel_id - slack_team_identity = org.slack_team_identity - - try: - slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity) - - org.default_slack_channel = slack_channel - organizations_to_update.append(org) - - updated_orgs += 1 - logger.info( - f"Organization {org.id} updated with SlackChannel {slack_channel.id} (slack_id: {slack_id})." - ) - except SlackChannel.DoesNotExist: - missing_channels += 1 - logger.warning( - f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} " - f"does not exist for Organization {org.id}." - ) - - if organizations_to_update: - Organization.objects.bulk_update(organizations_to_update, ["default_slack_channel"]) - logger.info(f"Bulk updated {len(organizations_to_update)} organizations with their default Slack channel.") - - logger.info( - f"Finished migration. Total organizations processed: {total_orgs}. " - f"Organizations updated: {updated_orgs}. Missing SlackChannels: {missing_channels}." - ) + logger.info(f"Bulk updated {updated_rows} organizations with their default Slack channel.") + logger.info("Finished migration to populate default_slack_channel field.") class Migration(migrations.Migration):