fix: improve performance of recent SlackChannel related migrations (#5233)
# What this PR does After deploying [`r439-v1.12.0`](https://github.com/grafana/oncall-private/releases/tag/r439-v1.12.0) to staging, I noticed that the migrations were taking a long time, and caused some wonkiness (see https://raintank-corp.slack.com/archives/C08063QES5N). ```bash Apply all migrations: [redacted secret grafana-admin-creds:admin-user], alerts, auth, auth_token, base, contenttypes, email, exotel, fcm_django, google, heartbeat, labels, mobile_app, oss_installation, phone_notifications, schedules, sessions, slack, social_django, telegram, twilioapp, user_management, webhooks, zvonok Running migrations: source=engine:app google_trace_id=none logger=apps.alerts.migrations.0063_migrate_channelfilter_slack_channel_id Starting migration to populate slack_channel field. source=engine:app google_trace_id=none logger=apps.alerts.migrations.0063_migrate_channelfilter_slack_channel_id Bulk updated 1 ChannelFilters with their Slack channel. source=engine:app google_trace_id=none logger=apps.alerts.migrations.0063_migrate_channelfilter_slack_channel_id Finished migration to populate slack_channel field. Applying alerts.0063_migrate_channelfilter_slack_channel_id... OK source=engine:app google_trace_id=none logger=apps.alerts.migrations.0064_migrate_resolutionnoteslackmessage_slack_channel_id Starting migration to populate slack_channel field. source=engine:app google_trace_id=none logger=apps.alerts.migrations.0064_migrate_resolutionnoteslackmessage_slack_channel_id Bulk updated 1 ResolutionNoteSlackMessage records with their Slack channel. source=engine:app google_trace_id=none logger=apps.alerts.migrations.0064_migrate_resolutionnoteslackmessage_slack_channel_id Finished migration to populate slack_channel field. Applying alerts.0064_migrate_resolutionnoteslackmessage_slack_channel_id... OK source=engine:app google_trace_id=none logger=apps.schedules.migrations.0019_auto_20241021_1735 Starting migration to populate slack_channel field. source=engine:app google_trace_id=none logger=apps.schedules.migrations.0019_auto_20241021_1735 Bulk updated 6 OnCallSchedules with their Slack channel. source=engine:app google_trace_id=none logger=apps.schedules.migrations.0019_auto_20241021_1735 Finished migration to populate slack_channel field. Applying schedules.0019_auto_20241021_1735... OK source=engine:app google_trace_id=none logger=apps.user_management.migrations.0026_auto_20241017_1919 Starting migration to populate default_slack_channel field. source=engine:app google_trace_id=none logger=apps.user_management.migrations.0026_auto_20241017_1919 Bulk updated 1 organizations with their default Slack channel. source=engine:app google_trace_id=none logger=apps.user_management.migrations.0026_auto_20241017_1919 Finished migration to populate default_slack_channel field. Applying user_management.0026_auto_20241017_1919... OK ``` **NOTE**: wrt these migrations already being run for certain OSS stacks; it shouldn't have much of an impact on OSS deployments, as it's really only an issue for _very large_ versions of these tables (particularly the `ResolutionNoteSlackMessage` table, which by its nature, has a tendency to generate a lot of data). ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] Added the relevant release notes label (see labels prefixed w/ `release:`). These labels dictate how your PR will show up in the autogenerated release notes.
This commit is contained in:
Joey Orlando
GitHub
parent
effaa0a330
commit
53ac2bcc12
4 changed files with 64 additions and 151 deletions
|
|
@ -9,49 +9,28 @@ logger = logging.getLogger(__name__)
|
||||||
def populate_slack_channel(apps, schema_editor):
|
def populate_slack_channel(apps, schema_editor):
|
||||||
ChannelFilter = apps.get_model("alerts", "ChannelFilter")
|
ChannelFilter = apps.get_model("alerts", "ChannelFilter")
|
||||||
SlackChannel = apps.get_model("slack", "SlackChannel")
|
SlackChannel = apps.get_model("slack", "SlackChannel")
|
||||||
|
AlertReceiveChannel = apps.get_model("alerts", "AlertReceiveChannel")
|
||||||
|
Organization = apps.get_model("user_management", "Organization")
|
||||||
|
|
||||||
logger.info("Starting migration to populate slack_channel field.")
|
logger.info("Starting migration to populate slack_channel field.")
|
||||||
|
|
||||||
queryset = ChannelFilter.objects.filter(
|
sql = f"""
|
||||||
_slack_channel_id__isnull=False,
|
UPDATE {ChannelFilter._meta.db_table} AS cf
|
||||||
alert_receive_channel__organization__slack_team_identity__isnull=False,
|
JOIN {AlertReceiveChannel._meta.db_table} AS arc ON arc.id = cf.alert_receive_channel_id
|
||||||
)
|
JOIN {Organization._meta.db_table} AS org ON org.id = arc.organization_id
|
||||||
total_channel_filters = queryset.count()
|
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = cf._slack_channel_id
|
||||||
updated_channel_filters = 0
|
AND sc.slack_team_identity_id = org.slack_team_identity_id
|
||||||
missing_channel_filters = 0
|
SET cf.slack_channel_id = sc.id
|
||||||
channel_filters_to_update = []
|
WHERE cf._slack_channel_id IS NOT NULL
|
||||||
|
AND org.slack_team_identity_id IS NOT NULL;
|
||||||
|
"""
|
||||||
|
|
||||||
logger.info(f"Total channel filters to process: {total_channel_filters}")
|
with schema_editor.connection.cursor() as cursor:
|
||||||
|
cursor.execute(sql)
|
||||||
|
updated_rows = cursor.rowcount # Number of rows updated
|
||||||
|
|
||||||
for channel_filter in queryset:
|
logger.info(f"Bulk updated {updated_rows} ChannelFilters with their Slack channel.")
|
||||||
slack_id = channel_filter._slack_channel_id
|
logger.info("Finished migration to populate slack_channel field.")
|
||||||
slack_team_identity = channel_filter.alert_receive_channel.organization.slack_team_identity
|
|
||||||
|
|
||||||
try:
|
|
||||||
slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity)
|
|
||||||
channel_filter.slack_channel = slack_channel
|
|
||||||
channel_filters_to_update.append(channel_filter)
|
|
||||||
|
|
||||||
updated_channel_filters += 1
|
|
||||||
logger.info(
|
|
||||||
f"ChannelFilter {channel_filter.id} updated with SlackChannel {slack_channel.id} "
|
|
||||||
f"(slack_id: {slack_id})."
|
|
||||||
)
|
|
||||||
except SlackChannel.DoesNotExist:
|
|
||||||
missing_channel_filters += 1
|
|
||||||
logger.warning(
|
|
||||||
f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} "
|
|
||||||
f"does not exist for ChannelFilter {channel_filter.id}."
|
|
||||||
)
|
|
||||||
|
|
||||||
if channel_filters_to_update:
|
|
||||||
ChannelFilter.objects.bulk_update(channel_filters_to_update, ["slack_channel"])
|
|
||||||
logger.info(f"Bulk updated {len(channel_filters_to_update)} ChannelFilters with their Slack channel.")
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
f"Finished migration. Total channel filters processed: {total_channel_filters}. "
|
|
||||||
f"Channel filters updated: {updated_channel_filters}. Missing SlackChannels: {missing_channel_filters}."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
class Migration(migrations.Migration):
|
||||||
|
|
|
||||||
|
|
@ -10,53 +10,30 @@ logger = logging.getLogger(__name__)
|
||||||
def populate_slack_channel(apps, schema_editor):
|
def populate_slack_channel(apps, schema_editor):
|
||||||
ResolutionNoteSlackMessage = apps.get_model("alerts", "ResolutionNoteSlackMessage")
|
ResolutionNoteSlackMessage = apps.get_model("alerts", "ResolutionNoteSlackMessage")
|
||||||
SlackChannel = apps.get_model("slack", "SlackChannel")
|
SlackChannel = apps.get_model("slack", "SlackChannel")
|
||||||
|
AlertGroup = apps.get_model("alerts", "AlertGroup")
|
||||||
|
AlertReceiveChannel = apps.get_model("alerts", "AlertReceiveChannel")
|
||||||
|
Organization = apps.get_model("user_management", "Organization")
|
||||||
|
|
||||||
logger.info("Starting migration to populate slack_channel field.")
|
logger.info("Starting migration to populate slack_channel field.")
|
||||||
|
|
||||||
queryset = ResolutionNoteSlackMessage.objects.filter(
|
sql = f"""
|
||||||
_slack_channel_id__isnull=False,
|
UPDATE {ResolutionNoteSlackMessage._meta.db_table} AS rsm
|
||||||
alert_group__channel__organization__slack_team_identity__isnull=False,
|
JOIN {AlertGroup._meta.db_table} AS ag ON ag.id = rsm.alert_group_id
|
||||||
)
|
JOIN {AlertReceiveChannel._meta.db_table} AS arc ON arc.id = ag.channel_id
|
||||||
total_resolution_notes = queryset.count()
|
JOIN {Organization._meta.db_table} AS org ON org.id = arc.organization_id
|
||||||
updated_resolution_notes = 0
|
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = rsm._slack_channel_id
|
||||||
missing_resolution_notes = 0
|
AND sc.slack_team_identity_id = org.slack_team_identity_id
|
||||||
resolution_notes_to_update = []
|
SET rsm.slack_channel_id = sc.id
|
||||||
|
WHERE rsm._slack_channel_id IS NOT NULL
|
||||||
|
AND org.slack_team_identity_id IS NOT NULL;
|
||||||
|
"""
|
||||||
|
|
||||||
logger.info(f"Total resolution note slack messages to process: {total_resolution_notes}")
|
with schema_editor.connection.cursor() as cursor:
|
||||||
|
cursor.execute(sql)
|
||||||
for resolution_note in queryset:
|
updated_rows = cursor.rowcount # Number of rows updated
|
||||||
slack_id = resolution_note._slack_channel_id
|
|
||||||
slack_team_identity = resolution_note.alert_group.channel.organization.slack_team_identity
|
|
||||||
|
|
||||||
try:
|
|
||||||
slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity)
|
|
||||||
resolution_note.slack_channel = slack_channel
|
|
||||||
resolution_notes_to_update.append(resolution_note)
|
|
||||||
|
|
||||||
updated_resolution_notes += 1
|
|
||||||
logger.info(
|
|
||||||
f"ResolutionNoteSlackMessage {resolution_note.id} updated with SlackChannel {slack_channel.id} "
|
|
||||||
f"(slack_id: {slack_id})."
|
|
||||||
)
|
|
||||||
except SlackChannel.DoesNotExist:
|
|
||||||
missing_resolution_notes += 1
|
|
||||||
logger.warning(
|
|
||||||
f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} "
|
|
||||||
f"does not exist for ResolutionNoteSlackMessage {resolution_note.id}."
|
|
||||||
)
|
|
||||||
|
|
||||||
if resolution_notes_to_update:
|
|
||||||
ResolutionNoteSlackMessage.objects.bulk_update(resolution_notes_to_update, ["slack_channel"])
|
|
||||||
logger.info(
|
|
||||||
f"Bulk updated {len(resolution_notes_to_update)} ResolutionNoteSlackMessage with their Slack channel."
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
f"Finished migration. Total resolution note slack messages processed: {total_resolution_notes}. "
|
|
||||||
f"Resolution note slack messages updated: {updated_resolution_notes}. "
|
|
||||||
f"Missing SlackChannels: {missing_resolution_notes}."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
logger.info(f"Bulk updated {updated_rows} ResolutionNoteSlackMessage records with their Slack channel.")
|
||||||
|
logger.info("Finished migration to populate slack_channel field.")
|
||||||
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
class Migration(migrations.Migration):
|
||||||
|
|
|
||||||
|
|
@ -9,47 +9,26 @@ logger = logging.getLogger(__name__)
|
||||||
def populate_slack_channel(apps, schema_editor):
|
def populate_slack_channel(apps, schema_editor):
|
||||||
OnCallSchedule = apps.get_model("schedules", "OnCallSchedule")
|
OnCallSchedule = apps.get_model("schedules", "OnCallSchedule")
|
||||||
SlackChannel = apps.get_model("slack", "SlackChannel")
|
SlackChannel = apps.get_model("slack", "SlackChannel")
|
||||||
|
Organization = apps.get_model("user_management", "Organization")
|
||||||
|
|
||||||
logger.info("Starting migration to populate slack_channel field.")
|
logger.info("Starting migration to populate slack_channel field.")
|
||||||
|
|
||||||
queryset = OnCallSchedule.objects.filter(channel__isnull=False, organization__slack_team_identity__isnull=False)
|
sql = f"""
|
||||||
total_schedules = queryset.count()
|
UPDATE {OnCallSchedule._meta.db_table} AS ocs
|
||||||
updated_schedules = 0
|
JOIN {Organization._meta.db_table} AS org ON org.id = ocs.organization_id
|
||||||
missing_channels = 0
|
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = ocs.channel
|
||||||
schedules_to_update = []
|
AND sc.slack_team_identity_id = org.slack_team_identity_id
|
||||||
|
SET ocs.slack_channel_id = sc.id
|
||||||
|
WHERE ocs.channel IS NOT NULL
|
||||||
|
AND org.slack_team_identity_id IS NOT NULL;
|
||||||
|
"""
|
||||||
|
|
||||||
logger.info(f"Total schedules to process: {total_schedules}")
|
with schema_editor.connection.cursor() as cursor:
|
||||||
|
cursor.execute(sql)
|
||||||
for schedule in queryset:
|
updated_rows = cursor.rowcount # Number of rows updated
|
||||||
slack_id = schedule.channel
|
|
||||||
slack_team_identity = schedule.organization.slack_team_identity
|
|
||||||
|
|
||||||
try:
|
|
||||||
slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity)
|
|
||||||
|
|
||||||
schedule.slack_channel = slack_channel
|
|
||||||
schedules_to_update.append(schedule)
|
|
||||||
|
|
||||||
updated_schedules += 1
|
|
||||||
logger.info(
|
|
||||||
f"Schedule {schedule.id} updated with SlackChannel {slack_channel.id} (slack_id: {slack_id})."
|
|
||||||
)
|
|
||||||
except SlackChannel.DoesNotExist:
|
|
||||||
missing_channels += 1
|
|
||||||
logger.warning(
|
|
||||||
f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} "
|
|
||||||
f"does not exist for Schedule {schedule.id}."
|
|
||||||
)
|
|
||||||
|
|
||||||
if schedules_to_update:
|
|
||||||
OnCallSchedule.objects.bulk_update(schedules_to_update, ["slack_channel"])
|
|
||||||
logger.info(f"Bulk updated {len(schedules_to_update)} OnCallSchedules with their Slack channel.")
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
f"Finished migration. Total schedules processed: {total_schedules}. "
|
|
||||||
f"Schedules updated: {updated_schedules}. Missing SlackChannels: {missing_channels}."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
logger.info(f"Bulk updated {updated_rows} OnCallSchedules with their Slack channel.")
|
||||||
|
logger.info("Finished migration to populate slack_channel field.")
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -13,43 +13,21 @@ def populate_default_slack_channel(apps, schema_editor):
|
||||||
|
|
||||||
logger.info("Starting migration to populate default_slack_channel field.")
|
logger.info("Starting migration to populate default_slack_channel field.")
|
||||||
|
|
||||||
queryset = Organization.objects.filter(general_log_channel_id__isnull=False, slack_team_identity__isnull=False)
|
sql = f"""
|
||||||
total_orgs = queryset.count()
|
UPDATE {Organization._meta.db_table} AS org
|
||||||
updated_orgs = 0
|
JOIN {SlackChannel._meta.db_table} AS sc ON sc.slack_id = org.general_log_channel_id
|
||||||
missing_channels = 0
|
AND sc.slack_team_identity_id = org.slack_team_identity_id
|
||||||
organizations_to_update = []
|
SET org.default_slack_channel_id = sc.id
|
||||||
|
WHERE org.general_log_channel_id IS NOT NULL
|
||||||
|
AND org.slack_team_identity_id IS NOT NULL;
|
||||||
|
"""
|
||||||
|
|
||||||
logger.info(f"Total organizations to process: {total_orgs}")
|
with schema_editor.connection.cursor() as cursor:
|
||||||
|
cursor.execute(sql)
|
||||||
|
updated_rows = cursor.rowcount # Number of rows updated
|
||||||
|
|
||||||
for org in queryset:
|
logger.info(f"Bulk updated {updated_rows} organizations with their default Slack channel.")
|
||||||
slack_id = org.general_log_channel_id
|
logger.info("Finished migration to populate default_slack_channel field.")
|
||||||
slack_team_identity = org.slack_team_identity
|
|
||||||
|
|
||||||
try:
|
|
||||||
slack_channel = SlackChannel.objects.get(slack_id=slack_id, slack_team_identity=slack_team_identity)
|
|
||||||
|
|
||||||
org.default_slack_channel = slack_channel
|
|
||||||
organizations_to_update.append(org)
|
|
||||||
|
|
||||||
updated_orgs += 1
|
|
||||||
logger.info(
|
|
||||||
f"Organization {org.id} updated with SlackChannel {slack_channel.id} (slack_id: {slack_id})."
|
|
||||||
)
|
|
||||||
except SlackChannel.DoesNotExist:
|
|
||||||
missing_channels += 1
|
|
||||||
logger.warning(
|
|
||||||
f"SlackChannel with slack_id {slack_id} and slack_team_identity {slack_team_identity} "
|
|
||||||
f"does not exist for Organization {org.id}."
|
|
||||||
)
|
|
||||||
|
|
||||||
if organizations_to_update:
|
|
||||||
Organization.objects.bulk_update(organizations_to_update, ["default_slack_channel"])
|
|
||||||
logger.info(f"Bulk updated {len(organizations_to_update)} organizations with their default Slack channel.")
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
f"Finished migration. Total organizations processed: {total_orgs}. "
|
|
||||||
f"Organizations updated: {updated_orgs}. Missing SlackChannels: {missing_channels}."
|
|
||||||
)
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue