From 30caa18f9d22dfe52b25f8922437a85362c67df0 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Fri, 1 Dec 2023 18:49:00 +0800 Subject: [PATCH] Make telegram on_alert_group_action_triggered asynchronous (#3471) # What this PR does [send_alert_group_signal](https://github.com/grafana/oncall/blob/dev/engine/apps/alerts/tasks/send_alert_group_signal.py#L12) task is not idempotent. It launches [on_alert_group_action_triggered_async](https://github.com/grafana/oncall/blob/a2851d3f813c04768ce825c8af99e26912d80b4e/engine/apps/slack/representatives/alert_group_representative.py#L158) for slack and then might fail on [on_alert_group_action_triggered](https://github.com/grafana/oncall/blob/b2f4ffb98a940b894042df9161e209dc8a781dc1/engine/apps/telegram/alert_group_representative.py#L79) (not async) due to database DoesNotExist exception. This PR makes telegram representative asyncronous ## Which issue(s) this PR fixes ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required) --- CHANGELOG.md | 1 + .../telegram/alert_group_representative.py | 20 +++++++++---------- engine/apps/telegram/tasks.py | 18 +++++++++++++++++ engine/settings/celery_task_routes.py | 1 + 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e741b717..198b709b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ Minor bugfixes + dependency updates :) - Filters polishing ([3183](https://github.com/grafana/oncall/issues/3183)) - Fixed permissions so User settings reader role included list users @mderynck ([#3419](https://github.com/grafana/oncall/pull/3419)) - Fixed alert group rendering when some links were broken because of replacing `-` to `_` @Ferril ([#3424](https://github.com/grafana/oncall/pull/3424)) +- Make telegram on_alert_group_action_triggered asynchronous([#3471](https://github.com/grafana/oncall/pull/3471)) ## v1.3.62 (2023-11-21) diff --git a/engine/apps/telegram/alert_group_representative.py b/engine/apps/telegram/alert_group_representative.py index a3b6933c..635420c6 100644 --- a/engine/apps/telegram/alert_group_representative.py +++ b/engine/apps/telegram/alert_group_representative.py @@ -3,7 +3,11 @@ import logging from apps.alerts.models import AlertGroup from apps.alerts.representative import AlertGroupAbstractRepresentative from apps.telegram.models import TelegramMessage -from apps.telegram.tasks import edit_message, on_create_alert_telegram_representative_async +from apps.telegram.tasks import ( + edit_message, + on_alert_group_action_triggered_async, + on_create_alert_telegram_representative_async, +) logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -78,15 +82,11 @@ class AlertGroupTelegramRepresentative(AlertGroupAbstractRepresentative): from apps.alerts.models import AlertGroupLogRecord log_record = kwargs["log_record"] - logger.info(f"AlertGroupTelegramRepresentative ACTION SIGNAL, log record {log_record}") - - if not isinstance(log_record, AlertGroupLogRecord): - log_record = AlertGroupLogRecord.objects.get(pk=log_record) - - instance = cls(log_record) - if instance.is_applicable(): - handler = instance.get_handler() - handler() + if isinstance(log_record, AlertGroupLogRecord): + log_record_id = log_record.pk + else: + log_record_id = log_record + on_alert_group_action_triggered_async.apply_async((log_record_id,)) @staticmethod def on_create_alert(**kwargs): diff --git a/engine/apps/telegram/tasks.py b/engine/apps/telegram/tasks.py index 54b84945..1c160562 100644 --- a/engine/apps/telegram/tasks.py +++ b/engine/apps/telegram/tasks.py @@ -215,3 +215,21 @@ def on_create_alert_telegram_representative_async(self, alert_pk): ) for message in messages_to_edit: edit_message.delay(message_pk=message.pk) + + +@shared_dedicated_queue_retry_task( + autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else None +) +def on_alert_group_action_triggered_async(log_record_id): + from apps.alerts.models import AlertGroupLogRecord + + from .alert_group_representative import AlertGroupTelegramRepresentative + + logger.info(f"AlertGroupTelegramRepresentative ACTION SIGNAL, log record {log_record_id}") + + log_record = AlertGroupLogRecord.objects.get(pk=log_record_id) + + instance = AlertGroupTelegramRepresentative(log_record) + if instance.is_applicable(): + handler = instance.get_handler() + handler() diff --git a/engine/settings/celery_task_routes.py b/engine/settings/celery_task_routes.py index b7803b46..a1f45836 100644 --- a/engine/settings/celery_task_routes.py +++ b/engine/settings/celery_task_routes.py @@ -161,6 +161,7 @@ CELERY_TASK_ROUTES = { "apps.telegram.tasks.register_telegram_webhook": {"queue": "telegram"}, "apps.telegram.tasks.send_link_to_channel_message_or_fallback_to_full_alert_group": {"queue": "telegram"}, "apps.telegram.tasks.send_log_and_actions_message": {"queue": "telegram"}, + "apps.telegram.tasks.on_alert_group_action_triggered_async": {"queue": "telegram"}, # WEBHOOK "apps.alerts.tasks.custom_button_result.custom_button_result": {"queue": "webhook"}, "apps.alerts.tasks.custom_webhook_result.custom_webhook_result": {"queue": "webhook"},