Make telegram on_alert_group_action_triggered asynchronous (#3471)

# What this PR does


[send_alert_group_signal](https://github.com/grafana/oncall/blob/dev/engine/apps/alerts/tasks/send_alert_group_signal.py#L12)
task is not idempotent. It launches
[on_alert_group_action_triggered_async](a2851d3f81/engine/apps/slack/representatives/alert_group_representative.py (L158))
for slack and then might fail on
[on_alert_group_action_triggered](b2f4ffb98a/engine/apps/telegram/alert_group_representative.py (L79))
(not async) due to database DoesNotExist exception.

This PR makes telegram representative asyncronous

## Which issue(s) this PR fixes

## Checklist

- [ ] Unit, integration, and e2e (if applicable) tests updated
- [ ] Documentation added (or `pr:no public docs` PR label added if not
required)
- [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not
required)

CHANGELOG.md

@@ -66,6 +66,7 @@ Minor bugfixes + dependency updates :)
 - Filters polishing ([3183](https://github.com/grafana/oncall/issues/3183))
 - Fixed permissions so User settings reader role included list users @mderynck ([#3419](https://github.com/grafana/oncall/pull/3419))
 - Fixed alert group rendering when some links were broken because of replacing `-` to `_` @Ferril ([#3424](https://github.com/grafana/oncall/pull/3424))
+- Make telegram on_alert_group_action_triggered asynchronous([#3471](https://github.com/grafana/oncall/pull/3471))
 
 ## v1.3.62 (2023-11-21)
 

engine/apps/telegram/alert_group_representative.py

@@ -3,7 +3,11 @@ import logging
 from apps.alerts.models import AlertGroup
 from apps.alerts.representative import AlertGroupAbstractRepresentative
 from apps.telegram.models import TelegramMessage
-from apps.telegram.tasks import edit_message, on_create_alert_telegram_representative_async
+from apps.telegram.tasks import (
+    edit_message,
+    on_alert_group_action_triggered_async,
+    on_create_alert_telegram_representative_async,
+)
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -78,15 +82,11 @@ class AlertGroupTelegramRepresentative(AlertGroupAbstractRepresentative):
         from apps.alerts.models import AlertGroupLogRecord
 
         log_record = kwargs["log_record"]
-        logger.info(f"AlertGroupTelegramRepresentative ACTION SIGNAL, log record {log_record}")
-
-        if not isinstance(log_record, AlertGroupLogRecord):
-            log_record = AlertGroupLogRecord.objects.get(pk=log_record)
-
-        instance = cls(log_record)
-        if instance.is_applicable():
-            handler = instance.get_handler()
-            handler()
+        if isinstance(log_record, AlertGroupLogRecord):
+            log_record_id = log_record.pk
+        else:
+            log_record_id = log_record
+        on_alert_group_action_triggered_async.apply_async((log_record_id,))
 
     @staticmethod
     def on_create_alert(**kwargs):

engine/apps/telegram/tasks.py

@@ -215,3 +215,21 @@ def on_create_alert_telegram_representative_async(self, alert_pk):
     )
     for message in messages_to_edit:
         edit_message.delay(message_pk=message.pk)
+
+
+@shared_dedicated_queue_retry_task(
+    autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else None
+)
+def on_alert_group_action_triggered_async(log_record_id):
+    from apps.alerts.models import AlertGroupLogRecord
+
+    from .alert_group_representative import AlertGroupTelegramRepresentative
+
+    logger.info(f"AlertGroupTelegramRepresentative ACTION SIGNAL, log record {log_record_id}")
+
+    log_record = AlertGroupLogRecord.objects.get(pk=log_record_id)
+
+    instance = AlertGroupTelegramRepresentative(log_record)
+    if instance.is_applicable():
+        handler = instance.get_handler()
+        handler()

engine/settings/celery_task_routes.py

@@ -161,6 +161,7 @@ CELERY_TASK_ROUTES = {
     "apps.telegram.tasks.register_telegram_webhook": {"queue": "telegram"},
     "apps.telegram.tasks.send_link_to_channel_message_or_fallback_to_full_alert_group": {"queue": "telegram"},
     "apps.telegram.tasks.send_log_and_actions_message": {"queue": "telegram"},
+    "apps.telegram.tasks.on_alert_group_action_triggered_async": {"queue": "telegram"},
     # WEBHOOK
     "apps.alerts.tasks.custom_button_result.custom_button_result": {"queue": "webhook"},
     "apps.alerts.tasks.custom_webhook_result.custom_webhook_result": {"queue": "webhook"},