Merge pull request #467 from grafana/retrying-tasks-fixes

Handle exceptions on some retrying tasks

engine/apps/alerts/escalation_snapshot/escalation_snapshot_mixin.py

@@ -7,6 +7,7 @@ from dateutil.parser import parse
 from django.apps import apps
 from django.utils import timezone
 from django.utils.functional import cached_property
+from rest_framework.exceptions import ValidationError
 
 from apps.alerts.constants import NEXT_ESCALATION_DELAY
 from apps.alerts.escalation_snapshot.snapshot_classes import (
@@ -189,7 +190,10 @@ class EscalationSnapshotMixin:
         escalation_snapshot_object = None
         raw_escalation_snapshot = self.raw_escalation_snapshot
         if raw_escalation_snapshot is not None:
-            escalation_snapshot_object = self._deserialize_escalation_snapshot(raw_escalation_snapshot)
+            try:
+                escalation_snapshot_object = self._deserialize_escalation_snapshot(raw_escalation_snapshot)
+            except ValidationError as e:
+                logger.error(f"Error trying to deserialize raw escalation snapshot: {e}")
         return escalation_snapshot_object
 
     def _deserialize_escalation_snapshot(self, raw_escalation_snapshot) -> EscalationSnapshot:

engine/apps/base/models/user_notification_policy_log_record.py

@@ -69,7 +69,8 @@ class UserNotificationPolicyLogRecord(models.Model):
         ERROR_NOTIFICATION_IN_SLACK_RATELIMIT,
         ERROR_NOTIFICATION_MESSAGING_BACKEND_ERROR,
         ERROR_NOTIFICATION_NOT_ALLOWED_USER_ROLE,
-    ) = range(26)
+        ERROR_NOTIFICATION_TELEGRAM_USER_IS_DEACTIVATED,
+    ) = range(27)
 
     # for this errors we want to send message to general log channel
     ERRORS_TO_SEND_IN_SLACK_CHANNEL = [
@@ -272,6 +273,11 @@ class UserNotificationPolicyLogRecord(models.Model):
                 self.notification_error_code == UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_NOT_ALLOWED_USER_ROLE
             ):
                 result += f"failed to notify {user_verbal}, not allowed role"
+            elif (
+                self.notification_error_code
+                == UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_TELEGRAM_USER_IS_DEACTIVATED
+            ):
+                result += f"failed to send telegram message to {user_verbal} because user has been deactivated"
             else:
                 # TODO: handle specific backend errors
                 try:

engine/apps/slack/scenarios/notification_delivery.py

@@ -87,12 +87,3 @@ class NotificationDeliveryStep(scenario_step.ScenarioStep):
                 print(e)
             else:
                 raise e
-
-    def get_color_id(self, color):
-        if color == "red":
-            color_id = "#FF0000"
-        elif color == "yellow":
-            color_id = "#c6c000"
-        else:
-            color_id = color
-        return color_id

engine/apps/telegram/models/connectors/personal.py

@@ -111,6 +111,13 @@ class TelegramToUserConnector(models.Model):
                         notification_policy,
                         UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_TELEGRAM_TOKEN_ERROR,
                     )
+                elif e.message == "Forbidden: user is deactivated":
+                    TelegramToUserConnector.create_telegram_notification_error(
+                        alert_group,
+                        self.user,
+                        notification_policy,
+                        UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_TELEGRAM_USER_IS_DEACTIVATED,
+                    )
                 else:
                     raise e
         else:

engine/apps/telegram/tasks.py

@@ -119,7 +119,17 @@ def send_link_to_channel_message_or_fallback_to_full_incident(
 @ignore_bot_deleted
 def send_log_and_actions_message(self, channel_chat_id, group_chat_id, channel_message_id, reply_to_message_id):
     with OkToRetry(task=self, exc=TelegramMessage.DoesNotExist, num_retries=5):
-        channel_message = TelegramMessage.objects.get(chat_id=channel_chat_id, message_id=channel_message_id)
+        try:
+            channel_message = TelegramMessage.objects.get(chat_id=channel_chat_id, message_id=channel_message_id)
+        except TelegramMessage.DoesNotExist:
+            if self.request.retries <= 5:
+                raise
+            else:
+                logger.warning(
+                    f"Could not send log and actions message, telegram message does not exist "
+                    f" chat_id={channel_chat_id} message_id={channel_message_id}"
+                )
+                return
 
         if channel_message.discussion_group_message_id is None:
             channel_message.discussion_group_message_id = reply_to_message_id