From 50dc7d62404c780e9dd9ef6e2ef87a527f49fc16 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Thu, 1 Sep 2022 11:39:47 -0600 Subject: [PATCH 1/3] Handle a exceptions on some retrying tasks --- .../escalation_snapshot/escalation_snapshot_mixin.py | 6 +++++- .../base/models/user_notification_policy_log_record.py | 8 +++++++- engine/apps/slack/scenarios/notification_delivery.py | 9 --------- engine/apps/telegram/models/connectors/personal.py | 7 +++++++ engine/apps/telegram/tasks.py | 9 ++++++++- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/engine/apps/alerts/escalation_snapshot/escalation_snapshot_mixin.py b/engine/apps/alerts/escalation_snapshot/escalation_snapshot_mixin.py index 0ec5e67c..40f983e5 100644 --- a/engine/apps/alerts/escalation_snapshot/escalation_snapshot_mixin.py +++ b/engine/apps/alerts/escalation_snapshot/escalation_snapshot_mixin.py @@ -7,6 +7,7 @@ from dateutil.parser import parse from django.apps import apps from django.utils import timezone from django.utils.functional import cached_property +from rest_framework.exceptions import ValidationError from apps.alerts.constants import NEXT_ESCALATION_DELAY from apps.alerts.escalation_snapshot.snapshot_classes import ( @@ -189,7 +190,10 @@ class EscalationSnapshotMixin: escalation_snapshot_object = None raw_escalation_snapshot = self.raw_escalation_snapshot if raw_escalation_snapshot is not None: - escalation_snapshot_object = self._deserialize_escalation_snapshot(raw_escalation_snapshot) + try: + escalation_snapshot_object = self._deserialize_escalation_snapshot(raw_escalation_snapshot) + except ValidationError as e: + logger.error(f"Error trying to deserialize raw escalation snapshot: {e}") return escalation_snapshot_object def _deserialize_escalation_snapshot(self, raw_escalation_snapshot) -> EscalationSnapshot: diff --git a/engine/apps/base/models/user_notification_policy_log_record.py b/engine/apps/base/models/user_notification_policy_log_record.py index ed261b2b..e29a9ec4 100644 --- a/engine/apps/base/models/user_notification_policy_log_record.py +++ b/engine/apps/base/models/user_notification_policy_log_record.py @@ -69,7 +69,8 @@ class UserNotificationPolicyLogRecord(models.Model): ERROR_NOTIFICATION_IN_SLACK_RATELIMIT, ERROR_NOTIFICATION_MESSAGING_BACKEND_ERROR, ERROR_NOTIFICATION_NOT_ALLOWED_USER_ROLE, - ) = range(26) + ERROR_NOTIFICATION_TELEGRAM_USER_IS_DEACTIVATED, + ) = range(27) # for this errors we want to send message to general log channel ERRORS_TO_SEND_IN_SLACK_CHANNEL = [ @@ -272,6 +273,11 @@ class UserNotificationPolicyLogRecord(models.Model): self.notification_error_code == UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_NOT_ALLOWED_USER_ROLE ): result += f"failed to notify {user_verbal}, not allowed role" + elif ( + self.notification_error_code + == UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_TELEGRAM_USER_IS_DEACTIVATED + ): + result += f"failed to send telegram message to {user_verbal} because user has been deactivated" else: # TODO: handle specific backend errors try: diff --git a/engine/apps/slack/scenarios/notification_delivery.py b/engine/apps/slack/scenarios/notification_delivery.py index 4b9805da..73ed7d05 100644 --- a/engine/apps/slack/scenarios/notification_delivery.py +++ b/engine/apps/slack/scenarios/notification_delivery.py @@ -87,12 +87,3 @@ class NotificationDeliveryStep(scenario_step.ScenarioStep): print(e) else: raise e - - def get_color_id(self, color): - if color == "red": - color_id = "#FF0000" - elif color == "yellow": - color_id = "#c6c000" - else: - color_id = color - return color_id diff --git a/engine/apps/telegram/models/connectors/personal.py b/engine/apps/telegram/models/connectors/personal.py index 895c6a62..8ac440ac 100644 --- a/engine/apps/telegram/models/connectors/personal.py +++ b/engine/apps/telegram/models/connectors/personal.py @@ -111,6 +111,13 @@ class TelegramToUserConnector(models.Model): notification_policy, UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_TELEGRAM_TOKEN_ERROR, ) + elif e.message == "Forbidden: user is deactivated": + TelegramToUserConnector.create_telegram_notification_error( + alert_group, + self.user, + notification_policy, + UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_TELEGRAM_USER_IS_DEACTIVATED, + ) else: raise e else: diff --git a/engine/apps/telegram/tasks.py b/engine/apps/telegram/tasks.py index 9f6cfd97..cd55aa27 100644 --- a/engine/apps/telegram/tasks.py +++ b/engine/apps/telegram/tasks.py @@ -119,7 +119,14 @@ def send_link_to_channel_message_or_fallback_to_full_incident( @ignore_bot_deleted def send_log_and_actions_message(self, channel_chat_id, group_chat_id, channel_message_id, reply_to_message_id): with OkToRetry(task=self, exc=TelegramMessage.DoesNotExist, num_retries=5): - channel_message = TelegramMessage.objects.get(chat_id=channel_chat_id, message_id=channel_message_id) + try: + channel_message = TelegramMessage.objects.get(chat_id=channel_chat_id, message_id=channel_message_id) + except TelegramMessage.DoesNotExist: + logger.warning( + f"Could not send log and actions message, telegram message does not exit " + f" chat_id={channel_chat_id} message_id={channel_message_id}" + ) + return if channel_message.discussion_group_message_id is None: channel_message.discussion_group_message_id = reply_to_message_id From 0ef2cbf77b440894b455218ce9dc669118d0e7d0 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Thu, 1 Sep 2022 11:41:52 -0600 Subject: [PATCH 2/3] Fix message typo --- engine/apps/telegram/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/apps/telegram/tasks.py b/engine/apps/telegram/tasks.py index cd55aa27..8d8c0e0b 100644 --- a/engine/apps/telegram/tasks.py +++ b/engine/apps/telegram/tasks.py @@ -123,7 +123,7 @@ def send_log_and_actions_message(self, channel_chat_id, group_chat_id, channel_m channel_message = TelegramMessage.objects.get(chat_id=channel_chat_id, message_id=channel_message_id) except TelegramMessage.DoesNotExist: logger.warning( - f"Could not send log and actions message, telegram message does not exit " + f"Could not send log and actions message, telegram message does not exist " f" chat_id={channel_chat_id} message_id={channel_message_id}" ) return From c3b054ae610651ad08cddbaddaf209cd10ee1045 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Fri, 2 Sep 2022 15:03:08 -0600 Subject: [PATCH 3/3] Retry on exception first 5 times --- engine/apps/telegram/tasks.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/engine/apps/telegram/tasks.py b/engine/apps/telegram/tasks.py index 8d8c0e0b..fff14616 100644 --- a/engine/apps/telegram/tasks.py +++ b/engine/apps/telegram/tasks.py @@ -122,11 +122,14 @@ def send_log_and_actions_message(self, channel_chat_id, group_chat_id, channel_m try: channel_message = TelegramMessage.objects.get(chat_id=channel_chat_id, message_id=channel_message_id) except TelegramMessage.DoesNotExist: - logger.warning( - f"Could not send log and actions message, telegram message does not exist " - f" chat_id={channel_chat_id} message_id={channel_message_id}" - ) - return + if self.request.retries <= 5: + raise + else: + logger.warning( + f"Could not send log and actions message, telegram message does not exist " + f" chat_id={channel_chat_id} message_id={channel_message_id}" + ) + return if channel_message.discussion_group_message_id is None: channel_message.discussion_group_message_id = reply_to_message_id