Handle Slack invalid_auth error when posting alert group notification (#4970)

Also, make telegram error check more flexible (case insensitive, e.g. we
got some of these recently: `telegram.error.Unauthorized: Forbidden: bot
was blocked by the user`)

engine/apps/alerts/migrations/0058_alter_alertgroup_reason_to_skip_escalation.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.15 on 2024-09-02 13:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('alerts', '0057_remove_alertgroup_slack_log_message_db'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='alertgroup',
+            name='reason_to_skip_escalation',
+            field=models.IntegerField(choices=[(0, 'account_inactive'), (1, 'is_archived'), (2, 'no_reason'), (3, 'rate_limited'), (4, 'channel_not_specified'), (5, 'restricted_action'), (6, 'invalid_auth')], default=2),
+        ),
+    ]

engine/apps/alerts/models/alert_group.py

@@ -365,14 +365,23 @@ class AlertGroup(AlertGroupSlackRenderingMixin, EscalationSnapshotMixin, models.
         else:
             return AlertGroup.NEW
 
-    ACCOUNT_INACTIVE, CHANNEL_ARCHIVED, NO_REASON, RATE_LIMITED, CHANNEL_NOT_SPECIFIED, RESTRICTED_ACTION = range(6)
+    (
+        ACCOUNT_INACTIVE,
+        CHANNEL_ARCHIVED,
+        NO_REASON,
+        RATE_LIMITED,
+        CHANNEL_NOT_SPECIFIED,
+        RESTRICTED_ACTION,
+        INVALID_AUTH,
+    ) = range(7)
     REASONS_TO_SKIP_ESCALATIONS = (
         (ACCOUNT_INACTIVE, "account_inactive"),
-        (CHANNEL_ARCHIVED, "channel_archived"),
+        (CHANNEL_ARCHIVED, "is_archived"),
         (NO_REASON, "no_reason"),
         (RATE_LIMITED, "rate_limited"),
         (CHANNEL_NOT_SPECIFIED, "channel_not_specified"),
         (RESTRICTED_ACTION, "restricted_action"),
+        (INVALID_AUTH, "invalid_auth"),
     )
     reason_to_skip_escalation = models.IntegerField(choices=REASONS_TO_SKIP_ESCALATIONS, default=NO_REASON)
 

engine/apps/slack/scenarios/distribute_alerts.py

@@ -16,6 +16,7 @@ from apps.slack.errors import (
     SlackAPIChannelArchivedError,
     SlackAPIChannelNotFoundError,
     SlackAPIError,
+    SlackAPIInvalidAuthError,
     SlackAPIMessageNotFoundError,
     SlackAPIRatelimitError,
     SlackAPIRestrictedActionError,
@@ -161,6 +162,10 @@ class AlertShootingStep(scenario_step.ScenarioStep):
             alert_group.reason_to_skip_escalation = AlertGroup.ACCOUNT_INACTIVE
             alert_group.save(update_fields=["reason_to_skip_escalation"])
             logger.info("Not delivering alert due to account_inactive.")
+        except SlackAPIInvalidAuthError:
+            alert_group.reason_to_skip_escalation = AlertGroup.INVALID_AUTH
+            alert_group.save(update_fields=["reason_to_skip_escalation"])
+            logger.info("Not delivering alert due to invalid_auth.")
         except SlackAPIChannelArchivedError:
             alert_group.reason_to_skip_escalation = AlertGroup.CHANNEL_ARCHIVED
             alert_group.save(update_fields=["reason_to_skip_escalation"])

engine/apps/slack/tests/test_scenario_steps/test_distribute_alerts.py

@@ -3,7 +3,7 @@ from unittest.mock import patch
 import pytest
 
 from apps.alerts.models import AlertGroup
-from apps.slack.errors import SlackAPIRestrictedActionError
+from apps.slack.errors import get_error_class
 from apps.slack.models import SlackMessage
 from apps.slack.scenarios.distribute_alerts import AlertShootingStep
 from apps.slack.scenarios.scenario_step import ScenarioStep
@@ -11,11 +11,21 @@ from apps.slack.tests.conftest import build_slack_response
 
 
 @pytest.mark.django_db
-def test_restricted_action_error(
+@pytest.mark.parametrize(
+    "reason,slack_error",
+    [
+        (reason, slack_error)
+        for reason, slack_error in AlertGroup.REASONS_TO_SKIP_ESCALATIONS
+        if reason != AlertGroup.NO_REASON
+    ],
+)
+def test_skip_escalations_error(
     make_organization_and_user_with_slack_identities,
     make_alert_receive_channel,
     make_alert_group,
     make_alert,
+    reason,
+    slack_error,
 ):
     SlackAlertShootingStep = ScenarioStep.get_step("distribute_alerts", "AlertShootingStep")
     organization, _, slack_team_identity, _ = make_organization_and_user_with_slack_identities()
@@ -26,14 +36,17 @@ def test_restricted_action_error(
     step = SlackAlertShootingStep(slack_team_identity)
 
     with patch.object(step._slack_client, "api_call") as mock_slack_api_call:
-        mock_slack_api_call.side_effect = SlackAPIRestrictedActionError(
-            response=build_slack_response({"error": "restricted_action"})
-        )
-        step._post_alert_group_to_slack(slack_team_identity, alert_group, alert, None, "channel-id", [])
+        error_response = build_slack_response({"error": slack_error})
+        error_class = get_error_class(error_response)
+        mock_slack_api_call.side_effect = error_class(error_response)
+        channel_id = "channel-id"
+        if reason == AlertGroup.CHANNEL_NOT_SPECIFIED:
+            channel_id = None
+        step._post_alert_group_to_slack(slack_team_identity, alert_group, alert, None, channel_id, [])
 
     alert_group.refresh_from_db()
     alert.refresh_from_db()
-    assert alert_group.reason_to_skip_escalation == AlertGroup.RESTRICTED_ACTION
+    assert alert_group.reason_to_skip_escalation == reason
     assert alert_group.slack_message is None
     assert SlackMessage.objects.count() == 0
     assert not alert.delivered

engine/apps/telegram/client.py

@@ -186,4 +186,4 @@ class TelegramClient:
 
     @staticmethod
     def error_message_is(error: TelegramError, messages: list[str]) -> bool:
-        return error.message in messages
+        return error.message.lower() in (m.lower() for m in messages)