From c87d3018b9135bc3b6b066d1dd8cfc8d2918c75a Mon Sep 17 00:00:00 2001
From: Yulya Artyukhina <Ferril.darkdiver@gmail.com>
Date: Mon, 26 Aug 2024 12:27:07 +0200
Subject: [PATCH] Handle TimeoutError on sending message to Slack (#4916)

# What this PR does

## Which issue(s) this PR closes

Related to https://github.com/grafana/oncall-private/issues/2758

<!--
*Note*: If you want the issue to be auto-closed once the PR is merged,
change "Related to" to "Closes" in the line above.
If you have more than one GitHub issue that this PR closes, be sure to
preface
each issue link with a [closing
keyword](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/using-keywords-in-issues-and-pull-requests#linking-a-pull-request-to-an-issue).
This ensures that the issue(s) are auto-closed once the PR has been
merged.
-->

## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] Added the relevant release notes label (see labels prefixed w/
`release:`). These labels dictate how your PR will
    show up in the autogenerated release notes.
---
 .../apps/slack/scenarios/distribute_alerts.py |  2 +-
 .../test_distribute_alerts.py                 | 32 +++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/engine/apps/slack/scenarios/distribute_alerts.py b/engine/apps/slack/scenarios/distribute_alerts.py
index d25fd193..52ded932 100644
--- a/engine/apps/slack/scenarios/distribute_alerts.py
+++ b/engine/apps/slack/scenarios/distribute_alerts.py
@@ -73,7 +73,7 @@ class AlertShootingStep(scenario_step.ScenarioStep):
                     else alert.group.channel.organization.general_log_channel_id
                 )
                 self._send_first_alert(alert, channel_id)
-            except SlackAPIError:
+            except (SlackAPIError, TimeoutError):
                 AlertGroup.objects.filter(pk=alert.group.pk).update(slack_message_sent=False)
                 raise
 
diff --git a/engine/apps/slack/tests/test_scenario_steps/test_distribute_alerts.py b/engine/apps/slack/tests/test_scenario_steps/test_distribute_alerts.py
index 1eba2b9a..7d73febe 100644
--- a/engine/apps/slack/tests/test_scenario_steps/test_distribute_alerts.py
+++ b/engine/apps/slack/tests/test_scenario_steps/test_distribute_alerts.py
@@ -39,6 +39,38 @@ def test_restricted_action_error(
     assert not alert.delivered
 
 
+@pytest.mark.django_db
+def test_timeout_error(
+    make_slack_team_identity,
+    make_organization,
+    make_alert_receive_channel,
+    make_alert_group,
+    make_alert,
+):
+    SlackAlertShootingStep = ScenarioStep.get_step("distribute_alerts", "AlertShootingStep")
+    slack_team_identity = make_slack_team_identity()
+    organization = make_organization(
+        slack_team_identity=slack_team_identity, general_log_channel_id="DEFAULT_CHANNEL_ID"
+    )
+    alert_receive_channel = make_alert_receive_channel(organization)
+    alert_group = make_alert_group(alert_receive_channel)
+    alert = make_alert(alert_group, raw_request_data="{}")
+
+    step = SlackAlertShootingStep(slack_team_identity)
+
+    with pytest.raises(TimeoutError):
+        with patch.object(step._slack_client, "api_call") as mock_slack_api_call:
+            mock_slack_api_call.side_effect = TimeoutError
+            step.process_signal(alert)
+
+    alert_group.refresh_from_db()
+    alert.refresh_from_db()
+    assert alert_group.slack_message is None
+    assert alert_group.slack_message_sent is False
+    assert SlackMessage.objects.count() == 0
+    assert not alert.delivered
+
+
 @patch.object(AlertShootingStep, "_post_alert_group_to_slack")
 @pytest.mark.django_db
 def test_alert_shooting_no_channel_filter(