Handle ag telegram message does not exist (#3830)

# What this PR does
Restart `send_link_to_channel_message_or_fallback_to_full_alert_group`
task without retry on the first run if alert group telegram message
doesn't exist. On the second run raise Exception

## Which issue(s) this PR fixes
https://github.com/grafana/oncall-private/issues/2492
## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not
required)

engine/apps/telegram/client.py

@@ -8,6 +8,7 @@ from telegram.utils.request import Request
 
 from apps.alerts.models import AlertGroup
 from apps.base.utils import live_settings
+from apps.telegram.exceptions import AlertGroupTelegramMessageDoesNotExist
 from apps.telegram.models import TelegramMessage
 from apps.telegram.renderers.keyboard import TelegramKeyboardRenderer
 from apps.telegram.renderers.message import TelegramMessageRenderer
@@ -157,7 +158,10 @@ class TelegramClient:
             ).first()
 
             if alert_group_message is None:
-                raise Exception("No alert group message found, probably it is not saved to database yet")
+                raise AlertGroupTelegramMessageDoesNotExist(
+                    f"No alert group message found, probably it is not saved to database yet, "
+                    f"alert group: {alert_group.id}"
+                )
 
             include_title = message_type == TelegramMessage.LINK_TO_CHANNEL_MESSAGE
             link = alert_group_message.link

engine/apps/telegram/exceptions.py

@@ -0,0 +1,2 @@
+class AlertGroupTelegramMessageDoesNotExist(Exception):
+    pass

engine/apps/telegram/models/connectors/personal.py

@@ -43,10 +43,15 @@ class TelegramToUserConnector(models.Model):
         telegram_channel = TelegramToOrganizationConnector.get_channel_for_alert_group(alert_group)
 
         if telegram_channel is not None:
-            send_link_to_channel_message_or_fallback_to_full_alert_group.delay(
-                alert_group_pk=alert_group.pk,
-                notification_policy_pk=notification_policy.pk,
-                user_connector_pk=self.pk,
+            # Call this task with a countdown to avoid unnecessary retry when alert group telegram message hasn't been
+            # created yet
+            send_link_to_channel_message_or_fallback_to_full_alert_group.apply_async(
+                kwargs={
+                    "alert_group_pk": alert_group.pk,
+                    "notification_policy_pk": notification_policy.pk,
+                    "user_connector_pk": self.pk,
+                },
+                countdown=3,
             )
         else:
             self.send_full_alert_group(alert_group=alert_group, notification_policy=notification_policy)