Fix task retries for deleted alert groups (#3553)

# What this PR does

## Which issue(s) this PR fixes

## Checklist

- [ ] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not
required)

engine/apps/alerts/tasks/send_update_log_report_signal.py

@@ -7,7 +7,7 @@ from .task_logger import task_logger
 
 
 @shared_dedicated_queue_retry_task(
-    autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else None
+    autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else 10
 )
 def send_update_log_report_signal(log_record_pk=None, alert_group_pk=None):
     from apps.alerts.models import AlertGroup, AlertReceiveChannel

engine/apps/telegram/tasks.py

@@ -184,8 +184,14 @@ def on_create_alert_telegram_representative_async(self, alert_pk):
     """
     It's async in order to prevent Telegram downtime or formatting issues causing delay with SMS and other destinations.
     """
-
-    alert = Alert.objects.get(pk=alert_pk)
+    try:
+        alert = Alert.objects.get(pk=alert_pk)
+    except Alert.DoesNotExist as e:
+        if on_create_alert_telegram_representative_async.request.retries >= 10:
+            logger.error(f"Alert {alert_pk} was not found. Probably it was deleted. Stop retrying")
+            return
+        else:
+            raise e
     alert_group = alert.group
 
     alert_group_messages = alert_group.telegram_messages.filter(