Handle a possible outdated cached integration error (#3741)

Related to
[logs](https://ops.grafana-ops.net/explore?schemaVersion=1&panes=%7B%22hum%22:%7B%22datasource%22:%22c-R8UWvVk%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bnamespace%3D%5C%22amixr-prod%5C%22,%20job%3D%5C%22amixr-prod%2Famixr-integrations%5C%22%7D%20%7C%3D%20%5C%22django.core.serializers.base.DeserializationError%5C%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22c-R8UWvVk%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%221706023840486%22,%22to%22:%221706024722486%22%7D%7D%7D&orgId=1)

engine/apps/integrations/mixins/alert_channel_defining_mixin.py

@@ -35,7 +35,13 @@ class AlertChannelDefiningMixin(object):
             cache_key_short_term = self.CACHE_KEY_SHORT_TERM + "_" + str(kwargs["alert_channel_key"])
             cached_alert_receive_channel_raw = cache.get(cache_key_short_term)
             if cached_alert_receive_channel_raw is not None:
-                alert_receive_channel = next(serializers.deserialize("json", cached_alert_receive_channel_raw)).object
+                try:
+                    alert_receive_channel = next(
+                        serializers.deserialize("json", cached_alert_receive_channel_raw)
+                    ).object
+                except serializers.base.DeserializationError:
+                    # cached object model is outdated
+                    alert_receive_channel = None
 
             if alert_receive_channel is None:
                 # Trying to define channel from DB

engine/apps/integrations/tests/test_views.py

@@ -1,6 +1,7 @@
 from unittest.mock import call, patch
 
 import pytest
+from django.core.cache import cache
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db import OperationalError
 from django.urls import reverse
@@ -459,3 +460,58 @@ def test_integration_grafana_endpoint_without_cache_has_alerts(
             call((alert_receive_channel.pk, data["alerts"][1]), kwargs={"received_at": now.isoformat()}),
         ]
     )
+
+
+@patch("apps.integrations.views.create_alert")
+@pytest.mark.parametrize(
+    "integration_type",
+    [
+        arc_type
+        for arc_type in AlertReceiveChannel.INTEGRATION_TYPES
+        if arc_type not in ["amazon_sns", "grafana", "alertmanager", "grafana_alerting", "maintenance"]
+    ],
+)
+@pytest.mark.django_db
+def test_integration_outdated_cached_model(
+    mock_create_alert,
+    make_organization_and_user,
+    make_alert_receive_channel,
+    integration_type,
+):
+    organization, user = make_organization_and_user()
+    alert_receive_channel = make_alert_receive_channel(
+        organization=organization,
+        author=user,
+        integration=integration_type,
+    )
+
+    # set an invalid cache value for the requested integration
+    cache_key = AlertChannelDefiningMixin.CACHE_KEY_SHORT_TERM + "_" + alert_receive_channel.token
+    cache.set(cache_key, '{"some": "invalid json model"}')
+
+    client = APIClient()
+    url = reverse(
+        "integrations:universal",
+        kwargs={"integration_type": integration_type, "alert_channel_key": alert_receive_channel.token},
+    )
+    data = {"foo": "bar"}
+    now = timezone.now()
+    with patch("django.utils.timezone.now") as mock_now:
+        mock_now.return_value = now
+        response = client.post(url, data, format="json")
+
+    assert response.status_code == status.HTTP_200_OK
+
+    mock_create_alert.apply_async.assert_called_once_with(
+        [],
+        {
+            "title": None,
+            "message": None,
+            "image_url": None,
+            "link_to_upstream_details": None,
+            "alert_receive_channel_pk": alert_receive_channel.pk,
+            "integration_unique_data": None,
+            "raw_request_data": data,
+            "received_at": now.isoformat(),
+        },
+    )