Make alert ingestion cache independent

Tiltfile

@@ -58,7 +58,7 @@ local_resource(
     allow_parallel=True,
 )
 
-yaml = helm("helm/oncall", name=HELM_PREFIX, values=["./dev/helm-local.yml"])
+yaml = helm("helm/oncall", name=HELM_PREFIX, values=["./dev/helm-local.yml", "./dev/helm-local.dev.yml"])
 
 k8s_yaml(yaml)
 

engine/apps/integrations/mixins/alert_channel_defining_mixin.py

@@ -5,6 +5,8 @@ from django.core import serializers
 from django.core.cache import cache
 from django.core.exceptions import PermissionDenied
 from django.db import OperationalError
+from django_redis.exceptions import ConnectionInterrupted as RedisConnectionInterrupted
+from redis.exceptions import ConnectionError as RedisConnectionError
 
 from apps.user_management.exceptions import OrganizationMovedException
 
@@ -33,21 +35,29 @@ class AlertChannelDefiningMixin(object):
         try:
             # Trying to define from short-term cache
             cache_key_short_term = self.CACHE_KEY_SHORT_TERM + "_" + str(kwargs["alert_channel_key"])
-            cached_alert_receive_channel_raw = cache.get(cache_key_short_term)
+            try:
+                cached_alert_receive_channel_raw = cache.get(cache_key_short_term)
+            except RedisConnectionError:
+                logger.error("Skip reading AlertReceiveChannel from cache as Redis is not available")
+                cached_alert_receive_channel_raw = None
+
             if cached_alert_receive_channel_raw is not None:
                 alert_receive_channel = next(serializers.deserialize("json", cached_alert_receive_channel_raw)).object
 
             if alert_receive_channel is None:
                 # Trying to define channel from DB
                 alert_receive_channel = AlertReceiveChannel.objects.get(token=kwargs["alert_channel_key"])
-                # Update short term cache
-                serialized = serializers.serialize("json", [alert_receive_channel])
-                cache.set(cache_key_short_term, serialized, self.CACHE_SHORT_TERM_TIMEOUT)
+                try:
+                    # Update short term cache
+                    serialized = serializers.serialize("json", [alert_receive_channel])
+                    cache.set(cache_key_short_term, serialized, self.CACHE_SHORT_TERM_TIMEOUT)
 
-                # Update cached channels
-                if cache.get(self.CACHE_DB_FALLBACK_OBSOLETE_KEY) is None:
-                    cache.set(self.CACHE_DB_FALLBACK_OBSOLETE_KEY, True, self.CACHE_DB_FALLBACK_REFRESH_INTERVAL)
-                    self.update_alert_receive_channel_cache()
+                    # Update cached channels
+                    if cache.get(self.CACHE_DB_FALLBACK_OBSOLETE_KEY) is None:
+                        cache.set(self.CACHE_DB_FALLBACK_OBSOLETE_KEY, True, self.CACHE_DB_FALLBACK_REFRESH_INTERVAL)
+                        self.update_alert_receive_channel_cache()
+                except (RedisConnectionError, RedisConnectionInterrupted):
+                    logger.error("Skip updating AlertReceiveChannel cache as Redis is not available")
 
         except AlertReceiveChannel.DoesNotExist:
             raise PermissionDenied("Integration key was not found. Permission denied.")

engine/apps/integrations/mixins/ratelimit_mixin.py

@@ -2,12 +2,14 @@ import logging
 from abc import ABC, abstractmethod
 from functools import wraps
 
+from django.conf import settings
 from django.core.cache import cache
 from django.http import HttpRequest, HttpResponse
 from django.views import View
 from ratelimit import ALL
 from ratelimit.exceptions import Ratelimited
 from ratelimit.utils import is_ratelimited
+from redis.exceptions import ConnectionError as RedisConnectionError
 
 from apps.integrations.tasks import start_notify_about_integration_ratelimit
 
@@ -54,9 +56,16 @@ def ratelimit(group=None, key=None, rate=None, method=ALL, block=False, reason=N
             request.limited = getattr(request, "limited", False)
             was_limited_before = request.limited
 
-            ratelimited = is_ratelimited(
-                request=request, group=group, fn=fn, key=key, rate=rate, method=method, increment=True
-            )
+            # Allow requests when redis cache backend fails and RATELIMIT_FAIL_OPEN setting is true
+            try:
+                ratelimited = is_ratelimited(
+                    request=request, group=group, fn=fn, key=key, rate=rate, method=method, increment=True
+                )
+            except RedisConnectionError as e:
+                if settings.RATELIMIT_FAIL_OPEN:
+                    ratelimited = False
+                else:
+                    raise e
 
             # We need to know if it's the first ratelimited request for notification purposes.
             request.is_first_rate_limited_request = getattr(request, "is_first_rate_limited_request", False)

engine/engine/views.py

@@ -1,10 +1,15 @@
+import logging
+
 from django.conf import settings
 from django.core.cache import cache
 from django.http import HttpResponse, JsonResponse
 from django.views.generic import View
+from redis.exceptions import ConnectionError as RedisConnectionError
 
 from apps.integrations.mixins import AlertChannelDefiningMixin
 
+logger = logging.getLogger(__name__)
+
 
 class HealthCheckView(View):
     """
@@ -43,11 +48,12 @@ class StartupProbeView(View):
     dangerously_bypass_middlewares = True
 
     def get(self, request):
-        if cache.get(AlertChannelDefiningMixin.CACHE_KEY_DB_FALLBACK) is None:
-            AlertChannelDefiningMixin().update_alert_receive_channel_cache()
+        try:
+            if cache.get(AlertChannelDefiningMixin.CACHE_KEY_DB_FALLBACK) is None:
+                AlertChannelDefiningMixin().update_alert_receive_channel_cache()
 
-        cache.set("healthcheck", "healthcheck", 30)  # Checking cache connectivity
-        assert cache.get("healthcheck") == "healthcheck"
+        except RedisConnectionError:
+            logger.error("Skip updating AlertReceiveChannel cache as Redis is not available")
 
         return HttpResponse("Ok")
 

engine/settings/base.py

@@ -838,3 +838,5 @@ ZVONOK_POSTBACK_USER_CHOICE = os.getenv("ZVONOK_POSTBACK_USER_CHOICE", None)
 ZVONOK_POSTBACK_USER_CHOICE_ACK = os.getenv("ZVONOK_POSTBACK_USER_CHOICE_ACK", None)
 
 DETACHED_INTEGRATIONS_SERVER = getenv_boolean("DETACHED_INTEGRATIONS_SERVER", default=False)
+
+RATELIMIT_FAIL_OPEN = getenv_boolean("RATELIMIT_FAIL_OPEN", default=True)