From 0d5ef785bfd14061043819e1be7b736e11277108 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Thu, 23 Nov 2023 11:27:47 +0800 Subject: [PATCH] Make alert ingestion cache independent --- Tiltfile | 2 +- .../mixins/alert_channel_defining_mixin.py | 26 +++++++++++++------ .../integrations/mixins/ratelimit_mixin.py | 15 ++++++++--- engine/engine/views.py | 14 +++++++--- engine/settings/base.py | 2 ++ 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/Tiltfile b/Tiltfile index 5a293c92..eb71dad6 100644 --- a/Tiltfile +++ b/Tiltfile @@ -58,7 +58,7 @@ local_resource( allow_parallel=True, ) -yaml = helm("helm/oncall", name=HELM_PREFIX, values=["./dev/helm-local.yml"]) +yaml = helm("helm/oncall", name=HELM_PREFIX, values=["./dev/helm-local.yml", "./dev/helm-local.dev.yml"]) k8s_yaml(yaml) diff --git a/engine/apps/integrations/mixins/alert_channel_defining_mixin.py b/engine/apps/integrations/mixins/alert_channel_defining_mixin.py index c621412f..15d24dbc 100644 --- a/engine/apps/integrations/mixins/alert_channel_defining_mixin.py +++ b/engine/apps/integrations/mixins/alert_channel_defining_mixin.py @@ -5,6 +5,8 @@ from django.core import serializers from django.core.cache import cache from django.core.exceptions import PermissionDenied from django.db import OperationalError +from django_redis.exceptions import ConnectionInterrupted as RedisConnectionInterrupted +from redis.exceptions import ConnectionError as RedisConnectionError from apps.user_management.exceptions import OrganizationMovedException @@ -33,21 +35,29 @@ class AlertChannelDefiningMixin(object): try: # Trying to define from short-term cache cache_key_short_term = self.CACHE_KEY_SHORT_TERM + "_" + str(kwargs["alert_channel_key"]) - cached_alert_receive_channel_raw = cache.get(cache_key_short_term) + try: + cached_alert_receive_channel_raw = cache.get(cache_key_short_term) + except RedisConnectionError: + logger.error("Skip reading AlertReceiveChannel from cache as Redis is not available") + cached_alert_receive_channel_raw = None + if cached_alert_receive_channel_raw is not None: alert_receive_channel = next(serializers.deserialize("json", cached_alert_receive_channel_raw)).object if alert_receive_channel is None: # Trying to define channel from DB alert_receive_channel = AlertReceiveChannel.objects.get(token=kwargs["alert_channel_key"]) - # Update short term cache - serialized = serializers.serialize("json", [alert_receive_channel]) - cache.set(cache_key_short_term, serialized, self.CACHE_SHORT_TERM_TIMEOUT) + try: + # Update short term cache + serialized = serializers.serialize("json", [alert_receive_channel]) + cache.set(cache_key_short_term, serialized, self.CACHE_SHORT_TERM_TIMEOUT) - # Update cached channels - if cache.get(self.CACHE_DB_FALLBACK_OBSOLETE_KEY) is None: - cache.set(self.CACHE_DB_FALLBACK_OBSOLETE_KEY, True, self.CACHE_DB_FALLBACK_REFRESH_INTERVAL) - self.update_alert_receive_channel_cache() + # Update cached channels + if cache.get(self.CACHE_DB_FALLBACK_OBSOLETE_KEY) is None: + cache.set(self.CACHE_DB_FALLBACK_OBSOLETE_KEY, True, self.CACHE_DB_FALLBACK_REFRESH_INTERVAL) + self.update_alert_receive_channel_cache() + except (RedisConnectionError, RedisConnectionInterrupted): + logger.error("Skip updating AlertReceiveChannel cache as Redis is not available") except AlertReceiveChannel.DoesNotExist: raise PermissionDenied("Integration key was not found. Permission denied.") diff --git a/engine/apps/integrations/mixins/ratelimit_mixin.py b/engine/apps/integrations/mixins/ratelimit_mixin.py index 808b4276..58bda952 100644 --- a/engine/apps/integrations/mixins/ratelimit_mixin.py +++ b/engine/apps/integrations/mixins/ratelimit_mixin.py @@ -2,12 +2,14 @@ import logging from abc import ABC, abstractmethod from functools import wraps +from django.conf import settings from django.core.cache import cache from django.http import HttpRequest, HttpResponse from django.views import View from ratelimit import ALL from ratelimit.exceptions import Ratelimited from ratelimit.utils import is_ratelimited +from redis.exceptions import ConnectionError as RedisConnectionError from apps.integrations.tasks import start_notify_about_integration_ratelimit @@ -54,9 +56,16 @@ def ratelimit(group=None, key=None, rate=None, method=ALL, block=False, reason=N request.limited = getattr(request, "limited", False) was_limited_before = request.limited - ratelimited = is_ratelimited( - request=request, group=group, fn=fn, key=key, rate=rate, method=method, increment=True - ) + # Allow requests when redis cache backend fails and RATELIMIT_FAIL_OPEN setting is true + try: + ratelimited = is_ratelimited( + request=request, group=group, fn=fn, key=key, rate=rate, method=method, increment=True + ) + except RedisConnectionError as e: + if settings.RATELIMIT_FAIL_OPEN: + ratelimited = False + else: + raise e # We need to know if it's the first ratelimited request for notification purposes. request.is_first_rate_limited_request = getattr(request, "is_first_rate_limited_request", False) diff --git a/engine/engine/views.py b/engine/engine/views.py index 7f3e2f5d..dca68143 100644 --- a/engine/engine/views.py +++ b/engine/engine/views.py @@ -1,10 +1,15 @@ +import logging + from django.conf import settings from django.core.cache import cache from django.http import HttpResponse, JsonResponse from django.views.generic import View +from redis.exceptions import ConnectionError as RedisConnectionError from apps.integrations.mixins import AlertChannelDefiningMixin +logger = logging.getLogger(__name__) + class HealthCheckView(View): """ @@ -43,11 +48,12 @@ class StartupProbeView(View): dangerously_bypass_middlewares = True def get(self, request): - if cache.get(AlertChannelDefiningMixin.CACHE_KEY_DB_FALLBACK) is None: - AlertChannelDefiningMixin().update_alert_receive_channel_cache() + try: + if cache.get(AlertChannelDefiningMixin.CACHE_KEY_DB_FALLBACK) is None: + AlertChannelDefiningMixin().update_alert_receive_channel_cache() - cache.set("healthcheck", "healthcheck", 30) # Checking cache connectivity - assert cache.get("healthcheck") == "healthcheck" + except RedisConnectionError: + logger.error("Skip updating AlertReceiveChannel cache as Redis is not available") return HttpResponse("Ok") diff --git a/engine/settings/base.py b/engine/settings/base.py index 50eee03d..17cba72b 100644 --- a/engine/settings/base.py +++ b/engine/settings/base.py @@ -838,3 +838,5 @@ ZVONOK_POSTBACK_USER_CHOICE = os.getenv("ZVONOK_POSTBACK_USER_CHOICE", None) ZVONOK_POSTBACK_USER_CHOICE_ACK = os.getenv("ZVONOK_POSTBACK_USER_CHOICE_ACK", None) DETACHED_INTEGRATIONS_SERVER = getenv_boolean("DETACHED_INTEGRATIONS_SERVER", default=False) + +RATELIMIT_FAIL_OPEN = getenv_boolean("RATELIMIT_FAIL_OPEN", default=True)