centralcloud/oncall-engine
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Revert "Make alert ingestion cache independent (#3414)" (#3430)

Browse source 
This reverts commit acfba47a81.

# What this PR does

## Which issue(s) this PR fixes

## Checklist

- [ ] Unit, integration, and e2e (if applicable) tests updated
- [ ] Documentation added (or `pr:no public docs` PR label added if not
required)
- [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not
required)
This commit is contained in:
Ildar Iskhakov 2023-11-27 20:23:57 +08:00 • committed by GitHub
parent aab49f0594
commit 2ddb47ea29
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 15 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
engine/apps/integrations/mixins/alert_channel_defining_mixin.py
View file

@ -5,8 +5,6 @@ from django.core import serializers
from django.core.cache import cache
from django.core.exceptions import PermissionDenied
from django.db import OperationalError
from django_redis.exceptions import ConnectionInterrupted as RedisConnectionInterrupted # type: ignore
from redis.exceptions import ConnectionError as RedisConnectionError
from apps.user_management.exceptions import OrganizationMovedException
@ -35,29 +33,21 @@ class AlertChannelDefiningMixin(object):
try:
# Trying to define from short-term cache
cache_key_short_term = self.CACHE_KEY_SHORT_TERM + "_" + str(kwargs["alert_channel_key"])
try:
cached_alert_receive_channel_raw = cache.get(cache_key_short_term)
except RedisConnectionError:
logger.error("Skip reading AlertReceiveChannel from cache as Redis is not available")
cached_alert_receive_channel_raw = None
cached_alert_receive_channel_raw = cache.get(cache_key_short_term)
if cached_alert_receive_channel_raw is not None:
alert_receive_channel = next(serializers.deserialize("json", cached_alert_receive_channel_raw)).object
if alert_receive_channel is None:
# Trying to define channel from DB
alert_receive_channel = AlertReceiveChannel.objects.get(token=kwargs["alert_channel_key"])
try:
# Update short term cache
serialized = serializers.serialize("json", [alert_receive_channel])
cache.set(cache_key_short_term, serialized, self.CACHE_SHORT_TERM_TIMEOUT)
# Update short term cache
serialized = serializers.serialize("json", [alert_receive_channel])
cache.set(cache_key_short_term, serialized, self.CACHE_SHORT_TERM_TIMEOUT)
# Update cached channels
if cache.get(self.CACHE_DB_FALLBACK_OBSOLETE_KEY) is None:
cache.set(self.CACHE_DB_FALLBACK_OBSOLETE_KEY, True, self.CACHE_DB_FALLBACK_REFRESH_INTERVAL)
self.update_alert_receive_channel_cache()
except (RedisConnectionError, RedisConnectionInterrupted):
logger.error("Skip updating AlertReceiveChannel cache as Redis is not available")
# Update cached channels
if cache.get(self.CACHE_DB_FALLBACK_OBSOLETE_KEY) is None:
cache.set(self.CACHE_DB_FALLBACK_OBSOLETE_KEY, True, self.CACHE_DB_FALLBACK_REFRESH_INTERVAL)
self.update_alert_receive_channel_cache()
except AlertReceiveChannel.DoesNotExist:
raise PermissionDenied("Integration key was not found. Permission denied.")

15
engine/apps/integrations/mixins/ratelimit_mixin.py
View file

@ -2,14 +2,12 @@ import logging
from abc import ABC, abstractmethod
from functools import wraps
from django.conf import settings
from django.core.cache import cache
from django.http import HttpRequest, HttpResponse
from django.views import View
from ratelimit import ALL
from ratelimit.exceptions import Ratelimited
from ratelimit.utils import is_ratelimited
from redis.exceptions import ConnectionError as RedisConnectionError
from apps.integrations.tasks import start_notify_about_integration_ratelimit
@ -56,16 +54,9 @@ def ratelimit(group=None, key=None, rate=None, method=ALL, block=False, reason=N
request.limited = getattr(request, "limited", False)
was_limited_before = request.limited
# Allow requests when redis cache backend fails and RATELIMIT_FAIL_OPEN setting is true
try:
ratelimited = is_ratelimited(
request=request, group=group, fn=fn, key=key, rate=rate, method=method, increment=True
)
except RedisConnectionError as e:
if settings.RATELIMIT_FAIL_OPEN:
ratelimited = False
else:
raise e
ratelimited = is_ratelimited(
request=request, group=group, fn=fn, key=key, rate=rate, method=method, increment=True
)
# We need to know if it's the first ratelimited request for notification purposes.
request.is_first_rate_limited_request = getattr(request, "is_first_rate_limited_request", False)

14
engine/engine/views.py
View file

@ -1,15 +1,10 @@
import logging
from django.conf import settings
from django.core.cache import cache
from django.http import HttpResponse, JsonResponse
from django.views.generic import View
from redis.exceptions import ConnectionError as RedisConnectionError
from apps.integrations.mixins import AlertChannelDefiningMixin
logger = logging.getLogger(__name__)
class HealthCheckView(View):
"""
@ -48,12 +43,11 @@ class StartupProbeView(View):
dangerously_bypass_middlewares = True
def get(self, request):
try:
if cache.get(AlertChannelDefiningMixin.CACHE_KEY_DB_FALLBACK) is None:
AlertChannelDefiningMixin().update_alert_receive_channel_cache()
if cache.get(AlertChannelDefiningMixin.CACHE_KEY_DB_FALLBACK) is None:
AlertChannelDefiningMixin().update_alert_receive_channel_cache()
except RedisConnectionError:
logger.error("Skip updating AlertReceiveChannel cache as Redis is not available")
cache.set("healthcheck", "healthcheck", 30) # Checking cache connectivity
assert cache.get("healthcheck") == "healthcheck"
return HttpResponse("Ok")

2
engine/settings/base.py
View file

@ -838,5 +838,3 @@ ZVONOK_POSTBACK_USER_CHOICE = os.getenv("ZVONOK_POSTBACK_USER_CHOICE", None)
ZVONOK_POSTBACK_USER_CHOICE_ACK = os.getenv("ZVONOK_POSTBACK_USER_CHOICE_ACK", None)
DETACHED_INTEGRATIONS_SERVER = getenv_boolean("DETACHED_INTEGRATIONS_SERVER", default=False)
RATELIMIT_FAIL_OPEN = getenv_boolean("RATELIMIT_FAIL_OPEN", default=True)