Merge pull request #620 from grafana/dev

Merge dev to main

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
         run: |
           docker run -v ${PWD}/docs/sources:/hugo/content/docs/oncall/latest -e HUGO_REFLINKSERRORLEVEL=ERROR --rm grafana/docs-base:latest /bin/bash -c 'make hugo'
 
-  unit-test-backend:
+  unit-test-backend-mysql-rabbitmq:
     runs-on: ubuntu-latest
     container: python:3.9
     env:
@@ -66,11 +66,11 @@ jobs:
           pip install -r requirements.txt
           ./wait_for_test_mysql_start.sh && pytest --ds=settings.ci-test -x
 
-  unit-test-backend-postgresql:
+  unit-test-backend-postgresql-rabbitmq:
     runs-on: ubuntu-latest
     container: python:3.9
     env:
-      DB_BACKEND: postgresql
+      DATABASE_TYPE: postgresql
       DJANGO_SETTINGS_MODULE: settings.ci-test
       SLACK_CLIENT_OAUTH_ID: 1
     services:
@@ -98,3 +98,29 @@ jobs:
           pip install -r requirements.txt
           pytest --ds=settings.ci-test -x
 
+  unit-test-backend-sqlite-redis:
+    runs-on: ubuntu-latest
+    container: python:3.9
+    env:
+      DATABASE_TYPE: sqlite3
+      BROKER_TYPE: redis
+      REDIS_URI: redis://redis_test:6379
+      DJANGO_SETTINGS_MODULE: settings.ci-test
+      SLACK_CLIENT_OAUTH_ID: 1
+    services:
+      redis_test:
+        image: redis:7.0.5
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Unit Test Backend
+        run: |
+          apt-get update && apt-get install -y netcat
+          cd engine/
+          pip install -r requirements.txt
+          pytest --ds=settings.ci-test -x

.gitignore

@@ -1,5 +1,6 @@
 # Backend
 */db.sqlite3
+engine/oncall_dev.db
 *.pyc
 venv
 .python-version

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change Log
 
+## v1.0.40 (2022-10-05)
+- Improved database and celery backends support
+- Added script to import PagerDuty users to Grafana
+- Bug fixes
+
 ## v1.0.39 (2022-10-03)
 
 - Fix issue in v1.0.38 blocking the creation of schedules and webhooks in the UI

DEVELOPER.md

@@ -59,8 +59,8 @@ pip install -U pip wheel
 # Copy and check .env.dev file.
 cp .env.dev.example .env.dev
 
-# NOTE: if you want to use the PostgreSQL db backend add DB_BACKEND=postgresql to your .env.dev file;
-#       currently allowed backend values are `mysql` (default) and `postgresql`
+# NOTE: if you want to use the PostgreSQL db backend add DATABASE_TYPE=postgresql to your .env.dev file;
+#       currently allowed backend values are `mysql` (default), `postgresql` and `sqlite3`
 
 # Apply .env.dev to current terminal.
 # For PyCharm it's better to use https://plugins.jetbrains.com/plugin/7861-envfile/

engine/Dockerfile

@@ -9,8 +9,11 @@ RUN pip install -r requirements.txt
 
 COPY ./ ./
 
-RUN DJANGO_SETTINGS_MODULE=settings.prod_without_db SECRET_KEY="ThEmUsTSecretKEYforBUILDstage123" TELEGRAM_TOKEN="0000000000:XXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXX" SLACK_CLIENT_OAUTH_ID=1 python manage.py collectstatic --no-input
-RUN rm db.sqlite3
+# Collect static files and create an SQLite database
+RUN mkdir -p /var/lib/oncall
+RUN DJANGO_SETTINGS_MODULE=settings.prod_without_db DATABASE_TYPE=sqlite3 DATABASE_NAME=/var/lib/oncall/oncall.db SECRET_KEY="ThEmUsTSecretKEYforBUILDstage123" python manage.py collectstatic --no-input
+RUN chown -R 1000:2000 /var/lib/oncall
+
 
 # This is required for prometheus_client to sync between uwsgi workers
 RUN mkdir -p /tmp/prometheus_django_metrics;

engine/apps/api/views/features.py

@@ -27,6 +27,7 @@ class FeaturesAPIView(APIView):
         return Response(self._get_enabled_features(request))
 
     def _get_enabled_features(self, request):
+        DynamicSetting = apps.get_model("base", "DynamicSetting")
         enabled_features = []
 
         if settings.FEATURE_SLACK_INTEGRATION_ENABLED:
@@ -36,7 +37,6 @@ class FeaturesAPIView(APIView):
             enabled_features.append(FEATURE_TELEGRAM)
 
         if settings.MOBILE_APP_PUSH_NOTIFICATIONS_ENABLED:
-            DynamicSetting = apps.get_model("base", "DynamicSetting")
             mobile_app_settings = DynamicSetting.objects.get_or_create(
                 name="mobile_app_settings",
                 defaults={
@@ -59,5 +59,17 @@ class FeaturesAPIView(APIView):
 
         if settings.FEATURE_WEB_SCHEDULES_ENABLED:
             enabled_features.append(FEATURE_WEB_SCHEDULES)
+        else:
+            # allow enabling web schedules per org, independently of global status flag
+            enabled_web_schedules_orgs = DynamicSetting.objects.get_or_create(
+                name="enabled_web_schedules_orgs",
+                defaults={
+                    "json_value": {
+                        "org_ids": [],
+                    }
+                },
+            )[0]
+            if request.auth.organization.pk in enabled_web_schedules_orgs.json_value["org_ids"]:
+                enabled_features.append(FEATURE_WEB_SCHEDULES)
 
         return enabled_features

engine/apps/integrations/tests/test_ratelimit.py

@@ -8,12 +8,9 @@ from django.urls import reverse
 from apps.alerts.models import AlertReceiveChannel
 
 
-# Ratelimit keys are stored in cache. Clean it before and after every test to make them idempotent.
-def setup_module(module):
-    cache.clear()
-
-
-def teardown_module(module):
+@pytest.fixture(autouse=True)
+def clear_cache():
+    # Ratelimit keys are stored in cache. Clean it before and after every test to make them idempotent.
     cache.clear()
 
 

engine/apps/schedules/models/custom_on_call_shift.py

@@ -420,8 +420,7 @@ class CustomOnCallShift(models.Model):
         repetitions = UnfoldableCalendar(current_event).RepeatedEvent(
             current_event, next_event_start.replace(microsecond=0)
         )
-        ical_iter = repetitions.__iter__()
-        for event in ical_iter:
+        for event in repetitions.__iter__():
             if end_date:  # end_date exists for long events with frequency weekly and monthly
                 if end_date >= event.start >= next_event_start:
                     if (
@@ -460,8 +459,7 @@ class CustomOnCallShift(models.Model):
         repetitions = UnfoldableCalendar(initial_event).RepeatedEvent(
             initial_event, initial_event_start.replace(microsecond=0)
         )
-        ical_iter = repetitions.__iter__()
-        for event in ical_iter:
+        for event in repetitions.__iter__():
             if event.start > date:
                 break
             last_event = event

engine/apps/slack/scenarios/alertgroup_appearance.py

@@ -57,7 +57,7 @@ class OpenAlertAppearanceDialogStep(
 
         # This is a special case for amazon sns notifications in str format CHEKED
         if (
-            AlertReceiveChannel.INTEGRATION_AMAZON_SNS is not None
+            hasattr(AlertReceiveChannel, "INTEGRATION_AMAZON_SNS")
             and alert_group.channel.integration == AlertReceiveChannel.INTEGRATION_AMAZON_SNS
             and raw_request_data == "{}"
         ):

engine/requirements.txt

@@ -5,8 +5,8 @@ whitenoise==5.3.0
 twilio~=6.37.0
 phonenumbers==8.10.0
 django-ordered-model==3.1.1
-celery==5.2.7
-redis==3.2.0
+celery[amqp,redis]==5.2.7
+redis==3.4.1
 humanize==0.5.1
 uwsgi==2.0.20
 django-cors-headers==3.7.0
@@ -24,7 +24,7 @@ slack-export-viewer==1.0.0
 beautifulsoup4==4.8.1
 social-auth-app-django==3.1.0
 sendgrid==6.1.2
-cryptography==3.2
+cryptography==3.3.2
 pytest==5.4.3
 pytest-django==3.9.0
 pytest_factoryboy==2.0.3

engine/settings/base.py

@@ -64,9 +64,6 @@ TWILIO_VERIFY_SERVICE_SID = os.environ.get("TWILIO_VERIFY_SERVICE_SID")
 TELEGRAM_WEBHOOK_HOST = os.environ.get("TELEGRAM_WEBHOOK_HOST", BASE_URL)
 TELEGRAM_TOKEN = os.environ.get("TELEGRAM_TOKEN")
 
-os.environ.setdefault("MYSQL_PASSWORD", "empty")
-os.environ.setdefault("RABBIT_URI", "empty")
-
 # For Sending email
 SENDGRID_API_KEY = os.environ.get("SENDGRID_API_KEY")
 SENDGRID_FROM_EMAIL = os.environ.get("SENDGRID_FROM_EMAIL")
@@ -84,21 +81,101 @@ GRAFANA_CLOUD_ONCALL_TOKEN = os.environ.get("GRAFANA_CLOUD_ONCALL_TOKEN", None)
 # Outgoing webhook settings
 DANGEROUS_WEBHOOKS_ENABLED = getenv_boolean("DANGEROUS_WEBHOOKS_ENABLED", default=False)
 
-# DB backend defaults
-DB_BACKEND = os.environ.get("DB_BACKEND", "mysql")
-DB_BACKEND_DEFAULT_VALUES = {
-    "mysql": {
+
+# Database
+class DatabaseTypes:
+    MYSQL = "mysql"
+    POSTGRESQL = "postgresql"
+    SQLITE3 = "sqlite3"
+
+
+DATABASE_DEFAULTS = {
+    DatabaseTypes.MYSQL: {
         "USER": "root",
-        "PORT": "3306",
+        "PORT": 3306,
+    },
+    DatabaseTypes.POSTGRESQL: {
+        "USER": "postgres",
+        "PORT": 5432,
+    },
+}
+
+DATABASE_NAME = os.getenv("DATABASE_NAME") or os.getenv("MYSQL_DB_NAME")
+DATABASE_USER = os.getenv("DATABASE_USER") or os.getenv("MYSQL_USER")
+DATABASE_PASSWORD = os.getenv("DATABASE_PASSWORD") or os.getenv("MYSQL_PASSWORD")
+DATABASE_HOST = os.getenv("DATABASE_HOST") or os.getenv("MYSQL_HOST")
+DATABASE_PORT = os.getenv("DATABASE_PORT") or os.getenv("MYSQL_PORT")
+
+DATABASE_TYPE = os.getenv("DATABASE_TYPE", DatabaseTypes.MYSQL).lower()
+assert DATABASE_TYPE in {DatabaseTypes.MYSQL, DatabaseTypes.POSTGRESQL, DatabaseTypes.SQLITE3}
+
+DATABASE_ENGINE = f"django.db.backends.{DATABASE_TYPE}"
+
+DATABASE_CONFIGS = {
+    DatabaseTypes.SQLITE3: {
+        "ENGINE": DATABASE_ENGINE,
+        "NAME": DATABASE_NAME or "/var/lib/oncall/oncall.db",
+    },
+    DatabaseTypes.MYSQL: {
+        "ENGINE": DATABASE_ENGINE,
+        "NAME": DATABASE_NAME,
+        "USER": DATABASE_USER,
+        "PASSWORD": DATABASE_PASSWORD,
+        "HOST": DATABASE_HOST,
+        "PORT": DATABASE_PORT,
         "OPTIONS": {
             "charset": "utf8mb4",
             "connect_timeout": 1,
         },
     },
-    "postgresql": {
-        "USER": "postgres",
-        "PORT": "5432",
-        "OPTIONS": {},
+    DatabaseTypes.POSTGRESQL: {
+        "ENGINE": DATABASE_ENGINE,
+        "NAME": DATABASE_NAME,
+        "USER": DATABASE_USER,
+        "PASSWORD": DATABASE_PASSWORD,
+        "HOST": DATABASE_HOST,
+        "PORT": DATABASE_PORT,
+    },
+}
+
+DATABASES = {
+    "default": DATABASE_CONFIGS[DATABASE_TYPE],
+}
+if DATABASE_TYPE == DatabaseTypes.MYSQL:
+    # Workaround to use pymysql instead of mysqlclient
+    import pymysql
+
+    pymysql.install_as_MySQLdb()
+
+# Redis
+REDIS_USERNAME = os.getenv("REDIS_USERNAME", "")
+REDIS_PASSWORD = os.getenv("REDIS_PASSWORD")
+REDIS_HOST = os.getenv("REDIS_HOST")
+REDIS_PORT = os.getenv("REDIS_PORT", 6379)
+REDIS_PROTOCOL = os.getenv("REDIS_PROTOCOL", "redis")
+
+REDIS_URI = os.getenv("REDIS_URI")
+if not REDIS_URI:
+    REDIS_URI = f"{REDIS_PROTOCOL}://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}"
+
+# Cache
+CACHES = {
+    "default": {
+        "BACKEND": "redis_cache.RedisCache",
+        "LOCATION": [
+            REDIS_URI,
+        ],
+        "OPTIONS": {
+            "DB": 1,
+            "PARSER_CLASS": "redis.connection.HiredisParser",
+            "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool",
+            "CONNECTION_POOL_CLASS_KWARGS": {
+                "max_connections": 50,
+                "timeout": 20,
+            },
+            "MAX_CONNECTIONS": 1000,
+            "PICKLE_VERSION": -1,
+        },
     },
 }
 
@@ -261,7 +338,34 @@ USE_TZ = True
 STATIC_URL = os.environ.get("STATIC_URL", "/static/")
 STATIC_ROOT = "./static/"
 
-CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@localhost:5672"
+# RabbitMQ
+RABBITMQ_USERNAME = os.getenv("RABBITMQ_USERNAME")
+RABBITMQ_PASSWORD = os.getenv("RABBITMQ_PASSWORD")
+RABBITMQ_HOST = os.getenv("RABBITMQ_HOST")
+RABBITMQ_PORT = os.getenv("RABBITMQ_PORT", 5672)
+RABBITMQ_PROTOCOL = os.getenv("RABBITMQ_PROTOCOL", "amqp")
+RABBITMQ_VHOST = os.getenv("RABBITMQ_VHOST", "")
+
+RABBITMQ_URI = os.getenv("RABBITMQ_URI") or os.getenv("RABBIT_URI")
+if not RABBITMQ_URI:
+    RABBITMQ_URI = f"{RABBITMQ_PROTOCOL}://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}/{RABBITMQ_VHOST}"
+
+
+# Celery
+class BrokerTypes:
+    RABBITMQ = "rabbitmq"
+    REDIS = "redis"
+
+
+BROKER_TYPE = os.getenv("BROKER_TYPE", BrokerTypes.RABBITMQ).lower()
+assert BROKER_TYPE in {BrokerTypes.RABBITMQ, BrokerTypes.REDIS}
+
+if BROKER_TYPE == BrokerTypes.RABBITMQ:
+    CELERY_BROKER_URL = RABBITMQ_URI
+elif BROKER_TYPE == BrokerTypes.REDIS:
+    CELERY_BROKER_URL = REDIS_URI
+else:
+    raise ValueError(f"Invalid BROKER_TYPE env variable: {BROKER_TYPE}")
 
 # By default, apply_async will just hang indefinitely trying to reach to RabbitMQ even if RabbitMQ is down.
 # This makes apply_async retry 3 times trying to reach to RabbitMQ, with some extra info on periods between retries.

engine/settings/ci-test.py

@@ -1,6 +1,6 @@
-# flake8: noqa: F405
+# flake8: noqa
 
-from .base import *  # noqa
+from .base import *
 
 SECRET_KEY = "u5/IIbuiJR3Y9FQMBActk+btReZ5oOxu+l8MIJQWLfVzESoan5REE6UNSYYEQdjBOcty9CDak2X"
 
@@ -9,27 +9,29 @@ MIRAGE_CIPHER_IV = "X+VFcDqtxJ5bbU+V"
 
 BASE_URL = "http://localhost"
 
-CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@rabbit_test:5672"
+if DATABASE_TYPE == DatabaseTypes.SQLITE3:
+    DATABASES["default"]["NAME"] = DATABASE_NAME or "oncall_ci.db"
+else:
+    DATABASES["default"] |= {
+        "NAME": DATABASE_NAME or "oncall_local_dev",
+        "USER": DATABASE_USER or DATABASE_DEFAULTS[DATABASE_TYPE]["USER"],
+        "PASSWORD": DATABASE_PASSWORD or "local_dev_pwd",
+        "HOST": DATABASE_HOST or f"{DATABASE_TYPE}_test",
+        "PORT": DATABASE_PORT or DATABASE_DEFAULTS[DATABASE_TYPE]["PORT"],
+    }
 
-if DB_BACKEND == "mysql":
-    # Workaround to use pymysql instead of mysqlclient
-    import pymysql
+if BROKER_TYPE == BrokerTypes.RABBITMQ:
+    CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@rabbit_test:5672"
+elif BROKER_TYPE == BrokerTypes.REDIS:
+    CELERY_BROKER_URL = REDIS_URI
 
-    pymysql.install_as_MySQLdb()
-    DB_BACKEND_DEFAULT_VALUES[DB_BACKEND]["OPTIONS"] = {"charset": "utf8mb4"}
-
-
-DATABASES = {
-    "default": {
-        "ENGINE": "django.db.backends.{}".format(DB_BACKEND),
-        "NAME": os.environ.get("DB_NAME", "oncall_local_dev"),
-        "USER": os.environ.get("DB_USER", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("USER", "root")),
-        "PASSWORD": "local_dev_pwd",
-        "HOST": "{}_test".format(DB_BACKEND),
-        "PORT": os.environ.get("DB_PORT", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("PORT", "3306")),
-        "OPTIONS": DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("OPTIONS", {}),
-    },
-}
+# use redis as cache and celery broker on CI tests
+if BROKER_TYPE != BrokerTypes.REDIS:
+    CACHES = {
+        "default": {
+            "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+        }
+    }
 
 # Dummy Telegram token (fake one)
 TELEGRAM_TOKEN = "0000000000:XXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXX"

engine/settings/dev.py

@@ -1,27 +1,28 @@
+# flake8: noqa
 import os
 import sys
 
-from .base import *  # noqa
-
-if DB_BACKEND == "mysql":  # noqa
-    # Workaround to use pymysql instead of mysqlclient
-    import pymysql
-
-    pymysql.install_as_MySQLdb()
+from .base import *
 
 DEBUG = True
 
-DATABASES = {
-    "default": {
-        "ENGINE": "django.db.backends.{}".format(DB_BACKEND),  # noqa
-        "NAME": os.environ.get("DB_NAME", "oncall_local_dev"),
-        "USER": os.environ.get("DB_USER", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("USER", "root")),  # noqa
-        "PASSWORD": os.environ.get("DB_PASSWORD", "empty"),
-        "HOST": os.environ.get("DB_HOST", "127.0.0.1"),
-        "PORT": os.environ.get("DB_PORT", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("PORT", "3306")),  # noqa
-        "OPTIONS": DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("OPTIONS", {}),  # noqa
-    },
-}
+if DATABASE_TYPE == DatabaseTypes.SQLITE3:
+    DATABASES["default"]["NAME"] = DATABASE_NAME or "oncall_dev.db"
+else:
+    DATABASES["default"] |= {
+        "NAME": DATABASE_NAME or "oncall_local_dev",
+        "USER": DATABASE_USER or DATABASE_DEFAULTS[DATABASE_TYPE]["USER"],
+        "PASSWORD": DATABASE_PASSWORD or "empty",
+        "HOST": DATABASE_HOST or "127.0.0.1",
+        "PORT": DATABASE_PORT or DATABASE_DEFAULTS[DATABASE_TYPE]["PORT"],
+    }
+
+if BROKER_TYPE == BrokerTypes.RABBITMQ:
+    CELERY_BROKER_URL = "pyamqp://rabbitmq:rabbitmq@localhost:5672"
+elif BROKER_TYPE == BrokerTypes.REDIS:
+    CELERY_BROKER_URL = "redis://localhost:6379"
+
+CACHES["default"]["LOCATION"] = ["localhost:6379"]
 
 SECRET_KEY = os.environ.get("SECRET_KEY", "osMsNM0PqlRHBlUvqmeJ7+ldU3IUETCrY9TrmiViaSmInBHolr1WUlS0OFS4AHrnnkp1vp9S9z1")
 
@@ -32,28 +33,6 @@ MIRAGE_CIPHER_IV = os.environ.get("MIRAGE_CIPHER_IV", "tZZa+60zTZO2NRcS")
 
 TESTING = "pytest" in sys.modules or "unittest" in sys.modules
 
-CACHES = {
-    "default": {
-        "BACKEND": "redis_cache.RedisCache",
-        "LOCATION": [
-            "localhost:6379",
-        ],
-        "OPTIONS": {
-            "DB": 1,
-            "PARSER_CLASS": "redis.connection.HiredisParser",
-            "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool",
-            "CONNECTION_POOL_CLASS_KWARGS": {
-                "max_connections": 50,
-                "timeout": 20,
-            },
-            "MAX_CONNECTIONS": 1000,
-            "PICKLE_VERSION": -1,
-        },
-    },
-}
-
-CELERY_BROKER_URL = "pyamqp://rabbitmq:rabbitmq@localhost:5672"
-
 SILKY_PYTHON_PROFILER = True
 
 # For any requests that come in with that header/value, request.is_secure() will return True.

engine/settings/helm.py

@@ -1,64 +1,4 @@
-import os
-
-# Workaround to use pymysql instead of mysqlclient
-import pymysql
-
-from .prod_without_db import *  # noqa
-
-pymysql.install_as_MySQLdb()
-
-DATABASES = {
-    "default": {
-        "ENGINE": "django.db.backends.mysql",
-        "NAME": os.environ.get("MYSQL_DB_NAME"),
-        "USER": os.environ.get("MYSQL_USER"),
-        "PASSWORD": os.environ["MYSQL_PASSWORD"],
-        "HOST": os.environ.get("MYSQL_HOST"),
-        "PORT": os.environ.get("MYSQL_PORT"),
-        "OPTIONS": {
-            "charset": "utf8mb4",
-            "connect_timeout": 1,
-        },
-    },
-}
-
-RABBITMQ_USERNAME = os.environ.get("RABBITMQ_USERNAME")
-RABBITMQ_PASSWORD = os.environ.get("RABBITMQ_PASSWORD")
-RABBITMQ_HOST = os.environ.get("RABBITMQ_HOST")
-RABBITMQ_PORT = os.environ.get("RABBITMQ_PORT")
-RABBITMQ_PROTOCOL = os.environ.get("RABBITMQ_PROTOCOL")
-RABBITMQ_VHOST = os.environ.get("RABBITMQ_VHOST", "")
-
-CELERY_BROKER_URL = (
-    f"{RABBITMQ_PROTOCOL}://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}/{RABBITMQ_VHOST}"
-)
-
-REDIS_USERNAME = os.environ.get("REDIS_USERNAME", "")
-REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD")
-REDIS_HOST = os.environ.get("REDIS_HOST")
-REDIS_PORT = os.environ.get("REDIS_PORT", "6379")
-REDIS_PROTOCOL = os.environ.get("REDIS_PROTOCOL", "redis")
-REDIS_URI = f"{REDIS_PROTOCOL}://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}"
-
-CACHES = {
-    "default": {
-        "BACKEND": "redis_cache.RedisCache",
-        "LOCATION": [
-            REDIS_URI,
-        ],
-        "OPTIONS": {
-            "DB": 1,
-            "PARSER_CLASS": "redis.connection.HiredisParser",
-            "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool",
-            "CONNECTION_POOL_CLASS_KWARGS": {
-                "max_connections": 50,
-                "timeout": 20,
-            },
-            "MAX_CONNECTIONS": 1000,
-            "PICKLE_VERSION": -1,
-        },
-    },
-}
+from .prod_without_db import *  # noqa: F401, F403
 
 APPEND_SLASH = False
 SECURE_SSL_REDIRECT = False

engine/settings/hobby.py

@@ -1,37 +1,6 @@
-# flake8: noqa: F405
+from .prod_without_db import *  # noqa: F403
 
-from random import randrange
-
-# Workaround to use pymysql instead of mysqlclient
-import pymysql
-
-from .prod_without_db import *  # noqa
-
-pymysql.install_as_MySQLdb()
-
-DATABASES = {
-    "default": {
-        "ENGINE": "django.db.backends.mysql",
-        "NAME": os.environ.get("MYSQL_DB_NAME"),
-        "USER": os.environ.get("MYSQL_USER"),
-        "PASSWORD": os.environ["MYSQL_PASSWORD"],
-        "HOST": os.environ.get("MYSQL_HOST"),
-        "PORT": os.environ.get("MYSQL_PORT"),
-        "OPTIONS": {
-            "charset": "utf8mb4",
-            "connect_timeout": 1,
-        },
-    },
-}
-
-RABBITMQ_USERNAME = os.environ.get("RABBITMQ_USERNAME")
-RABBITMQ_PASSWORD = os.environ.get("RABBITMQ_PASSWORD")
-RABBITMQ_HOST = os.environ.get("RABBITMQ_HOST")
-RABBITMQ_PORT = os.environ.get("RABBITMQ_PORT")
-
-CELERY_BROKER_URL = f"amqp://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}"
-
-MIRAGE_SECRET_KEY = SECRET_KEY
+MIRAGE_SECRET_KEY = SECRET_KEY  # noqa: F405
 MIRAGE_CIPHER_IV = "1234567890abcdef"  # use default
 
 APPEND_SLASH = False

engine/settings/prod_without_db.py

@@ -15,36 +15,6 @@ except ModuleNotFoundError:
 
 from .base import *  # noqa
 
-# It's required for collectstatic to avoid connecting it to MySQL
-
-# Primary database must have the name "default"
-DATABASES = {
-    "default": {
-        "ENGINE": "django.db.backends.sqlite3",
-        "NAME": os.path.join(BASE_DIR, "db.sqlite3"),  # noqa
-    }
-}
-
-CACHES = {
-    "default": {
-        "BACKEND": "redis_cache.RedisCache",
-        "LOCATION": [
-            os.environ.get("REDIS_URI"),
-        ],
-        "OPTIONS": {
-            "DB": 1,
-            "PARSER_CLASS": "redis.connection.HiredisParser",
-            "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool",
-            "CONNECTION_POOL_CLASS_KWARGS": {
-                "max_connections": 50,
-                "timeout": 20,
-            },
-            "MAX_CONNECTIONS": 1000,
-            "PICKLE_VERSION": -1,
-        },
-    },
-}
-
 SLACK_SIGNING_SECRET = os.environ.get("SLACK_SIGNING_SECRET")
 SLACK_SIGNING_SECRET_LIVE = os.environ.get("SLACK_SIGNING_SECRET_LIVE", "")
 
@@ -56,8 +26,6 @@ STATIC_ROOT = "./collected_static/"
 
 DEBUG = False
 
-CELERY_BROKER_URL = os.environ["RABBIT_URI"]
-
 SECURE_SSL_REDIRECT = True
 SECURE_REDIRECT_EXEMPT = [
     "^health/",

grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx

@@ -130,7 +130,7 @@ export const PluginConfigPage = (props: Props) => {
 
   const handleSyncException = useCallback((e) => {
     const buildErrMsg = (msg: string): string =>
-      constructSyncErrorMessage(msg, plugin.meta.jsonData.onCallApiUrl);
+      constructSyncErrorMessage(msg, plugin.meta.jsonData?.onCallApiUrl);
 
     if (plugin.meta.jsonData?.onCallApiUrl) {
       const { status: statusCode  } = e.response;

helm/oncall/templates/engine/deployment.yaml

@@ -73,3 +73,15 @@ spec:
             timeoutSeconds: 3
           resources:
             {{- toYaml .Values.engine.resources | nindent 12 }}
+      {{- with .Values.engine.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.engine.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.engine.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

helm/oncall/values.yaml

@@ -29,6 +29,18 @@ engine:
     #   cpu: 100m
     #   memory: 128Mi
 
+  ## Affinity for pod assignment
+  ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+  affinity: {}
+
+  ## Node labels for pod assignment
+  ## ref: https://kubernetes.io/docs/user-guide/node-selection/
+  nodeSelector: {}
+
+  ## Tolerations for pod assignment
+  ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+  tolerations: []
+
 # Celery workers pods configuration
 celery:
   replicaCount: 1

tools/pagerduty-migrator/scripts/README.md

@@ -0,0 +1,12 @@
+# PagerDuty migrator scripts
+
+When we run MODE="plan" we can notice that there is escalation, integration in pagerduty that needs to be linked to a user.
+
+To solve this problem, we can run the  add_users_pagerduty_to_grafana.py script
+
+```bash
+docker run -it --rm -e PAGERDUTY_API_TOKEN="mytoken" -e GRAFANA_URL="http://localhost:3000" -e GRAFANA_USERNAME="admin" -e GRAFANA_PASSWORD="admin" pd-oncall-migrator python /app/scripts/add_users_pagerduty_to_grafana.py
+```
+
+It is worth remembering that this script will create a user with a random password.
+To access with the user created, it will be necessary to change the password in grafana web.

tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py

@@ -0,0 +1,42 @@
+import os
+import secrets
+import sys
+import requests
+
+from urllib.parse import urljoin
+from pdpyras import APISession
+
+PAGERDUTY_API_TOKEN = os.environ["PAGERDUTY_API_TOKEN"]
+PATH_USERS_GRAFANA = "/api/admin/users"
+GRAFANA_URL = os.environ["GRAFANA_URL"] # Example: http://localhost:3000
+GRAFANA_USERNAME = os.environ["GRAFANA_USERNAME"]
+GRAFANA_PASSWORD = os.environ["GRAFANA_PASSWORD"]
+SUCCESS_SIGN = "✅"
+ERROR_SIGN = "❌"
+
+def list_pagerduty_users():
+    session = APISession(PAGERDUTY_API_TOKEN)
+
+    users = session.list_all("users")
+
+    for user in users:
+        password = secrets.token_urlsafe(15)
+        username = user["email"].split("@")[0]
+        json = {"name": user["name"], "email": user["email"], "login": username, "password": password}
+        create_grafana_user(json)
+
+def create_grafana_user(data):
+    url = urljoin(GRAFANA_URL, PATH_USERS_GRAFANA)
+    response = requests.request("POST", url, auth=(GRAFANA_USERNAME, GRAFANA_PASSWORD), json=data)
+
+    if response.status_code == 200:
+        print(SUCCESS_SIGN + " User created: " + data["login"])
+    elif response.status_code == 401:
+        sys.exit(ERROR_SIGN + " Invalid username or password.")
+    elif response.status_code == 412:
+        print(ERROR_SIGN + " User " + data["login"] + " already exists." )
+    else: 
+        print("{} {}".format(ERROR_SIGN, response.text))
+
+if __name__ == "__main__":
+    list_pagerduty_users()