Allow multiple database and celery broker types (#582)

* add libs for celery + redis

* move redis & cache config to settings/base.py

* move rmq & celery config to settings/base.py

* BROKER -> BROKER_TYPE

* allow multiple database types

* flake8

* add sqlite db creation to dockerfile

* fix ci

* fix ci

* debug

* remove some defaults

* remove prints

* use local memory as cache on ci

* debug

* add DATABASE_DEFAULTS

* add ci test for sqlite + redis

* add ci test for sqlite + redis

* add ci test for sqlite + redis

* debug

* add redis healthcheck

* fix sqlite

* fix dev settings

* refactor dev settings

* tweak ci settings

* clear cache properly between tests

* move db and broker types to constants

* add librabbitmq deps

* use amqp instead of librabbitmq
This commit is contained in:
Vadim Stepanov 2022-10-04 09:25:53 +01:00 committed by GitHub
parent 1d239fac52
commit b84b174e20
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 204 additions and 215 deletions

View file

@ -39,7 +39,7 @@ jobs:
run: |
docker run -v ${PWD}/docs/sources:/hugo/content/docs/oncall/latest -e HUGO_REFLINKSERRORLEVEL=ERROR --rm grafana/docs-base:latest /bin/bash -c 'make hugo'
unit-test-backend:
unit-test-backend-mysql-rabbitmq:
runs-on: ubuntu-latest
container: python:3.9
env:
@ -66,11 +66,11 @@ jobs:
pip install -r requirements.txt
./wait_for_test_mysql_start.sh && pytest --ds=settings.ci-test -x
unit-test-backend-postgresql:
unit-test-backend-postgresql-rabbitmq:
runs-on: ubuntu-latest
container: python:3.9
env:
DB_BACKEND: postgresql
DATABASE_TYPE: postgresql
DJANGO_SETTINGS_MODULE: settings.ci-test
SLACK_CLIENT_OAUTH_ID: 1
services:
@ -98,3 +98,29 @@ jobs:
pip install -r requirements.txt
pytest --ds=settings.ci-test -x
unit-test-backend-sqlite-redis:
runs-on: ubuntu-latest
container: python:3.9
env:
DATABASE_TYPE: sqlite3
BROKER_TYPE: redis
REDIS_URI: redis://redis_test:6379
DJANGO_SETTINGS_MODULE: settings.ci-test
SLACK_CLIENT_OAUTH_ID: 1
services:
redis_test:
image: redis:7.0.5
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v2
- name: Unit Test Backend
run: |
apt-get update && apt-get install -y netcat
cd engine/
pip install -r requirements.txt
pytest --ds=settings.ci-test -x

1
.gitignore vendored
View file

@ -1,5 +1,6 @@
# Backend
*/db.sqlite3
engine/oncall_dev.db
*.pyc
venv
.python-version

View file

@ -59,8 +59,8 @@ pip install -U pip wheel
# Copy and check .env.dev file.
cp .env.dev.example .env.dev
# NOTE: if you want to use the PostgreSQL db backend add DB_BACKEND=postgresql to your .env.dev file;
# currently allowed backend values are `mysql` (default) and `postgresql`
# NOTE: if you want to use the PostgreSQL db backend add DATABASE_TYPE=postgresql to your .env.dev file;
# currently allowed backend values are `mysql` (default), `postgresql` and `sqlite3`
# Apply .env.dev to current terminal.
# For PyCharm it's better to use https://plugins.jetbrains.com/plugin/7861-envfile/

View file

@ -9,8 +9,11 @@ RUN pip install -r requirements.txt
COPY ./ ./
RUN DJANGO_SETTINGS_MODULE=settings.prod_without_db SECRET_KEY="ThEmUsTSecretKEYforBUILDstage123" TELEGRAM_TOKEN="0000000000:XXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXX" SLACK_CLIENT_OAUTH_ID=1 python manage.py collectstatic --no-input
RUN rm db.sqlite3
# Collect static files and create an SQLite database
RUN mkdir -p /var/lib/oncall
RUN DJANGO_SETTINGS_MODULE=settings.prod_without_db DATABASE_TYPE=sqlite3 DATABASE_NAME=/var/lib/oncall/oncall.db SECRET_KEY="ThEmUsTSecretKEYforBUILDstage123" python manage.py collectstatic --no-input
RUN chown -R 1000:2000 /var/lib/oncall
# This is required for prometheus_client to sync between uwsgi workers
RUN mkdir -p /tmp/prometheus_django_metrics;

View file

@ -8,12 +8,9 @@ from django.urls import reverse
from apps.alerts.models import AlertReceiveChannel
# Ratelimit keys are stored in cache. Clean it before and after every test to make them idempotent.
def setup_module(module):
cache.clear()
def teardown_module(module):
@pytest.fixture(autouse=True)
def clear_cache():
# Ratelimit keys are stored in cache. Clean it before and after every test to make them idempotent.
cache.clear()

View file

@ -5,8 +5,8 @@ whitenoise==5.3.0
twilio~=6.37.0
phonenumbers==8.10.0
django-ordered-model==3.1.1
celery==5.2.7
redis==3.2.0
celery[amqp,redis]==5.2.7
redis==3.4.1
humanize==0.5.1
uwsgi==2.0.20
django-cors-headers==3.7.0

View file

@ -64,9 +64,6 @@ TWILIO_VERIFY_SERVICE_SID = os.environ.get("TWILIO_VERIFY_SERVICE_SID")
TELEGRAM_WEBHOOK_HOST = os.environ.get("TELEGRAM_WEBHOOK_HOST", BASE_URL)
TELEGRAM_TOKEN = os.environ.get("TELEGRAM_TOKEN")
os.environ.setdefault("MYSQL_PASSWORD", "empty")
os.environ.setdefault("RABBIT_URI", "empty")
# For Sending email
SENDGRID_API_KEY = os.environ.get("SENDGRID_API_KEY")
SENDGRID_FROM_EMAIL = os.environ.get("SENDGRID_FROM_EMAIL")
@ -84,21 +81,101 @@ GRAFANA_CLOUD_ONCALL_TOKEN = os.environ.get("GRAFANA_CLOUD_ONCALL_TOKEN", None)
# Outgoing webhook settings
DANGEROUS_WEBHOOKS_ENABLED = getenv_boolean("DANGEROUS_WEBHOOKS_ENABLED", default=False)
# DB backend defaults
DB_BACKEND = os.environ.get("DB_BACKEND", "mysql")
DB_BACKEND_DEFAULT_VALUES = {
"mysql": {
# Database
class DatabaseTypes:
MYSQL = "mysql"
POSTGRESQL = "postgresql"
SQLITE3 = "sqlite3"
DATABASE_DEFAULTS = {
DatabaseTypes.MYSQL: {
"USER": "root",
"PORT": "3306",
"PORT": 3306,
},
DatabaseTypes.POSTGRESQL: {
"USER": "postgres",
"PORT": 5432,
},
}
DATABASE_NAME = os.getenv("DATABASE_NAME") or os.getenv("MYSQL_DB_NAME")
DATABASE_USER = os.getenv("DATABASE_USER") or os.getenv("MYSQL_USER")
DATABASE_PASSWORD = os.getenv("DATABASE_PASSWORD") or os.getenv("MYSQL_PASSWORD")
DATABASE_HOST = os.getenv("DATABASE_HOST") or os.getenv("MYSQL_HOST")
DATABASE_PORT = os.getenv("DATABASE_PORT") or os.getenv("MYSQL_PORT")
DATABASE_TYPE = os.getenv("DATABASE_TYPE", DatabaseTypes.MYSQL).lower()
assert DATABASE_TYPE in {DatabaseTypes.MYSQL, DatabaseTypes.POSTGRESQL, DatabaseTypes.SQLITE3}
DATABASE_ENGINE = f"django.db.backends.{DATABASE_TYPE}"
DATABASE_CONFIGS = {
DatabaseTypes.SQLITE3: {
"ENGINE": DATABASE_ENGINE,
"NAME": DATABASE_NAME or "/var/lib/oncall/oncall.db",
},
DatabaseTypes.MYSQL: {
"ENGINE": DATABASE_ENGINE,
"NAME": DATABASE_NAME,
"USER": DATABASE_USER,
"PASSWORD": DATABASE_PASSWORD,
"HOST": DATABASE_HOST,
"PORT": DATABASE_PORT,
"OPTIONS": {
"charset": "utf8mb4",
"connect_timeout": 1,
},
},
"postgresql": {
"USER": "postgres",
"PORT": "5432",
"OPTIONS": {},
DatabaseTypes.POSTGRESQL: {
"ENGINE": DATABASE_ENGINE,
"NAME": DATABASE_NAME,
"USER": DATABASE_USER,
"PASSWORD": DATABASE_PASSWORD,
"HOST": DATABASE_HOST,
"PORT": DATABASE_PORT,
},
}
DATABASES = {
"default": DATABASE_CONFIGS[DATABASE_TYPE],
}
if DATABASE_TYPE == DatabaseTypes.MYSQL:
# Workaround to use pymysql instead of mysqlclient
import pymysql
pymysql.install_as_MySQLdb()
# Redis
REDIS_USERNAME = os.getenv("REDIS_USERNAME", "")
REDIS_PASSWORD = os.getenv("REDIS_PASSWORD")
REDIS_HOST = os.getenv("REDIS_HOST")
REDIS_PORT = os.getenv("REDIS_PORT", 6379)
REDIS_PROTOCOL = os.getenv("REDIS_PROTOCOL", "redis")
REDIS_URI = os.getenv("REDIS_URI")
if not REDIS_URI:
REDIS_URI = f"{REDIS_PROTOCOL}://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}"
# Cache
CACHES = {
"default": {
"BACKEND": "redis_cache.RedisCache",
"LOCATION": [
REDIS_URI,
],
"OPTIONS": {
"DB": 1,
"PARSER_CLASS": "redis.connection.HiredisParser",
"CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool",
"CONNECTION_POOL_CLASS_KWARGS": {
"max_connections": 50,
"timeout": 20,
},
"MAX_CONNECTIONS": 1000,
"PICKLE_VERSION": -1,
},
},
}
@ -261,7 +338,34 @@ USE_TZ = True
STATIC_URL = os.environ.get("STATIC_URL", "/static/")
STATIC_ROOT = "./static/"
CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@localhost:5672"
# RabbitMQ
RABBITMQ_USERNAME = os.getenv("RABBITMQ_USERNAME")
RABBITMQ_PASSWORD = os.getenv("RABBITMQ_PASSWORD")
RABBITMQ_HOST = os.getenv("RABBITMQ_HOST")
RABBITMQ_PORT = os.getenv("RABBITMQ_PORT", 5672)
RABBITMQ_PROTOCOL = os.getenv("RABBITMQ_PROTOCOL", "amqp")
RABBITMQ_VHOST = os.getenv("RABBITMQ_VHOST", "")
RABBITMQ_URI = os.getenv("RABBITMQ_URI") or os.getenv("RABBIT_URI")
if not RABBITMQ_URI:
RABBITMQ_URI = f"{RABBITMQ_PROTOCOL}://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}/{RABBITMQ_VHOST}"
# Celery
class BrokerTypes:
RABBITMQ = "rabbitmq"
REDIS = "redis"
BROKER_TYPE = os.getenv("BROKER_TYPE", BrokerTypes.RABBITMQ).lower()
assert BROKER_TYPE in {BrokerTypes.RABBITMQ, BrokerTypes.REDIS}
if BROKER_TYPE == BrokerTypes.RABBITMQ:
CELERY_BROKER_URL = RABBITMQ_URI
elif BROKER_TYPE == BrokerTypes.REDIS:
CELERY_BROKER_URL = REDIS_URI
else:
raise ValueError(f"Invalid BROKER_TYPE env variable: {BROKER_TYPE}")
# By default, apply_async will just hang indefinitely trying to reach to RabbitMQ even if RabbitMQ is down.
# This makes apply_async retry 3 times trying to reach to RabbitMQ, with some extra info on periods between retries.

View file

@ -1,6 +1,6 @@
# flake8: noqa: F405
# flake8: noqa
from .base import * # noqa
from .base import *
SECRET_KEY = "u5/IIbuiJR3Y9FQMBActk+btReZ5oOxu+l8MIJQWLfVzESoan5REE6UNSYYEQdjBOcty9CDak2X"
@ -9,27 +9,29 @@ MIRAGE_CIPHER_IV = "X+VFcDqtxJ5bbU+V"
BASE_URL = "http://localhost"
CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@rabbit_test:5672"
if DATABASE_TYPE == DatabaseTypes.SQLITE3:
DATABASES["default"]["NAME"] = DATABASE_NAME or "oncall_ci.db"
else:
DATABASES["default"] |= {
"NAME": DATABASE_NAME or "oncall_local_dev",
"USER": DATABASE_USER or DATABASE_DEFAULTS[DATABASE_TYPE]["USER"],
"PASSWORD": DATABASE_PASSWORD or "local_dev_pwd",
"HOST": DATABASE_HOST or f"{DATABASE_TYPE}_test",
"PORT": DATABASE_PORT or DATABASE_DEFAULTS[DATABASE_TYPE]["PORT"],
}
if DB_BACKEND == "mysql":
# Workaround to use pymysql instead of mysqlclient
import pymysql
if BROKER_TYPE == BrokerTypes.RABBITMQ:
CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@rabbit_test:5672"
elif BROKER_TYPE == BrokerTypes.REDIS:
CELERY_BROKER_URL = REDIS_URI
pymysql.install_as_MySQLdb()
DB_BACKEND_DEFAULT_VALUES[DB_BACKEND]["OPTIONS"] = {"charset": "utf8mb4"}
DATABASES = {
"default": {
"ENGINE": "django.db.backends.{}".format(DB_BACKEND),
"NAME": os.environ.get("DB_NAME", "oncall_local_dev"),
"USER": os.environ.get("DB_USER", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("USER", "root")),
"PASSWORD": "local_dev_pwd",
"HOST": "{}_test".format(DB_BACKEND),
"PORT": os.environ.get("DB_PORT", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("PORT", "3306")),
"OPTIONS": DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("OPTIONS", {}),
},
}
# use redis as cache and celery broker on CI tests
if BROKER_TYPE != BrokerTypes.REDIS:
CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
}
}
# Dummy Telegram token (fake one)
TELEGRAM_TOKEN = "0000000000:XXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXX"

View file

@ -1,27 +1,28 @@
# flake8: noqa
import os
import sys
from .base import * # noqa
if DB_BACKEND == "mysql": # noqa
# Workaround to use pymysql instead of mysqlclient
import pymysql
pymysql.install_as_MySQLdb()
from .base import *
DEBUG = True
DATABASES = {
"default": {
"ENGINE": "django.db.backends.{}".format(DB_BACKEND), # noqa
"NAME": os.environ.get("DB_NAME", "oncall_local_dev"),
"USER": os.environ.get("DB_USER", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("USER", "root")), # noqa
"PASSWORD": os.environ.get("DB_PASSWORD", "empty"),
"HOST": os.environ.get("DB_HOST", "127.0.0.1"),
"PORT": os.environ.get("DB_PORT", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("PORT", "3306")), # noqa
"OPTIONS": DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("OPTIONS", {}), # noqa
},
}
if DATABASE_TYPE == DatabaseTypes.SQLITE3:
DATABASES["default"]["NAME"] = DATABASE_NAME or "oncall_dev.db"
else:
DATABASES["default"] |= {
"NAME": DATABASE_NAME or "oncall_local_dev",
"USER": DATABASE_USER or DATABASE_DEFAULTS[DATABASE_TYPE]["USER"],
"PASSWORD": DATABASE_PASSWORD or "empty",
"HOST": DATABASE_HOST or "127.0.0.1",
"PORT": DATABASE_PORT or DATABASE_DEFAULTS[DATABASE_TYPE]["PORT"],
}
if BROKER_TYPE == BrokerTypes.RABBITMQ:
CELERY_BROKER_URL = "pyamqp://rabbitmq:rabbitmq@localhost:5672"
elif BROKER_TYPE == BrokerTypes.REDIS:
CELERY_BROKER_URL = "redis://localhost:6379"
CACHES["default"]["LOCATION"] = ["localhost:6379"]
SECRET_KEY = os.environ.get("SECRET_KEY", "osMsNM0PqlRHBlUvqmeJ7+ldU3IUETCrY9TrmiViaSmInBHolr1WUlS0OFS4AHrnnkp1vp9S9z1")
@ -32,28 +33,6 @@ MIRAGE_CIPHER_IV = os.environ.get("MIRAGE_CIPHER_IV", "tZZa+60zTZO2NRcS")
TESTING = "pytest" in sys.modules or "unittest" in sys.modules
CACHES = {
"default": {
"BACKEND": "redis_cache.RedisCache",
"LOCATION": [
"localhost:6379",
],
"OPTIONS": {
"DB": 1,
"PARSER_CLASS": "redis.connection.HiredisParser",
"CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool",
"CONNECTION_POOL_CLASS_KWARGS": {
"max_connections": 50,
"timeout": 20,
},
"MAX_CONNECTIONS": 1000,
"PICKLE_VERSION": -1,
},
},
}
CELERY_BROKER_URL = "pyamqp://rabbitmq:rabbitmq@localhost:5672"
SILKY_PYTHON_PROFILER = True
# For any requests that come in with that header/value, request.is_secure() will return True.

View file

@ -1,64 +1,4 @@
import os
# Workaround to use pymysql instead of mysqlclient
import pymysql
from .prod_without_db import * # noqa
pymysql.install_as_MySQLdb()
DATABASES = {
"default": {
"ENGINE": "django.db.backends.mysql",
"NAME": os.environ.get("MYSQL_DB_NAME"),
"USER": os.environ.get("MYSQL_USER"),
"PASSWORD": os.environ["MYSQL_PASSWORD"],
"HOST": os.environ.get("MYSQL_HOST"),
"PORT": os.environ.get("MYSQL_PORT"),
"OPTIONS": {
"charset": "utf8mb4",
"connect_timeout": 1,
},
},
}
RABBITMQ_USERNAME = os.environ.get("RABBITMQ_USERNAME")
RABBITMQ_PASSWORD = os.environ.get("RABBITMQ_PASSWORD")
RABBITMQ_HOST = os.environ.get("RABBITMQ_HOST")
RABBITMQ_PORT = os.environ.get("RABBITMQ_PORT")
RABBITMQ_PROTOCOL = os.environ.get("RABBITMQ_PROTOCOL")
RABBITMQ_VHOST = os.environ.get("RABBITMQ_VHOST", "")
CELERY_BROKER_URL = (
f"{RABBITMQ_PROTOCOL}://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}/{RABBITMQ_VHOST}"
)
REDIS_USERNAME = os.environ.get("REDIS_USERNAME", "")
REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD")
REDIS_HOST = os.environ.get("REDIS_HOST")
REDIS_PORT = os.environ.get("REDIS_PORT", "6379")
REDIS_PROTOCOL = os.environ.get("REDIS_PROTOCOL", "redis")
REDIS_URI = f"{REDIS_PROTOCOL}://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}"
CACHES = {
"default": {
"BACKEND": "redis_cache.RedisCache",
"LOCATION": [
REDIS_URI,
],
"OPTIONS": {
"DB": 1,
"PARSER_CLASS": "redis.connection.HiredisParser",
"CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool",
"CONNECTION_POOL_CLASS_KWARGS": {
"max_connections": 50,
"timeout": 20,
},
"MAX_CONNECTIONS": 1000,
"PICKLE_VERSION": -1,
},
},
}
from .prod_without_db import * # noqa: F401, F403
APPEND_SLASH = False
SECURE_SSL_REDIRECT = False

View file

@ -1,37 +1,6 @@
# flake8: noqa: F405
from .prod_without_db import * # noqa: F403
from random import randrange
# Workaround to use pymysql instead of mysqlclient
import pymysql
from .prod_without_db import * # noqa
pymysql.install_as_MySQLdb()
DATABASES = {
"default": {
"ENGINE": "django.db.backends.mysql",
"NAME": os.environ.get("MYSQL_DB_NAME"),
"USER": os.environ.get("MYSQL_USER"),
"PASSWORD": os.environ["MYSQL_PASSWORD"],
"HOST": os.environ.get("MYSQL_HOST"),
"PORT": os.environ.get("MYSQL_PORT"),
"OPTIONS": {
"charset": "utf8mb4",
"connect_timeout": 1,
},
},
}
RABBITMQ_USERNAME = os.environ.get("RABBITMQ_USERNAME")
RABBITMQ_PASSWORD = os.environ.get("RABBITMQ_PASSWORD")
RABBITMQ_HOST = os.environ.get("RABBITMQ_HOST")
RABBITMQ_PORT = os.environ.get("RABBITMQ_PORT")
CELERY_BROKER_URL = f"amqp://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}"
MIRAGE_SECRET_KEY = SECRET_KEY
MIRAGE_SECRET_KEY = SECRET_KEY # noqa: F405
MIRAGE_CIPHER_IV = "1234567890abcdef" # use default
APPEND_SLASH = False

View file

@ -15,36 +15,6 @@ except ModuleNotFoundError:
from .base import * # noqa
# It's required for collectstatic to avoid connecting it to MySQL
# Primary database must have the name "default"
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": os.path.join(BASE_DIR, "db.sqlite3"), # noqa
}
}
CACHES = {
"default": {
"BACKEND": "redis_cache.RedisCache",
"LOCATION": [
os.environ.get("REDIS_URI"),
],
"OPTIONS": {
"DB": 1,
"PARSER_CLASS": "redis.connection.HiredisParser",
"CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool",
"CONNECTION_POOL_CLASS_KWARGS": {
"max_connections": 50,
"timeout": 20,
},
"MAX_CONNECTIONS": 1000,
"PICKLE_VERSION": -1,
},
},
}
SLACK_SIGNING_SECRET = os.environ.get("SLACK_SIGNING_SECRET")
SLACK_SIGNING_SECRET_LIVE = os.environ.get("SLACK_SIGNING_SECRET_LIVE", "")
@ -56,8 +26,6 @@ STATIC_ROOT = "./collected_static/"
DEBUG = False
CELERY_BROKER_URL = os.environ["RABBIT_URI"]
SECURE_SSL_REDIRECT = True
SECURE_REDIRECT_EXEMPT = [
"^health/",