diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d122096..e4b913c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: run: | docker run -v ${PWD}/docs/sources:/hugo/content/docs/oncall/latest -e HUGO_REFLINKSERRORLEVEL=ERROR --rm grafana/docs-base:latest /bin/bash -c 'make hugo' - unit-test-backend: + unit-test-backend-mysql-rabbitmq: runs-on: ubuntu-latest container: python:3.9 env: @@ -66,11 +66,11 @@ jobs: pip install -r requirements.txt ./wait_for_test_mysql_start.sh && pytest --ds=settings.ci-test -x - unit-test-backend-postgresql: + unit-test-backend-postgresql-rabbitmq: runs-on: ubuntu-latest container: python:3.9 env: - DB_BACKEND: postgresql + DATABASE_TYPE: postgresql DJANGO_SETTINGS_MODULE: settings.ci-test SLACK_CLIENT_OAUTH_ID: 1 services: @@ -98,3 +98,29 @@ jobs: pip install -r requirements.txt pytest --ds=settings.ci-test -x + unit-test-backend-sqlite-redis: + runs-on: ubuntu-latest + container: python:3.9 + env: + DATABASE_TYPE: sqlite3 + BROKER_TYPE: redis + REDIS_URI: redis://redis_test:6379 + DJANGO_SETTINGS_MODULE: settings.ci-test + SLACK_CLIENT_OAUTH_ID: 1 + services: + redis_test: + image: redis:7.0.5 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v2 + - name: Unit Test Backend + run: | + apt-get update && apt-get install -y netcat + cd engine/ + pip install -r requirements.txt + pytest --ds=settings.ci-test -x diff --git a/.gitignore b/.gitignore index 308f671f..d0748610 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Backend */db.sqlite3 +engine/oncall_dev.db *.pyc venv .python-version diff --git a/CHANGELOG.md b/CHANGELOG.md index ebfa6fb2..01e27157 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log +## v1.0.40 (2022-10-05) +- Improved database and celery backends support +- Added script to import PagerDuty users to Grafana +- Bug fixes + ## v1.0.39 (2022-10-03) - Fix issue in v1.0.38 blocking the creation of schedules and webhooks in the UI diff --git a/DEVELOPER.md b/DEVELOPER.md index fd71c96b..347f7e95 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -59,8 +59,8 @@ pip install -U pip wheel # Copy and check .env.dev file. cp .env.dev.example .env.dev -# NOTE: if you want to use the PostgreSQL db backend add DB_BACKEND=postgresql to your .env.dev file; -# currently allowed backend values are `mysql` (default) and `postgresql` +# NOTE: if you want to use the PostgreSQL db backend add DATABASE_TYPE=postgresql to your .env.dev file; +# currently allowed backend values are `mysql` (default), `postgresql` and `sqlite3` # Apply .env.dev to current terminal. # For PyCharm it's better to use https://plugins.jetbrains.com/plugin/7861-envfile/ diff --git a/engine/Dockerfile b/engine/Dockerfile index 4a736620..8a72ef39 100644 --- a/engine/Dockerfile +++ b/engine/Dockerfile @@ -9,8 +9,11 @@ RUN pip install -r requirements.txt COPY ./ ./ -RUN DJANGO_SETTINGS_MODULE=settings.prod_without_db SECRET_KEY="ThEmUsTSecretKEYforBUILDstage123" TELEGRAM_TOKEN="0000000000:XXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXX" SLACK_CLIENT_OAUTH_ID=1 python manage.py collectstatic --no-input -RUN rm db.sqlite3 +# Collect static files and create an SQLite database +RUN mkdir -p /var/lib/oncall +RUN DJANGO_SETTINGS_MODULE=settings.prod_without_db DATABASE_TYPE=sqlite3 DATABASE_NAME=/var/lib/oncall/oncall.db SECRET_KEY="ThEmUsTSecretKEYforBUILDstage123" python manage.py collectstatic --no-input +RUN chown -R 1000:2000 /var/lib/oncall + # This is required for prometheus_client to sync between uwsgi workers RUN mkdir -p /tmp/prometheus_django_metrics; diff --git a/engine/apps/api/views/features.py b/engine/apps/api/views/features.py index fc56ca93..cc69514f 100644 --- a/engine/apps/api/views/features.py +++ b/engine/apps/api/views/features.py @@ -27,6 +27,7 @@ class FeaturesAPIView(APIView): return Response(self._get_enabled_features(request)) def _get_enabled_features(self, request): + DynamicSetting = apps.get_model("base", "DynamicSetting") enabled_features = [] if settings.FEATURE_SLACK_INTEGRATION_ENABLED: @@ -36,7 +37,6 @@ class FeaturesAPIView(APIView): enabled_features.append(FEATURE_TELEGRAM) if settings.MOBILE_APP_PUSH_NOTIFICATIONS_ENABLED: - DynamicSetting = apps.get_model("base", "DynamicSetting") mobile_app_settings = DynamicSetting.objects.get_or_create( name="mobile_app_settings", defaults={ @@ -59,5 +59,17 @@ class FeaturesAPIView(APIView): if settings.FEATURE_WEB_SCHEDULES_ENABLED: enabled_features.append(FEATURE_WEB_SCHEDULES) + else: + # allow enabling web schedules per org, independently of global status flag + enabled_web_schedules_orgs = DynamicSetting.objects.get_or_create( + name="enabled_web_schedules_orgs", + defaults={ + "json_value": { + "org_ids": [], + } + }, + )[0] + if request.auth.organization.pk in enabled_web_schedules_orgs.json_value["org_ids"]: + enabled_features.append(FEATURE_WEB_SCHEDULES) return enabled_features diff --git a/engine/apps/integrations/tests/test_ratelimit.py b/engine/apps/integrations/tests/test_ratelimit.py index 75e3d903..97b56937 100644 --- a/engine/apps/integrations/tests/test_ratelimit.py +++ b/engine/apps/integrations/tests/test_ratelimit.py @@ -8,12 +8,9 @@ from django.urls import reverse from apps.alerts.models import AlertReceiveChannel -# Ratelimit keys are stored in cache. Clean it before and after every test to make them idempotent. -def setup_module(module): - cache.clear() - - -def teardown_module(module): +@pytest.fixture(autouse=True) +def clear_cache(): + # Ratelimit keys are stored in cache. Clean it before and after every test to make them idempotent. cache.clear() diff --git a/engine/apps/schedules/models/custom_on_call_shift.py b/engine/apps/schedules/models/custom_on_call_shift.py index 7b24676a..db9ca5d3 100644 --- a/engine/apps/schedules/models/custom_on_call_shift.py +++ b/engine/apps/schedules/models/custom_on_call_shift.py @@ -420,8 +420,7 @@ class CustomOnCallShift(models.Model): repetitions = UnfoldableCalendar(current_event).RepeatedEvent( current_event, next_event_start.replace(microsecond=0) ) - ical_iter = repetitions.__iter__() - for event in ical_iter: + for event in repetitions.__iter__(): if end_date: # end_date exists for long events with frequency weekly and monthly if end_date >= event.start >= next_event_start: if ( @@ -460,8 +459,7 @@ class CustomOnCallShift(models.Model): repetitions = UnfoldableCalendar(initial_event).RepeatedEvent( initial_event, initial_event_start.replace(microsecond=0) ) - ical_iter = repetitions.__iter__() - for event in ical_iter: + for event in repetitions.__iter__(): if event.start > date: break last_event = event diff --git a/engine/apps/slack/scenarios/alertgroup_appearance.py b/engine/apps/slack/scenarios/alertgroup_appearance.py index 7b772a0e..8a335fcd 100644 --- a/engine/apps/slack/scenarios/alertgroup_appearance.py +++ b/engine/apps/slack/scenarios/alertgroup_appearance.py @@ -57,7 +57,7 @@ class OpenAlertAppearanceDialogStep( # This is a special case for amazon sns notifications in str format CHEKED if ( - AlertReceiveChannel.INTEGRATION_AMAZON_SNS is not None + hasattr(AlertReceiveChannel, "INTEGRATION_AMAZON_SNS") and alert_group.channel.integration == AlertReceiveChannel.INTEGRATION_AMAZON_SNS and raw_request_data == "{}" ): diff --git a/engine/requirements.txt b/engine/requirements.txt index 1bf66e51..950c1d1e 100644 --- a/engine/requirements.txt +++ b/engine/requirements.txt @@ -5,8 +5,8 @@ whitenoise==5.3.0 twilio~=6.37.0 phonenumbers==8.10.0 django-ordered-model==3.1.1 -celery==5.2.7 -redis==3.2.0 +celery[amqp,redis]==5.2.7 +redis==3.4.1 humanize==0.5.1 uwsgi==2.0.20 django-cors-headers==3.7.0 @@ -24,7 +24,7 @@ slack-export-viewer==1.0.0 beautifulsoup4==4.8.1 social-auth-app-django==3.1.0 sendgrid==6.1.2 -cryptography==3.2 +cryptography==3.3.2 pytest==5.4.3 pytest-django==3.9.0 pytest_factoryboy==2.0.3 diff --git a/engine/settings/base.py b/engine/settings/base.py index 3f893246..d9ec9f36 100644 --- a/engine/settings/base.py +++ b/engine/settings/base.py @@ -64,9 +64,6 @@ TWILIO_VERIFY_SERVICE_SID = os.environ.get("TWILIO_VERIFY_SERVICE_SID") TELEGRAM_WEBHOOK_HOST = os.environ.get("TELEGRAM_WEBHOOK_HOST", BASE_URL) TELEGRAM_TOKEN = os.environ.get("TELEGRAM_TOKEN") -os.environ.setdefault("MYSQL_PASSWORD", "empty") -os.environ.setdefault("RABBIT_URI", "empty") - # For Sending email SENDGRID_API_KEY = os.environ.get("SENDGRID_API_KEY") SENDGRID_FROM_EMAIL = os.environ.get("SENDGRID_FROM_EMAIL") @@ -84,21 +81,101 @@ GRAFANA_CLOUD_ONCALL_TOKEN = os.environ.get("GRAFANA_CLOUD_ONCALL_TOKEN", None) # Outgoing webhook settings DANGEROUS_WEBHOOKS_ENABLED = getenv_boolean("DANGEROUS_WEBHOOKS_ENABLED", default=False) -# DB backend defaults -DB_BACKEND = os.environ.get("DB_BACKEND", "mysql") -DB_BACKEND_DEFAULT_VALUES = { - "mysql": { + +# Database +class DatabaseTypes: + MYSQL = "mysql" + POSTGRESQL = "postgresql" + SQLITE3 = "sqlite3" + + +DATABASE_DEFAULTS = { + DatabaseTypes.MYSQL: { "USER": "root", - "PORT": "3306", + "PORT": 3306, + }, + DatabaseTypes.POSTGRESQL: { + "USER": "postgres", + "PORT": 5432, + }, +} + +DATABASE_NAME = os.getenv("DATABASE_NAME") or os.getenv("MYSQL_DB_NAME") +DATABASE_USER = os.getenv("DATABASE_USER") or os.getenv("MYSQL_USER") +DATABASE_PASSWORD = os.getenv("DATABASE_PASSWORD") or os.getenv("MYSQL_PASSWORD") +DATABASE_HOST = os.getenv("DATABASE_HOST") or os.getenv("MYSQL_HOST") +DATABASE_PORT = os.getenv("DATABASE_PORT") or os.getenv("MYSQL_PORT") + +DATABASE_TYPE = os.getenv("DATABASE_TYPE", DatabaseTypes.MYSQL).lower() +assert DATABASE_TYPE in {DatabaseTypes.MYSQL, DatabaseTypes.POSTGRESQL, DatabaseTypes.SQLITE3} + +DATABASE_ENGINE = f"django.db.backends.{DATABASE_TYPE}" + +DATABASE_CONFIGS = { + DatabaseTypes.SQLITE3: { + "ENGINE": DATABASE_ENGINE, + "NAME": DATABASE_NAME or "/var/lib/oncall/oncall.db", + }, + DatabaseTypes.MYSQL: { + "ENGINE": DATABASE_ENGINE, + "NAME": DATABASE_NAME, + "USER": DATABASE_USER, + "PASSWORD": DATABASE_PASSWORD, + "HOST": DATABASE_HOST, + "PORT": DATABASE_PORT, "OPTIONS": { "charset": "utf8mb4", "connect_timeout": 1, }, }, - "postgresql": { - "USER": "postgres", - "PORT": "5432", - "OPTIONS": {}, + DatabaseTypes.POSTGRESQL: { + "ENGINE": DATABASE_ENGINE, + "NAME": DATABASE_NAME, + "USER": DATABASE_USER, + "PASSWORD": DATABASE_PASSWORD, + "HOST": DATABASE_HOST, + "PORT": DATABASE_PORT, + }, +} + +DATABASES = { + "default": DATABASE_CONFIGS[DATABASE_TYPE], +} +if DATABASE_TYPE == DatabaseTypes.MYSQL: + # Workaround to use pymysql instead of mysqlclient + import pymysql + + pymysql.install_as_MySQLdb() + +# Redis +REDIS_USERNAME = os.getenv("REDIS_USERNAME", "") +REDIS_PASSWORD = os.getenv("REDIS_PASSWORD") +REDIS_HOST = os.getenv("REDIS_HOST") +REDIS_PORT = os.getenv("REDIS_PORT", 6379) +REDIS_PROTOCOL = os.getenv("REDIS_PROTOCOL", "redis") + +REDIS_URI = os.getenv("REDIS_URI") +if not REDIS_URI: + REDIS_URI = f"{REDIS_PROTOCOL}://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}" + +# Cache +CACHES = { + "default": { + "BACKEND": "redis_cache.RedisCache", + "LOCATION": [ + REDIS_URI, + ], + "OPTIONS": { + "DB": 1, + "PARSER_CLASS": "redis.connection.HiredisParser", + "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool", + "CONNECTION_POOL_CLASS_KWARGS": { + "max_connections": 50, + "timeout": 20, + }, + "MAX_CONNECTIONS": 1000, + "PICKLE_VERSION": -1, + }, }, } @@ -261,7 +338,34 @@ USE_TZ = True STATIC_URL = os.environ.get("STATIC_URL", "/static/") STATIC_ROOT = "./static/" -CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@localhost:5672" +# RabbitMQ +RABBITMQ_USERNAME = os.getenv("RABBITMQ_USERNAME") +RABBITMQ_PASSWORD = os.getenv("RABBITMQ_PASSWORD") +RABBITMQ_HOST = os.getenv("RABBITMQ_HOST") +RABBITMQ_PORT = os.getenv("RABBITMQ_PORT", 5672) +RABBITMQ_PROTOCOL = os.getenv("RABBITMQ_PROTOCOL", "amqp") +RABBITMQ_VHOST = os.getenv("RABBITMQ_VHOST", "") + +RABBITMQ_URI = os.getenv("RABBITMQ_URI") or os.getenv("RABBIT_URI") +if not RABBITMQ_URI: + RABBITMQ_URI = f"{RABBITMQ_PROTOCOL}://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}/{RABBITMQ_VHOST}" + + +# Celery +class BrokerTypes: + RABBITMQ = "rabbitmq" + REDIS = "redis" + + +BROKER_TYPE = os.getenv("BROKER_TYPE", BrokerTypes.RABBITMQ).lower() +assert BROKER_TYPE in {BrokerTypes.RABBITMQ, BrokerTypes.REDIS} + +if BROKER_TYPE == BrokerTypes.RABBITMQ: + CELERY_BROKER_URL = RABBITMQ_URI +elif BROKER_TYPE == BrokerTypes.REDIS: + CELERY_BROKER_URL = REDIS_URI +else: + raise ValueError(f"Invalid BROKER_TYPE env variable: {BROKER_TYPE}") # By default, apply_async will just hang indefinitely trying to reach to RabbitMQ even if RabbitMQ is down. # This makes apply_async retry 3 times trying to reach to RabbitMQ, with some extra info on periods between retries. diff --git a/engine/settings/ci-test.py b/engine/settings/ci-test.py index b3d39d4e..7af883d3 100644 --- a/engine/settings/ci-test.py +++ b/engine/settings/ci-test.py @@ -1,6 +1,6 @@ -# flake8: noqa: F405 +# flake8: noqa -from .base import * # noqa +from .base import * SECRET_KEY = "u5/IIbuiJR3Y9FQMBActk+btReZ5oOxu+l8MIJQWLfVzESoan5REE6UNSYYEQdjBOcty9CDak2X" @@ -9,27 +9,29 @@ MIRAGE_CIPHER_IV = "X+VFcDqtxJ5bbU+V" BASE_URL = "http://localhost" -CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@rabbit_test:5672" +if DATABASE_TYPE == DatabaseTypes.SQLITE3: + DATABASES["default"]["NAME"] = DATABASE_NAME or "oncall_ci.db" +else: + DATABASES["default"] |= { + "NAME": DATABASE_NAME or "oncall_local_dev", + "USER": DATABASE_USER or DATABASE_DEFAULTS[DATABASE_TYPE]["USER"], + "PASSWORD": DATABASE_PASSWORD or "local_dev_pwd", + "HOST": DATABASE_HOST or f"{DATABASE_TYPE}_test", + "PORT": DATABASE_PORT or DATABASE_DEFAULTS[DATABASE_TYPE]["PORT"], + } -if DB_BACKEND == "mysql": - # Workaround to use pymysql instead of mysqlclient - import pymysql +if BROKER_TYPE == BrokerTypes.RABBITMQ: + CELERY_BROKER_URL = "amqp://rabbitmq:rabbitmq@rabbit_test:5672" +elif BROKER_TYPE == BrokerTypes.REDIS: + CELERY_BROKER_URL = REDIS_URI - pymysql.install_as_MySQLdb() - DB_BACKEND_DEFAULT_VALUES[DB_BACKEND]["OPTIONS"] = {"charset": "utf8mb4"} - - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.{}".format(DB_BACKEND), - "NAME": os.environ.get("DB_NAME", "oncall_local_dev"), - "USER": os.environ.get("DB_USER", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("USER", "root")), - "PASSWORD": "local_dev_pwd", - "HOST": "{}_test".format(DB_BACKEND), - "PORT": os.environ.get("DB_PORT", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("PORT", "3306")), - "OPTIONS": DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("OPTIONS", {}), - }, -} +# use redis as cache and celery broker on CI tests +if BROKER_TYPE != BrokerTypes.REDIS: + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + } + } # Dummy Telegram token (fake one) TELEGRAM_TOKEN = "0000000000:XXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXX" diff --git a/engine/settings/dev.py b/engine/settings/dev.py index fb7ddc3e..9c418ae4 100644 --- a/engine/settings/dev.py +++ b/engine/settings/dev.py @@ -1,27 +1,28 @@ +# flake8: noqa import os import sys -from .base import * # noqa - -if DB_BACKEND == "mysql": # noqa - # Workaround to use pymysql instead of mysqlclient - import pymysql - - pymysql.install_as_MySQLdb() +from .base import * DEBUG = True -DATABASES = { - "default": { - "ENGINE": "django.db.backends.{}".format(DB_BACKEND), # noqa - "NAME": os.environ.get("DB_NAME", "oncall_local_dev"), - "USER": os.environ.get("DB_USER", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("USER", "root")), # noqa - "PASSWORD": os.environ.get("DB_PASSWORD", "empty"), - "HOST": os.environ.get("DB_HOST", "127.0.0.1"), - "PORT": os.environ.get("DB_PORT", DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("PORT", "3306")), # noqa - "OPTIONS": DB_BACKEND_DEFAULT_VALUES.get(DB_BACKEND, {}).get("OPTIONS", {}), # noqa - }, -} +if DATABASE_TYPE == DatabaseTypes.SQLITE3: + DATABASES["default"]["NAME"] = DATABASE_NAME or "oncall_dev.db" +else: + DATABASES["default"] |= { + "NAME": DATABASE_NAME or "oncall_local_dev", + "USER": DATABASE_USER or DATABASE_DEFAULTS[DATABASE_TYPE]["USER"], + "PASSWORD": DATABASE_PASSWORD or "empty", + "HOST": DATABASE_HOST or "127.0.0.1", + "PORT": DATABASE_PORT or DATABASE_DEFAULTS[DATABASE_TYPE]["PORT"], + } + +if BROKER_TYPE == BrokerTypes.RABBITMQ: + CELERY_BROKER_URL = "pyamqp://rabbitmq:rabbitmq@localhost:5672" +elif BROKER_TYPE == BrokerTypes.REDIS: + CELERY_BROKER_URL = "redis://localhost:6379" + +CACHES["default"]["LOCATION"] = ["localhost:6379"] SECRET_KEY = os.environ.get("SECRET_KEY", "osMsNM0PqlRHBlUvqmeJ7+ldU3IUETCrY9TrmiViaSmInBHolr1WUlS0OFS4AHrnnkp1vp9S9z1") @@ -32,28 +33,6 @@ MIRAGE_CIPHER_IV = os.environ.get("MIRAGE_CIPHER_IV", "tZZa+60zTZO2NRcS") TESTING = "pytest" in sys.modules or "unittest" in sys.modules -CACHES = { - "default": { - "BACKEND": "redis_cache.RedisCache", - "LOCATION": [ - "localhost:6379", - ], - "OPTIONS": { - "DB": 1, - "PARSER_CLASS": "redis.connection.HiredisParser", - "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool", - "CONNECTION_POOL_CLASS_KWARGS": { - "max_connections": 50, - "timeout": 20, - }, - "MAX_CONNECTIONS": 1000, - "PICKLE_VERSION": -1, - }, - }, -} - -CELERY_BROKER_URL = "pyamqp://rabbitmq:rabbitmq@localhost:5672" - SILKY_PYTHON_PROFILER = True # For any requests that come in with that header/value, request.is_secure() will return True. diff --git a/engine/settings/helm.py b/engine/settings/helm.py index 00fa96c3..6ae28e8a 100644 --- a/engine/settings/helm.py +++ b/engine/settings/helm.py @@ -1,64 +1,4 @@ -import os - -# Workaround to use pymysql instead of mysqlclient -import pymysql - -from .prod_without_db import * # noqa - -pymysql.install_as_MySQLdb() - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.mysql", - "NAME": os.environ.get("MYSQL_DB_NAME"), - "USER": os.environ.get("MYSQL_USER"), - "PASSWORD": os.environ["MYSQL_PASSWORD"], - "HOST": os.environ.get("MYSQL_HOST"), - "PORT": os.environ.get("MYSQL_PORT"), - "OPTIONS": { - "charset": "utf8mb4", - "connect_timeout": 1, - }, - }, -} - -RABBITMQ_USERNAME = os.environ.get("RABBITMQ_USERNAME") -RABBITMQ_PASSWORD = os.environ.get("RABBITMQ_PASSWORD") -RABBITMQ_HOST = os.environ.get("RABBITMQ_HOST") -RABBITMQ_PORT = os.environ.get("RABBITMQ_PORT") -RABBITMQ_PROTOCOL = os.environ.get("RABBITMQ_PROTOCOL") -RABBITMQ_VHOST = os.environ.get("RABBITMQ_VHOST", "") - -CELERY_BROKER_URL = ( - f"{RABBITMQ_PROTOCOL}://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}/{RABBITMQ_VHOST}" -) - -REDIS_USERNAME = os.environ.get("REDIS_USERNAME", "") -REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD") -REDIS_HOST = os.environ.get("REDIS_HOST") -REDIS_PORT = os.environ.get("REDIS_PORT", "6379") -REDIS_PROTOCOL = os.environ.get("REDIS_PROTOCOL", "redis") -REDIS_URI = f"{REDIS_PROTOCOL}://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}" - -CACHES = { - "default": { - "BACKEND": "redis_cache.RedisCache", - "LOCATION": [ - REDIS_URI, - ], - "OPTIONS": { - "DB": 1, - "PARSER_CLASS": "redis.connection.HiredisParser", - "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool", - "CONNECTION_POOL_CLASS_KWARGS": { - "max_connections": 50, - "timeout": 20, - }, - "MAX_CONNECTIONS": 1000, - "PICKLE_VERSION": -1, - }, - }, -} +from .prod_without_db import * # noqa: F401, F403 APPEND_SLASH = False SECURE_SSL_REDIRECT = False diff --git a/engine/settings/hobby.py b/engine/settings/hobby.py index 3bd73c13..ca7299b0 100644 --- a/engine/settings/hobby.py +++ b/engine/settings/hobby.py @@ -1,37 +1,6 @@ -# flake8: noqa: F405 +from .prod_without_db import * # noqa: F403 -from random import randrange - -# Workaround to use pymysql instead of mysqlclient -import pymysql - -from .prod_without_db import * # noqa - -pymysql.install_as_MySQLdb() - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.mysql", - "NAME": os.environ.get("MYSQL_DB_NAME"), - "USER": os.environ.get("MYSQL_USER"), - "PASSWORD": os.environ["MYSQL_PASSWORD"], - "HOST": os.environ.get("MYSQL_HOST"), - "PORT": os.environ.get("MYSQL_PORT"), - "OPTIONS": { - "charset": "utf8mb4", - "connect_timeout": 1, - }, - }, -} - -RABBITMQ_USERNAME = os.environ.get("RABBITMQ_USERNAME") -RABBITMQ_PASSWORD = os.environ.get("RABBITMQ_PASSWORD") -RABBITMQ_HOST = os.environ.get("RABBITMQ_HOST") -RABBITMQ_PORT = os.environ.get("RABBITMQ_PORT") - -CELERY_BROKER_URL = f"amqp://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}" - -MIRAGE_SECRET_KEY = SECRET_KEY +MIRAGE_SECRET_KEY = SECRET_KEY # noqa: F405 MIRAGE_CIPHER_IV = "1234567890abcdef" # use default APPEND_SLASH = False diff --git a/engine/settings/prod_without_db.py b/engine/settings/prod_without_db.py index 6b7c20d8..0c583483 100644 --- a/engine/settings/prod_without_db.py +++ b/engine/settings/prod_without_db.py @@ -15,36 +15,6 @@ except ModuleNotFoundError: from .base import * # noqa -# It's required for collectstatic to avoid connecting it to MySQL - -# Primary database must have the name "default" -DATABASES = { - "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": os.path.join(BASE_DIR, "db.sqlite3"), # noqa - } -} - -CACHES = { - "default": { - "BACKEND": "redis_cache.RedisCache", - "LOCATION": [ - os.environ.get("REDIS_URI"), - ], - "OPTIONS": { - "DB": 1, - "PARSER_CLASS": "redis.connection.HiredisParser", - "CONNECTION_POOL_CLASS": "redis.BlockingConnectionPool", - "CONNECTION_POOL_CLASS_KWARGS": { - "max_connections": 50, - "timeout": 20, - }, - "MAX_CONNECTIONS": 1000, - "PICKLE_VERSION": -1, - }, - }, -} - SLACK_SIGNING_SECRET = os.environ.get("SLACK_SIGNING_SECRET") SLACK_SIGNING_SECRET_LIVE = os.environ.get("SLACK_SIGNING_SECRET_LIVE", "") @@ -56,8 +26,6 @@ STATIC_ROOT = "./collected_static/" DEBUG = False -CELERY_BROKER_URL = os.environ["RABBIT_URI"] - SECURE_SSL_REDIRECT = True SECURE_REDIRECT_EXEMPT = [ "^health/", diff --git a/grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx b/grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx index e199bf02..ab5eb863 100644 --- a/grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx +++ b/grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx @@ -130,7 +130,7 @@ export const PluginConfigPage = (props: Props) => { const handleSyncException = useCallback((e) => { const buildErrMsg = (msg: string): string => - constructSyncErrorMessage(msg, plugin.meta.jsonData.onCallApiUrl); + constructSyncErrorMessage(msg, plugin.meta.jsonData?.onCallApiUrl); if (plugin.meta.jsonData?.onCallApiUrl) { const { status: statusCode } = e.response; diff --git a/helm/oncall/templates/engine/deployment.yaml b/helm/oncall/templates/engine/deployment.yaml index 391e0077..216d3055 100644 --- a/helm/oncall/templates/engine/deployment.yaml +++ b/helm/oncall/templates/engine/deployment.yaml @@ -73,3 +73,15 @@ spec: timeoutSeconds: 3 resources: {{- toYaml .Values.engine.resources | nindent 12 }} + {{- with .Values.engine.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.engine.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.engine.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/oncall/values.yaml b/helm/oncall/values.yaml index 324498c7..f0a02773 100644 --- a/helm/oncall/values.yaml +++ b/helm/oncall/values.yaml @@ -29,6 +29,18 @@ engine: # cpu: 100m # memory: 128Mi + ## Affinity for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + # Celery workers pods configuration celery: replicaCount: 1 diff --git a/tools/pagerduty-migrator/scripts/README.md b/tools/pagerduty-migrator/scripts/README.md new file mode 100644 index 00000000..c1aab511 --- /dev/null +++ b/tools/pagerduty-migrator/scripts/README.md @@ -0,0 +1,12 @@ +# PagerDuty migrator scripts + +When we run MODE="plan" we can notice that there is escalation, integration in pagerduty that needs to be linked to a user. + +To solve this problem, we can run the add_users_pagerduty_to_grafana.py script + +```bash +docker run -it --rm -e PAGERDUTY_API_TOKEN="mytoken" -e GRAFANA_URL="http://localhost:3000" -e GRAFANA_USERNAME="admin" -e GRAFANA_PASSWORD="admin" pd-oncall-migrator python /app/scripts/add_users_pagerduty_to_grafana.py +``` + +It is worth remembering that this script will create a user with a random password. +To access with the user created, it will be necessary to change the password in grafana web. \ No newline at end of file diff --git a/tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py b/tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py new file mode 100644 index 00000000..28aa542e --- /dev/null +++ b/tools/pagerduty-migrator/scripts/add_users_pagerduty_to_grafana.py @@ -0,0 +1,42 @@ +import os +import secrets +import sys +import requests + +from urllib.parse import urljoin +from pdpyras import APISession + +PAGERDUTY_API_TOKEN = os.environ["PAGERDUTY_API_TOKEN"] +PATH_USERS_GRAFANA = "/api/admin/users" +GRAFANA_URL = os.environ["GRAFANA_URL"] # Example: http://localhost:3000 +GRAFANA_USERNAME = os.environ["GRAFANA_USERNAME"] +GRAFANA_PASSWORD = os.environ["GRAFANA_PASSWORD"] +SUCCESS_SIGN = "✅" +ERROR_SIGN = "❌" + +def list_pagerduty_users(): + session = APISession(PAGERDUTY_API_TOKEN) + + users = session.list_all("users") + + for user in users: + password = secrets.token_urlsafe(15) + username = user["email"].split("@")[0] + json = {"name": user["name"], "email": user["email"], "login": username, "password": password} + create_grafana_user(json) + +def create_grafana_user(data): + url = urljoin(GRAFANA_URL, PATH_USERS_GRAFANA) + response = requests.request("POST", url, auth=(GRAFANA_USERNAME, GRAFANA_PASSWORD), json=data) + + if response.status_code == 200: + print(SUCCESS_SIGN + " User created: " + data["login"]) + elif response.status_code == 401: + sys.exit(ERROR_SIGN + " Invalid username or password.") + elif response.status_code == 412: + print(ERROR_SIGN + " User " + data["login"] + " already exists." ) + else: + print("{} {}".format(ERROR_SIGN, response.text)) + +if __name__ == "__main__": + list_pagerduty_users() \ No newline at end of file