From c0708902d2583415c6d2712ced0a7e93c2b954f5 Mon Sep 17 00:00:00 2001 From: Matias Bordese Date: Fri, 10 Jun 2022 11:09:05 -0300 Subject: [PATCH 01/28] Remove unused messaging backends feature flag --- engine/apps/api/serializers/organization.py | 2 -- engine/apps/api/tests/test_organization.py | 24 ------------------- .../tests/test_user_notification_policy.py | 10 ++------ engine/apps/base/messaging.py | 8 +------ engine/apps/base/tests/test_messaging.py | 10 -------- engine/settings/base.py | 1 - engine/settings/ci-test.py | 1 - engine/settings/dev.py | 1 - grafana-plugin/src/models/team/team.types.ts | 1 - 9 files changed, 3 insertions(+), 55 deletions(-) diff --git a/engine/apps/api/serializers/organization.py b/engine/apps/api/serializers/organization.py index 4dc69402..85d63fdf 100644 --- a/engine/apps/api/serializers/organization.py +++ b/engine/apps/api/serializers/organization.py @@ -3,7 +3,6 @@ from datetime import timedelta import humanize import pytz from django.apps import apps -from django.conf import settings from django.utils import timezone from rest_framework import fields, serializers @@ -121,7 +120,6 @@ class CurrentOrganizationSerializer(OrganizationSerializer): return { "telegram_configured": telegram_configured, "twilio_configured": twilio_configured, - "extra_messaging_backends_enabled": settings.FEATURE_EXTRA_MESSAGING_BACKENDS_ENABLED, } def get_stats(self, obj): diff --git a/engine/apps/api/tests/test_organization.py b/engine/apps/api/tests/test_organization.py index 0b97701e..ed13fb2c 100644 --- a/engine/apps/api/tests/test_organization.py +++ b/engine/apps/api/tests/test_organization.py @@ -80,30 +80,6 @@ def test_current_team_update_permissions( assert response.status_code == expected_status -@pytest.mark.django_db -@pytest.mark.parametrize("feature_flag_enabled", [False, True]) -def test_current_team_messaging_backend_status( - settings, - make_organization, - make_user_for_organization, - make_token_for_organization, - make_user_auth_headers, - feature_flag_enabled, -): - org = make_organization() - tester = make_user_for_organization(org, role=Role.ADMIN) - _, token = make_token_for_organization(org) - - client = APIClient() - - settings.FEATURE_EXTRA_MESSAGING_BACKENDS_ENABLED = feature_flag_enabled - url = reverse("api-internal:api-current-team") - response = client.get(url, format="json", **make_user_auth_headers(tester, token)) - - assert response.status_code == status.HTTP_200_OK - assert response.json()["env_status"]["extra_messaging_backends_enabled"] == bool(feature_flag_enabled) - - @pytest.mark.django_db @pytest.mark.parametrize( "role,expected_status", diff --git a/engine/apps/api/tests/test_user_notification_policy.py b/engine/apps/api/tests/test_user_notification_policy.py index 4760b47c..1eb39e61 100644 --- a/engine/apps/api/tests/test_user_notification_policy.py +++ b/engine/apps/api/tests/test_user_notification_policy.py @@ -450,22 +450,16 @@ def test_switch_wait_delay( @pytest.mark.django_db -@pytest.mark.parametrize("feature_flag_enabled", [False, True]) def test_notification_policy_backends_enabled( - user_notification_policy_internal_api_setup, settings, make_user_auth_headers, feature_flag_enabled + user_notification_policy_internal_api_setup, settings, make_user_auth_headers ): token, _, users = user_notification_policy_internal_api_setup admin, _ = users - settings.FEATURE_EXTRA_MESSAGING_BACKENDS_ENABLED = feature_flag_enabled - client = APIClient() url = reverse("api-internal:notification_policy-notify-by-options") response = client.get(url, **make_user_auth_headers(admin, token)) assert response.status_code == status.HTTP_200_OK options = [opt["display_name"] for opt in response.json()] - if feature_flag_enabled: - assert "Test Only Backend" in options - else: - assert "Test Only Backend" not in options + assert "Test Only Backend" in options diff --git a/engine/apps/base/messaging.py b/engine/apps/base/messaging.py index 694bb221..3b288540 100644 --- a/engine/apps/base/messaging.py +++ b/engine/apps/base/messaging.py @@ -52,9 +52,6 @@ def load_backend(path): def get_messaging_backends(): global _messaging_backends - if not settings.FEATURE_EXTRA_MESSAGING_BACKENDS_ENABLED: - return {} - if _messaging_backends is None: _messaging_backends = {} for backend_path in settings.EXTRA_MESSAGING_BACKENDS: @@ -64,10 +61,7 @@ def get_messaging_backends(): def get_messaging_backend_from_id(backend_id): - backend = None - if settings.FEATURE_EXTRA_MESSAGING_BACKENDS_ENABLED: - backend = _messaging_backends.get(backend_id) - return backend + return _messaging_backends.get(backend_id) _messaging_backends = None diff --git a/engine/apps/base/tests/test_messaging.py b/engine/apps/base/tests/test_messaging.py index 542a8250..ed12b819 100644 --- a/engine/apps/base/tests/test_messaging.py +++ b/engine/apps/base/tests/test_messaging.py @@ -3,17 +3,7 @@ import pytest from apps.base.messaging import get_messaging_backend_from_id, get_messaging_backends -@pytest.mark.django_db -def test_messaging_backends_disabled(settings): - settings.FEATURE_EXTRA_MESSAGING_BACKENDS_ENABLED = False - - assert get_messaging_backends() == {} - assert get_messaging_backend_from_id("TESTONLY") is None - - @pytest.mark.django_db def test_messaging_backends_enabled(settings): - settings.FEATURE_EXTRA_MESSAGING_BACKENDS_ENABLED = True - assert get_messaging_backends() != {} assert get_messaging_backend_from_id("TESTONLY") is not None diff --git a/engine/settings/base.py b/engine/settings/base.py index 0495440e..c0af0a7b 100644 --- a/engine/settings/base.py +++ b/engine/settings/base.py @@ -424,7 +424,6 @@ DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880 # Log inbound/outbound calls as slow=1 if they exceed threshold SLOW_THRESHOLD_SECONDS = 2.0 -FEATURE_EXTRA_MESSAGING_BACKENDS_ENABLED = getenv_boolean("FEATURE_EXTRA_MESSAGING_BACKENDS_ENABLED", default=False) EXTRA_MESSAGING_BACKENDS = [] INSTALLED_ONCALL_INTEGRATIONS = [ diff --git a/engine/settings/ci-test.py b/engine/settings/ci-test.py index 5389cbd5..15ece71d 100644 --- a/engine/settings/ci-test.py +++ b/engine/settings/ci-test.py @@ -25,7 +25,6 @@ SENDGRID_SECRET_KEY = "dummy_sendgrid_secret_key" TWILIO_ACCOUNT_SID = "dummy_twilio_account_sid" TWILIO_AUTH_TOKEN = "dummy_twilio_auth_token" -FEATURE_EXTRA_MESSAGING_BACKENDS_ENABLED = True EXTRA_MESSAGING_BACKENDS = ["apps.base.tests.messaging_backend.TestOnlyBackend"] OSS_INSTALLATION = True INSTALLED_APPS += ["apps.oss_installation"] # noqa diff --git a/engine/settings/dev.py b/engine/settings/dev.py index aff8ca9d..21ed4576 100644 --- a/engine/settings/dev.py +++ b/engine/settings/dev.py @@ -86,7 +86,6 @@ SWAGGER_SETTINGS = { } if TESTING: - FEATURE_EXTRA_MESSAGING_BACKENDS_ENABLED = True EXTRA_MESSAGING_BACKENDS = ["apps.base.tests.messaging_backend.TestOnlyBackend"] TELEGRAM_TOKEN = "0000000000:XXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXX" TWILIO_AUTH_TOKEN = "twilio_auth_token" diff --git a/grafana-plugin/src/models/team/team.types.ts b/grafana-plugin/src/models/team/team.types.ts index 8c75b4b8..505052db 100644 --- a/grafana-plugin/src/models/team/team.types.ts +++ b/grafana-plugin/src/models/team/team.types.ts @@ -68,6 +68,5 @@ export interface Team { env_status: { twilio_configured: boolean; telegram_configured: boolean; - extra_messaging_backends_enabled: boolean; }; } From 7a2ebf16abccd34f70369350432ccd02a0278472 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 14 Jun 2022 20:05:29 +0300 Subject: [PATCH 02/28] Edit settings/helm.py --- engine/settings/helm.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/engine/settings/helm.py b/engine/settings/helm.py index 5e35613b..25b4497f 100644 --- a/engine/settings/helm.py +++ b/engine/settings/helm.py @@ -57,29 +57,3 @@ CACHES = { APPEND_SLASH = False SECURE_SSL_REDIRECT = False - -TESTING = "pytest" in sys.modules or "unittest" in sys.modules - - -if TESTING: - TELEGRAM_TOKEN = "0000000000:XXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXX" - TWILIO_AUTH_TOKEN = "twilio_auth_token" - -# TODO: OSS: Add these setting to oss settings file. Add Version there too. -OSS_INSTALLATION_FEATURES_ENABLED = True - -INSTALLED_APPS += ["apps.oss_installation"] # noqa - -CELERY_BEAT_SCHEDULE["send_usage_stats"] = { # noqa - "task": "apps.oss_installation.tasks.send_usage_stats_report", - "schedule": crontab(hour=0, minute=randrange(0, 59)), # Send stats report at a random minute past midnight # noqa - "args": (), -} # noqa - -CELERY_BEAT_SCHEDULE["send_cloud_heartbeat"] = { # noqa - "task": "apps.oss_installation.tasks.send_cloud_heartbeat", - "schedule": crontab(minute="*/3"), # noqa - "args": (), -} # noqa - -SEND_ANONYMOUS_USAGE_STATS = True From ff22836ec5a2cf6a25111154fd883d42d0db161b Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 14 Jun 2022 20:47:34 +0300 Subject: [PATCH 03/28] Edit README.md, fix linting --- engine/settings/helm.py | 1 - helm/oncall/README.md | 14 +++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/engine/settings/helm.py b/engine/settings/helm.py index 25b4497f..6fcabd85 100644 --- a/engine/settings/helm.py +++ b/engine/settings/helm.py @@ -1,5 +1,4 @@ import os -import sys # Workaround to use pymysql instead of mysqlclient import pymysql diff --git a/helm/oncall/README.md b/helm/oncall/README.md index 2aa7845d..c5abedb1 100644 --- a/helm/oncall/README.md +++ b/helm/oncall/README.md @@ -21,7 +21,7 @@ helm install \ --wait \ --set base_url=example.com \ --set grafana."grafana\.ini".server.domain=example.com \ - oncall \ + release-oncall \ . ``` @@ -36,7 +36,7 @@ helm upgrade \ --wait \ --set base_url=example.com \ --set grafana."grafana\.ini".server.domain=example.com \ - oncall \ + release-oncall \ . ``` @@ -104,17 +104,17 @@ externalRabbitmq: ## Uninstall ### Uninstalling the helm chart ```bash -helm delete oncall +helm delete release-oncall ``` ### Clean up PVC's ```bash -kubectl delete pvc data-oncall-mariadb-0 data-oncall-rabbitmq-0 \ -redis-data-oncall-redis-master-0 redis-data-oncall-redis-replicas-0 \ -redis-data-oncall-redis-replicas-1 redis-data-oncall-redis-replicas-2 +kubectl delete pvc data-release-oncall-mariadb-0 data-release-oncall-rabbitmq-0 \ +redis-data-release-oncall-redis-master-0 redis-data-release-oncall-redis-replicas-0 \ +redis-data-release-oncall-redis-replicas-1 redis-data-release-oncall-redis-replicas-2 ``` ### Clean up secrets ```bash -kubectl delete secrets certificate-tls oncall-cert-manager-webhook-ca oncall-ingress-nginx-admission +kubectl delete secrets certificate-tls release-oncall-cert-manager-webhook-ca release-oncall-ingress-nginx-admission ``` From 2fc8aa1ce0ac30aa71d9882dfd64911a2c901012 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Tue, 14 Jun 2022 11:58:02 -0600 Subject: [PATCH 04/28] Update README.md (#78) --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 6148dba6..fc40640a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ +[![Latest Release](https://img.shields.io/github/v/release/grafana/oncall?display_name=tag&sort=semver)](https://github.com/grafana/oncall/releases) +[![License](https://img.shields.io/github/license/grafana/oncall)](https://github.com/grafana/oncall/blob/dev/LICENSE) +[![Docker Pulls](https://img.shields.io/docker/pulls/grafana/oncall)](https://hub.docker.com/r/grafana/oncall/tags) +[![Slack](https://img.shields.io/badge/join%20slack-%23grafana-%2Doncall-brightgreen.svg)](https://grafana.slack.com/archives/C02LSUUSE2G) +[![Discussion](https://img.shields.io/badge/discuss-oncall%20forum-orange.svg)](https://github.com/grafana/oncall/discussions) +[![Build Status](https://drone.grafana.net/api/badges/grafana/oncall/status.svg?ref=refs/heads/dev)](https://drone.grafana.net/grafana/oncall) + Developer-friendly incident response with brilliant Slack integration. From dd0e4bf3cd2e6ff4c2164f3f12d134b2fa614120 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Tue, 14 Jun 2022 14:40:57 -0600 Subject: [PATCH 05/28] Update README.md (#81) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fc40640a..604762e3 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Grafana Url: http://grafana:3000 - + ## Further Reading - *Migration from the PagerDuty* - [Migrator](https://github.com/grafana/oncall/tree/dev/tools/pagerduty-migrator) From a865914dca73c338cfe3ebd645d35bbdc5828f34 Mon Sep 17 00:00:00 2001 From: Nate Childers Date: Wed, 15 Jun 2022 05:07:26 -0400 Subject: [PATCH 06/28] Update open-source.md not sure if this should link to dev or main branch but the original url here was a 404 --- docs/sources/open-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/open-source.md b/docs/sources/open-source.md index a95c21eb..51b9eb6a 100644 --- a/docs/sources/open-source.md +++ b/docs/sources/open-source.md @@ -17,7 +17,7 @@ We prepared three environments for OSS users: ## Production Environment -We prepared the helm chart for production environment: https://github.com/grafana/oncall/helm +We prepared the helm chart for production environment: https://github.com/grafana/oncall/tree/dev/helm/oncall ## Slack Setup From e0ff1e29add5bd5c054373fbaa549a2ae708f3fc Mon Sep 17 00:00:00 2001 From: Matvey Kukuy Date: Wed, 15 Jun 2022 12:24:44 +0300 Subject: [PATCH 07/28] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 604762e3..3572b91f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Latest Release](https://img.shields.io/github/v/release/grafana/oncall?display_name=tag&sort=semver)](https://github.com/grafana/oncall/releases) [![License](https://img.shields.io/github/license/grafana/oncall)](https://github.com/grafana/oncall/blob/dev/LICENSE) [![Docker Pulls](https://img.shields.io/docker/pulls/grafana/oncall)](https://hub.docker.com/r/grafana/oncall/tags) -[![Slack](https://img.shields.io/badge/join%20slack-%23grafana-%2Doncall-brightgreen.svg)](https://grafana.slack.com/archives/C02LSUUSE2G) +[![Slack](https://img.shields.io/badge/join%20slack-%23grafana-%2Doncall-brightgreen.svg)](https://slack.grafana.com/) [![Discussion](https://img.shields.io/badge/discuss-oncall%20forum-orange.svg)](https://github.com/grafana/oncall/discussions) [![Build Status](https://drone.grafana.net/api/badges/grafana/oncall/status.svg?ref=refs/heads/dev)](https://drone.grafana.net/grafana/oncall) @@ -60,7 +60,7 @@ Grafana Url: http://grafana:3000 - + ## Further Reading - *Migration from the PagerDuty* - [Migrator](https://github.com/grafana/oncall/tree/dev/tools/pagerduty-migrator) From 0a8649fa0e282f0f1ca3ea64a9168ccd3d1fc2e0 Mon Sep 17 00:00:00 2001 From: Vadim Stepanov Date: Wed, 15 Jun 2022 13:52:25 +0300 Subject: [PATCH 08/28] Fix Telegram message template for AlertManager --- engine/config_integrations/alertmanager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/config_integrations/alertmanager.py b/engine/config_integrations/alertmanager.py index 948927a4..cb2fa9b6 100644 --- a/engine/config_integrations/alertmanager.py +++ b/engine/config_integrations/alertmanager.py @@ -96,7 +96,7 @@ telegram_message = """\ {%- if "status" in payload -%} Status: {{ payload.status }} {% endif -%} -Labels:** {% for k, v in payload["labels"].items() %} +Labels: {% for k, v in payload["labels"].items() %} {{ k }}: {{ v }}{% endfor %} Annotations: {%- for k, v in payload.get("annotations", {}).items() %} @@ -211,7 +211,7 @@ tests = { "title": "KubeJobCompletion", "message": ( "Status: firing\n" - "Labels:** \n" + "Labels: \n" "job: kube-state-metrics\n" "instance: 10.143.139.7:8443\n" "job_name: email-tracking-perform-initialization-1.0.50\n" From 365af4d545beea1e60469c869a3d3379b585b003 Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Wed, 15 Jun 2022 14:51:12 +0100 Subject: [PATCH 09/28] Add CI test to ensure that broken docs links are not merged to dev or main This is the same workflow that gates the publishing of docs to the website in the publish-technical-documentation-*.yml workflows. Signed-off-by: Jack Baldry --- .github/workflows/ci.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38f061d3..84bd05bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,18 @@ jobs: - name: Lint All run: | pre-commit run --all-files - + + test-technical-documentation: + runs-on: ubuntu-latest + steps: + - name: "Check out code" + uses: "actions/checkout@v3" + - name: "Build website" + # -e HUGO_REFLINKSERRORLEVEL=ERROR prevents merging broken refs with the downside + # that no refs to external content can be used as these refs will not resolve in the + # docs-base image. + run: | + docker run -v ${PWD}/docs/sources:/hugo/content/docs/oncall/latest -e HUGO_REFLINKSERRORLEVEL=ERROR --rm grafana/docs-base:latest /bin/bash -c 'make hugo' unit-test-backend: runs-on: ubuntu-latest From faaa9988fcae27cba00f97654fbf4932f9921061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Bauer?= Date: Wed, 15 Jun 2022 16:24:24 +0200 Subject: [PATCH 10/28] fix link to documentation (#92) --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index b6a557c7..2a78a922 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # Grafana Cloud Documentation -Source for documentation at https://grafana.com/docs/amixr/ +Source for documentation at https://grafana.com/docs/oncall/ ## Preview the website From 3a69ddcc706b8bd8fb10d2a8b785734bb51db4d2 Mon Sep 17 00:00:00 2001 From: Matias Bordese Date: Wed, 15 Jun 2022 11:39:12 -0300 Subject: [PATCH 11/28] Add missing slack settings to live settings mapping --- engine/settings/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/engine/settings/base.py b/engine/settings/base.py index baba3861..ad942c30 100644 --- a/engine/settings/base.py +++ b/engine/settings/base.py @@ -350,6 +350,8 @@ SOCIAL_AUTH_SLACK_LOGIN_KEY = SLACK_CLIENT_OAUTH_ID SOCIAL_AUTH_SLACK_LOGIN_SECRET = SLACK_CLIENT_OAUTH_SECRET SOCIAL_AUTH_SETTING_NAME_TO_LIVE_SETTING_NAME = { + "SOCIAL_AUTH_SLACK_LOGIN_KEY": "SLACK_CLIENT_OAUTH_ID", + "SOCIAL_AUTH_SLACK_LOGIN_SECRET": "SLACK_CLIENT_OAUTH_SECRET", "SOCIAL_AUTH_SLACK_INSTALL_FREE_KEY": "SLACK_CLIENT_OAUTH_ID", "SOCIAL_AUTH_SLACK_INSTALL_FREE_SECRET": "SLACK_CLIENT_OAUTH_SECRET", } From 471d29d50d6e0922d96170c2ea1ca623481ecba4 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Wed, 15 Jun 2022 19:13:50 +0300 Subject: [PATCH 12/28] Update uwsgi.ini --- engine/uwsgi.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/uwsgi.ini b/engine/uwsgi.ini index 6f612248..80590f05 100644 --- a/engine/uwsgi.ini +++ b/engine/uwsgi.ini @@ -11,7 +11,6 @@ harakiri=620 max-requests=5000 vacuum=True buffer-size=65535 -listen=1024 http-auto-chunked=True http-timeout=620 post-buffering=1 From e6462938855d7bcc8bdfd328b6ac4e3fb5e42e10 Mon Sep 17 00:00:00 2001 From: Aleksey Date: Thu, 16 Jun 2022 11:39:13 +0400 Subject: [PATCH 13/28] Fix externalRedis for correct template `externalRedis` in https://github.com/grafana/oncall/blob/dev/helm/oncall/templates/_env.tpl#L141 but here `external_redis` --- helm/oncall/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/oncall/values.yaml b/helm/oncall/values.yaml index 6c781718..98b82347 100644 --- a/helm/oncall/values.yaml +++ b/helm/oncall/values.yaml @@ -120,7 +120,7 @@ externalRabbitmq: redis: enabled: true -external_redis: +externalRedis: host: password: From 948ed5ff6c1c46efc8a2c438c560aba809b1d669 Mon Sep 17 00:00:00 2001 From: Aditya C S Date: Thu, 16 Jun 2022 13:36:42 +0530 Subject: [PATCH 14/28] fix(helm): fix password key in external redis secret --- helm/oncall/templates/secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/oncall/templates/secrets.yaml b/helm/oncall/templates/secrets.yaml index 2a1ecba9..88b4eaed 100644 --- a/helm/oncall/templates/secrets.yaml +++ b/helm/oncall/templates/secrets.yaml @@ -38,6 +38,6 @@ metadata: name: {{ include "oncall.fullname" . }}-redis-external type: Opaque data: - rabbitmq-password: {{ required "externalRedis.password is required if not redis.enabled" .Values.externalRedis.password | b64enc | quote }} + redis-password: {{ required "externalRedis.password is required if not redis.enabled" .Values.externalRedis.password | b64enc | quote }} {{- end }} From d7492bb943d8acd2f36cac5ecd451545c0aec56b Mon Sep 17 00:00:00 2001 From: Julia Date: Thu, 16 Jun 2022 17:16:31 +0300 Subject: [PATCH 15/28] Fix creation contact points for grafana alerting integration --- .../grafana_alerting_sync.py | 95 +++++++++++++++---- ...fanaalertingcontactpoint_datasource_uid.py | 18 ++++ .../models/grafana_alerting_contact_point.py | 3 +- .../create_contact_points_for_datasource.py | 35 +++++-- engine/apps/grafana_plugin/helpers/client.py | 8 +- 5 files changed, 133 insertions(+), 26 deletions(-) create mode 100644 engine/apps/alerts/migrations/0003_grafanaalertingcontactpoint_datasource_uid.py diff --git a/engine/apps/alerts/grafana_alerting_sync_manager/grafana_alerting_sync.py b/engine/apps/alerts/grafana_alerting_sync_manager/grafana_alerting_sync.py index a9ca08fb..34d8272a 100644 --- a/engine/apps/alerts/grafana_alerting_sync_manager/grafana_alerting_sync.py +++ b/engine/apps/alerts/grafana_alerting_sync_manager/grafana_alerting_sync.py @@ -54,6 +54,31 @@ class GrafanaAlertingSyncManager: ) return + def alerting_config_with_respect_to_grafana_version( + self, is_grafana_datasource, datasource_id, datasource_uid, client_method, *args + ): + """ Fast fix for deprecated grafana alerting api endpoints""" + + if is_grafana_datasource: + datasource_attr = GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT + config, response_info = client_method(datasource_attr, *args) + else: + # Get config by datasource id for Grafana version < 9 + datasource_attr = datasource_id + config, response_info = client_method(datasource_attr, *args) + + if response_info["status_code"] == status.HTTP_400_BAD_REQUEST: + # Get config by datasource uid for Grafana version >= 9 + datasource_attr = datasource_uid + config, response_info = client_method(datasource_attr, *args) + if config is None: + logger.warning( + f"Got config None in alerting_config_with_respect_to_grafana_version with method " + f"{client_method.__name__} for is_grafana_datasource {is_grafana_datasource} for integration " + f"{self.alert_receive_channel.pk}; response: {response_info}" + ) + return config, response_info + def create_contact_points(self) -> None: """ Get all alertmanager datasources and try to create contact points for them. @@ -84,6 +109,10 @@ class GrafanaAlertingSyncManager: datasources_to_create.append(datasource) if datasources_to_create: + logger.warning( + f"Some contact points were not created for integration {self.alert_receive_channel.pk}, " + f"trying to create async" + ) # create other contact points async schedule_create_contact_points_for_datasource(self.alert_receive_channel.pk, datasources_to_create) else: @@ -98,13 +127,14 @@ class GrafanaAlertingSyncManager: if datasource is None: datasource = {} - datasource_id_or_grafana = datasource.get("id") or GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT datasource_type = datasource.get("type") or GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT is_grafana_datasource = datasource.get("id") is None logger.info( f"Create contact point for {datasource_type} datasource, integration {self.alert_receive_channel.pk}" ) - config, response_info = self.client.get_alerting_config(datasource_id_or_grafana) + config, response_info = self.alerting_config_with_respect_to_grafana_version( + is_grafana_datasource, datasource.get("id"), datasource.get("uid"), self.client.get_alerting_config + ) if config is None: logger.warning( @@ -116,7 +146,12 @@ class GrafanaAlertingSyncManager: updated_config = copy.deepcopy(config) if config["alertmanager_config"] is None: - default_config, response_info = self.client.get_alertmanager_status_with_config(datasource_id_or_grafana) + default_config, response_info = self.alerting_config_with_respect_to_grafana_version( + is_grafana_datasource, + datasource.get("id"), + datasource.get("uid"), + self.client.get_alertmanager_status_with_config, + ) if default_config is None: logger.warning( f"Failed to create contact point (alertmanager_config is None) for integration " @@ -144,7 +179,13 @@ class GrafanaAlertingSyncManager: ) updated_config["alertmanager_config"]["receivers"] = receivers + [new_receiver] - response, response_info = self.client.update_alerting_config(updated_config, datasource_id_or_grafana) + response, response_info = self.alerting_config_with_respect_to_grafana_version( + is_grafana_datasource, + datasource.get("id"), + datasource.get("uid"), + self.client.update_alerting_config, + updated_config, + ) if response is None: logger.warning( f"Failed to create contact point for integration {self.alert_receive_channel.pk} (POST): {response_info}" @@ -153,7 +194,9 @@ class GrafanaAlertingSyncManager: logger.warning(f"Config: {config}\nUpdated config: {updated_config}") return - config, response_info = self.client.get_alerting_config(datasource_id_or_grafana) + config, response_info = self.alerting_config_with_respect_to_grafana_version( + is_grafana_datasource, datasource.get("id"), datasource.get("uid"), self.client.get_alerting_config + ) contact_point = self._create_contact_point_from_payload(config, receiver_name, datasource) contact_point_created_text = "created" if contact_point else "not created, creation will be retried" logger.info( @@ -232,6 +275,7 @@ class GrafanaAlertingSyncManager: uid=receiver_config.get("uid"), # uid is None for non-Grafana datasource datasource_name=datasource.get("name") or GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT, datasource_id=datasource.get("id"), # id is None for Grafana datasource + datasource_uid=datasource.get("uid"), # uid is None for Grafana datasource ) contact_point.save() return contact_point @@ -268,14 +312,21 @@ class GrafanaAlertingSyncManager: def sync_contact_point(self, contact_point) -> None: """Update name of contact point and related routes or delete it if integration was deleted""" - datasource_id = contact_point.datasource_id or GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT - datasource_type = "grafana" if not contact_point.datasource_id else "nongrafana" + datasource_type = GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT \ + if not (contact_point.datasource_id or contact_point.datasource_uid) \ + else "nongrafana" + is_grafana_datasource = datasource_type == GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT logger.info( f"Sync contact point for {datasource_type} (name: {contact_point.datasource_name}) datasource, integration " f"{self.alert_receive_channel.pk}" ) - config, response_info = self.client.get_alerting_config(datasource_id) + config, response_info = self.alerting_config_with_respect_to_grafana_version( + is_grafana_datasource, + contact_point.datasource_id, + contact_point.datasource_uid, + self.client.get_alerting_config, + ) if config is None: logger.warning( f"Failed to update contact point (GET) for integration {self.alert_receive_channel.pk}: Is unified " @@ -286,7 +337,7 @@ class GrafanaAlertingSyncManager: receivers = config["alertmanager_config"]["receivers"] name_in_alerting = self.find_name_of_contact_point( contact_point.uid, - datasource_id, + is_grafana_datasource, receivers, ) @@ -300,8 +351,8 @@ class GrafanaAlertingSyncManager: new_name, ) contact_point.name = new_name - if datasource_id != GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT: - datasource_name = self.get_datasource_name(datasource_id) + if not is_grafana_datasource: + datasource_name = self.get_datasource_name(contact_point) contact_point.datasource_name = datasource_name contact_point.save(update_fields=["name", "datasource_name"]) # if integration was deleted, delete contact point and related routes @@ -310,8 +361,13 @@ class GrafanaAlertingSyncManager: updated_config, name_in_alerting, ) - - response, response_info = self.client.update_alerting_config(updated_config, datasource_id) + response, response_info = self.alerting_config_with_respect_to_grafana_version( + is_grafana_datasource, + contact_point.datasource_id, + contact_point.datasource_uid, + self.client.update_alerting_config, + updated_config, + ) if response is None: logger.warning( f"Failed to update contact point for integration {self.alert_receive_channel.pk} " @@ -379,8 +435,8 @@ class GrafanaAlertingSyncManager: return alerting_route - def find_name_of_contact_point(self, contact_point_uid, datasource_id, receivers) -> str: - if datasource_id == GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT: + def find_name_of_contact_point(self, contact_point_uid, is_grafana_datasource, receivers) -> str: + if is_grafana_datasource: name_in_alerting = self._find_name_of_contact_point_by_uid(contact_point_uid, receivers) else: name_in_alerting = self._find_name_of_contact_point_by_integration_url(receivers) @@ -415,6 +471,11 @@ class GrafanaAlertingSyncManager: break return name_in_alerting - def get_datasource_name(self, datasource_id) -> str: - datasource, _ = self.client.get_datasource(datasource_id) + def get_datasource_name(self, contact_point) -> str: + datasource_id = contact_point.datasource_id + datasource_uid = contact_point.datasource_uid + datasource, response_info = self.client.get_datasource(datasource_uid) + if response_info["status_code"] != 200: + # For old Grafana versions (< 9) try to use deprecated endpoint + datasource, _ = self.client.get_datasource_by_id(datasource_id) return datasource["name"] diff --git a/engine/apps/alerts/migrations/0003_grafanaalertingcontactpoint_datasource_uid.py b/engine/apps/alerts/migrations/0003_grafanaalertingcontactpoint_datasource_uid.py new file mode 100644 index 00000000..4bdcec63 --- /dev/null +++ b/engine/apps/alerts/migrations/0003_grafanaalertingcontactpoint_datasource_uid.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-06-14 15:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('alerts', '0002_squashed_initial'), + ] + + operations = [ + migrations.AddField( + model_name='grafanaalertingcontactpoint', + name='datasource_uid', + field=models.CharField(default=None, max_length=100, null=True), + ), + ] diff --git a/engine/apps/alerts/models/grafana_alerting_contact_point.py b/engine/apps/alerts/models/grafana_alerting_contact_point.py index d4cee24c..00f28981 100644 --- a/engine/apps/alerts/models/grafana_alerting_contact_point.py +++ b/engine/apps/alerts/models/grafana_alerting_contact_point.py @@ -16,7 +16,8 @@ class GrafanaAlertingContactPoint(models.Model): default=None, related_name="contact_points", ) - uid = models.CharField(max_length=100, null=True, default=None) # uid is None for non-Grafana datasource + uid = models.CharField(max_length=100, null=True, default=None) # receiver uid is None for non-Grafana datasource name = models.CharField(max_length=100) datasource_name = models.CharField(max_length=100, default="grafana") datasource_id = models.IntegerField(null=True, default=None) # id is None for Grafana datasource + datasource_uid = models.CharField(max_length=100, null=True, default=None) # uid is None for Grafana datasource diff --git a/engine/apps/alerts/tasks/create_contact_points_for_datasource.py b/engine/apps/alerts/tasks/create_contact_points_for_datasource.py index a447a39c..7532d187 100644 --- a/engine/apps/alerts/tasks/create_contact_points_for_datasource.py +++ b/engine/apps/alerts/tasks/create_contact_points_for_datasource.py @@ -42,7 +42,14 @@ def create_contact_points_for_datasource(alert_receive_channel_id, datasource_li AlertReceiveChannel = apps.get_model("alerts", "AlertReceiveChannel") - alert_receive_channel = AlertReceiveChannel.objects.get(pk=alert_receive_channel_id) + alert_receive_channel = AlertReceiveChannel.objects.filter(pk=alert_receive_channel_id).first() + if not alert_receive_channel: + logger.debug( + f"Cannot create contact point for integration {alert_receive_channel_id}: integration does not exist" + ) + return + + grafana_alerting_sync_manager = alert_receive_channel.grafana_alerting_sync_manager client = GrafanaAPIClient( api_url=alert_receive_channel.organization.grafana_url, @@ -52,11 +59,23 @@ def create_contact_points_for_datasource(alert_receive_channel_id, datasource_li datasources_to_create = [] for datasource in datasource_list: contact_point = None - config, response_info = client.get_alerting_config(datasource["id"]) + is_grafana_datasource = not (datasource.get("id") or datasource.get("uid")) + config, response_info = grafana_alerting_sync_manager.alerting_config_with_respect_to_grafana_version( + is_grafana_datasource, datasource.get("id"), datasource.get("uid"), client.get_alerting_config + ) if config is None: + logger.debug( + f"Got config None for is_grafana_datasource {is_grafana_datasource} " + f"for integration {alert_receive_channel_id}; response: {response_info}" + ) if response_info.get("status_code") == status.HTTP_404_NOT_FOUND: - client.get_alertmanager_status_with_config(datasource["id"]) - contact_point = alert_receive_channel.grafana_alerting_sync_manager.create_contact_point(datasource) + grafana_alerting_sync_manager.alerting_config_with_respect_to_grafana_version( + is_grafana_datasource, + datasource.get("id"), + datasource.get("uid"), + client.get_alertmanager_status_with_config, + ) + contact_point = grafana_alerting_sync_manager.create_contact_point(datasource) elif response_info.get("status_code") == status.HTTP_400_BAD_REQUEST: logger.warning( f"Failed to create contact point for integration {alert_receive_channel_id}, " @@ -64,9 +83,13 @@ def create_contact_points_for_datasource(alert_receive_channel_id, datasource_li ) continue else: - contact_point = alert_receive_channel.grafana_alerting_sync_manager.create_contact_point(datasource) + contact_point = grafana_alerting_sync_manager.create_contact_point(datasource) if contact_point is None: - # Failed to create contact point duo to getting wrong alerting config. + logger.warning( + f"Failed to create contact point for integration {alert_receive_channel_id} due to getting wrong " + f"config, datasource info: {datasource}; response: {response_info}. Retrying" + ) + # Failed to create contact point due to getting wrong alerting config. # Add datasource to list and retry to create contact point for it again datasources_to_create.append(datasource) diff --git a/engine/apps/grafana_plugin/helpers/client.py b/engine/apps/grafana_plugin/helpers/client.py index bb4586da..7c864383 100644 --- a/engine/apps/grafana_plugin/helpers/client.py +++ b/engine/apps/grafana_plugin/helpers/client.py @@ -103,16 +103,20 @@ class GrafanaAPIClient(APIClient): def get_datasources(self): return self.api_get("api/datasources") - def get_datasource(self, datasource_id): + def get_datasource_by_id(self, datasource_id): + # This endpoint is deprecated for Grafana version >= 9. Use get_datasource instead return self.api_get(f"api/datasources/{datasource_id}") + def get_datasource(self, datasource_uid): + return self.api_get(f"api/datasources/uid/{datasource_uid}") + def get_alertmanager_status_with_config(self, recipient): return self.api_get(f"api/alertmanager/{recipient}/api/v2/status") def get_alerting_config(self, recipient): return self.api_get(f"api/alertmanager/{recipient}/config/api/v1/alerts") - def update_alerting_config(self, config, recipient): + def update_alerting_config(self, recipient, config): return self.api_post(f"api/alertmanager/{recipient}/config/api/v1/alerts", config) From 49116519b5f3fd46f902af6258a90bbe392fc30b Mon Sep 17 00:00:00 2001 From: Julia Date: Thu, 16 Jun 2022 17:30:34 +0300 Subject: [PATCH 16/28] lint fix --- .../grafana_alerting_sync.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/engine/apps/alerts/grafana_alerting_sync_manager/grafana_alerting_sync.py b/engine/apps/alerts/grafana_alerting_sync_manager/grafana_alerting_sync.py index 34d8272a..d72524bf 100644 --- a/engine/apps/alerts/grafana_alerting_sync_manager/grafana_alerting_sync.py +++ b/engine/apps/alerts/grafana_alerting_sync_manager/grafana_alerting_sync.py @@ -55,9 +55,9 @@ class GrafanaAlertingSyncManager: return def alerting_config_with_respect_to_grafana_version( - self, is_grafana_datasource, datasource_id, datasource_uid, client_method, *args + self, is_grafana_datasource, datasource_id, datasource_uid, client_method, *args ): - """ Fast fix for deprecated grafana alerting api endpoints""" + """Quick fix for deprecated grafana alerting api endpoints""" if is_grafana_datasource: datasource_attr = GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT @@ -312,9 +312,11 @@ class GrafanaAlertingSyncManager: def sync_contact_point(self, contact_point) -> None: """Update name of contact point and related routes or delete it if integration was deleted""" - datasource_type = GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT \ - if not (contact_point.datasource_id or contact_point.datasource_uid) \ + datasource_type = ( + GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT + if not (contact_point.datasource_id or contact_point.datasource_uid) else "nongrafana" + ) is_grafana_datasource = datasource_type == GrafanaAlertingSyncManager.GRAFANA_CONTACT_POINT logger.info( f"Sync contact point for {datasource_type} (name: {contact_point.datasource_name}) datasource, integration " From ca9f907f86c91d871a9f34074f2cf8800626737a Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Fri, 17 Jun 2022 14:08:09 +0300 Subject: [PATCH 17/28] Add restart always policy to the services --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 3642e441..894b26fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: engine: image: grafana/oncall + restart: always ports: - 8080:8080 command: > @@ -35,6 +36,7 @@ services: celery: # TODO: change to the public image once it's public image: grafana/oncall + restart: always command: sh -c "./celery_with_exporter.sh" environment: BASE_URL: $DOMAIN @@ -122,6 +124,7 @@ services: rabbitmq: image: "rabbitmq:3.7.15-management" + restart: always hostname: rabbitmq mem_limit: 1000m cpus: 0.5 @@ -144,6 +147,7 @@ services: grafana: image: "grafana/grafana:9.0.0-beta3" + restart: always mem_limit: 500m ports: - 3000:3000 From f04f4eaa3fb29e78998c4fc741390ed66b5eb305 Mon Sep 17 00:00:00 2001 From: Julia Date: Fri, 17 Jun 2022 15:34:59 +0300 Subject: [PATCH 18/28] Update public endpoint for outgoing webhooks - Add abilities to create, update and delete outgoing webhooks by public api endpoint --- engine/apps/public_api/serializers/action.py | 75 +++++++++++++++++++- engine/apps/public_api/views/action.py | 37 ++++++++-- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/engine/apps/public_api/serializers/action.py b/engine/apps/public_api/serializers/action.py index db202b22..963aacbc 100644 --- a/engine/apps/public_api/serializers/action.py +++ b/engine/apps/public_api/serializers/action.py @@ -1,17 +1,88 @@ +import json + +from django.core.validators import URLValidator, ValidationError +from jinja2 import Template, TemplateError from rest_framework import serializers +from rest_framework.validators import UniqueTogetherValidator from apps.alerts.models import CustomButton from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField +from common.api_helpers.utils import CurrentOrganizationDefault -class ActionSerializer(serializers.ModelSerializer): +class ActionCreateSerializer(serializers.ModelSerializer): id = serializers.CharField(read_only=True, source="public_primary_key") - team_id = TeamPrimaryKeyRelatedField(allow_null=True, source="team") + organization = serializers.HiddenField(default=CurrentOrganizationDefault()) + team_id = TeamPrimaryKeyRelatedField(required=False, allow_null=True, source="team") class Meta: model = CustomButton fields = [ "id", "name", + "organization", "team_id", + "webhook", + "data", + "user", + "password", + "authorization_header", + "forward_whole_payload", ] + extra_kwargs = { + "name": {"required": True, "allow_null": False, "allow_blank": False}, + "webhook": {"required": True, "allow_null": False, "allow_blank": False}, + "data": {"required": False, "allow_null": True, "allow_blank": False}, + "user": {"required": False, "allow_null": True, "allow_blank": False}, + "password": {"required": False, "allow_null": True, "allow_blank": False}, + "authorization_header": {"required": False, "allow_null": True, "allow_blank": False}, + "forward_whole_payload": {"required": False, "allow_null": True}, + } + + validators = [UniqueTogetherValidator(queryset=CustomButton.objects.all(), fields=["name", "organization"])] + + def validate_webhook(self, webhook): + if webhook: + try: + URLValidator()(webhook) + except ValidationError: + raise serializers.ValidationError("Webhook is incorrect") + return webhook + return None + + def validate_data(self, data): + if not data: + return None + + try: + json.loads(data) + except ValueError: + raise serializers.ValidationError("Data has incorrect format") + + try: + Template(data) + except TemplateError: + raise serializers.ValidationError("Data has incorrect template") + + return data + + def validate_forward_whole_payload(self, data): + if data is None: + return False + return data + + +class ActionUpdateSerializer(ActionCreateSerializer): + team_id = TeamPrimaryKeyRelatedField(source="team", read_only=True) + + class Meta(ActionCreateSerializer.Meta): + + extra_kwargs = { + "name": {"required": False, "allow_null": False, "allow_blank": False}, + "webhook": {"required": False, "allow_null": False, "allow_blank": False}, + "data": {"required": False, "allow_null": True, "allow_blank": False}, + "user": {"required": False, "allow_null": True, "allow_blank": False}, + "password": {"required": False, "allow_null": True, "allow_blank": False}, + "authorization_header": {"required": False, "allow_null": True, "allow_blank": False}, + "forward_whole_payload": {"required": False, "allow_null": True}, + } diff --git a/engine/apps/public_api/views/action.py b/engine/apps/public_api/views/action.py index 60ca1465..0e5944eb 100644 --- a/engine/apps/public_api/views/action.py +++ b/engine/apps/public_api/views/action.py @@ -1,25 +1,26 @@ from django_filters import rest_framework as filters -from rest_framework import mixins from rest_framework.permissions import IsAuthenticated -from rest_framework.viewsets import GenericViewSet +from rest_framework.viewsets import ModelViewSet from apps.alerts.models import CustomButton from apps.auth_token.auth import ApiTokenAuthentication -from apps.public_api.serializers.action import ActionSerializer +from apps.public_api.serializers.action import ActionCreateSerializer, ActionUpdateSerializer from apps.public_api.throttlers.user_throttle import UserThrottle +from apps.user_management.organization_log_creator import OrganizationLogType, create_organization_log from common.api_helpers.filters import ByTeamFilter -from common.api_helpers.mixins import RateLimitHeadersMixin +from common.api_helpers.mixins import PublicPrimaryKeyMixin, RateLimitHeadersMixin, UpdateSerializerMixin from common.api_helpers.paginators import FiftyPageSizePaginator -class ActionView(RateLimitHeadersMixin, mixins.ListModelMixin, GenericViewSet): +class ActionView(RateLimitHeadersMixin, PublicPrimaryKeyMixin, UpdateSerializerMixin, ModelViewSet): authentication_classes = (ApiTokenAuthentication,) permission_classes = (IsAuthenticated,) pagination_class = FiftyPageSizePaginator throttle_classes = [UserThrottle] model = CustomButton - serializer_class = ActionSerializer + serializer_class = ActionCreateSerializer + update_serializer_class = ActionUpdateSerializer filter_backends = (filters.DjangoFilterBackend,) filterset_class = ByTeamFilter @@ -32,3 +33,27 @@ class ActionView(RateLimitHeadersMixin, mixins.ListModelMixin, GenericViewSet): queryset = queryset.filter(name=action_name) return queryset + + def perform_create(self, serializer): + serializer.save() + instance = serializer.instance + organization = self.request.auth.organization + user = self.request.user + description = f"Custom action {instance.name} was created" + create_organization_log(organization, user, OrganizationLogType.TYPE_CUSTOM_ACTION_CREATED, description) + + def perform_update(self, serializer): + organization = self.request.auth.organization + user = self.request.user + old_state = serializer.instance.repr_settings_for_client_side_logging + serializer.save() + new_state = serializer.instance.repr_settings_for_client_side_logging + description = f"Custom action {serializer.instance.name} was changed " f"from:\n{old_state}\nto:\n{new_state}" + create_organization_log(organization, user, OrganizationLogType.TYPE_CUSTOM_ACTION_CHANGED, description) + + def perform_destroy(self, instance): + organization = self.request.auth.organization + user = self.request.user + description = f"Custom action {instance.name} was deleted" + create_organization_log(organization, user, OrganizationLogType.TYPE_CUSTOM_ACTION_DELETED, description) + instance.delete() From 43bc8c2fe5366930004ae6821e00765e6a004dda Mon Sep 17 00:00:00 2001 From: Julia Date: Fri, 17 Jun 2022 15:41:46 +0300 Subject: [PATCH 19/28] Add tests for outgoing webhooks public api endpoint --- .../public_api/tests/test_custom_actions.py | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) diff --git a/engine/apps/public_api/tests/test_custom_actions.py b/engine/apps/public_api/tests/test_custom_actions.py index 2fc39f92..ee0e5f67 100644 --- a/engine/apps/public_api/tests/test_custom_actions.py +++ b/engine/apps/public_api/tests/test_custom_actions.py @@ -3,6 +3,8 @@ from django.urls import reverse from rest_framework import status from rest_framework.test import APIClient +from apps.alerts.models import CustomButton + @pytest.mark.django_db def test_get_custom_actions( @@ -28,6 +30,12 @@ def test_get_custom_actions( "id": custom_action.public_primary_key, "name": custom_action.name, "team_id": None, + "webhook": custom_action.webhook, + "data": custom_action.data, + "user": custom_action.user, + "password": custom_action.password, + "authorization_header": custom_action.authorization_header, + "forward_whole_payload": custom_action.forward_whole_payload, } ], } @@ -60,6 +68,12 @@ def test_get_custom_actions_filter_by_name( "id": custom_action.public_primary_key, "name": custom_action.name, "team_id": None, + "webhook": custom_action.webhook, + "data": custom_action.data, + "user": custom_action.user, + "password": custom_action.password, + "authorization_header": custom_action.authorization_header, + "forward_whole_payload": custom_action.forward_whole_payload, } ], } @@ -87,3 +101,171 @@ def test_get_custom_actions_filter_by_name_empty_result( assert response.status_code == status.HTTP_200_OK assert response.data == expected_payload + + +@pytest.mark.django_db +def test_get_custom_action( + make_organization_and_user_with_token, + make_custom_action, +): + + organization, user, token = make_organization_and_user_with_token() + client = APIClient() + + custom_action = make_custom_action(organization=organization) + + url = reverse("api-public:actions-detail", kwargs={"pk": custom_action.public_primary_key}) + + response = client.get(url, format="json", HTTP_AUTHORIZATION=f"{token}") + + expected_payload = { + "id": custom_action.public_primary_key, + "name": custom_action.name, + "team_id": None, + "webhook": custom_action.webhook, + "data": custom_action.data, + "user": custom_action.user, + "password": custom_action.password, + "authorization_header": custom_action.authorization_header, + "forward_whole_payload": custom_action.forward_whole_payload, + } + + assert response.status_code == status.HTTP_200_OK + assert response.data == expected_payload + + +@pytest.mark.django_db +def test_create_custom_action(make_organization_and_user_with_token): + + organization, user, token = make_organization_and_user_with_token() + client = APIClient() + + url = reverse("api-public:actions-list") + + data = { + "name": "Test outgoing webhook", + "webhook": "https://example.com", + } + + response = client.post(url, data=data, format="json", HTTP_AUTHORIZATION=f"{token}") + + custom_action = CustomButton.objects.get(public_primary_key=response.data["id"]) + + expected_result = { + "id": custom_action.public_primary_key, + "name": custom_action.name, + "team_id": None, + "webhook": custom_action.webhook, + "data": custom_action.data, + "user": custom_action.user, + "password": custom_action.password, + "authorization_header": custom_action.authorization_header, + "forward_whole_payload": custom_action.forward_whole_payload, + } + + assert response.status_code == status.HTTP_201_CREATED + assert response.data == expected_result + + +@pytest.mark.django_db +def test_create_custom_action_invalid_data( + make_organization_and_user_with_token, +): + + organization, user, token = make_organization_and_user_with_token() + client = APIClient() + + url = reverse("api-public:actions-list") + + data = { + "name": "Test outgoing webhook", + "webhook": "invalid_url", + } + + response = client.post(url, data=data, format="json", HTTP_AUTHORIZATION=f"{token}") + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.data["webhook"][0] == "Webhook is incorrect" + + data = { + "name": "Test outgoing webhook", + } + + response = client.post(url, data=data, format="json", HTTP_AUTHORIZATION=f"{token}") + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.data["webhook"][0] == "This field is required." + + data = { + "webhook": "https://example.com", + } + + response = client.post(url, data=data, format="json", HTTP_AUTHORIZATION=f"{token}") + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.data["name"][0] == "This field is required." + + +@pytest.mark.django_db +def test_update_custom_action( + make_organization_and_user_with_token, + make_custom_action, +): + + organization, user, token = make_organization_and_user_with_token() + client = APIClient() + + custom_action = make_custom_action(organization=organization) + + url = reverse("api-public:actions-detail", kwargs={"pk": custom_action.public_primary_key}) + + data = { + "name": "RENAMED", + } + + assert custom_action.name != data["name"] + + response = client.put(url, data=data, format="json", HTTP_AUTHORIZATION=f"{token}") + + expected_result = { + "id": custom_action.public_primary_key, + "name": data["name"], + "team_id": None, + "webhook": custom_action.webhook, + "data": custom_action.data, + "user": custom_action.user, + "password": custom_action.password, + "authorization_header": custom_action.authorization_header, + "forward_whole_payload": custom_action.forward_whole_payload, + } + + assert response.status_code == status.HTTP_200_OK + custom_action.refresh_from_db() + assert custom_action.name == expected_result["name"] + assert response.data == expected_result + + +@pytest.mark.django_db +def test_delete_custom_action( + make_organization_and_user_with_token, + make_custom_action, +): + + organization, user, token = make_organization_and_user_with_token() + client = APIClient() + + custom_action = make_custom_action(organization=organization) + url = reverse("api-public:actions-detail", kwargs={"pk": custom_action.public_primary_key}) + + assert custom_action.deleted_at is None + + response = client.delete(url, format="json", HTTP_AUTHORIZATION=f"{token}") + assert response.status_code == status.HTTP_204_NO_CONTENT + + custom_action.refresh_from_db() + assert custom_action.deleted_at is not None + + response = client.get(url, format="json", HTTP_AUTHORIZATION=f"{token}") + + assert response.status_code == status.HTTP_404_NOT_FOUND + assert response.data["detail"] == "Not found." From 856c6d5302d5f05b62693feb062e2b5c70f3b941 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Fri, 17 Jun 2022 10:23:38 -0600 Subject: [PATCH 20/28] Update changelog for v1.0.2 (#107) --- CHANGELOG.md | 11 ++++++++++- grafana-plugin/CHANGELOG.md | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8893332c..6711b92e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Change Log +## 1.0.2 (2022-06-17) + +- Fix Grafana Alerting integration to handle API changes in Grafana 9 +- Improve public api endpoint for for outgoing webhooks (/actions) by adding ability to create, update and delete outgoing webhook instance + +## 1.0.0 (2022-06-14) + +- First Public Release + ## 0.0.71 (2022-06-06) -- Initial Release \ No newline at end of file +- Initial Commit Release \ No newline at end of file diff --git a/grafana-plugin/CHANGELOG.md b/grafana-plugin/CHANGELOG.md index 8893332c..e48e4082 100644 --- a/grafana-plugin/CHANGELOG.md +++ b/grafana-plugin/CHANGELOG.md @@ -1,5 +1,14 @@ # Change Log +## 1.0.2 (2022-06-17) + +- Fix Grafana Alerting integration to handle API changes in Grafana 9 +- Improve public API endpoint for outgoing webhooks (/actions) by adding ability to create, update and delete + +## 1.0.0 (2022-06-14) + +- First Public Release + ## 0.0.71 (2022-06-06) -- Initial Release \ No newline at end of file +- Initial Commit Release \ No newline at end of file From c307bc56ce044ca41f26480537906d4c08375178 Mon Sep 17 00:00:00 2001 From: Alexandre Chaussier Date: Sat, 18 Jun 2022 15:40:58 +0200 Subject: [PATCH 21/28] fix: fix ingress-nginx dependency management in values.yaml --- helm/oncall/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/oncall/values.yaml b/helm/oncall/values.yaml index 6c781718..7ebecdc9 100644 --- a/helm/oncall/values.yaml +++ b/helm/oncall/values.yaml @@ -55,7 +55,7 @@ ingress: cert-manager.io/issuer: "letsencrypt-prod" # Whether to install ingress controller -nginx-ingress: +ingress-nginx: enabled: true # Install cert-manager as a part of the release From ce982ae1c2ce4b022aa0ab5615d956bb40925336 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Mon, 20 Jun 2022 09:29:37 -0600 Subject: [PATCH 22/28] Tweak docker-compose (#104) * Remove env var causing celery container to exit, put containers in their own network * Remove unnecessary network, remove version since we are mixing, make DB and redis ports internal * Restore property for CELERY_WORKER_SHUTDOWN_INVERVAL since restart policy added --- docker-compose.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 894b26fe..bf44777b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,6 @@ services: condition: service_started celery: - # TODO: change to the public image once it's public image: grafana/oncall restart: always command: sh -c "./celery_with_exporter.sh" @@ -102,8 +101,8 @@ services: cpus: 0.5 command: --default-authentication-plugin=mysql_native_password --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci restart: always - ports: - - 3306:3306 + expose: + - 3306 volumes: - dbdata:/var/lib/mysql environment: @@ -119,8 +118,8 @@ services: mem_limit: 100m cpus: 0.1 restart: always - ports: - - 6379:6379 + expose: + - 6379 rabbitmq: image: "rabbitmq:3.7.15-management" From a4d9bc99a8efe639c45c303e2733179b441bbd51 Mon Sep 17 00:00:00 2001 From: Julia Date: Tue, 21 Jun 2022 12:53:22 +0300 Subject: [PATCH 23/28] Rename field `webhook` to `url` for outgoing webhook public api endpoint, update tests --- engine/apps/public_api/serializers/action.py | 16 +++++++-------- .../public_api/tests/test_custom_actions.py | 20 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/engine/apps/public_api/serializers/action.py b/engine/apps/public_api/serializers/action.py index 963aacbc..f652bc7c 100644 --- a/engine/apps/public_api/serializers/action.py +++ b/engine/apps/public_api/serializers/action.py @@ -14,6 +14,7 @@ class ActionCreateSerializer(serializers.ModelSerializer): id = serializers.CharField(read_only=True, source="public_primary_key") organization = serializers.HiddenField(default=CurrentOrganizationDefault()) team_id = TeamPrimaryKeyRelatedField(required=False, allow_null=True, source="team") + url = serializers.CharField(required=True, allow_null=False, allow_blank=False, source="webhook") class Meta: model = CustomButton @@ -22,7 +23,7 @@ class ActionCreateSerializer(serializers.ModelSerializer): "name", "organization", "team_id", - "webhook", + "url", "data", "user", "password", @@ -31,7 +32,6 @@ class ActionCreateSerializer(serializers.ModelSerializer): ] extra_kwargs = { "name": {"required": True, "allow_null": False, "allow_blank": False}, - "webhook": {"required": True, "allow_null": False, "allow_blank": False}, "data": {"required": False, "allow_null": True, "allow_blank": False}, "user": {"required": False, "allow_null": True, "allow_blank": False}, "password": {"required": False, "allow_null": True, "allow_blank": False}, @@ -41,13 +41,13 @@ class ActionCreateSerializer(serializers.ModelSerializer): validators = [UniqueTogetherValidator(queryset=CustomButton.objects.all(), fields=["name", "organization"])] - def validate_webhook(self, webhook): - if webhook: + def validate_url(self, url): + if url: try: - URLValidator()(webhook) + URLValidator()(url) except ValidationError: - raise serializers.ValidationError("Webhook is incorrect") - return webhook + raise serializers.ValidationError("URL is incorrect") + return url return None def validate_data(self, data): @@ -74,12 +74,12 @@ class ActionCreateSerializer(serializers.ModelSerializer): class ActionUpdateSerializer(ActionCreateSerializer): team_id = TeamPrimaryKeyRelatedField(source="team", read_only=True) + url = serializers.CharField(required=False, allow_null=False, allow_blank=False, source="webhook") class Meta(ActionCreateSerializer.Meta): extra_kwargs = { "name": {"required": False, "allow_null": False, "allow_blank": False}, - "webhook": {"required": False, "allow_null": False, "allow_blank": False}, "data": {"required": False, "allow_null": True, "allow_blank": False}, "user": {"required": False, "allow_null": True, "allow_blank": False}, "password": {"required": False, "allow_null": True, "allow_blank": False}, diff --git a/engine/apps/public_api/tests/test_custom_actions.py b/engine/apps/public_api/tests/test_custom_actions.py index ee0e5f67..9fb4ebb6 100644 --- a/engine/apps/public_api/tests/test_custom_actions.py +++ b/engine/apps/public_api/tests/test_custom_actions.py @@ -30,7 +30,7 @@ def test_get_custom_actions( "id": custom_action.public_primary_key, "name": custom_action.name, "team_id": None, - "webhook": custom_action.webhook, + "url": custom_action.webhook, "data": custom_action.data, "user": custom_action.user, "password": custom_action.password, @@ -68,7 +68,7 @@ def test_get_custom_actions_filter_by_name( "id": custom_action.public_primary_key, "name": custom_action.name, "team_id": None, - "webhook": custom_action.webhook, + "url": custom_action.webhook, "data": custom_action.data, "user": custom_action.user, "password": custom_action.password, @@ -122,7 +122,7 @@ def test_get_custom_action( "id": custom_action.public_primary_key, "name": custom_action.name, "team_id": None, - "webhook": custom_action.webhook, + "url": custom_action.webhook, "data": custom_action.data, "user": custom_action.user, "password": custom_action.password, @@ -144,7 +144,7 @@ def test_create_custom_action(make_organization_and_user_with_token): data = { "name": "Test outgoing webhook", - "webhook": "https://example.com", + "url": "https://example.com", } response = client.post(url, data=data, format="json", HTTP_AUTHORIZATION=f"{token}") @@ -155,7 +155,7 @@ def test_create_custom_action(make_organization_and_user_with_token): "id": custom_action.public_primary_key, "name": custom_action.name, "team_id": None, - "webhook": custom_action.webhook, + "url": custom_action.webhook, "data": custom_action.data, "user": custom_action.user, "password": custom_action.password, @@ -179,13 +179,13 @@ def test_create_custom_action_invalid_data( data = { "name": "Test outgoing webhook", - "webhook": "invalid_url", + "url": "invalid_url", } response = client.post(url, data=data, format="json", HTTP_AUTHORIZATION=f"{token}") assert response.status_code == status.HTTP_400_BAD_REQUEST - assert response.data["webhook"][0] == "Webhook is incorrect" + assert response.data["url"][0] == "URL is incorrect" data = { "name": "Test outgoing webhook", @@ -194,10 +194,10 @@ def test_create_custom_action_invalid_data( response = client.post(url, data=data, format="json", HTTP_AUTHORIZATION=f"{token}") assert response.status_code == status.HTTP_400_BAD_REQUEST - assert response.data["webhook"][0] == "This field is required." + assert response.data["url"][0] == "This field is required." data = { - "webhook": "https://example.com", + "url": "https://example.com", } response = client.post(url, data=data, format="json", HTTP_AUTHORIZATION=f"{token}") @@ -231,7 +231,7 @@ def test_update_custom_action( "id": custom_action.public_primary_key, "name": data["name"], "team_id": None, - "webhook": custom_action.webhook, + "url": custom_action.webhook, "data": custom_action.data, "user": custom_action.user, "password": custom_action.password, From 1fc68ec871ce0718b00f9fd1c34510f7cae56cc4 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 21 Jun 2022 13:13:42 +0300 Subject: [PATCH 24/28] Release helm chart from grafana/oncall to grafana/helm-charts using common workflow --- .github/workflows/helm_release.yml | 17 +++++++++++++++++ helm/cr.yaml | 5 +++++ helm/oncall/Chart.yaml | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/helm_release.yml create mode 100644 helm/cr.yaml diff --git a/.github/workflows/helm_release.yml b/.github/workflows/helm_release.yml new file mode 100644 index 00000000..de0cca58 --- /dev/null +++ b/.github/workflows/helm_release.yml @@ -0,0 +1,17 @@ +name: helm-release + +on: + push: + branches: + - main + +jobs: + call-update-helm-repo: + uses: grafana/helm-charts/.github/workflows/update-helm-repo.yaml@main + with: + charts_dir: helm/ + cr_configfile: helm/cr.yaml + secrets: + helm_repo_token: ${{ secrets.GH_BOT_ACCESS_TOKEN }} + # See https://github.com/grafana/helm-charts/blob/main/INTERNAL.md about this key + gpg_key_base64: ${{ secrets.HELM_SIGN_KEY_BASE64 }} diff --git a/helm/cr.yaml b/helm/cr.yaml new file mode 100644 index 00000000..39265199 --- /dev/null +++ b/helm/cr.yaml @@ -0,0 +1,5 @@ +git-repo: helm-charts +key: Grafana Loki +owner: grafana +sign: true +skip-existing: true \ No newline at end of file diff --git a/helm/oncall/Chart.yaml b/helm/oncall/Chart.yaml index 81051591..23a91cd5 100644 --- a/helm/oncall/Chart.yaml +++ b/helm/oncall/Chart.yaml @@ -8,7 +8,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 1.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From d872a0e939eb0a35b79b41ea4c73490af660135e Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 21 Jun 2022 15:01:46 +0300 Subject: [PATCH 25/28] Add helm tests --- .github/workflows/helm_release.yml | 3 ++- helm/ct.yaml | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 helm/ct.yaml diff --git a/.github/workflows/helm_release.yml b/.github/workflows/helm_release.yml index de0cca58..7f0f9672 100644 --- a/.github/workflows/helm_release.yml +++ b/.github/workflows/helm_release.yml @@ -9,8 +9,9 @@ jobs: call-update-helm-repo: uses: grafana/helm-charts/.github/workflows/update-helm-repo.yaml@main with: - charts_dir: helm/ + charts_dir: helm cr_configfile: helm/cr.yaml + ct_configfile: helm/ct.yaml secrets: helm_repo_token: ${{ secrets.GH_BOT_ACCESS_TOKEN }} # See https://github.com/grafana/helm-charts/blob/main/INTERNAL.md about this key diff --git a/helm/ct.yaml b/helm/ct.yaml new file mode 100644 index 00000000..c0297aa0 --- /dev/null +++ b/helm/ct.yaml @@ -0,0 +1,12 @@ +# See https://github.com/helm/chart-testing#configuration +remote: origin +target-branch: main +chart-dirs: + - helm/ +chart-repos: + - jetstack=https://charts.jetstack.io + - bitnami=https://charts.bitnami.com/bitnami + - grafana=https://grafana.github.io/helm-charts + - ingress-nginx=https://kubernetes.github.io/ingress-nginx +helm-extra-args: --timeout 600s +validate-maintainers: false \ No newline at end of file From 26395011aabd0eeafec60a35411dc55b755b4aca Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 21 Jun 2022 15:03:42 +0300 Subject: [PATCH 26/28] Remove trailing slash --- helm/ct.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/ct.yaml b/helm/ct.yaml index c0297aa0..ac6439b6 100644 --- a/helm/ct.yaml +++ b/helm/ct.yaml @@ -2,7 +2,7 @@ remote: origin target-branch: main chart-dirs: - - helm/ + - helm chart-repos: - jetstack=https://charts.jetstack.io - bitnami=https://charts.bitnami.com/bitnami From 8783a3aa6e81ba46ad7bf0d0931e22b89e0eff45 Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Tue, 21 Jun 2022 13:46:56 +0100 Subject: [PATCH 27/28] Allow workflow to silently succeed if nothing is to be committed Signed-off-by: Jack Baldry --- .github/workflows/publish-technical-documentation-release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/publish-technical-documentation-release.yml b/.github/workflows/publish-technical-documentation-release.yml index ace2f629..707e20c7 100644 --- a/.github/workflows/publish-technical-documentation-release.yml +++ b/.github/workflows/publish-technical-documentation-release.yml @@ -72,3 +72,7 @@ jobs: source_folder: "docs/sources" # Append ".x" to target to produce a v..x directory. target_folder: "content/docs/oncall/${{ steps.target.outputs.target }}.x" + # Allow the workflow to succeed if there are no changes to commit. + # This is only going to be true on tags as those events ignore the path + # filter in the workflow `on.push` section. + allow_no_changes: "true" From 622bc4eb06b8a3f6c25c88358edc27923cbe0807 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 21 Jun 2022 16:13:29 +0300 Subject: [PATCH 28/28] Edit envs and remove tests --- helm/oncall/templates/_env.tpl | 2 ++ helm/oncall/templates/tests/test-connection.yaml | 15 --------------- 2 files changed, 2 insertions(+), 15 deletions(-) delete mode 100644 helm/oncall/templates/tests/test-connection.yaml diff --git a/helm/oncall/templates/_env.tpl b/helm/oncall/templates/_env.tpl index db8b3e14..d5b881f2 100644 --- a/helm/oncall/templates/_env.tpl +++ b/helm/oncall/templates/_env.tpl @@ -19,6 +19,8 @@ value: "admin" - name: OSS value: "True" +- name: UWSGI_LISTEN + value: "1024" {{- end }} {{- define "snippet.celery.env" -}} diff --git a/helm/oncall/templates/tests/test-connection.yaml b/helm/oncall/templates/tests/test-connection.yaml deleted file mode 100644 index fc82b110..00000000 --- a/helm/oncall/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "oncall.fullname" . }}-test-connection" - labels: - {{- include "oncall.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "oncall.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never