From 6a5e75e083aaac089edfd70321c4d52a6178cd52 Mon Sep 17 00:00:00 2001 From: Innokentii Konstantinov Date: Wed, 1 Mar 2023 16:32:15 +0800 Subject: [PATCH] Fix of templates api behaviour for public and private api (#1408) # What this PR does This PR fixes templates behaviour for public and private api. It fix "reset to default" for templates from messaging backends and some minor bugs. Also added acknowledge signal and source link templates ## Checklist - [x] Tests updated - [x] Documentation added - [x] `CHANGELOG.md` updated --- CHANGELOG.md | 4 + .../oncall-api-reference/integrations.md | 47 +++- .../alerts/models/alert_receive_channel.py | 30 +-- .../api/serializers/alert_receive_channel.py | 19 +- .../test_alert_receive_channel_template.py | 127 ++++++--- .../api/views/preview_template_options.py | 4 +- engine/apps/base/messaging.py | 11 + .../base/models/user_notification_policy.py | 2 +- engine/apps/mobile_app/backend.py | 7 + .../public_api/serializers/integrations.py | 245 ++++++++++++++---- engine/apps/public_api/serializers/routes.py | 6 +- .../public_api/tests/test_integrations.py | 74 ++++++ engine/common/api_helpers/mixins.py | 24 +- engine/settings/dev.py | 6 +- 14 files changed, 451 insertions(+), 155 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6713759..753db01e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add pagination to schedule listing - Show 100 latest alerts on alert group page ([1417](https://github.com/grafana/oncall/pull/1417)) +### Added + +- Add acknowledge_signal and source link to public api + ## v1.1.29 (2023-02-23) ### Changed diff --git a/docs/sources/oncall-api-reference/integrations.md b/docs/sources/oncall-api-reference/integrations.md index b792e459..507841db 100644 --- a/docs/sources/oncall-api-reference/integrations.md +++ b/docs/sources/oncall-api-reference/integrations.md @@ -34,9 +34,11 @@ The above command returns JSON structured in the following way: "channel_id": "CH23212D" } }, - "templates": { + "templates": { "grouping_key": null, "resolve_signal": null, + "acknowledge_signal": null, + "source_link": null, "slack": { "title": null, "message": null, @@ -47,10 +49,6 @@ The above command returns JSON structured in the following way: "message": null, "image_url": null }, - "email": { - "title": null, - "message": null - }, "sms": { "title": null }, @@ -61,6 +59,15 @@ The above command returns JSON structured in the following way: "title": null, "message": null, "image_url": null + }, + "email": { + "title": null, + "message": null + }, + "msteams": { + "title": null, + "message": null, + "image_url": null } } } @@ -102,6 +109,8 @@ The above command returns JSON structured in the following way: "templates": { "grouping_key": null, "resolve_signal": null, + "acknowledge_signal": null, + "source_link": null, "slack": { "title": null, "message": null, @@ -112,10 +121,6 @@ The above command returns JSON structured in the following way: "message": null, "image_url": null }, - "email": { - "title": null, - "message": null - }, "sms": { "title": null }, @@ -126,6 +131,15 @@ The above command returns JSON structured in the following way: "title": null, "message": null, "image_url": null + }, + "email": { + "title": null, + "message": null + }, + "msteams": { + "title": null, + "message": null, + "image_url": null } } } @@ -170,6 +184,8 @@ The above command returns JSON structured in the following way: "templates": { "grouping_key": null, "resolve_signal": null, + "acknowledge_signal": null, + "source_link": null, "slack": { "title": null, "message": null, @@ -180,10 +196,6 @@ The above command returns JSON structured in the following way: "message": null, "image_url": null }, - "email": { - "title": null, - "message": null - }, "sms": { "title": null }, @@ -194,6 +206,15 @@ The above command returns JSON structured in the following way: "title": null, "message": null, "image_url": null + }, + "email": { + "title": null, + "message": null + }, + "msteams": { + "title": null, + "message": null, + "image_url": null } } } diff --git a/engine/apps/alerts/models/alert_receive_channel.py b/engine/apps/alerts/models/alert_receive_channel.py index 28c9247e..ac04b37c 100644 --- a/engine/apps/alerts/models/alert_receive_channel.py +++ b/engine/apps/alerts/models/alert_receive_channel.py @@ -149,6 +149,8 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): is_finished_alerting_setup = models.BooleanField(default=False) + # *_*_template fields are legacy way of storing templates + # messaging_backends_templates for new integrations' templates slack_title_template = models.TextField(null=True, default=None) slack_message_template = models.TextField(null=True, default=None) slack_image_url_template = models.TextField(null=True, default=None) @@ -176,33 +178,6 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): resolve_condition_template = models.TextField(null=True, default=None) acknowledge_condition_template = models.TextField(null=True, default=None) - PUBLIC_TEMPLATES_FIELDS = { - "grouping_key": "grouping_id_template", - "resolve_signal": "resolve_condition_template", - "acknowledge_signal": "acknowledge_condition_template", - "slack": { - "title": "slack_title_template", - "message": "slack_message_template", - "image_url": "slack_image_url_template", - }, - "web": { - "title": "web_title_template", - "message": "web_message_template", - "image_url": "web_image_url_template", - }, - "sms": { - "title": "sms_title_template", - }, - "phone_call": { - "title": "phone_call_title_template", - }, - "telegram": { - "title": "telegram_title_template", - "message": "telegram_message_template", - "image_url": "telegram_image_url_template", - }, - } - # additional messaging backends templates # e.g. {'': {'title': 'title template', 'message': 'message template', 'image_url': 'url template'}} messaging_backends_templates = models.JSONField(null=True, default=None) @@ -459,6 +434,7 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): "grouping_key": self.grouping_id_template, "resolve_signal": self.resolve_condition_template, "acknowledge_signal": self.acknowledge_condition_template, + "source_link": self.source_link_template, "slack": { "title": self.slack_title_template, "message": self.slack_message_template, diff --git a/engine/apps/api/serializers/alert_receive_channel.py b/engine/apps/api/serializers/alert_receive_channel.py index b0e52c50..7bd2ced4 100644 --- a/engine/apps/api/serializers/alert_receive_channel.py +++ b/engine/apps/api/serializers/alert_receive_channel.py @@ -5,7 +5,6 @@ from django.apps import apps from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError as DjangoValidationError -from django.core.validators import URLValidator from django.template.loader import render_to_string from django.utils import timezone from jinja2 import TemplateSyntaxError @@ -19,7 +18,7 @@ from apps.alerts.models import AlertReceiveChannel from apps.base.messaging import get_messaging_backends from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField, WritableSerializerMethodField from common.api_helpers.exceptions import BadRequest -from common.api_helpers.mixins import IMAGE_URL, TEMPLATE_NAMES_ONLY_WITH_NOTIFICATION_CHANNEL, EagerLoadingMixin +from common.api_helpers.mixins import APPEARANCE_TEMPLATE_NAMES, EagerLoadingMixin from common.api_helpers.utils import CurrentTeamDefault from common.jinja_templater import apply_jinja_template, jinja_template_env from common.jinja_templater.apply_jinja_template import JinjaTemplateWarning @@ -552,17 +551,19 @@ class AlertReceiveChannelTemplatesSerializer(EagerLoadingMixin, serializers.Mode def _handle_messaging_backend_updates(self, data, ret): """Update additional messaging backend templates if needed.""" errors = {} - for backend_id, _ in get_messaging_backends(): + for backend_id, backend in get_messaging_backends(): + if not backend.customizable_templates: + continue # fetch existing templates if any backend_templates = {} if self.instance.messaging_backends_templates is not None: backend_templates = self.instance.messaging_backends_templates.get(backend_id, {}) # validate updated templates if any backend_updates = {} - for field in TEMPLATE_NAMES_ONLY_WITH_NOTIFICATION_CHANNEL: - field_name = f"{backend_id.lower()}_{field}_template" + for field in APPEARANCE_TEMPLATE_NAMES: + field_name = f"{backend.slug}_{field}_template" value = data.get(field_name) - validator = jinja_template_env.from_string if field != IMAGE_URL else URLValidator() + validator = jinja_template_env.from_string if value is not None: try: if value: @@ -616,12 +617,14 @@ class AlertReceiveChannelTemplatesSerializer(EagerLoadingMixin, serializers.Mode """Return additional messaging backend templates if any.""" templates = {} for backend_id, backend in get_messaging_backends(): + if not backend.customizable_templates: + continue for field in backend.template_fields: value = None if obj.messaging_backends_templates: value = obj.messaging_backends_templates.get(backend_id, {}).get(field) - if value is None: + if not value: value = obj.get_default_template_attribute(backend_id, field) - field_name = f"{backend_id.lower()}_{field}_template" + field_name = f"{backend.slug}_{field}_template" templates[field_name] = value return templates diff --git a/engine/apps/api/tests/test_alert_receive_channel_template.py b/engine/apps/api/tests/test_alert_receive_channel_template.py index 0c2d658d..1b2a1d6a 100644 --- a/engine/apps/api/tests/test_alert_receive_channel_template.py +++ b/engine/apps/api/tests/test_alert_receive_channel_template.py @@ -8,6 +8,7 @@ from rest_framework.test import APIClient from apps.api.permissions import LegacyAccessControlRole from apps.base.messaging import BaseMessagingBackend +from apps.base.tests.messaging_backend import TestOnlyBackend @pytest.mark.django_db @@ -155,51 +156,114 @@ def test_update_alert_receive_channel_backend_template_invalid_template( @pytest.mark.django_db -def test_update_alert_receive_channel_backend_template_invalid_url( +def test_update_alert_receive_channel_backend_template_set_default_template( make_organization_and_user_with_plugin_token, make_user_auth_headers, make_alert_receive_channel, ): organization, user, token = make_organization_and_user_with_plugin_token() - alert_receive_channel = make_alert_receive_channel(organization, messaging_backends_templates=None) - client = APIClient() - - url = reverse( - "api-internal:alert_receive_channel_template-detail", kwargs={"pk": alert_receive_channel.public_primary_key} - ) - - response = client.put( - url, format="json", data={"testonly_image_url_template": "not-url"}, **make_user_auth_headers(user, token) - ) - - assert response.status_code == status.HTTP_400_BAD_REQUEST - assert response.json() == {"testonly_image_url_template": "invalid URL"} - - -@pytest.mark.django_db -def test_update_alert_receive_channel_backend_template_empty_values_allowed( - make_organization_and_user_with_plugin_token, - make_user_auth_headers, - make_alert_receive_channel, -): - organization, user, token = make_organization_and_user_with_plugin_token() - alert_receive_channel = make_alert_receive_channel(organization, messaging_backends_templates=None) + # create alert_receive_channel with non-default values for TESTONLY messaging backend templates + testonly_templates = {"TESTONLY": {"title": "non-default", "message": "non-default", "image_url": "non-default"}} + alert_receive_channel = make_alert_receive_channel(organization, messaging_backends_templates=testonly_templates) + client = APIClient() url = reverse( "api-internal:alert_receive_channel_template-detail", kwargs={"pk": alert_receive_channel.public_primary_key} ) + # update templates with empty string, which mean templates are default response = client.put( url, format="json", - data={"testonly_title_template": "", "testonly_image_url_template": ""}, + data={"testonly_title_template": "", "testonly_message_template": "", "testonly_image_url_template": ""}, **make_user_auth_headers(user, token), ) assert response.status_code == status.HTTP_200_OK alert_receive_channel.refresh_from_db() - assert alert_receive_channel.messaging_backends_templates["TESTONLY"] == {"title": "", "image_url": ""} + assert alert_receive_channel.messaging_backends_templates["TESTONLY"] == { + "title": "", + "message": "", + "image_url": "", + } + + # check if internal api returns default values + response = client.get( + url, + format="json", + **make_user_auth_headers(user, token), + ) + + assert response.status_code == status.HTTP_200_OK + # WEB_TEMPLATE is default for templates from messaging backends + default_title = alert_receive_channel.INTEGRATION_TO_DEFAULT_WEB_TITLE_TEMPLATE[alert_receive_channel.integration] + default_message = alert_receive_channel.INTEGRATION_TO_DEFAULT_WEB_MESSAGE_TEMPLATE[ + alert_receive_channel.integration + ] + default_image_url = alert_receive_channel.INTEGRATION_TO_DEFAULT_WEB_IMAGE_URL_TEMPLATE[ + alert_receive_channel.integration + ] + + assert response.json()["testonly_title_template"] == default_title + assert response.json()["testonly_message_template"] == default_message + assert response.json()["testonly_image_url_template"] == default_image_url + + +@pytest.mark.django_db +def test_update_alert_receive_channel_legacy_template_set_default_template( + make_organization_and_user_with_plugin_token, + make_user_auth_headers, + make_alert_receive_channel, +): + organization, user, token = make_organization_and_user_with_plugin_token() + alert_receive_channel = make_alert_receive_channel(organization, messaging_backends_templates=None) + client = APIClient() + + url = reverse( + "api-internal:alert_receive_channel_template-detail", kwargs={"pk": alert_receive_channel.public_primary_key} + ) + + # set non-default templates + alert_receive_channel.slack_title_template = "non-default-template" + alert_receive_channel.slack_message_template = "non-default-template" + alert_receive_channel.slack_image_url_template = "non-default-template" + alert_receive_channel.save() + + # update templates with empty string, which mean templates are default + response = client.put( + url, + format="json", + data={"slack_title_template": "", "slack_message_template": "", "slack_image_url_template": ""}, + **make_user_auth_headers(user, token), + ) + + assert response.status_code == status.HTTP_200_OK + alert_receive_channel.refresh_from_db() + assert alert_receive_channel.slack_title_template == "" + assert alert_receive_channel.slack_message_template == "" + assert alert_receive_channel.slack_image_url_template == "" + + # check if internal api returns default values + response = client.get( + url, + format="json", + **make_user_auth_headers(user, token), + ) + + assert response.status_code == status.HTTP_200_OK + + default_title = alert_receive_channel.INTEGRATION_TO_DEFAULT_SLACK_TITLE_TEMPLATE[alert_receive_channel.integration] + default_message = alert_receive_channel.INTEGRATION_TO_DEFAULT_SLACK_MESSAGE_TEMPLATE[ + alert_receive_channel.integration + ] + default_image_url = alert_receive_channel.INTEGRATION_TO_DEFAULT_SLACK_IMAGE_URL_TEMPLATE[ + alert_receive_channel.integration + ] + + assert response.json()["slack_title_template"] == default_title + assert response.json()["slack_message_template"] == default_message + assert response.json()["slack_image_url_template"] == default_image_url @pytest.mark.django_db @@ -225,7 +289,7 @@ def test_update_alert_receive_channel_backend_template_update_values( # patch messaging backends to add OTHER as a valid backend with patch( "apps.api.serializers.alert_receive_channel.get_messaging_backends", - return_value=[("TESTONLY", BaseMessagingBackend), ("OTHER", BaseMessagingBackend)], + return_value=[("TESTONLY", TestOnlyBackend()), ("OTHER", BaseMessagingBackend())], ): response = client.put( url, format="json", data={"testonly_title_template": "updated-title"}, **make_user_auth_headers(user, token) @@ -277,8 +341,7 @@ def test_update_alert_receive_channel_templates( make_alert_receive_channel, ): def template_update_func(template): - # set url here to pass *_url templates validation - return "https://grafana.com" + return f"{template}_updated" organization, user, token = make_organization_and_user_with_plugin_token() alert_receive_channel = make_alert_receive_channel( @@ -290,23 +353,27 @@ def test_update_alert_receive_channel_templates( url = reverse( "api-internal:alert_receive_channel_template-detail", kwargs={"pk": alert_receive_channel.public_primary_key} ) - + # Get response from templates endpoint to get initial templates data response = client.get(url, format="json", **make_user_auth_headers(user, token)) assert response.status_code == status.HTTP_200_OK existing_templates_data = response.json() + # build data for PUT request from data we received + # leave only templates-related fields del existing_templates_data["id"] del existing_templates_data["verbal_name"] del existing_templates_data["payload_example"] + # update each template new_templates_data = {} for template_name, template_value in existing_templates_data.items(): new_templates_data[template_name] = template_update_func(template_value) response = client.put(url, format="json", data=new_templates_data, **make_user_auth_headers(user, token)) + # check if updated templates are applied updated_templates_data = response.json() for template_name, prev_template_value in existing_templates_data.items(): assert updated_templates_data[template_name] == template_update_func(prev_template_value) diff --git a/engine/apps/api/views/preview_template_options.py b/engine/apps/api/views/preview_template_options.py index 1f972913..172d0a17 100644 --- a/engine/apps/api/views/preview_template_options.py +++ b/engine/apps/api/views/preview_template_options.py @@ -3,7 +3,7 @@ from rest_framework.response import Response from rest_framework.views import APIView from apps.auth_token.auth import PluginAuthentication -from common.api_helpers.mixins import NOTIFICATION_CHANNEL_OPTIONS, TEMPLATE_NAME_OPTIONS +from common.api_helpers.mixins import ALL_TEMPLATE_NAMES, NOTIFICATION_CHANNEL_OPTIONS class PreviewTemplateOptionsView(APIView): @@ -14,6 +14,6 @@ class PreviewTemplateOptionsView(APIView): return Response( { "notification_channel_options": NOTIFICATION_CHANNEL_OPTIONS, - "template_name_options": TEMPLATE_NAME_OPTIONS, + "template_name_options": ALL_TEMPLATE_NAMES, } ) diff --git a/engine/apps/base/messaging.py b/engine/apps/base/messaging.py index 6c66277d..367fe5e8 100644 --- a/engine/apps/base/messaging.py +++ b/engine/apps/base/messaging.py @@ -54,6 +54,17 @@ class BaseMessagingBackend: """ raise NotImplementedError("notify_user method missing implementation") + @property + def slug(self): + return self.backend_id.lower() + + @property + def customizable_templates(self): + """ + customizable_templates indicates if templates for messaging backend can be changes by user + """ + return True + def load_backend(path, *args, **kwargs): return import_string(path)(*args, **kwargs) diff --git a/engine/apps/base/models/user_notification_policy.py b/engine/apps/base/models/user_notification_policy.py index 1a4a3526..a8c3e34e 100644 --- a/engine/apps/base/models/user_notification_policy.py +++ b/engine/apps/base/models/user_notification_policy.py @@ -248,7 +248,7 @@ class NotificationChannelPublicAPIOptions(NotificationChannelAPIOptions): } LABELS.update( { - getattr(UserNotificationPolicy.NotificationChannel, backend_id): "notify_by_{}".format(b.backend_id.lower()) + getattr(UserNotificationPolicy.NotificationChannel, backend_id): "notify_by_{}".format(b.slug) for backend_id, b in get_messaging_backends() } ) diff --git a/engine/apps/mobile_app/backend.py b/engine/apps/mobile_app/backend.py index bbf1d220..bec8c180 100644 --- a/engine/apps/mobile_app/backend.py +++ b/engine/apps/mobile_app/backend.py @@ -50,6 +50,13 @@ class MobileAppBackend(BaseMessagingBackend): critical=critical, ) + @property + def customizable_templates(self): + """ + Disable customization if templates for mobile app + """ + return False + class MobileAppCriticalBackend(MobileAppBackend): """ diff --git a/engine/apps/public_api/serializers/integrations.py b/engine/apps/public_api/serializers/integrations.py index b1860b9a..fb55dcd5 100644 --- a/engine/apps/public_api/serializers/integrations.py +++ b/engine/apps/public_api/serializers/integrations.py @@ -8,7 +8,7 @@ from apps.alerts.models import AlertReceiveChannel from apps.base.messaging import get_messaging_backends from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField from common.api_helpers.exceptions import BadRequest -from common.api_helpers.mixins import NOTIFICATION_CHANNEL_OPTIONS, EagerLoadingMixin +from common.api_helpers.mixins import PHONE_CALL, SLACK, SMS, TELEGRAM, WEB, EagerLoadingMixin from common.jinja_templater import jinja_template_env from common.utils import timed_lru_cache @@ -16,6 +16,46 @@ from .integtration_heartbeat import IntegrationHeartBeatSerializer from .maintenance import MaintainableObjectSerializerMixin from .routes import DefaultChannelFilterSerializer +# Behaviour templates are named differently in public api +PUBLIC_BEHAVIOUR_TEMPLATES_FIELDS = ["resolve_signal", "grouping_key", "acknowledge_signal", "source_link"] + +# TEMPLATE_PUBLIC_API_NAME_TO_DB_FIELD is map from template name in public api to its db field. +# It's applied only for legacy messengers, which are not using messaging backend system +TEMPLATE_PUBLIC_API_NAME_TO_DB_FIELD = { + "grouping_key": "grouping_id_template", + "resolve_signal": "resolve_condition_template", + "acknowledge_signal": "acknowledge_condition_template", + "source_link": "source_link_template", + "slack": { + "title": "slack_title_template", + "message": "slack_message_template", + "image_url": "slack_image_url_template", + }, + "web": { + "title": "web_title_template", + "message": "web_message_template", + "image_url": "web_image_url_template", + }, + "sms": { + "title": "sms_title_template", + }, + "phone_call": { + "title": "phone_call_title_template", + }, + "telegram": { + "title": "telegram_title_template", + "message": "telegram_message_template", + "image_url": "telegram_image_url_template", + }, +} + +TEMPLATES_WITH_SEPARATE_DB_FIELD = [SLACK, WEB, PHONE_CALL, SMS, TELEGRAM] + PUBLIC_BEHAVIOUR_TEMPLATES_FIELDS + +PUBLIC_API_CUSTOMIZABLE_NOTIFICATION_CHANNEL_TEMPLATES = [SLACK, WEB, PHONE_CALL, SMS, TELEGRAM] +for backend_id, backend in get_messaging_backends(): + if backend.customizable_templates: + PUBLIC_API_CUSTOMIZABLE_NOTIFICATION_CHANNEL_TEMPLATES.append(backend.slug) + class IntegrationTypeField(fields.CharField): def to_representation(self, value): @@ -105,42 +145,11 @@ class IntegrationSerializer(EagerLoadingMixin, serializers.ModelSerializer, Main else: raise BadRequest(detail="Integration with this name already exists") - def _correct_validated_data(self, validated_data): - validated_data = self._correct_validated_data_for_messaging_backends(validated_data) - - templates = validated_data.pop("templates", {}) - for template_name, templates_for_notification_channel in templates.items(): - if type(templates_for_notification_channel) is dict: - for attr, template in templates_for_notification_channel.items(): - try: - validated_data[AlertReceiveChannel.PUBLIC_TEMPLATES_FIELDS[template_name][attr]] = template - except KeyError: - raise BadRequest(detail="Invalid template data") - elif type(templates_for_notification_channel) is str: - try: - validated_data[ - AlertReceiveChannel.PUBLIC_TEMPLATES_FIELDS[template_name] - ] = templates_for_notification_channel - except KeyError: - raise BadRequest(detail="Invalid template data") - elif templates_for_notification_channel is None: - try: - template_to_set_to_default = AlertReceiveChannel.PUBLIC_TEMPLATES_FIELDS[template_name] - if type(template_to_set_to_default) is str: - validated_data[AlertReceiveChannel.PUBLIC_TEMPLATES_FIELDS[template_name]] = None - elif type(template_to_set_to_default) is dict: - for key in template_to_set_to_default.keys(): - validated_data[AlertReceiveChannel.PUBLIC_TEMPLATES_FIELDS[template_name][key]] = None - except KeyError: - raise BadRequest(detail="Invalid template data") - - return validated_data - def validate_templates(self, templates): if not isinstance(templates, dict): raise BadRequest(detail="Invalid template data") - for notification_channel in NOTIFICATION_CHANNEL_OPTIONS: + for notification_channel in PUBLIC_API_CUSTOMIZABLE_NOTIFICATION_CHANNEL_TEMPLATES: template_data = templates.get(notification_channel, {}) if template_data is None: continue @@ -154,44 +163,165 @@ class IntegrationSerializer(EagerLoadingMixin, serializers.ModelSerializer, Main except TemplateSyntaxError: raise BadRequest(detail=f"invalid {notification_channel} {attr} template") - for common_template in ["resolve_signal", "grouping_key"]: - template_data = templates.get(common_template, "") + for template_name in PUBLIC_BEHAVIOUR_TEMPLATES_FIELDS: + template_data = templates.get(template_name, "") if template_data is None: continue if not isinstance(template_data, str): - raise BadRequest(detail=f"Invalid {common_template} template data") + raise BadRequest(detail=f"Invalid {template_name} template data") try: jinja_template_env.from_string(template_data) except TemplateSyntaxError: - raise BadRequest(detail=f"Invalid {common_template} template data") + raise BadRequest(detail=f"Invalid {template_name} template data") return templates - def _correct_validated_data_for_messaging_backends(self, validated_data): - templates = validated_data.get("templates", {}) + def _correct_validated_data(self, validated_data): + """ + Process input templates data. + 1. Reshapes it. + 1.1 We are receiving templates in dict format + { + resolve_signal: "resolve me!" + slack: { + title: "title", + message: "message", + image_url: "image_url", + }, + ... + } + but store them in separate fields: slack_title_template, slack_message_template. + See _correct_validated_data_for_legacy_template method - messaging_backends_templates = self.instance.messaging_backends_templates if self.instance else None + 1.2 We are storing templates from messaging backends in separate messaging_backends_templates field. + So, we need to shape input data related to messaging_backends_templates also. + 2. Handle None templates. + Public API set template to default value in two cases: (This behaviour is required by terraform plugin). + 2.1 None for the whole template: + { + slack: None, + ... + } + In that case all slack templates will be set to default. + + 2.2 One particular field is None: + { + slack: { + title: "My custom title: + message: None, + }, + ... + } + In that case slack message template will be set to default. + + TODO: System described above is too complicated, should be simplified. + It can be simplified via unification all chatops integrations and messaging_backends + and/or by introducing unified templates + """ + + validated_data = self._correct_validated_data_for_messaging_backends_templates(validated_data) + + validated_data = self._correct_validated_data_for_legacy_templates(validated_data) + + validated_data.pop("templates", {}) + return validated_data + + def _correct_validated_data_for_legacy_templates(self, validated_data): + """ + _correct_validated_data_for_legacy_template reshapes validated data to store them. + It converts data from "templates" dict to db fields, which were used before messaging backends. + Example: + { + "slack": { + "title": Hello + } + } + Will be converted to + + slack_title_template=Hello + """ + templates_data_from_request = validated_data.get("templates", {}) + for template_backend_name, template_from_request in templates_data_from_request.items(): + # correct_validated_data for templates with its own db fields. + if template_backend_name in TEMPLATES_WITH_SEPARATE_DB_FIELD: + if type(template_from_request) is str: # if it's plain template: {"resolve_signal": "resolve me"} + try: + validated_data[ + TEMPLATE_PUBLIC_API_NAME_TO_DB_FIELD[template_backend_name] + ] = template_from_request + except KeyError: + raise BadRequest(detail="Invalid template data") + elif type(template_from_request) is dict: # if it's nested template: {slack: {"title": "some title"}} + for attr, template in template_from_request.items(): + try: + validated_data[TEMPLATE_PUBLIC_API_NAME_TO_DB_FIELD[template_backend_name][attr]] = template + except KeyError: + raise BadRequest(detail="Invalid template data") + elif template_from_request is None: + # if it's we receive None, it's needed to set template to default value + try: + template_to_set_to_default = TEMPLATE_PUBLIC_API_NAME_TO_DB_FIELD[template_backend_name] + if type(template_to_set_to_default) is str: + # if we receive None for plain template just set it to None + validated_data[TEMPLATE_PUBLIC_API_NAME_TO_DB_FIELD[template_backend_name]] = None + elif type(template_to_set_to_default) is dict: + # if we receive None for nested template set all it's fields to None + for key in template_to_set_to_default.keys(): + validated_data[TEMPLATE_PUBLIC_API_NAME_TO_DB_FIELD[template_backend_name][key]] = None + except KeyError: + raise BadRequest(detail="Invalid template data") + + return validated_data + + def _correct_validated_data_for_messaging_backends_templates(self, validated_data): + """ + _correct_validated_data_for_messaging_backends_templates reshapes validated data to store them. + It converts data from "templates" dict to messaging_backends_templates field format. + Example: + { + "msteams": { + "title": Hello + } + } + Will be converted to + + messaging_backends={"MSTEAMS": {"title": "Hello"}, + """ + templates_data_from_request = validated_data.get("templates", {}) + + messaging_backends_templates = self.instance.messaging_backends_templates if self.instance else {} + if messaging_backends_templates is None: + messaging_backends_templates = {} for backend_id, backend in get_messaging_backends(): - backend_templates = {} - if messaging_backends_templates is not None: - backend_templates = messaging_backends_templates.get(backend_id, {}) + if not backend.customizable_templates: + continue + backend_template = {} + if backend.slug in templates_data_from_request: # check to modify only templates from request data + template_from_request = templates_data_from_request[backend.slug] + else: + continue + if template_from_request is None: + # If we receive None backend template, like {"msteams": None }, set all template fields to none. + for field in backend.template_fields: + backend_template[field] = None + elif type(template_from_request) is dict: + # go through existing backend_template and update with values from request + backend_template = messaging_backends_templates.get(backend_id, {}) + for field in backend.template_fields: + try: + updated_field_template = template_from_request[field] + except KeyError: + continue - for field in backend.template_fields: - try: - template = templates[backend_id.lower()][field] - except KeyError: - continue - - backend_templates[field] = template + backend_template[field] = updated_field_template # remove backend-specific template from payload - templates.pop(backend_id.lower(), None) + templates_data_from_request.pop(backend.slug, None) - if backend_templates: - validated_data["messaging_backends_templates"] = messaging_backends_templates or {} | { - backend_id: backend_templates - } + if backend_template: + messaging_backends_templates[backend_id] = backend_template + validated_data["messaging_backends_templates"] = messaging_backends_templates return validated_data @staticmethod @@ -200,10 +330,11 @@ class IntegrationSerializer(EagerLoadingMixin, serializers.ModelSerializer, Main messaging_backends_templates = instance.messaging_backends_templates or {} for backend_id, backend in get_messaging_backends(): + if not backend.customizable_templates: + continue if not backend.template_fields: continue - - result[backend_id.lower()] = { + result[backend.slug] = { field: messaging_backends_templates.get(backend_id, {}).get(field) for field in backend.template_fields } diff --git a/engine/apps/public_api/serializers/routes.py b/engine/apps/public_api/serializers/routes.py index 04fa1821..c083da02 100644 --- a/engine/apps/public_api/serializers/routes.py +++ b/engine/apps/public_api/serializers/routes.py @@ -18,7 +18,7 @@ class BaseChannelFilterSerializer(OrderedModelSerializerMixin, serializers.Model for backend_id, backend in get_messaging_backends(): if backend is None: continue - field = backend_id.lower() + field = backend.slug self._declared_fields[field] = serializers.DictField(required=False) self.Meta.fields.append(field) @@ -33,7 +33,7 @@ class BaseChannelFilterSerializer(OrderedModelSerializerMixin, serializers.Model for backend_id, backend in get_messaging_backends(): if backend is None: continue - field = backend_id.lower() + field = backend.slug channel_id = None notification_enabled = False if instance.notification_backends and instance.notification_backends.get(backend_id): @@ -63,7 +63,7 @@ class BaseChannelFilterSerializer(OrderedModelSerializerMixin, serializers.Model for backend_id, backend in get_messaging_backends(): if backend is None: continue - field = backend_id.lower() + field = backend.slug backend_field = validated_data.pop(field, {}) if backend_field: notification_backend = {} diff --git a/engine/apps/public_api/tests/test_integrations.py b/engine/apps/public_api/tests/test_integrations.py index ac5968e7..d508e15a 100644 --- a/engine/apps/public_api/tests/test_integrations.py +++ b/engine/apps/public_api/tests/test_integrations.py @@ -47,6 +47,7 @@ def test_get_list_integrations( "grouping_key": None, "resolve_signal": None, "acknowledge_signal": None, + "source_link": None, "slack": {"title": None, "message": None, "image_url": None}, "web": {"title": None, "message": None, "image_url": None}, "sms": { @@ -177,6 +178,7 @@ def test_update_integration_template( "grouping_key": "ip_addr", "resolve_signal": None, "acknowledge_signal": None, + "source_link": None, "slack": {"title": "Incident", "message": None, "image_url": None}, "web": {"title": None, "message": None, "image_url": None}, "sms": { @@ -237,6 +239,7 @@ def test_update_integration_template_messaging_backend( "grouping_key": "ip_addr", "resolve_signal": None, "acknowledge_signal": None, + "source_link": None, "slack": {"title": None, "message": None, "image_url": None}, "web": {"title": None, "message": None, "image_url": None}, "sms": { @@ -313,6 +316,7 @@ def test_update_resolve_signal_template( "grouping_key": None, "resolve_signal": "resig", "acknowledge_signal": None, + "source_link": None, "slack": {"title": None, "message": None, "image_url": None}, "web": {"title": None, "message": None, "image_url": None}, "sms": { @@ -421,6 +425,7 @@ def test_update_sms_template_with_empty_dict( "grouping_key": None, "resolve_signal": None, "acknowledge_signal": None, + "source_link": None, "slack": {"title": None, "message": None, "image_url": None}, "web": {"title": None, "message": None, "image_url": None}, "sms": { @@ -481,6 +486,7 @@ def test_update_integration_name( "grouping_key": None, "resolve_signal": None, "acknowledge_signal": None, + "source_link": None, "slack": {"title": None, "message": None, "image_url": None}, "web": {"title": None, "message": None, "image_url": None}, "sms": { @@ -544,6 +550,7 @@ def test_set_default_template( "grouping_key": None, "resolve_signal": None, "acknowledge_signal": None, + "source_link": None, "slack": {"title": None, "message": None, "image_url": None}, "web": {"title": None, "message": None, "image_url": None}, "sms": { @@ -573,6 +580,73 @@ def test_set_default_template( assert response.data == expected_response +@pytest.mark.django_db +def test_set_default_messaging_backend_template( + make_organization_and_user_with_token, make_alert_receive_channel, make_channel_filter, make_integration_heartbeat +): + organization, user, token = make_organization_and_user_with_token() + integration = make_alert_receive_channel( + organization, + verbal_name="grafana", + messaging_backends_templates={ + "TESTONLY": {"title": "the-title", "message": "the-message", "image_url": "the-image-url"} + }, + ) + default_channel_filter = make_channel_filter(integration, is_default=True) + make_integration_heartbeat(integration) + + client = APIClient() + data_for_update = {"templates": {"testonly": {"title": None}}} + expected_response = { + "id": integration.public_primary_key, + "team_id": None, + "name": "grafana", + "link": integration.integration_url, + "type": "grafana", + "default_route": { + "escalation_chain_id": None, + "id": default_channel_filter.public_primary_key, + "slack": {"channel_id": None, "enabled": True}, + "telegram": {"id": None, "enabled": False}, + TEST_MESSAGING_BACKEND_FIELD: {"id": None, "enabled": False}, + }, + "heartbeat": { + "link": f"{integration.integration_url}heartbeat/", + }, + "templates": { + "grouping_key": None, + "resolve_signal": None, + "acknowledge_signal": None, + "source_link": None, + "slack": {"title": None, "message": None, "image_url": None}, + "web": {"title": None, "message": None, "image_url": None}, + "sms": { + "title": None, + }, + "phone_call": { + "title": None, + }, + "telegram": { + "title": None, + "message": None, + "image_url": None, + }, + TEST_MESSAGING_BACKEND_FIELD: { + "title": None, + "message": "the-message", + "image_url": "the-image-url", + }, + }, + "maintenance_mode": None, + "maintenance_started_at": None, + "maintenance_end_at": None, + } + url = reverse("api-public:integrations-detail", args=[integration.public_primary_key]) + response = client.put(url, data=data_for_update, format="json", HTTP_AUTHORIZATION=f"{token}") + assert response.status_code == status.HTTP_200_OK + assert response.data == expected_response + + @pytest.mark.django_db def test_get_list_integrations_direct_paging_hidden( make_organization_and_user_with_token, diff --git a/engine/common/api_helpers/mixins.py b/engine/common/api_helpers/mixins.py index c3a0991d..b32af10b 100644 --- a/engine/common/api_helpers/mixins.py +++ b/engine/common/api_helpers/mixins.py @@ -257,7 +257,9 @@ WEB = "web" PHONE_CALL = "phone_call" SMS = "sms" TELEGRAM = "telegram" +# templates with its own field in db, this concept replaced by messaging_backend_templates field NOTIFICATION_CHANNEL_OPTIONS = [SLACK, WEB, PHONE_CALL, SMS, TELEGRAM] + TITLE = "title" MESSAGE = "message" IMAGE_URL = "image_url" @@ -265,7 +267,7 @@ RESOLVE_CONDITION = "resolve_condition" ACKNOWLEDGE_CONDITION = "acknowledge_condition" GROUPING_ID = "grouping_id" SOURCE_LINK = "source_link" -TEMPLATE_NAME_OPTIONS = [TITLE, MESSAGE, IMAGE_URL, RESOLVE_CONDITION, ACKNOWLEDGE_CONDITION, GROUPING_ID, SOURCE_LINK] + NOTIFICATION_CHANNEL_TO_TEMPLATER_MAP = { SLACK: AlertSlackTemplater, WEB: AlertWebTemplater, @@ -277,12 +279,12 @@ NOTIFICATION_CHANNEL_TO_TEMPLATER_MAP = { # add additionally supported messaging backends for backend_id, backend in get_messaging_backends(): if backend.templater is not None: - backend_slug = backend_id.lower() - NOTIFICATION_CHANNEL_OPTIONS.append(backend_slug) - NOTIFICATION_CHANNEL_TO_TEMPLATER_MAP[backend_slug] = backend.get_templater_class() + NOTIFICATION_CHANNEL_OPTIONS.append(backend.slug) + NOTIFICATION_CHANNEL_TO_TEMPLATER_MAP[backend.slug] = backend.get_templater_class() -TEMPLATE_NAMES_ONLY_WITH_NOTIFICATION_CHANNEL = [TITLE, MESSAGE, IMAGE_URL] -TEMPLATE_NAMES_WITHOUT_NOTIFICATION_CHANNEL = [RESOLVE_CONDITION, ACKNOWLEDGE_CONDITION, GROUPING_ID, SOURCE_LINK] +APPEARANCE_TEMPLATE_NAMES = [TITLE, MESSAGE, IMAGE_URL] +BEHAVIOUR_TEMPLATE_NAMES = [RESOLVE_CONDITION, ACKNOWLEDGE_CONDITION, GROUPING_ID, SOURCE_LINK] +ALL_TEMPLATE_NAMES = APPEARANCE_TEMPLATE_NAMES + BEHAVIOUR_TEMPLATE_NAMES class PreviewTemplateMixin: @@ -298,9 +300,9 @@ class PreviewTemplateMixin: notification_channel, attr_name = self.parse_name_and_notification_channel(template_name) if attr_name is None: raise BadRequest(detail={"template_name": "Attr name is required"}) - if attr_name not in TEMPLATE_NAME_OPTIONS: + if attr_name not in ALL_TEMPLATE_NAMES: raise BadRequest(detail={"template_name": "Unknown attr name"}) - if attr_name in TEMPLATE_NAMES_ONLY_WITH_NOTIFICATION_CHANNEL: + if attr_name in APPEARANCE_TEMPLATE_NAMES: if notification_channel is None: raise BadRequest(detail={"notification_channel": "notification_channel is required"}) if notification_channel not in NOTIFICATION_CHANNEL_OPTIONS: @@ -310,7 +312,7 @@ class PreviewTemplateMixin: if alert_to_template is None: raise BadRequest(detail="Alert to preview does not exist") - if attr_name in TEMPLATE_NAMES_ONLY_WITH_NOTIFICATION_CHANNEL: + if attr_name in APPEARANCE_TEMPLATE_NAMES: class PreviewTemplateLoader(TemplateLoader): def get_attr_template(self, attr, alert_receive_channel, render_for=None): @@ -329,7 +331,7 @@ class PreviewTemplateMixin: templated_attr = getattr(templated_alert, attr_name) - elif attr_name in TEMPLATE_NAMES_WITHOUT_NOTIFICATION_CHANNEL: + elif attr_name in BEHAVIOUR_TEMPLATE_NAMES: try: templated_attr = apply_jinja_template(template_body, payload=alert_to_template.raw_request_data) except (JinjaTemplateError, JinjaTemplateWarning) as e: @@ -347,7 +349,7 @@ class PreviewTemplateMixin: template_param = template_param.replace("_template", "") attr_name = None destination = None - if template_param.startswith(tuple(TEMPLATE_NAMES_WITHOUT_NOTIFICATION_CHANNEL)): + if template_param.startswith(tuple(BEHAVIOUR_TEMPLATE_NAMES)): attr_name = template_param elif template_param.startswith(tuple(NOTIFICATION_CHANNEL_OPTIONS)): for notification_channel in NOTIFICATION_CHANNEL_OPTIONS: diff --git a/engine/settings/dev.py b/engine/settings/dev.py index 63c503a8..48a40ba5 100644 --- a/engine/settings/dev.py +++ b/engine/settings/dev.py @@ -70,8 +70,8 @@ INTERNAL_IPS = [ "127.0.0.1", ] -# the below two lines make it possible to use django-debug-toolbar inside of docker locally -# https://knasmueller.net/fix-djangos-debug-toolbar-not-showing-inside-docker -# https://stackoverflow.com/questions/10517765/django-debug-toolbar-not-showing-up +# # the below two lines make it possible to use django-debug-toolbar inside of docker locally +# # https://knasmueller.net/fix-djangos-debug-toolbar-not-showing-inside-docker +# # https://stackoverflow.com/questions/10517765/django-debug-toolbar-not-showing-up hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) INTERNAL_IPS += [".".join(ip.split(".")[:-1] + ["1"]) for ip in ips]