From 92ed22645cd0a2cbf95b6c3fbf7214d4b93688f8 Mon Sep 17 00:00:00 2001 From: n3x14 <30580487+N3X14@users.noreply.github.com> Date: Tue, 21 Nov 2023 19:54:25 +0100 Subject: [PATCH 01/18] [Docs] add details inbound email integration setup (#3198) # What this PR does This PR adds some more depth to the documentation regarding the setup of the "Inbound Email" integration. In particular it references the setup requirement in the integration documentation and also outlines the requirements for the secret syntax and usage in the process. ## Which issue(s) this PR fixes https://github.com/grafana/oncall/issues/3015 ## Checklist - [x] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required) --- docs/sources/integrations/inbound-email/index.md | 4 ++++ docs/sources/open-source/_index.md | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/sources/integrations/inbound-email/index.md b/docs/sources/integrations/inbound-email/index.md index ef6add71..f5c78036 100644 --- a/docs/sources/integrations/inbound-email/index.md +++ b/docs/sources/integrations/inbound-email/index.md @@ -16,6 +16,10 @@ weight: 500 Inbound Email integration will consume emails from dedicated email address and make alert groups from them. +## Configure required environment variables + +See [Inbound Email Setup]({{< relref "../../open-source/_index.md#inbound-email-setup" >}}) for details. + ## Configure Inbound Email integration for Grafana OnCall You must have an Admin role to create integrations in Grafana OnCall. diff --git a/docs/sources/open-source/_index.md b/docs/sources/open-source/_index.md index ae0ca4bb..6c9487b6 100644 --- a/docs/sources/open-source/_index.md +++ b/docs/sources/open-source/_index.md @@ -265,7 +265,9 @@ To configure Inbound Email integration for Grafana OnCall OSS populate env varia - `INBOUND_EMAIL_DOMAIN` - Inbound email domain - `INBOUND_EMAIL_WEBHOOK_SECRET` - Inbound email webhook secret -You will also need to configure your ESP to forward messages to the following URL: `/integrations/v1/inbound_email_webhook`. +Required secret syntax: `part1ofsecret:part2ofsecret` (The colon `:` is a mandatory delimiter separating both parts of your secret.) + +You will also need to configure your ESP to forward messages to the following URL: `scheme://@/integrations/v1/inbound_email_webhook`. ## Limits From 9628bdc51f10320d54bda924b6e2c1e035788e20 Mon Sep 17 00:00:00 2001 From: Innokentii Konstantinov Date: Wed, 22 Nov 2023 19:17:41 +0800 Subject: [PATCH 02/18] Webhook labels (#3383) This PR add labels for webhooks. 1. Make webhook "labelable" with ability to filter by labels. 2. Add labels to the webhook payload. It contain new field webhook with it's name, id and labels. Field integration and alert_group has a corresponding label field as well. See example of a new payload below: ``` { "event": { "type": "escalation" }, "user": null, "alert_group": { "id": "IRFN6ZD31N31B", "integration_id": "CTWM7U4A2QG97", "route_id": "RUE7U7Z46SKGY", "alerts_count": 1, "state": "firing", "created_at": "2023-11-22T08:54:55.178243Z", "resolved_at": null, "acknowledged_at": null, "title": "Incident", "permalinks": { "slack": null, "telegram": null, "web": "http://grafana:3000/a/grafana-oncall-app/alert-groups/IRFN6ZD31N31B" }, "labels": { "severity": "critical" } }, "alert_group_id": "IRFN6ZD31N31B", "alert_payload": { "message": "This alert was sent by user for demonstration purposes" }, "integration": { "id": "CTWM7U4A2QG97", "type": "webhook", "name": "hi - Webhook", "team": null, "labels": { "hello": "world", "severity": "critical" } }, "notified_users": [], "users_to_be_notified": [], "webhook": { "id": "WHAXK4BTC7TAEQ", "name": "test", "labels": { "hello": "kesha" } } } ``` I feel that there is an opportunity to make code cleaner - remove all label logic from serializers, views and utils to models or dedicated LabelerService and introduce Labelable interface with something like label_verbal, update_labels methods. However, I don't want to tie webhook labels with a refactoring. --------- Co-authored-by: Dominik --- Tiltfile | 2 +- engine/apps/api/label_filtering.py | 15 ++ engine/apps/api/serializers/webhook.py | 20 +- .../api/tests/test_alert_receive_channel.py | 2 - engine/apps/api/tests/test_labels.py | 9 - engine/apps/api/tests/test_webhook_presets.py | 1 + engine/apps/api/tests/test_webhooks.py | 187 ++++++++++++++++++ engine/apps/api/views/alert_group.py | 17 +- .../apps/api/views/alert_receive_channel.py | 22 ++- engine/apps/api/views/labels.py | 33 +--- engine/apps/api/views/webhooks.py | 27 +++ .../migrations/0004_webhookassociatedlabel.py | 29 +++ engine/apps/labels/models.py | 21 ++ engine/apps/labels/tasks.py | 13 +- engine/apps/labels/tests/factories.py | 6 + engine/apps/labels/tests/test_labels.py | 13 +- engine/apps/labels/utils.py | 20 ++ engine/apps/webhooks/tasks/trigger_webhook.py | 2 +- .../webhooks/tests/test_trigger_webhook.py | 8 +- engine/apps/webhooks/utils.py | 9 +- engine/conftest.py | 10 + grafana-plugin/.eslintrc.js | 2 + grafana-plugin/package.json | 2 +- grafana-plugin/src/components/GForm/GForm.tsx | 11 +- .../LabelsTooltipBadge/LabelsTooltipBadge.tsx | 40 ++++ .../IntegrationForm/IntegrationForm.tsx | 11 +- .../src/containers/Labels/Labels.tsx | 16 +- .../OutgoingWebhookForm.config.tsx | 98 +++++---- .../OutgoingWebhookForm.tsx | 50 ++++- .../OutgoingWebhookForm.types.ts | 18 ++ .../RemoteFilters/RemoteFilters.helpers.ts | 5 +- .../RemoteFilters/RemoteFilters.tsx | 2 +- grafana-plugin/src/models/filters/filters.ts | 19 ++ .../outgoing_webhook.types.ts | 2 + .../src/pages/incidents/Incidents.tsx | 73 ++----- .../src/pages/integrations/Integrations.tsx | 69 +------ .../outgoing_webhooks/OutgoingWebhooks.tsx | 42 ++-- .../src/state/rootBaseStore/index.ts | 6 +- 38 files changed, 653 insertions(+), 279 deletions(-) create mode 100644 engine/apps/api/label_filtering.py create mode 100644 engine/apps/labels/migrations/0004_webhookassociatedlabel.py create mode 100644 grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx create mode 100644 grafana-plugin/src/containers/OutgoingWebhookForm/OutgoingWebhookForm.types.ts diff --git a/Tiltfile b/Tiltfile index 5a293c92..eb71dad6 100644 --- a/Tiltfile +++ b/Tiltfile @@ -58,7 +58,7 @@ local_resource( allow_parallel=True, ) -yaml = helm("helm/oncall", name=HELM_PREFIX, values=["./dev/helm-local.yml"]) +yaml = helm("helm/oncall", name=HELM_PREFIX, values=["./dev/helm-local.yml", "./dev/helm-local.dev.yml"]) k8s_yaml(yaml) diff --git a/engine/apps/api/label_filtering.py b/engine/apps/api/label_filtering.py new file mode 100644 index 00000000..10173c4f --- /dev/null +++ b/engine/apps/api/label_filtering.py @@ -0,0 +1,15 @@ +from typing import List, Tuple + + +def parse_label_query(label_query: List[str]) -> List[Tuple[str, str]]: + """ + parse_label_query returns list of key-value tuples from a list of "raw" labels – key-value pairs separated with ':'. + """ + kv_pairs = [] + for label in label_query: + label_data = label.split(":") + # Check if label_data is a valid key-value label pair]: ["key1", "value1"] + if len(label_data) != 2: + continue + kv_pairs.append((label_data[0], label_data[1])) + return kv_pairs diff --git a/engine/apps/api/serializers/webhook.py b/engine/apps/api/serializers/webhook.py index 832292ce..f49f7b8f 100644 --- a/engine/apps/api/serializers/webhook.py +++ b/engine/apps/api/serializers/webhook.py @@ -3,6 +3,7 @@ from collections import defaultdict from rest_framework import serializers from rest_framework.validators import UniqueTogetherValidator +from apps.api.serializers.labels import LabelsSerializerMixin from apps.webhooks.models import Webhook, WebhookResponse from apps.webhooks.models.webhook import PUBLIC_WEBHOOK_HTTP_METHODS, WEBHOOK_FIELD_PLACEHOLDER from apps.webhooks.presets.preset_options import WebhookPresetOptions @@ -27,7 +28,7 @@ class WebhookResponseSerializer(serializers.ModelSerializer): ] -class WebhookSerializer(serializers.ModelSerializer): +class WebhookSerializer(LabelsSerializerMixin, serializers.ModelSerializer): id = serializers.CharField(read_only=True, source="public_primary_key") organization = serializers.HiddenField(default=CurrentOrganizationDefault()) team = TeamPrimaryKeyRelatedField(allow_null=True, default=CurrentTeamDefault()) @@ -37,6 +38,8 @@ class WebhookSerializer(serializers.ModelSerializer): trigger_type = serializers.CharField(allow_null=True) trigger_type_name = serializers.SerializerMethodField() + PREFETCH_RELATED = ["labels", "labels__key", "labels__value"] + class Meta: model = Webhook fields = [ @@ -61,10 +64,25 @@ class WebhookSerializer(serializers.ModelSerializer): "last_response_log", "integration_filter", "preset", + "labels", ] validators = [UniqueTogetherValidator(queryset=Webhook.objects.all(), fields=["name", "organization"])] + def create(self, validated_data): + organization = self.context["request"].auth.organization + labels = validated_data.pop("labels", None) + + instance = super().create(validated_data) + self.update_labels_association_if_needed(labels, instance, organization) + return instance + + def update(self, instance, validated_data): + labels = validated_data.pop("labels", None) + organization = self.context["request"].auth.organization + self.update_labels_association_if_needed(labels, instance, organization) + return super().update(instance, validated_data) + def to_representation(self, instance): result = super().to_representation(instance) if instance.password: diff --git a/engine/apps/api/tests/test_alert_receive_channel.py b/engine/apps/api/tests/test_alert_receive_channel.py index e5c1ece0..989171db 100644 --- a/engine/apps/api/tests/test_alert_receive_channel.py +++ b/engine/apps/api/tests/test_alert_receive_channel.py @@ -1310,7 +1310,6 @@ def test_integration_filter_by_labels( def test_update_alert_receive_channel_labels( make_organization_and_user_with_plugin_token, make_alert_receive_channel, - make_integration_label_association, make_user_auth_headers, ): organization, user, token = make_organization_and_user_with_plugin_token() @@ -1353,7 +1352,6 @@ def test_update_alert_receive_channel_labels( def test_update_alert_receive_channel_labels_duplicate_key( make_organization_and_user_with_plugin_token, make_alert_receive_channel, - make_integration_label_association, make_user_auth_headers, ): organization, user, token = make_organization_and_user_with_plugin_token() diff --git a/engine/apps/api/tests/test_labels.py b/engine/apps/api/tests/test_labels.py index e47b7218..41e6be3f 100644 --- a/engine/apps/api/tests/test_labels.py +++ b/engine/apps/api/tests/test_labels.py @@ -43,7 +43,6 @@ def test_get_update_key_get( mocked_get_values, make_organization_and_user_with_plugin_token, make_user_auth_headers, - make_alert_receive_channel, ): organization, user, token = make_organization_and_user_with_plugin_token() client = APIClient() @@ -68,7 +67,6 @@ def test_get_update_key_put( mocked_rename_key, make_organization_and_user_with_plugin_token, make_user_auth_headers, - make_alert_receive_channel, ): organization, user, token = make_organization_and_user_with_plugin_token() client = APIClient() @@ -94,7 +92,6 @@ def test_add_value( mocked_add_value, make_organization_and_user_with_plugin_token, make_user_auth_headers, - make_alert_receive_channel, ): organization, user, token = make_organization_and_user_with_plugin_token() client = APIClient() @@ -120,7 +117,6 @@ def test_rename_value( mocked_rename_value, make_organization_and_user_with_plugin_token, make_user_auth_headers, - make_alert_receive_channel, ): organization, user, token = make_organization_and_user_with_plugin_token() client = APIClient() @@ -146,7 +142,6 @@ def test_get_value( mocked_get_value, make_organization_and_user_with_plugin_token, make_user_auth_headers, - make_alert_receive_channel, ): organization, user, token = make_organization_and_user_with_plugin_token() client = APIClient() @@ -171,7 +166,6 @@ def test_labels_create_label( mocked_create_label, make_organization_and_user_with_plugin_token, make_user_auth_headers, - make_alert_receive_channel, ): organization, user, token = make_organization_and_user_with_plugin_token() client = APIClient() @@ -189,7 +183,6 @@ def test_labels_create_label( def test_labels_feature_false( make_organization_and_user_with_plugin_token, make_user_auth_headers, - make_alert_receive_channel, settings, ): setattr(settings, "FEATURE_LABELS_ENABLED_FOR_ALL", False) @@ -239,7 +232,6 @@ def test_labels_feature_false( def test_labels_permissions_get_actions( make_organization_and_user_with_plugin_token, make_user_auth_headers, - make_alert_receive_channel, role, expected_status, ): @@ -274,7 +266,6 @@ def test_labels_permissions_get_actions( def test_labels_permissions_create_update_actions( make_organization_and_user_with_plugin_token, make_user_auth_headers, - make_alert_receive_channel, role, expected_status, ): diff --git a/engine/apps/api/tests/test_webhook_presets.py b/engine/apps/api/tests/test_webhook_presets.py index e87f7587..b2ce4df7 100644 --- a/engine/apps/api/tests/test_webhook_presets.py +++ b/engine/apps/api/tests/test_webhook_presets.py @@ -63,6 +63,7 @@ def test_create_webhook_from_preset( "http_method": "GET", "integration_filter": None, "is_webhook_enabled": True, + "labels": [], "is_legacy": False, "last_response_log": { "request_data": "", diff --git a/engine/apps/api/tests/test_webhooks.py b/engine/apps/api/tests/test_webhooks.py index f3162515..4c5bb35f 100644 --- a/engine/apps/api/tests/test_webhooks.py +++ b/engine/apps/api/tests/test_webhooks.py @@ -52,6 +52,7 @@ def test_get_list_webhooks(webhook_internal_api_setup, make_user_auth_headers): "http_method": "POST", "integration_filter": None, "is_webhook_enabled": True, + "labels": [], "is_legacy": False, "last_response_log": { "request_data": "", @@ -95,6 +96,7 @@ def test_get_detail_webhook(webhook_internal_api_setup, make_user_auth_headers): "http_method": "POST", "integration_filter": None, "is_webhook_enabled": True, + "labels": [], "is_legacy": False, "last_response_log": { "request_data": "", @@ -143,6 +145,7 @@ def test_create_webhook(webhook_internal_api_setup, make_user_auth_headers): "http_method": "POST", "integration_filter": None, "is_webhook_enabled": True, + "labels": [], "is_legacy": False, "last_response_log": { "request_data": "", @@ -203,6 +206,7 @@ def test_create_valid_templated_field(webhook_internal_api_setup, make_user_auth "http_method": "POST", "integration_filter": None, "is_webhook_enabled": True, + "labels": [], "is_legacy": False, "last_response_log": { "request_data": "", @@ -583,6 +587,7 @@ def test_webhook_field_masking(webhook_internal_api_setup, make_user_auth_header "http_method": "POST", "integration_filter": None, "is_webhook_enabled": True, + "labels": [], "is_legacy": False, "last_response_log": { "request_data": "", @@ -642,6 +647,7 @@ def test_webhook_copy(webhook_internal_api_setup, make_user_auth_headers): "http_method": "POST", "integration_filter": None, "is_webhook_enabled": True, + "labels": [], "is_legacy": False, "last_response_log": { "request_data": "", @@ -711,3 +717,184 @@ def test_create_invalid_missing_fields(webhook_internal_api_setup, make_user_aut response = client.post(url, data, format="json", **make_user_auth_headers(user, token)) assert response.status_code == status.HTTP_400_BAD_REQUEST assert response.json()["trigger_type"][0] == "This field is required." + + +@pytest.mark.django_db +def test_webhook_filter_by_labels( + make_organization_and_user_with_plugin_token, + make_custom_webhook, + make_webhook_label_association, + make_label_key_and_value, + make_user_auth_headers, +): + organization, user, token = make_organization_and_user_with_plugin_token() + webhook_with_label = make_custom_webhook(organization) + label = make_webhook_label_association(organization, webhook_with_label) + + webhook_with_another_label = make_custom_webhook(organization) + another_label = make_webhook_label_association(organization, webhook_with_another_label) + + not_attached_key, not_attached_value = make_label_key_and_value(organization) + + client = APIClient() + + # test filter by label, which is attached to only one webhook + url = reverse("api-internal:webhooks-list") + response = client.get( + f"{url}?label={label.key_id}:{label.value_id}", + content_type="application/json", + **make_user_auth_headers(user, token), + ) + + assert response.status_code == status.HTTP_200_OK + assert len(response.json()) == 1 + assert response.json()[0]["id"] == webhook_with_label.public_primary_key + + url = reverse("api-internal:webhooks-list") + response = client.get( + f"{url}?label={another_label.key_id}:{another_label.value_id}", + content_type="application/json", + **make_user_auth_headers(user, token), + ) + + assert response.status_code == status.HTTP_200_OK + assert len(response.json()) == 1 + assert response.json()[0]["id"] == webhook_with_another_label.public_primary_key + + # test filter by label which is not attached to any webhooks + response = client.get( + f"{url}?label={not_attached_key.id}:{not_attached_value.id}", + content_type="application/json", + **make_user_auth_headers(user, token), + ) + assert len(response.json()) == 0 + + +@pytest.mark.django_db +def test_update_webhook_labels( + webhook_internal_api_setup, + make_user_auth_headers, +): + user, token, webhook = webhook_internal_api_setup + client = APIClient() + + url = reverse("api-internal:webhooks-detail", kwargs={"pk": webhook.public_primary_key}) + key_id = "testkey" + value_id = "testvalue" + data = {"labels": [{"key": {"id": key_id, "name": "test"}, "value": {"id": value_id, "name": "testv"}}]} + response = client.patch( + url, + data=json.dumps(data), + content_type="application/json", + **make_user_auth_headers(user, token), + ) + + webhook.refresh_from_db() + + assert response.status_code == status.HTTP_200_OK + assert webhook.labels.count() == 1 + label = webhook.labels.first() + assert label.key_id == key_id + assert label.value_id == value_id + + response = client.patch( + url, + data=json.dumps({"labels": []}), + content_type="application/json", + **make_user_auth_headers(user, token), + ) + + webhook.refresh_from_db() + + assert response.status_code == status.HTTP_200_OK + assert webhook.labels.count() == 0 + + +@pytest.mark.django_db +def test_create_webhook_with_labels( + make_organization_and_user_with_plugin_token, + make_user_auth_headers, +): + organization, user, token = make_organization_and_user_with_plugin_token() + client = APIClient() + + url = reverse("api-internal:webhooks-list") + + key_id = "testkey" + value_id = "testvalue" + data = { + "name": "the_webhook", + "url": TEST_URL, + "trigger_type": Webhook.TRIGGER_ALERT_GROUP_CREATED, + "http_method": "POST", + "labels": [{"key": {"id": key_id, "name": "test"}, "value": {"id": value_id, "name": "testv"}}], + "team": None, + } + + response = client.post( + url, + data=json.dumps(data), + content_type="application/json", + **make_user_auth_headers(user, token), + ) + + assert response.status_code == 201 + webhook = Webhook.objects.get(public_primary_key=response.json()["id"]) + expected_response = data | { + "id": webhook.public_primary_key, + "data": None, + "username": None, + "password": None, + "authorization_header": None, + "forward_all": True, + "headers": None, + "http_method": "POST", + "integration_filter": None, + "is_webhook_enabled": True, + "is_legacy": False, + "last_response_log": { + "request_data": "", + "request_headers": "", + "timestamp": None, + "content": "", + "status_code": None, + "request_trigger": "", + "url": "", + "event_data": "", + }, + "trigger_template": None, + "trigger_type": str(data["trigger_type"]), + "trigger_type_name": "Alert Group Created", + "preset": None, + } + assert response.status_code == status.HTTP_201_CREATED + assert response.json() == expected_response + + +@pytest.mark.django_db +def test_update_webhook_labels_duplicate_key( + webhook_internal_api_setup, + make_user_auth_headers, +): + user, token, webhook = webhook_internal_api_setup + client = APIClient() + + url = reverse("api-internal:webhooks-detail", kwargs={"pk": webhook.public_primary_key}) + key_id = "testkey" + data = { + "labels": [ + {"key": {"id": key_id, "name": "test"}, "value": {"id": "testvalue1", "name": "testv1"}}, + {"key": {"id": key_id, "name": "test"}, "value": {"id": "testvalue2", "name": "testv2"}}, + ] + } + response = client.patch( + url, + data=json.dumps(data), + content_type="application/json", + **make_user_auth_headers(user, token), + ) + + webhook.refresh_from_db() + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert webhook.labels.count() == 0 diff --git a/engine/apps/api/views/alert_group.py b/engine/apps/api/views/alert_group.py index 81a26704..c5daecbd 100644 --- a/engine/apps/api/views/alert_group.py +++ b/engine/apps/api/views/alert_group.py @@ -18,6 +18,7 @@ from apps.alerts.models import Alert, AlertGroup, AlertReceiveChannel, Escalatio from apps.alerts.paging import unpage_user from apps.alerts.tasks import delete_alert_group, send_update_resolution_note_signal from apps.api.errors import AlertGroupAPIError +from apps.api.label_filtering import parse_label_query from apps.api.permissions import RBACPermission from apps.api.serializers.alert_group import AlertGroupListSerializer, AlertGroupSerializer from apps.api.serializers.team import TeamSerializer @@ -339,19 +340,15 @@ class AlertGroupView( alert_receive_channels_ids = list(alert_receive_channels_qs.values_list("id", flat=True)) queryset = AlertGroup.objects.filter(channel__in=alert_receive_channels_ids) - # filter by labels - labels = self.request.query_params.getlist("label") - for label in labels: - label_split = label.split(":") - if len(label_split) != 2: - continue - key_name, value_name = label_split - + # Filter by labels. Since alert group labels are "static" filter by names, not IDs. + label_query = self.request.query_params.getlist("label", []) + kv_pairs = parse_label_query(label_query) + for key, value in kv_pairs: # Utilize (organization, key_name, value_name, alert_group) index on AlertGroupAssociatedLabel queryset = queryset.filter( labels__organization=self.request.auth.organization, - labels__key_name=key_name, - labels__value_name=value_name, + labels__key_name=key, + labels__value_name=value, ) queryset = queryset.only("id") diff --git a/engine/apps/api/views/alert_receive_channel.py b/engine/apps/api/views/alert_receive_channel.py index aa79da6d..950af614 100644 --- a/engine/apps/api/views/alert_receive_channel.py +++ b/engine/apps/api/views/alert_receive_channel.py @@ -11,6 +11,7 @@ from rest_framework.viewsets import ModelViewSet from apps.alerts.grafana_alerting_sync_manager.grafana_alerting_sync import GrafanaAlertingSyncManager from apps.alerts.models import Alert, AlertGroup, AlertReceiveChannel from apps.alerts.models.maintainable_object import MaintainableObject +from apps.api.label_filtering import parse_label_query from apps.api.permissions import RBACPermission from apps.api.serializers.alert_receive_channel import ( AlertReceiveChannelSerializer, @@ -18,7 +19,7 @@ from apps.api.serializers.alert_receive_channel import ( FilterAlertReceiveChannelSerializer, ) from apps.api.throttlers import DemoAlertThrottler -from apps.api.views.labels import LabelsAssociatingMixin +from apps.api.views.labels import schedule_update_label_cache from apps.auth_token.auth import PluginAuthentication from apps.integrations.legacy_prefix import has_legacy_prefix, remove_legacy_prefix from apps.labels.utils import is_labels_feature_enabled @@ -76,7 +77,6 @@ class AlertReceiveChannelView( PublicPrimaryKeyMixin, FilterSerializerMixin, UpdateSerializerMixin, - LabelsAssociatingMixin, ModelViewSet, ): authentication_classes = ( @@ -159,7 +159,17 @@ class AlertReceiveChannelView( if not ignore_filtering_by_available_teams: queryset = queryset.filter(*self.available_teams_lookup_args).distinct() - queryset = self.filter_by_labels(queryset) + # filter labels + label_query = self.request.query_params.getlist("label", []) + kv_pairs = parse_label_query(label_query) + for key, value in kv_pairs: + queryset = queryset.filter( + labels__key_id=key, + labels__value_id=value, + ) + + # distinct to remove duplicates after alert_receive_channels X labels join + queryset = queryset.distinct() return queryset @@ -170,7 +180,11 @@ class AlertReceiveChannelView( """ if self.request.query_params.get("skip_pagination", "false").lower() == "true": return None - return super().paginate_queryset(queryset) + page = super().paginate_queryset(queryset) + if page is not None: + ids = [d.id for d in queryset] + schedule_update_label_cache(self.model.__name__, self.request.auth.organization, ids) + return page @action(detail=True, methods=["post"], throttle_classes=[DemoAlertThrottler]) def send_demo_alert(self, request, pk): diff --git a/engine/apps/api/views/labels.py b/engine/apps/api/views/labels.py index 9b3cd36c..dc1df6a4 100644 --- a/engine/apps/api/views/labels.py +++ b/engine/apps/api/views/labels.py @@ -6,7 +6,6 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.viewsets import ViewSet -from apps.alerts.models import AlertReceiveChannel from apps.api.permissions import BasicRolePermission, LegacyAccessControlRole from apps.api.serializers.labels import ( LabelKeySerializer, @@ -172,30 +171,8 @@ class AlertGroupLabelsViewSet(LabelsFeatureFlagViewSet): ) -class LabelsAssociatingMixin: # use for labelable objects views (ex. AlertReceiveChannelView) - def filter_by_labels(self, queryset): - """Call this method in `get_queryset()` to add filtering by labels""" - if not is_labels_feature_enabled(self.request.auth.organization): - return queryset - labels = self.request.query_params.getlist("label") # ["key1:value1", "key2:value2"] - if not labels: - return queryset - for label in labels: - label_data = label.split(":") - if len(label_data) != 2: # ["key1", "value1"] - continue - key_id, value_id = label_data - queryset &= AlertReceiveChannel.objects_with_deleted.filter( - labels__key_id=key_id, labels__value_id=value_id - ).distinct() - return queryset - - def paginate_queryset(self, queryset): - organization = self.request.auth.organization - data = super().paginate_queryset(queryset) - if not is_labels_feature_enabled(self.request.auth.organization): - return data - ids = [d.id for d in data] - logger.info(f"start update_instances_labels_cache for ids: {ids}") - update_instances_labels_cache.apply_async((organization.id, ids, self.model.__name__)) - return data +def schedule_update_label_cache(model_name, org, ids): + if not is_labels_feature_enabled(org): + return + logger.info(f"start update_instances_labels_cache for ids: {ids}") + update_instances_labels_cache.apply_async((org.id, ids, model_name)) diff --git a/engine/apps/api/views/webhooks.py b/engine/apps/api/views/webhooks.py index bd7dc8d7..2aa1258f 100644 --- a/engine/apps/api/views/webhooks.py +++ b/engine/apps/api/views/webhooks.py @@ -11,9 +11,12 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet +from apps.api.label_filtering import parse_label_query from apps.api.permissions import RBACPermission from apps.api.serializers.webhook import WebhookResponseSerializer, WebhookSerializer +from apps.api.views.labels import schedule_update_label_cache from apps.auth_token.auth import PluginAuthentication +from apps.labels.utils import is_labels_feature_enabled from apps.webhooks.models import Webhook, WebhookResponse from apps.webhooks.presets.preset_options import WebhookPresetOptions from apps.webhooks.utils import apply_jinja_template_for_json @@ -94,6 +97,21 @@ class WebhooksView(TeamFilteringMixin, PublicPrimaryKeyMixin, ModelViewSet): ).prefetch_related("responses") if not ignore_filtering_by_available_teams: queryset = queryset.filter(*self.available_teams_lookup_args).distinct() + + # filter by labels + label_query = self.request.query_params.getlist("label", []) + kv_pairs = parse_label_query(label_query) + for key, value in kv_pairs: + queryset = queryset.filter( + labels__key_id=key, + labels__value_id=value, + ) + # distinct to remove duplicates after webhooks X labels join + queryset = queryset.distinct() + # schedule update of labels cache + ids = [d.id for d in queryset] + schedule_update_label_cache(self.model.__name__, self.request.auth.organization, ids) + return queryset def get_object(self): @@ -132,6 +150,15 @@ class WebhooksView(TeamFilteringMixin, PublicPrimaryKeyMixin, ModelViewSet): }, ] + if is_labels_feature_enabled(self.request.auth.organization): + filter_options.append( + { + "name": "label", + "display_name": "Label", + "type": "labels", + } + ) + if filter_name is not None: filter_options = list(filter(lambda f: filter_name in f["name"], filter_options)) diff --git a/engine/apps/labels/migrations/0004_webhookassociatedlabel.py b/engine/apps/labels/migrations/0004_webhookassociatedlabel.py new file mode 100644 index 00000000..7c7f645b --- /dev/null +++ b/engine/apps/labels/migrations/0004_webhookassociatedlabel.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.7 on 2023-11-22 06:10 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('webhooks', '0011_auto_20230920_1813'), + ('user_management', '0017_alter_organization_maintenance_author'), + ('labels', '0003_alertreceivechannelassociatedlabel_inherit'), + ] + + operations = [ + migrations.CreateModel( + name='WebhookAssociatedLabel', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='labels.labelkeycache')), + ('organization', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='webhook_labels', to='user_management.organization')), + ('value', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='labels.labelvaluecache')), + ('webhook', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='labels', to='webhooks.webhook')), + ], + options={ + 'unique_together': {('key_id', 'value_id', 'webhook_id')}, + }, + ), + ] diff --git a/engine/apps/labels/models.py b/engine/apps/labels/models.py index f9861a72..28947e75 100644 --- a/engine/apps/labels/models.py +++ b/engine/apps/labels/models.py @@ -139,3 +139,24 @@ class AlertGroupAssociatedLabel(models.Model): name="unique_alert_group_label", ) ] + + +class WebhookAssociatedLabel(AssociatedLabel): + """Keeps information about label association with outgoing webhooks instances""" + + webhook = models.ForeignKey( + "webhooks.Webhook", + on_delete=models.CASCADE, + related_name="labels", + ) + organization = models.ForeignKey( + "user_management.Organization", on_delete=models.CASCADE, related_name="webhook_labels" + ) + + class Meta: + unique_together = ["key_id", "value_id", "webhook_id"] + + @staticmethod + def get_associating_label_field_name() -> str: + """Returns ForeignKey field name for the associated model""" + return "webhook" diff --git a/engine/apps/labels/tasks.py b/engine/apps/labels/tasks.py index 65c7c550..2ac22ba1 100644 --- a/engine/apps/labels/tasks.py +++ b/engine/apps/labels/tasks.py @@ -6,7 +6,13 @@ from django.conf import settings from django.utils import timezone from apps.labels.client import LabelsAPIClient -from apps.labels.utils import LABEL_OUTDATED_TIMEOUT_MINUTES, LabelKeyData, LabelsData, get_associating_label_model +from apps.labels.utils import ( + LABEL_OUTDATED_TIMEOUT_MINUTES, + LabelKeyData, + LabelsData, + ValueData, + get_associating_label_model, +) from apps.user_management.models import Organization from common.custom_celery_tasks import shared_dedicated_queue_retry_task @@ -14,11 +20,6 @@ logger = get_task_logger(__name__) logger.setLevel(logging.DEBUG) -class ValueData(typing.TypedDict): - value_name: str - key_name: str - - def unify_labels_data(labels_data: LabelsData | LabelKeyData) -> typing.Dict[str, ValueData]: values_data: typing.Dict[str, ValueData] if isinstance(labels_data, list): # LabelsData diff --git a/engine/apps/labels/tests/factories.py b/engine/apps/labels/tests/factories.py index 5f910989..aa4052f7 100644 --- a/engine/apps/labels/tests/factories.py +++ b/engine/apps/labels/tests/factories.py @@ -5,6 +5,7 @@ from apps.labels.models import ( AlertReceiveChannelAssociatedLabel, LabelKeyCache, LabelValueCache, + WebhookAssociatedLabel, ) from common.utils import UniqueFaker @@ -33,3 +34,8 @@ class AlertReceiveChannelAssociatedLabelFactory(factory.DjangoModelFactory): class AlertGroupAssociatedLabelFactory(factory.DjangoModelFactory): class Meta: model = AlertGroupAssociatedLabel + + +class WebhookAssociatedLabelFactory(factory.DjangoModelFactory): + class Meta: + model = WebhookAssociatedLabel diff --git a/engine/apps/labels/tests/test_labels.py b/engine/apps/labels/tests/test_labels.py index 9d32583a..b4d116c2 100644 --- a/engine/apps/labels/tests/test_labels.py +++ b/engine/apps/labels/tests/test_labels.py @@ -1,8 +1,14 @@ import pytest from apps.alerts.models import AlertReceiveChannel -from apps.labels.models import AlertReceiveChannelAssociatedLabel, AssociatedLabel, LabelValueCache +from apps.labels.models import ( + AlertReceiveChannelAssociatedLabel, + AssociatedLabel, + LabelValueCache, + WebhookAssociatedLabel, +) from apps.labels.utils import get_associating_label_model, is_labels_feature_enabled +from apps.webhooks.models import Webhook @pytest.mark.django_db @@ -104,6 +110,11 @@ def test_get_associating_label_model(): result = get_associating_label_model(model_name) assert result == expected_result + model_name = Webhook.__name__ + expected_result = WebhookAssociatedLabel + result = get_associating_label_model(model_name) + assert result == expected_result + wrong_model_name = "SomeModel" with pytest.raises(LookupError): get_associating_label_model(wrong_model_name) diff --git a/engine/apps/labels/utils.py b/engine/apps/labels/utils.py index 98d1bf95..46b0583a 100644 --- a/engine/apps/labels/utils.py +++ b/engine/apps/labels/utils.py @@ -27,6 +27,11 @@ class LabelData(typing.TypedDict): value: LabelParams +class ValueData(typing.TypedDict): + value_name: str + key_name: str + + class LabelKeyData(typing.TypedDict): key: LabelParams values: typing.List[LabelParams] @@ -66,3 +71,18 @@ def assign_labels(alert_group: "AlertGroup", alert_receive_channel: "AlertReceiv for label in alert_receive_channel.labels.filter(inheritable=True).select_related("key", "value") ] AlertGroupAssociatedLabel.objects.bulk_create(alert_group_labels) + + +def get_label_verbal(labelable) -> typing.Dict[str, str]: + """ + label_verbal returns dict of labels' key and values names for the given object + """ + return {label.key.name: label.value.name for label in labelable.labels.all().select_related("key", "value")} + + +def get_alert_group_label_verbal(alert_group: "AlertGroup") -> typing.Dict[str, str]: + """ + get_alert_group_label_verbal returns dict of labels' key and values names for the given alert group. + It's different from get_label_verbal, because AlertGroupAssociated labels store key/value_name, not key/value_id + """ + return {label.key_name: label.value_name for label in alert_group.labels.all()} diff --git a/engine/apps/webhooks/tasks/trigger_webhook.py b/engine/apps/webhooks/tasks/trigger_webhook.py index f7f71f0a..0e43f82c 100644 --- a/engine/apps/webhooks/tasks/trigger_webhook.py +++ b/engine/apps/webhooks/tasks/trigger_webhook.py @@ -91,7 +91,7 @@ def _build_payload(webhook, alert_group, user): response_data = r.content responses_data[r.webhook.public_primary_key] = response_data - data = serialize_event(event, alert_group, user, responses_data) + data = serialize_event(event, alert_group, user, webhook, responses_data) return data diff --git a/engine/apps/webhooks/tests/test_trigger_webhook.py b/engine/apps/webhooks/tests/test_trigger_webhook.py index 95483224..4da2e6ed 100644 --- a/engine/apps/webhooks/tests/test_trigger_webhook.py +++ b/engine/apps/webhooks/tests/test_trigger_webhook.py @@ -302,6 +302,7 @@ def test_execute_webhook_ok_forward_all( "type": alert_receive_channel.integration, "name": alert_receive_channel.short_name, "team": None, + "labels": {}, }, "notified_users": [ { @@ -310,10 +311,15 @@ def test_execute_webhook_ok_forward_all( "email": notified_user.email, } ], - "alert_group": IncidentSerializer(alert_group).data, + "alert_group": {**IncidentSerializer(alert_group).data, "labels": {}}, "alert_group_id": alert_group.public_primary_key, "alert_payload": "", "users_to_be_notified": [], + "webhook": { + "id": webhook.public_primary_key, + "name": webhook.name, + "labels": {}, + }, } expected_call = call( "https://something/{}/".format(alert_group.public_primary_key), diff --git a/engine/apps/webhooks/utils.py b/engine/apps/webhooks/utils.py index 5b9cb92a..feafe93e 100644 --- a/engine/apps/webhooks/utils.py +++ b/engine/apps/webhooks/utils.py @@ -7,6 +7,7 @@ from urllib.parse import urlparse from django.conf import settings from apps.base.utils import live_settings +from apps.labels.utils import get_alert_group_label_verbal, get_label_verbal, is_labels_feature_enabled from apps.schedules.ical_utils import list_users_to_notify_from_ical from common.jinja_templater import apply_jinja_template @@ -150,7 +151,7 @@ def _extract_users_from_escalation_snapshot(escalation_snapshot): return list({u["id"]: u for u in users if u}.values()) -def serialize_event(event, alert_group, user, responses=None): +def serialize_event(event, alert_group, user, webhook, responses=None): from apps.public_api.serializers import IncidentSerializer alert_payload = alert_group.alerts.first() @@ -179,4 +180,10 @@ def serialize_event(event, alert_group, user, responses=None): if responses: data["responses"] = responses + # Enrich webhook data with labels payloads if labels feature is enabled + # TODO: once feature flag will be removed this code should go to the 'data' dict declaration + if is_labels_feature_enabled(alert_group.channel.organization): + data["webhook"] = {"id": webhook.public_primary_key, "name": webhook.name, "labels": get_label_verbal(webhook)} + data["integration"]["labels"] = get_label_verbal(alert_group.channel) + data["alert_group"]["labels"] = get_alert_group_label_verbal(alert_group) return data diff --git a/engine/conftest.py b/engine/conftest.py index 200f3a0d..4f30e34a 100644 --- a/engine/conftest.py +++ b/engine/conftest.py @@ -62,6 +62,7 @@ from apps.labels.tests.factories import ( AlertReceiveChannelAssociatedLabelFactory, LabelKeyFactory, LabelValueFactory, + WebhookAssociatedLabelFactory, ) from apps.mobile_app.models import MobileAppAuthToken, MobileAppVerificationToken from apps.phone_notifications.phone_backend import PhoneBackend @@ -994,3 +995,12 @@ def make_alert_group_label_association(): return AlertGroupAssociatedLabelFactory(alert_group=alert_group, organization=organization, **kwargs) return _make_alert_group_label_association + + +@pytest.fixture +def make_webhook_label_association(make_label_key_and_value): + def _make_integration_label_association(organization, webhook, **kwargs): + key, value = make_label_key_and_value(organization) + return WebhookAssociatedLabelFactory(webhook=webhook, organization=organization, key=key, value=value, **kwargs) + + return _make_integration_label_association diff --git a/grafana-plugin/.eslintrc.js b/grafana-plugin/.eslintrc.js index 23cbc3ab..a09cde51 100644 --- a/grafana-plugin/.eslintrc.js +++ b/grafana-plugin/.eslintrc.js @@ -49,6 +49,8 @@ module.exports = { ], 'no-duplicate-imports': 'error', 'no-restricted-imports': 'warn', + // https://eslint.org/docs/latest/rules/no-redeclare#handled_by_typescript + 'no-redeclare': 0, 'react/display-name': 'warn', /** * It appears as though the react/prop-types rule has a bug in it diff --git a/grafana-plugin/package.json b/grafana-plugin/package.json index 02cd81fa..3b7bda98 100644 --- a/grafana-plugin/package.json +++ b/grafana-plugin/package.json @@ -4,7 +4,7 @@ "description": "Grafana OnCall Plugin", "scripts": { "lint": "eslint --cache --ext .js,.jsx,.ts,.tsx --max-warnings=0 ./src ./e2e-tests", - "lint:fix": "eslint --fix --cache --ext .js,.jsx,.ts,.tsx --max-warnings=0 --quiet ./src ./e2e-tests", + "lint:fix": "eslint --fix --cache --ext .js,.jsx,.ts,.tsx --quiet ./src ./e2e-tests", "stylelint": "stylelint ./src/**/*.{css,scss,module.css,module.scss}", "stylelint:fix": "stylelint --fix ./src/**/*.{css,scss,module.css,module.scss}", "build": "grafana-toolkit plugin:build", diff --git a/grafana-plugin/src/components/GForm/GForm.tsx b/grafana-plugin/src/components/GForm/GForm.tsx index cbba9571..fb1c77ec 100644 --- a/grafana-plugin/src/components/GForm/GForm.tsx +++ b/grafana-plugin/src/components/GForm/GForm.tsx @@ -10,13 +10,21 @@ import { FormItem, FormItemType } from 'components/GForm/GForm.types'; import MonacoEditor from 'components/MonacoEditor/MonacoEditor'; import { MONACO_READONLY_CONFIG } from 'components/MonacoEditor/MonacoEditor.config'; import GSelect from 'containers/GSelect/GSelect'; -import { CustomFieldSectionRendererProps } from 'containers/IntegrationForm/IntegrationForm'; import RemoteSelect from 'containers/RemoteSelect/RemoteSelect'; import styles from './GForm.module.scss'; const cx = cn.bind(styles); +export interface CustomFieldSectionRendererProps { + control: any; + formItem: FormItem; + errors: any; + register: any; + setValue: (fieldName: string, fieldValue: any) => void; + getValues: (fieldName: string | string[]) => T; +} + interface GFormProps { form: { name: string; fields: FormItem[] }; data: any; @@ -211,6 +219,7 @@ class GForm extends React.Component { }} errors={errors} register={register} + getValues={getValues} /> ); } diff --git a/grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx b/grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx new file mode 100644 index 00000000..5ec9d084 --- /dev/null +++ b/grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx @@ -0,0 +1,40 @@ +import React, { FC } from 'react'; + +import { LabelTag } from '@grafana/labels'; +import { VerticalGroup, HorizontalGroup, Button } from '@grafana/ui'; + +import TooltipBadge from 'components/TooltipBadge/TooltipBadge'; +import { LabelKeyValue } from 'models/label/label.types'; + +interface LabelsTooltipBadgeProps { + labels: LabelKeyValue[]; + onClick: (label: LabelKeyValue) => void; +} + +const LabelsTooltipBadge: FC = ({ labels, onClick }) => + labels.length ? ( + + {labels.map((label) => ( + + + - - - - - + + +
+ + + + +
+ + + {customLabelIndexToShowTemplateEditor !== undefined && ( + setCustomLabelIndexToShowTemplateEditor(undefined)} + onUpdateTemplates={({ alert_group_labels }) => { + const newCustom = [...alertGroupLabels.custom]; + newCustom[customLabelIndexToShowTemplateEditor].value.name = alert_group_labels; + + setAlertGroupLabels({ + ...alertGroupLabels, + custom: newCustom, + }); + + setCustomLabelIndexToShowTemplateEditor(undefined); + }} + /> + )} + {showTemplateEditor && ( + setShowTemplateEditor(false)} + onUpdateTemplates={({ alert_group_labels }) => { + setAlertGroupLabels({ + ...alertGroupLabels, + template: alert_group_labels, + }); + + setShowTemplateEditor(undefined); + }} + /> + )} + ); }); +interface CustomLabelsProps { + alertGroupLabels: AlertReceiveChannel['alert_group_labels']; + onChange: (value: AlertReceiveChannel['alert_group_labels']) => void; + onShowTemplateEditor: (index: number) => void; +} + +const CustomLabels = (props: CustomLabelsProps) => { + const { alertGroupLabels, onChange, onShowTemplateEditor } = props; + + const { labelsStore } = useStore(); + + const handlePlainLabelAdd = () => { + onChange({ + ...alertGroupLabels, + custom: [ + ...alertGroupLabels.custom, + { + key: { id: undefined, name: undefined }, + value: { id: undefined, name: undefined }, + }, + ], + }); + }; + const handleTemplatedLabelAdd = () => { + onChange({ + ...alertGroupLabels, + custom: [ + ...alertGroupLabels.custom, + { + key: { id: undefined, name: undefined }, + value: { id: null, name: undefined }, // id = null means it's a templated value + }, + ], + }); + }; + + const cachedOnLoadKeys = useCallback(() => { + let result = undefined; + return async (search?: string) => { + if (!result) { + try { + result = await labelsStore.loadKeys(); + } catch (error) { + openErrorNotification('There was an error processing your request. Please try again'); + } + } + + return result.filter((k) => k.name.toLowerCase().includes(search.toLowerCase())); + }; + }, []); + + const cachedOnLoadValuesForKey = useCallback(() => { + let result = undefined; + return async (key: string, search?: string) => { + if (!result) { + try { + const { values } = await labelsStore.loadValuesForKey(key, search); + result = values; + } catch (error) { + openErrorNotification('There was an error processing your request. Please try again'); + } + } + + return result.filter((k) => k.name.toLowerCase().includes(search.toLowerCase())); + }; + }, []); + + return ( + + + + + { + if (res?.response?.status === 409) { + openErrorNotification(`Duplicate values are not allowed`); + } else { + openErrorNotification('An error has occurred. Please try again'); + } + }} + renderValue={(option, index, renderValueDefault) => { + if (option.value.id === null) { + return ( + { + onShowTemplateEditor(index); + }} + /> + } + onChange={(e: ChangeEvent) => { + const newCustom = [...alertGroupLabels.custom]; + newCustom[index].value.name = e.currentTarget.value; + + onChange({ ...alertGroupLabels, custom: newCustom }); + }} + /> + ); + } else { + return renderValueDefault(option, index); + } + }} + onDataUpdate={(value) => { + onChange({ + ...alertGroupLabels, + custom: value, + }); + }} + /> + + + + + } + > + + + + ); +}; + export default IntegrationLabelsForm; diff --git a/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx b/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx index 7376ee96..98e98b71 100644 --- a/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx +++ b/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx @@ -5,6 +5,7 @@ import cn from 'classnames/bind'; import { debounce } from 'lodash-es'; import { observer } from 'mobx-react'; +import { templateForEdit } from 'components/AlertTemplates/AlertTemplatesForm.config'; import { TemplateForEdit } from 'components/AlertTemplates/CommonAlertTemplatesForm.config'; import CheatSheet from 'components/CheatSheet/CheatSheet'; import { @@ -38,7 +39,7 @@ interface IntegrationTemplateProps { templates: AlertTemplatesDTO[]; onHide: () => void; onUpdateTemplates: (values: any) => void; - onUpdateRoute: (values: any, channelFilterId?: ChannelFilter['id']) => void; + onUpdateRoute?: (values: any, channelFilterId?: ChannelFilter['id']) => void; } const IntegrationTemplate = observer((props: IntegrationTemplateProps) => { @@ -53,11 +54,13 @@ const IntegrationTemplate = observer((props: IntegrationTemplateProps) => { const [isRecentAlertGroupExisting, setIsRecentAlertGroupExisting] = useState(false); useEffect(() => { - const locationParams: any = { template: template.name }; - if (template.isRoute) { - locationParams.routeId = channelFilterId; + if (templateForEdit[template.name]) { + const locationParams: any = { template: template.name }; + if (template.isRoute) { + locationParams.routeId = channelFilterId; + } + LocationHelper.update(locationParams, 'partial'); } - LocationHelper.update(locationParams, 'partial'); }, []); useEffect(() => { diff --git a/grafana-plugin/src/containers/Labels/Labels.tsx b/grafana-plugin/src/containers/Labels/Labels.tsx index 21c4d46a..60fc1324 100644 --- a/grafana-plugin/src/containers/Labels/Labels.tsx +++ b/grafana-plugin/src/containers/Labels/Labels.tsx @@ -1,6 +1,6 @@ import React, { forwardRef, useCallback, useImperativeHandle, useState } from 'react'; -import ServiceLabels, { ServiceLabelsProps } from '@grafana/labels'; +import { ServiceLabels, ServiceLabelsProps } from '@grafana/labels'; import { Field } from '@grafana/ui'; import cn from 'classnames/bind'; import { isEmpty } from 'lodash-es'; diff --git a/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts b/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts index ec366bf2..fd3ea806 100644 --- a/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts +++ b/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts @@ -49,7 +49,11 @@ export interface AlertReceiveChannel { allow_delete: boolean; deleted?: boolean; labels: LabelKeyValue[]; - alert_group_labels: { inheritable: Record }; + alert_group_labels: { + inheritable: Record; + custom: LabelKeyValue[]; + template: string; + }; } export interface AlertReceiveChannelChoice { diff --git a/grafana-plugin/yarn.lock b/grafana-plugin/yarn.lock index 2b225b09..3f55fb20 100644 --- a/grafana-plugin/yarn.lock +++ b/grafana-plugin/yarn.lock @@ -1966,10 +1966,10 @@ "@opentelemetry/sdk-trace-web" "^1.8.0" "@opentelemetry/semantic-conventions" "^1.8.0" -"@grafana/labels@~1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@grafana/labels/-/labels-1.2.1.tgz#4113d584bf5cd826d011f957cb69c90bd0416ea8" - integrity sha512-Nlqqvjwh0MjWsqnfpYbKdYwByeKSmEpiit5mKd6Mnnbc5Hxb8ORIruMr40lTxxWLEnDfhENcAs6pvlBuIMG7tQ== +"@grafana/labels@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@grafana/labels/-/labels-1.3.4.tgz#8d9cdd215a80a1da1045d402c037be85d7efd6f5" + integrity sha512-YYCuLGvtrMz7KkbMc6qoNJQr6drDLo6mMI27LcqsTDMHCNO3uJWpzC1Q2Y9MIwctIuTFYhbgfLvIunEegCx6PQ== dependencies: "@emotion/css" "^11.11.2" "@grafana/ui" "^10.0.0" From e09422a07da12665bcc07a926e3d9331d5924869 Mon Sep 17 00:00:00 2001 From: Vadim Stepanov Date: Mon, 27 Nov 2023 17:28:34 +0000 Subject: [PATCH 16/18] Revert "Alert group payload labels" (#3433) Reverts grafana/oncall#3385 --- ...nnel_alert_group_labels_custom_and_more.py | 23 -- engine/apps/alerts/models/alert.py | 4 +- engine/apps/alerts/models/alert_group.py | 2 - .../alerts/models/alert_receive_channel.py | 32 +- .../api/serializers/alert_receive_channel.py | 198 +---------- engine/apps/api/serializers/channel_filter.py | 2 +- .../api/tests/test_alert_receive_channel.py | 106 +----- .../test_alert_receive_channel_template.py | 31 -- engine/apps/labels/alert_group_labels.py | 158 --------- engine/apps/labels/models.py | 12 +- engine/apps/labels/tests/test_alert_group.py | 59 +--- engine/apps/labels/utils.py | 38 +- engine/apps/public_api/serializers/routes.py | 2 +- engine/common/api_helpers/mixins.py | 20 +- engine/common/api_helpers/utils.py | 11 - engine/conftest.py | 30 +- grafana-plugin/package.json | 2 +- grafana-plugin/playwright.config.ts | 2 +- .../LabelsTooltipBadge/LabelsTooltipBadge.tsx | 2 +- .../components/MonacoEditor/MonacoEditor.tsx | 1 - .../IntegrationLabelsForm.tsx | 333 +++--------------- .../IntegrationTemplate.tsx | 13 +- .../src/containers/Labels/Labels.tsx | 2 +- .../alert_receive_channel.types.ts | 6 +- grafana-plugin/yarn.lock | 8 +- 25 files changed, 162 insertions(+), 935 deletions(-) delete mode 100644 engine/apps/alerts/migrations/0040_alertreceivechannel_alert_group_labels_custom_and_more.py delete mode 100644 engine/apps/labels/alert_group_labels.py diff --git a/engine/apps/alerts/migrations/0040_alertreceivechannel_alert_group_labels_custom_and_more.py b/engine/apps/alerts/migrations/0040_alertreceivechannel_alert_group_labels_custom_and_more.py deleted file mode 100644 index d08dbe4a..00000000 --- a/engine/apps/alerts/migrations/0040_alertreceivechannel_alert_group_labels_custom_and_more.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 4.2.7 on 2023-11-22 12:14 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('alerts', '0039_remove_alertreceivechannel_unique_integration_name'), - ] - - operations = [ - migrations.AddField( - model_name='alertreceivechannel', - name='alert_group_labels_custom', - field=models.JSONField(default=list, null=True), - ), - migrations.AddField( - model_name='alertreceivechannel', - name='alert_group_labels_template', - field=models.TextField(default=None, null=True), - ), - ] diff --git a/engine/apps/alerts/models/alert.py b/engine/apps/alerts/models/alert.py index 203f208a..79f458db 100644 --- a/engine/apps/alerts/models/alert.py +++ b/engine/apps/alerts/models/alert.py @@ -12,7 +12,7 @@ from django.db.models import JSONField from apps.alerts import tasks from apps.alerts.constants import TASK_DELAY_SECONDS from apps.alerts.incident_appearance.templaters import TemplateLoader -from apps.labels.alert_group_labels import assign_labels +from apps.labels.utils import assign_labels from common.jinja_templater import apply_jinja_template from common.jinja_templater.apply_jinja_template import JinjaTemplateError, JinjaTemplateWarning from common.public_primary_keys import generate_public_primary_key, increase_public_primary_key_length @@ -108,7 +108,7 @@ class Alert(models.Model): ) if group_created: - assign_labels(group, alert_receive_channel, raw_request_data) + assign_labels(group, alert_receive_channel) group.log_records.create(type=AlertGroupLogRecord.TYPE_REGISTERED) group.log_records.create(type=AlertGroupLogRecord.TYPE_ROUTE_ASSIGNED) diff --git a/engine/apps/alerts/models/alert_group.py b/engine/apps/alerts/models/alert_group.py index cbdb587a..6f6aec12 100644 --- a/engine/apps/alerts/models/alert_group.py +++ b/engine/apps/alerts/models/alert_group.py @@ -43,7 +43,6 @@ if typing.TYPE_CHECKING: ResolutionNoteSlackMessage, ) from apps.base.models import UserNotificationPolicyLogRecord - from apps.labels.models import AlertGroupAssociatedLabel from apps.slack.models import SlackMessage logger = logging.getLogger(__name__) @@ -195,7 +194,6 @@ class AlertGroup(AlertGroupSlackRenderingMixin, EscalationSnapshotMixin, models. slack_log_message: typing.Optional["SlackMessage"] slack_messages: "RelatedManager['SlackMessage']" users: "RelatedManager['User']" - labels: "RelatedManager['AlertGroupAssociatedLabel']" objects: models.Manager["AlertGroup"] = AlertGroupQuerySet.as_manager() diff --git a/engine/apps/alerts/models/alert_receive_channel.py b/engine/apps/alerts/models/alert_receive_channel.py index 2688b4c8..fb465193 100644 --- a/engine/apps/alerts/models/alert_receive_channel.py +++ b/engine/apps/alerts/models/alert_receive_channel.py @@ -42,7 +42,6 @@ if typing.TYPE_CHECKING: from django.db.models.manager import RelatedManager from apps.alerts.models import AlertGroup, ChannelFilter - from apps.labels.models import AlertReceiveChannelAssociatedLabel from apps.user_management.models import Organization, Team logger = logging.getLogger(__name__) @@ -88,6 +87,10 @@ def number_to_smiles_translator(number): return "".join(reversed(smileset)) +class IntegrationAlertGroupLabels(typing.TypedDict): + inheritable: typing.Dict[str, bool] + + class AlertReceiveChannelQueryset(models.QuerySet): def delete(self): self.update(deleted_at=timezone.now()) @@ -120,7 +123,6 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): channel_filters: "RelatedManager['ChannelFilter']" organization: "Organization" team: typing.Optional["Team"] - labels: "RelatedManager['AlertReceiveChannelAssociatedLabel']" objects = AlertReceiveChannelManager() objects_with_maintenance = AlertReceiveChannelManagerWithMaintenance() @@ -204,17 +206,6 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): rate_limited_in_slack_at = models.DateTimeField(null=True, default=None) rate_limit_message_task_id = models.CharField(max_length=100, null=True, default=None) - AlertGroupCustomLabels = list[tuple[str, str | None, str | None]] - alert_group_labels_custom: AlertGroupCustomLabels = models.JSONField(null=True, default=list) - """ - Stores "custom labels" for alert group labels. Custom labels can be either "plain" or "templated". - For plain labels, the format is: [, , None] - For templated labels, the format is: [, None, ] - """ - - alert_group_labels_template: str | None = models.TextField(null=True, default=None) - """Stores a Jinja2 template for "advanced label templating" for alert group labels.""" - def __str__(self): short_name_with_emojis = emojize(self.short_name, language="alias") return f"{self.pk}: {short_name_with_emojis}" @@ -644,6 +635,21 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): result["team"] = "General" return result + @property + def alert_group_labels(self) -> IntegrationAlertGroupLabels: + """ + Alert group labels configuration for the integration used by AlertReceiveChannelSerializer. + See AlertReceiveChannelAssociatedLabel.inheritable for more details. + """ + return {"inheritable": {label.key_id: label.inheritable for label in self.labels.all()}} + + @alert_group_labels.setter + def alert_group_labels(self, value: IntegrationAlertGroupLabels) -> None: + """Setter for alert_group_labels used by AlertReceiveChannelSerializer""" + inheritable_key_ids = [key_id for key_id, inheritable in value["inheritable"].items() if inheritable] + self.labels.filter(key_id__in=inheritable_key_ids).update(inheritable=True) + self.labels.filter(~Q(key_id__in=inheritable_key_ids)).update(inheritable=False) + @receiver(post_save, sender=AlertReceiveChannel) def listen_for_alertreceivechannel_model_save( diff --git a/engine/apps/api/serializers/alert_receive_channel.py b/engine/apps/api/serializers/alert_receive_channel.py index 45390996..ad9ddaf5 100644 --- a/engine/apps/api/serializers/alert_receive_channel.py +++ b/engine/apps/api/serializers/alert_receive_channel.py @@ -4,7 +4,6 @@ from collections import OrderedDict from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError as DjangoValidationError -from django.db.models import Q from jinja2 import TemplateSyntaxError from rest_framework import serializers from rest_framework.exceptions import ValidationError @@ -15,188 +14,30 @@ from apps.alerts.models import AlertReceiveChannel from apps.alerts.models.channel_filter import ChannelFilter from apps.base.messaging import get_messaging_backends from apps.integrations.legacy_prefix import has_legacy_prefix -from apps.labels.models import LabelKeyCache, LabelValueCache -from apps.user_management.models import Organization from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField from common.api_helpers.exceptions import BadRequest from common.api_helpers.mixins import APPEARANCE_TEMPLATE_NAMES, EagerLoadingMixin from common.api_helpers.utils import CurrentTeamDefault -from common.jinja_templater import jinja_template_env +from common.jinja_templater import apply_jinja_template, jinja_template_env +from common.jinja_templater.apply_jinja_template import JinjaTemplateWarning from .integration_heartbeat import IntegrationHeartBeatSerializer from .labels import LabelsSerializerMixin -class AlertGroupCustomLabelKey(typing.TypedDict): - id: str - name: str - - -class AlertGroupCustomLabelValue(typing.TypedDict): - id: str | None # None for templated labels, label value ID for plain labels - name: str # Jinja template for templated labels, label value name for plain labels - - -class AlertGroupCustomLabel(typing.TypedDict): - key: AlertGroupCustomLabelKey - value: AlertGroupCustomLabelValue - - -AlertGroupCustomLabels = list[AlertGroupCustomLabel] - - -class IntegrationAlertGroupLabels(typing.TypedDict): - inheritable: dict[str, bool] - custom: AlertGroupCustomLabels - template: str | None - - -class CustomLabelSerializer(serializers.Serializer): - """This serializer is consistent with apps.api.serializers.labels.LabelSerializer, but allows null for value ID.""" - - class KeySerializer(serializers.Serializer): - id = serializers.CharField() - name = serializers.CharField() - - class ValueSerializer(serializers.Serializer): - # ID is null for templated labels. For such labels, the "name" value is a Jinja2 template. - id = serializers.CharField(allow_null=True) - name = serializers.CharField() - - key = KeySerializer() - value = ValueSerializer() +def valid_jinja_template_for_serializer_method_field(template): + for _, val in template.items(): + try: + apply_jinja_template(val, payload={}) + except JinjaTemplateWarning: + # Suppress warnings, template may be valid with payload + pass class IntegrationAlertGroupLabelsSerializer(serializers.Serializer): """Alert group labels configuration for the integration. See AlertReceiveChannel.alert_group_labels for details.""" inheritable = serializers.DictField(child=serializers.BooleanField()) - custom = CustomLabelSerializer(many=True) - template = serializers.CharField(allow_null=True) - - @staticmethod - def pop_alert_group_labels(validated_data: dict) -> IntegrationAlertGroupLabels | None: - """Get alert group labels from validated data.""" - - # the "alert_group_labels" field is optional, so either all 3 fields are present or none - if "inheritable" not in validated_data: - return None - - return { - "inheritable": validated_data.pop("inheritable"), - "custom": validated_data.pop("custom"), - "template": validated_data.pop("template"), - } - - @classmethod - def update( - cls, instance: AlertReceiveChannel, alert_group_labels: IntegrationAlertGroupLabels | None - ) -> AlertReceiveChannel: - if alert_group_labels is None: - return instance - - # update inheritable labels - inheritable_key_ids = [ - key_id for key_id, inheritable in alert_group_labels["inheritable"].items() if inheritable - ] - instance.labels.filter(key_id__in=inheritable_key_ids).update(inheritable=True) - instance.labels.filter(~Q(key_id__in=inheritable_key_ids)).update(inheritable=False) - - # update DB cache for custom labels - cls._create_custom_labels(instance.organization, alert_group_labels["custom"]) - # update custom labels - instance.alert_group_labels_custom = cls._custom_labels_to_internal_value(alert_group_labels["custom"]) - - # update template - instance.alert_group_labels_template = alert_group_labels["template"] - - instance.save(update_fields=["alert_group_labels_custom", "alert_group_labels_template"]) - return instance - - @staticmethod - def _create_custom_labels(organization: Organization, labels: AlertGroupCustomLabels) -> None: - """Create LabelKeyCache and LabelValueCache objects for custom labels.""" - - label_keys = [ - LabelKeyCache(id=label["key"]["id"], name=label["key"]["name"], organization=organization) - for label in labels - ] - - label_values = [ - LabelValueCache(id=label["value"]["id"], name=label["value"]["name"], key_id=label["key"]["id"]) - for label in labels - if label["value"]["id"] # don't create LabelValueCache objects for templated labels - ] - - LabelKeyCache.objects.bulk_create(label_keys, ignore_conflicts=True, batch_size=5000) - LabelValueCache.objects.bulk_create(label_values, ignore_conflicts=True, batch_size=5000) - - @classmethod - def to_representation(cls, instance: AlertReceiveChannel) -> IntegrationAlertGroupLabels: - """ - The API representation of alert group labels is very different from the underlying model. - - "inheritable" is based on AlertReceiveChannelAssociatedLabel.inheritable, a property of another model. - "custom" is based on AlertReceiveChannel.alert_group_labels_custom, a JSONField with a different schema. - "template" is based on AlertReceiveChannel.alert_group_labels_template, this one is straightforward. - """ - - return { - "inheritable": {label.key_id: label.inheritable for label in instance.labels.all()}, - "custom": cls._custom_labels_to_representation(instance.alert_group_labels_custom), - "template": instance.alert_group_labels_template, - } - - @staticmethod - def _custom_labels_to_internal_value( - custom_labels: AlertGroupCustomLabels, - ) -> AlertReceiveChannel.AlertGroupCustomLabels: - """Convert custom labels from API representation to the schema used by the JSONField on the model.""" - - return [ - [label["key"]["id"], label["value"]["id"], None if label["value"]["id"] else label["value"]["name"]] - for label in custom_labels - ] - - @staticmethod - def _custom_labels_to_representation( - custom_labels: AlertReceiveChannel.AlertGroupCustomLabels, - ) -> AlertGroupCustomLabels: - """ - Inverse of the _custom_labels_to_internal_value method above. - Fetches label names from DB cache, so the API response schema is consistent with other label endpoints. - """ - - from apps.labels.models import LabelKeyCache, LabelValueCache - - # get up-to-date label key names - label_key_names = { - k.id: k.name - for k in LabelKeyCache.objects.filter(id__in=[label[0] for label in custom_labels]).only("id", "name") - } - - # get up-to-date label value names - label_value_names = { - v.id: v.name - for v in LabelValueCache.objects.filter(id__in=[label[1] for label in custom_labels if label[1]]).only( - "id", "name" - ) - } - - return [ - { - "key": { - "id": key_id, - "name": label_key_names[key_id], - }, - "value": { - "id": value_id if value_id else None, - "name": label_value_names[value_id] if value_id else typing.cast(str, template), - }, - } - for key_id, value_id, template in custom_labels - if key_id in label_key_names and (value_id in label_value_names or not value_id) - ] class AlertReceiveChannelSerializer( @@ -223,7 +64,7 @@ class AlertReceiveChannelSerializer( connected_escalations_chains_count = serializers.SerializerMethodField() inbound_email = serializers.CharField(required=False) is_legacy = serializers.SerializerMethodField() - alert_group_labels = IntegrationAlertGroupLabelsSerializer(source="*", required=False) + alert_group_labels = IntegrationAlertGroupLabelsSerializer(required=False) # integration heartbeat is in PREFETCH_RELATED not by mistake. # With using of select_related ORM builds strange join @@ -297,10 +138,8 @@ class AlertReceiveChannelSerializer( if _integration.slug == integration: is_able_to_autoresolve = _integration.is_able_to_autoresolve - # pop associated labels and alert group labels, so they are not passed to AlertReceiveChannel.create labels = validated_data.pop("labels", None) - alert_group_labels = IntegrationAlertGroupLabelsSerializer.pop_alert_group_labels(validated_data) - + alert_group_labels = validated_data.pop("alert_group_labels", None) try: instance = AlertReceiveChannel.create( **validated_data, @@ -311,22 +150,17 @@ class AlertReceiveChannelSerializer( except AlertReceiveChannel.DuplicateDirectPagingError: raise BadRequest(detail=AlertReceiveChannel.DuplicateDirectPagingError.DETAIL) - # Create label associations first, then update alert group labels + # Create label associations first, then update inheritable labels self.update_labels_association_if_needed(labels, instance, organization) - instance = IntegrationAlertGroupLabelsSerializer.update(instance, alert_group_labels) + if alert_group_labels: + instance.alert_group_labels = alert_group_labels return instance def update(self, instance, validated_data): - # update associated labels labels = validated_data.pop("labels", None) - self.update_labels_association_if_needed(labels, instance, self.context["request"].auth.organization) - - # update alert group labels - instance = IntegrationAlertGroupLabelsSerializer.update( - instance, IntegrationAlertGroupLabelsSerializer.pop_alert_group_labels(validated_data) - ) - + organization = self.context["request"].auth.organization + self.update_labels_association_if_needed(labels, instance, organization) try: return super().update(instance, validated_data) except AlertReceiveChannel.DuplicateDirectPagingError: diff --git a/engine/apps/api/serializers/channel_filter.py b/engine/apps/api/serializers/channel_filter.py index 7815a584..b9239d6d 100644 --- a/engine/apps/api/serializers/channel_filter.py +++ b/engine/apps/api/serializers/channel_filter.py @@ -3,12 +3,12 @@ import typing from rest_framework import serializers from apps.alerts.models import AlertReceiveChannel, ChannelFilter, EscalationChain +from apps.api.serializers.alert_receive_channel import valid_jinja_template_for_serializer_method_field from apps.base.messaging import get_messaging_backend_from_id from apps.telegram.models import TelegramToOrganizationConnector from common.api_helpers.custom_fields import OrganizationFilteredPrimaryKeyRelatedField from common.api_helpers.exceptions import BadRequest from common.api_helpers.mixins import EagerLoadingMixin -from common.api_helpers.utils import valid_jinja_template_for_serializer_method_field from common.jinja_templater.apply_jinja_template import JinjaTemplateError from common.utils import is_regex_valid diff --git a/engine/apps/api/tests/test_alert_receive_channel.py b/engine/apps/api/tests/test_alert_receive_channel.py index 4855a87c..989171db 100644 --- a/engine/apps/api/tests/test_alert_receive_channel.py +++ b/engine/apps/api/tests/test_alert_receive_channel.py @@ -9,7 +9,6 @@ from rest_framework.test import APIClient from apps.alerts.models import AlertReceiveChannel, EscalationPolicy from apps.api.permissions import LegacyAccessControlRole -from apps.labels.models import LabelKeyCache, LabelValueCache @pytest.fixture() @@ -1384,49 +1383,23 @@ def test_update_alert_receive_channel_labels_duplicate_key( def test_alert_group_labels_get( make_organization_and_user_with_plugin_token, make_alert_receive_channel, - make_label_key_and_value, make_integration_label_association, make_user_auth_headers, ): organization, user, token = make_organization_and_user_with_plugin_token() alert_receive_channel = make_alert_receive_channel(organization) - label_key, label_value = make_label_key_and_value(organization) - label_key_1, _ = make_label_key_and_value(organization) client = APIClient() url = reverse("api-internal:alert_receive_channel-detail", kwargs={"pk": alert_receive_channel.public_primary_key}) response = client.get(url, **make_user_auth_headers(user, token)) assert response.status_code == status.HTTP_200_OK - assert response.json()["alert_group_labels"] == {"inheritable": {}, "custom": [], "template": None} + assert response.json()["alert_group_labels"] == {"inheritable": {}} label = make_integration_label_association(organization, alert_receive_channel) - - template = "{{ payload.labels | tojson }}" - alert_receive_channel.alert_group_labels_template = template - - alert_receive_channel.alert_group_labels_custom = [ - (label_key.id, label_value.id, None), - (label_key_1.id, None, "{{ payload.foo }}"), - ] - alert_receive_channel.save(update_fields=["alert_group_labels_custom", "alert_group_labels_template"]) - response = client.get(url, **make_user_auth_headers(user, token)) assert response.status_code == status.HTTP_200_OK - assert response.json()["alert_group_labels"] == { - "inheritable": {label.key_id: True}, - "custom": [ - { - "key": {"id": label_key.id, "name": label_key.name}, - "value": {"id": label_value.id, "name": label_value.name}, - }, - { - "key": {"id": label_key_1.id, "name": label_key_1.name}, - "value": {"id": None, "name": "{{ payload.foo }}"}, - }, - ], - "template": template, - } + assert response.json()["alert_group_labels"] == {"inheritable": {label.key_id: True}} @pytest.mark.django_db @@ -1440,75 +1413,14 @@ def test_alert_group_labels_put( alert_receive_channel = make_alert_receive_channel(organization) label_1 = make_integration_label_association(organization, alert_receive_channel) label_2 = make_integration_label_association(organization, alert_receive_channel, inheritable=False) - label_3 = make_integration_label_association(organization, alert_receive_channel, inheritable=False) - - custom = [ - # plain label - { - "key": {"id": label_2.key.id, "name": label_2.key.name}, - "value": {"id": label_2.value.id, "name": label_2.value.name}, - }, - # plain label not present in DB cache - { - "key": {"id": "hello", "name": "world"}, - "value": {"id": "foo", "name": "bar"}, - }, - # templated label - { - "key": {"id": label_3.key.id, "name": label_3.key.name}, - "value": {"id": None, "name": "{{ payload.foo }}"}, - }, - ] - template = "{{ payload.labels | tojson }}" # advanced template client = APIClient() url = reverse("api-internal:alert_receive_channel-detail", kwargs={"pk": alert_receive_channel.public_primary_key}) - data = { - "alert_group_labels": { - "inheritable": {label_1.key_id: False, label_2.key_id: True, label_3.key_id: False}, - "custom": custom, - "template": template, - } - } + data = {"alert_group_labels": {"inheritable": {label_1.key_id: False, label_2.key_id: True}}} response = client.put(url, data, format="json", **make_user_auth_headers(user, token)) assert response.status_code == status.HTTP_200_OK - assert response.json()["alert_group_labels"] == { - "inheritable": {label_1.key_id: False, label_2.key_id: True, label_3.key_id: False}, - "custom": custom, - "template": template, - } - - alert_receive_channel.refresh_from_db() - assert alert_receive_channel.alert_group_labels_custom == [ - [label_2.key_id, label_2.value_id, None], - ["hello", "foo", None], - [label_3.key_id, None, "{{ payload.foo }}"], - ] - assert alert_receive_channel.alert_group_labels_template == template - - # check label keys & values are created - key = LabelKeyCache.objects.filter(id="hello", name="world", organization=organization).first() - assert key is not None - assert LabelValueCache.objects.filter(key=key, id="foo", name="bar").exists() - - -@pytest.mark.django_db -def test_alert_group_labels_put_none( - make_organization_and_user_with_plugin_token, - make_alert_receive_channel, - make_user_auth_headers, -): - organization, user, token = make_organization_and_user_with_plugin_token() - alert_receive_channel = make_alert_receive_channel(organization) - - client = APIClient() - url = reverse("api-internal:alert_receive_channel-detail", kwargs={"pk": alert_receive_channel.public_primary_key}) - response = client.put(url, {"verbal_name": "123"}, format="json", **make_user_auth_headers(user, token)) - - assert response.status_code == status.HTTP_200_OK - assert response.json()["verbal_name"] == "123" - assert response.json()["alert_group_labels"] == {"inheritable": {}, "custom": [], "template": None} + assert response.json()["alert_group_labels"] == {"inheritable": {label_1.key_id: False, label_2.key_id: True}} @pytest.mark.django_db @@ -1516,11 +1428,7 @@ def test_alert_group_labels_post(alert_receive_channel_internal_api_setup, make_ user, token, _ = alert_receive_channel_internal_api_setup labels = [{"key": {"id": "test", "name": "test"}, "value": {"id": "123", "name": "123"}}] - alert_group_labels = { - "inheritable": {"test": False}, - "custom": [{"key": {"id": "test", "name": "test"}, "value": {"id": "123", "name": "123"}}], - "template": "{{ payload.labels | tojson }}", - } + alert_group_labels = {"inheritable": {"test": False}} data = { "integration": AlertReceiveChannel.INTEGRATION_GRAFANA, "team": None, @@ -1535,7 +1443,3 @@ def test_alert_group_labels_post(alert_receive_channel_internal_api_setup, make_ assert response.status_code == status.HTTP_201_CREATED assert response.json()["labels"] == labels assert response.json()["alert_group_labels"] == alert_group_labels - - alert_receive_channel = AlertReceiveChannel.objects.get(public_primary_key=response.json()["id"]) - assert alert_receive_channel.alert_group_labels_custom == [["test", "123", None]] - assert alert_receive_channel.alert_group_labels_template == "{{ payload.labels | tojson }}" diff --git a/engine/apps/api/tests/test_alert_receive_channel_template.py b/engine/apps/api/tests/test_alert_receive_channel_template.py index f494d776..111696cd 100644 --- a/engine/apps/api/tests/test_alert_receive_channel_template.py +++ b/engine/apps/api/tests/test_alert_receive_channel_template.py @@ -337,37 +337,6 @@ def test_preview_alert_receive_channel_backend_templater( assert response.json() == {"preview": "title: alert!"} -@pytest.mark.django_db -def test_preview_alert_group_labels( - make_organization_and_user_with_plugin_token, - make_user_auth_headers, - make_alert_receive_channel, - make_channel_filter, - make_alert_group, - make_alert, -): - organization, user, token = make_organization_and_user_with_plugin_token() - alert_receive_channel = make_alert_receive_channel(organization) - default_channel_filter = make_channel_filter(alert_receive_channel, is_default=True) - alert_group = make_alert_group(alert_receive_channel, channel_filter=default_channel_filter) - make_alert(alert_group=alert_group, raw_request_data={"labels": {"1": "2"}}) - - client = APIClient() - url = reverse( - "api-internal:alert_receive_channel-preview-template", - kwargs={"pk": alert_receive_channel.public_primary_key}, - ) - - data = { - "template_body": "{{ payload.labels | tojson }}", - "template_name": "alert_group_labels", - } - response = client.post(url, format="json", data=data, **make_user_auth_headers(user, token)) - - assert response.status_code == status.HTTP_200_OK - assert response.json() == {"preview": '{"1": "2"}'} - - @pytest.mark.django_db def test_update_alert_receive_channel_templates( make_organization_and_user_with_plugin_token, diff --git a/engine/apps/labels/alert_group_labels.py b/engine/apps/labels/alert_group_labels.py deleted file mode 100644 index df735ca1..00000000 --- a/engine/apps/labels/alert_group_labels.py +++ /dev/null @@ -1,158 +0,0 @@ -import json -import logging -import typing - -from apps.labels.utils import is_labels_feature_enabled -from common.jinja_templater import apply_jinja_template -from common.jinja_templater.apply_jinja_template import JinjaTemplateError, JinjaTemplateWarning - -if typing.TYPE_CHECKING: - from apps.alerts.models import AlertGroup, AlertReceiveChannel - - -logger = logging.getLogger(__name__) - - -# What can be used as a label key/value coming out from the template -LABEL_VALUE_TYPES = (str, int, float, bool) - - -def assign_labels( - alert_group: "AlertGroup", alert_receive_channel: "AlertReceiveChannel", raw_request_data: typing.Any -) -> None: - from apps.labels.models import AlertGroupAssociatedLabel - - if not is_labels_feature_enabled(alert_receive_channel.organization): - return - - # inherit labels from the integration - labels = { - label.key.name: label.value.name - for label in alert_receive_channel.labels.filter(inheritable=True).select_related("key", "value") - } - - # apply custom labels - labels.update(_custom_labels(alert_receive_channel, raw_request_data)) - - # apply template labels - labels.update(_template_labels(alert_receive_channel, raw_request_data)) - - # create associated labels - alert_group_labels = [ - AlertGroupAssociatedLabel( - alert_group=alert_group, - organization=alert_receive_channel.organization, - key_name=key, - value_name=value, - ) - for key, value in labels.items() - ] - # sort associated labels by key and value - alert_group_labels.sort(key=lambda label: (label.key_name, label.value_name)) - # bulk create associated labels - AlertGroupAssociatedLabel.objects.bulk_create(alert_group_labels) - - -def _custom_labels(alert_receive_channel: "AlertReceiveChannel", raw_request_data: typing.Any) -> dict[str, str]: - from apps.labels.models import MAX_VALUE_NAME_LENGTH, LabelKeyCache, LabelValueCache - - # fetch up-to-date label key names - label_key_names = { - k.id: k.name - for k in LabelKeyCache.objects.filter( - id__in=[label[0] for label in alert_receive_channel.alert_group_labels_custom] - ).only("id", "name") - } - - # fetch up-to-date label value names - label_value_names = { - v.id: v.name - for v in LabelValueCache.objects.filter( - id__in=[label[1] for label in alert_receive_channel.alert_group_labels_custom if label[1]] - ).only("id", "name") - } - - rendered_labels = {} - for label in alert_receive_channel.alert_group_labels_custom: - key_id, value_id, template = label - - if key_id in label_key_names: - key = label_key_names[key_id] - else: - logger.warning("Label key cache not found. %s", key_id) - continue - - if value_id: - if value_id in label_value_names: - rendered_labels[key] = label_value_names[value_id] - else: - logger.warning("Label value cache not found. %s", value_id) - continue - else: - try: - rendered_labels[key] = apply_jinja_template(template, raw_request_data) - except (JinjaTemplateError, JinjaTemplateWarning) as e: - logger.warning("Failed to apply template. %s", e.fallback_message) - continue - - labels = {} - for key in rendered_labels: - value = rendered_labels[key] - - # check value length - if len(value) > MAX_VALUE_NAME_LENGTH: - logger.warning("Template result value is too long. %s", value) - continue - - labels[key] = value - - return labels - - -def _template_labels(alert_receive_channel: "AlertReceiveChannel", raw_request_data: typing.Any) -> dict[str, str]: - from apps.labels.models import MAX_KEY_NAME_LENGTH, MAX_VALUE_NAME_LENGTH - - if not alert_receive_channel.alert_group_labels_template: - return {} - - try: - rendered = apply_jinja_template(alert_receive_channel.alert_group_labels_template, raw_request_data) - except (JinjaTemplateError, JinjaTemplateWarning) as e: - logger.warning("Failed to apply template. %s", e.fallback_message) - return {} - - try: - rendered_labels = json.loads(rendered) - except (TypeError, json.JSONDecodeError): - logger.warning("Failed to parse template result. %s", rendered) - return {} - - if not isinstance(rendered_labels, dict): - logger.warning("Template result is not a dict. %s", rendered_labels) - return {} - - labels = {} - for key in rendered_labels: - value = rendered_labels[key] - - # check value type - if not isinstance(value, LABEL_VALUE_TYPES): - logger.warning("Template result value has invalid type. %s", value) - continue - - # convert value to string - value = str(value) - - # check key length - if len(key) > MAX_KEY_NAME_LENGTH: - logger.warning("Template result key is too long. %s", key) - continue - - # check value length - if len(value) > MAX_VALUE_NAME_LENGTH: - logger.warning("Template result value is too long. %s", value) - continue - - labels[key] = value - - return labels diff --git a/engine/apps/labels/models.py b/engine/apps/labels/models.py index 14cd446c..28947e75 100644 --- a/engine/apps/labels/models.py +++ b/engine/apps/labels/models.py @@ -10,13 +10,9 @@ if typing.TYPE_CHECKING: from apps.user_management.models import Organization -MAX_KEY_NAME_LENGTH = 200 -MAX_VALUE_NAME_LENGTH = 200 - - class LabelKeyCache(models.Model): id = models.CharField(primary_key=True, editable=False, max_length=36) - name = models.CharField(max_length=MAX_KEY_NAME_LENGTH) + name = models.CharField(max_length=200) organization = models.ForeignKey("user_management.Organization", on_delete=models.CASCADE) last_synced = models.DateTimeField(auto_now=True) @@ -27,7 +23,7 @@ class LabelKeyCache(models.Model): class LabelValueCache(models.Model): id = models.CharField(primary_key=True, editable=False, max_length=36) - name = models.CharField(max_length=MAX_VALUE_NAME_LENGTH) + name = models.CharField(max_length=200) key = models.ForeignKey("labels.LabelKeyCache", on_delete=models.CASCADE, related_name="values") last_synced = models.DateTimeField(auto_now=True) @@ -133,8 +129,8 @@ class AlertGroupAssociatedLabel(models.Model): "user_management.Organization", on_delete=models.CASCADE, related_name="alert_group_labels" ) - key_name = models.CharField(max_length=MAX_KEY_NAME_LENGTH) - value_name = models.CharField(max_length=MAX_VALUE_NAME_LENGTH) + key_name = models.CharField(max_length=200) + value_name = models.CharField(max_length=200) class Meta: constraints = [ diff --git a/engine/apps/labels/tests/test_alert_group.py b/engine/apps/labels/tests/test_alert_group.py index 32144377..a5ac35ca 100644 --- a/engine/apps/labels/tests/test_alert_group.py +++ b/engine/apps/labels/tests/test_alert_group.py @@ -3,13 +3,9 @@ from unittest import mock import pytest from apps.alerts.models import Alert -from apps.labels.models import MAX_KEY_NAME_LENGTH, MAX_VALUE_NAME_LENGTH - -TOO_LONG_KEY_NAME = "k" * (MAX_KEY_NAME_LENGTH + 1) -TOO_LONG_VALUE_NAME = "v" * (MAX_VALUE_NAME_LENGTH + 1) -@mock.patch("apps.labels.alert_group_labels.is_labels_feature_enabled", return_value=False) +@mock.patch("apps.labels.utils.is_labels_feature_enabled", return_value=False) @pytest.mark.django_db def test_assign_labels_feature_flag_disabled( _, make_organization, make_alert_receive_channel, make_integration_label_association @@ -32,59 +28,22 @@ def test_assign_labels_feature_flag_disabled( @pytest.mark.django_db -def test_assign_labels( - make_organization, - make_alert_receive_channel, - make_label_key_and_value, - make_label_key, - make_integration_label_association, -): +def test_assign_labels(make_organization, make_alert_receive_channel, make_integration_label_association): organization = make_organization() + alert_receive_channel = make_alert_receive_channel(organization) + label = make_integration_label_association(organization, alert_receive_channel) + make_integration_label_association(organization, alert_receive_channel, inheritable=False) - # create label repo labels - label_key, label_value = make_label_key_and_value(organization, key_name="a", value_name="b") - label_key_1 = make_label_key(organization=organization, key_name="c") - label_key_2 = make_label_key(organization=organization) - label_key_3 = make_label_key(organization=organization) - - # create alert receive channel with all 3 types of labels - alert_receive_channel = make_alert_receive_channel( - organization, - alert_group_labels_custom=[ - [label_key.id, label_value.id, None], # plain label - ["nonexistent", label_value.id, None], # plain label with nonexistent key ID - [label_key_2.id, "nonexistent", None], # plain label with nonexistent value ID - [label_key_1.id, None, "{{ payload.c }}"], # templated label - [label_key_3.id, None, TOO_LONG_VALUE_NAME], # templated label too long - ], - alert_group_labels_template="{{ payload.advanced_template | tojson }}", - ) - make_integration_label_association(organization, alert_receive_channel, key_name="e", value_name="f") - - # create alert group alert = Alert.create( title="the title", message="the message", alert_receive_channel=alert_receive_channel, - raw_request_data={ - "c": "d", - "advanced_template": { - "g": 123, - "too_long": TOO_LONG_VALUE_NAME, - TOO_LONG_KEY_NAME: "too_long", - "invalid_type": {"test": "test"}, - }, - "extra": "hi", - }, + raw_request_data={}, integration_unique_data={}, image_url=None, link_to_upstream_details=None, ) - # check alert group labels are assigned correctly, in the lexicographical order - assert [(label.key_name, label.value_name) for label in alert.group.labels.all()] == [ - ("a", "b"), - ("c", "d"), - ("e", "f"), - ("g", "123"), - ] + assert alert.group.labels.count() == 1 + assert alert.group.labels.first().key_name == label.key.name + assert alert.group.labels.first().value_name == label.value.name diff --git a/engine/apps/labels/utils.py b/engine/apps/labels/utils.py index bacf86c0..46b0583a 100644 --- a/engine/apps/labels/utils.py +++ b/engine/apps/labels/utils.py @@ -1,16 +1,13 @@ -import logging import typing from django.apps import apps # noqa: I251 from django.conf import settings if typing.TYPE_CHECKING: - from apps.alerts.models import AlertGroup + from apps.alerts.models import AlertGroup, AlertReceiveChannel from apps.labels.models import AssociatedLabel from apps.user_management.models import Organization -logger = logging.getLogger(__name__) - LABEL_OUTDATED_TIMEOUT_MINUTES = 30 ASSOCIATED_MODEL_NAME = "AssociatedLabel" @@ -57,10 +54,35 @@ def is_labels_feature_enabled(organization: "Organization") -> bool: ) -def get_label_verbal(obj: typing.Any) -> dict[str, str]: - return {label.key.name: label.value.name for label in obj.labels.all().select_related("key", "value")} +def assign_labels(alert_group: "AlertGroup", alert_receive_channel: "AlertReceiveChannel") -> None: + from apps.labels.models import AlertGroupAssociatedLabel + + if not is_labels_feature_enabled(alert_receive_channel.organization): + return + + # inherit labels from the integration + alert_group_labels = [ + AlertGroupAssociatedLabel( + alert_group=alert_group, + organization=alert_receive_channel.organization, + key_name=label.key.name, + value_name=label.value.name, + ) + for label in alert_receive_channel.labels.filter(inheritable=True).select_related("key", "value") + ] + AlertGroupAssociatedLabel.objects.bulk_create(alert_group_labels) -def get_alert_group_label_verbal(alert_group: "AlertGroup") -> dict[str, str]: - """This is different from get_label_verbal because alert group labels store key/value names, not IDs""" +def get_label_verbal(labelable) -> typing.Dict[str, str]: + """ + label_verbal returns dict of labels' key and values names for the given object + """ + return {label.key.name: label.value.name for label in labelable.labels.all().select_related("key", "value")} + + +def get_alert_group_label_verbal(alert_group: "AlertGroup") -> typing.Dict[str, str]: + """ + get_alert_group_label_verbal returns dict of labels' key and values names for the given alert group. + It's different from get_label_verbal, because AlertGroupAssociated labels store key/value_name, not key/value_id + """ return {label.key_name: label.value_name for label in alert_group.labels.all()} diff --git a/engine/apps/public_api/serializers/routes.py b/engine/apps/public_api/serializers/routes.py index bbe9ff55..1907c9e9 100644 --- a/engine/apps/public_api/serializers/routes.py +++ b/engine/apps/public_api/serializers/routes.py @@ -1,10 +1,10 @@ from rest_framework import fields, serializers from apps.alerts.models import AlertReceiveChannel, ChannelFilter, EscalationChain +from apps.api.serializers.alert_receive_channel import valid_jinja_template_for_serializer_method_field from apps.base.messaging import get_messaging_backend_from_id, get_messaging_backends from common.api_helpers.custom_fields import OrganizationFilteredPrimaryKeyRelatedField from common.api_helpers.exceptions import BadRequest -from common.api_helpers.utils import valid_jinja_template_for_serializer_method_field from common.jinja_templater.apply_jinja_template import JinjaTemplateError from common.ordered_model.serializer import OrderedModelSerializer from common.utils import is_regex_valid diff --git a/engine/common/api_helpers/mixins.py b/engine/common/api_helpers/mixins.py index 5f15f137..4204095f 100644 --- a/engine/common/api_helpers/mixins.py +++ b/engine/common/api_helpers/mixins.py @@ -248,7 +248,6 @@ ACKNOWLEDGE_CONDITION = "acknowledge_condition" GROUPING_ID = "grouping_id" SOURCE_LINK = "source_link" ROUTE = "route" -ALERT_GROUP_LABELS = "alert_group_labels" NOTIFICATION_CHANNEL_TO_TEMPLATER_MAP = { SLACK: AlertSlackTemplater, @@ -265,15 +264,9 @@ for backend_id, backend in get_messaging_backends(): NOTIFICATION_CHANNEL_TO_TEMPLATER_MAP[backend.slug] = backend.get_templater_class() APPEARANCE_TEMPLATE_NAMES = [TITLE, MESSAGE, IMAGE_URL] -BEHAVIOUR_TEMPLATE_NAMES = [ - RESOLVE_CONDITION, - ACKNOWLEDGE_CONDITION, - GROUPING_ID, - SOURCE_LINK, - ROUTE, - ALERT_GROUP_LABELS, -] -ALL_TEMPLATE_NAMES = APPEARANCE_TEMPLATE_NAMES + BEHAVIOUR_TEMPLATE_NAMES +BEHAVIOUR_TEMPLATE_NAMES = [RESOLVE_CONDITION, ACKNOWLEDGE_CONDITION, GROUPING_ID, SOURCE_LINK] +ROUTE_TEMPLATE_NAMES = [ROUTE] +ALL_TEMPLATE_NAMES = APPEARANCE_TEMPLATE_NAMES + BEHAVIOUR_TEMPLATE_NAMES + ROUTE_TEMPLATE_NAMES class PreviewTemplateException(Exception): @@ -333,6 +326,11 @@ class PreviewTemplateMixin: templated_attr = apply_jinja_template(template_body, payload=alert_to_template.raw_request_data) except (JinjaTemplateError, JinjaTemplateWarning) as e: return Response({"preview": e.fallback_message}, status.HTTP_200_OK) + elif attr_name in ROUTE_TEMPLATE_NAMES: + try: + templated_attr = apply_jinja_template(template_body, payload=alert_to_template.raw_request_data) + except (JinjaTemplateError, JinjaTemplateWarning) as e: + return Response({"preview": e.fallback_message}, status.HTTP_200_OK) else: templated_attr = None response = {"preview": templated_attr} @@ -348,6 +346,8 @@ class PreviewTemplateMixin: destination = None if template_param.startswith(tuple(BEHAVIOUR_TEMPLATE_NAMES)): attr_name = template_param + if template_param.startswith(tuple(ROUTE_TEMPLATE_NAMES)): + attr_name = template_param elif template_param.startswith(tuple(NOTIFICATION_CHANNEL_OPTIONS)): for notification_channel in NOTIFICATION_CHANNEL_OPTIONS: if template_param.startswith(notification_channel): diff --git a/engine/common/api_helpers/utils.py b/engine/common/api_helpers/utils.py index 0ec76200..1233c4b0 100644 --- a/engine/common/api_helpers/utils.py +++ b/engine/common/api_helpers/utils.py @@ -14,8 +14,6 @@ from rest_framework.request import Request from apps.schedules.ical_utils import fetch_ical_file from common.api_helpers.exceptions import BadRequest -from common.jinja_templater import apply_jinja_template -from common.jinja_templater.apply_jinja_template import JinjaTemplateWarning from common.timezones import raise_exception_if_not_valid_timezone @@ -167,12 +165,3 @@ def check_phone_number_is_valid(phone_number): def serialize_datetime_as_utc_timestamp(dt: datetime.datetime) -> str: return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ") - - -def valid_jinja_template_for_serializer_method_field(template): - for _, val in template.items(): - try: - apply_jinja_template(val, payload={}) - except JinjaTemplateWarning: - # Suppress warnings, template may be valid with payload - pass diff --git a/engine/conftest.py b/engine/conftest.py index 1698befe..4f30e34a 100644 --- a/engine/conftest.py +++ b/engine/conftest.py @@ -954,13 +954,7 @@ def webhook_preset_api_setup(): @pytest.fixture def make_label_key(): - def _make_label_key(organization, key_id=None, key_name=None, **kwargs): - if key_id is not None: - kwargs["id"] = key_id - - if key_name is not None: - kwargs["name"] = key_name - + def _make_label_key(organization, **kwargs): return LabelKeyFactory(organization=organization, **kwargs) return _make_label_key @@ -968,13 +962,7 @@ def make_label_key(): @pytest.fixture def make_label_value(): - def _make_label_value(key, value_id=None, value_name=None, **kwargs): - if value_id is not None: - kwargs["id"] = value_id - - if value_name is not None: - kwargs["name"] = value_name - + def _make_label_value(key, **kwargs): return LabelValueFactory(key=key, **kwargs) return _make_label_value @@ -982,9 +970,9 @@ def make_label_value(): @pytest.fixture def make_label_key_and_value(make_label_key, make_label_value): - def _make_label_key_and_value(organization, key_id=None, key_name=None, value_id=None, value_name=None): - key = make_label_key(organization=organization, key_id=key_id, key_name=key_name) - value = make_label_value(key=key, value_id=value_id, value_name=value_name) + def _make_label_key_and_value(organization): + key = make_label_key(organization=organization) + value = make_label_value(key=key) return key, value return _make_label_key_and_value @@ -992,12 +980,8 @@ def make_label_key_and_value(make_label_key, make_label_value): @pytest.fixture def make_integration_label_association(make_label_key_and_value): - def _make_integration_label_association( - organization, alert_receive_channel, key_id=None, key_name=None, value_id=None, value_name=None, **kwargs - ): - key, value = make_label_key_and_value( - organization, key_id=key_id, key_name=key_name, value_id=value_id, value_name=value_name - ) + def _make_integration_label_association(organization, alert_receive_channel, **kwargs): + key, value = make_label_key_and_value(organization) return AlertReceiveChannelAssociatedLabelFactory( alert_receive_channel=alert_receive_channel, organization=organization, key=key, value=value, **kwargs ) diff --git a/grafana-plugin/package.json b/grafana-plugin/package.json index 0178bc57..3b7bda98 100644 --- a/grafana-plugin/package.json +++ b/grafana-plugin/package.json @@ -115,7 +115,7 @@ "@grafana/data": "^9.2.4", "@grafana/faro-web-sdk": "^1.0.0-beta4", "@grafana/faro-web-tracing": "^1.0.0-beta4", - "@grafana/labels": "1.3.4", + "@grafana/labels": "~1.2.1", "@grafana/runtime": "9.3.0-beta1", "@grafana/ui": "^9.4.7", "@opentelemetry/api": "^1.3.0", diff --git a/grafana-plugin/playwright.config.ts b/grafana-plugin/playwright.config.ts index 835639e5..689cd6cc 100644 --- a/grafana-plugin/playwright.config.ts +++ b/grafana-plugin/playwright.config.ts @@ -1,4 +1,4 @@ -import { PlaywrightTestProject, defineConfig, devices } from '@playwright/test'; +import { PlaywrightTestConfig, PlaywrightTestProject, defineConfig, devices } from '@playwright/test'; import path from 'path'; /** diff --git a/grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx b/grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx index a2b9262d..5ec9d084 100644 --- a/grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx +++ b/grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx @@ -22,7 +22,7 @@ const LabelsTooltipBadge: FC = ({ labels, onClick }) => {labels.map((label) => ( - + - - - - - - {customLabelIndexToShowTemplateEditor !== undefined && ( - setCustomLabelIndexToShowTemplateEditor(undefined)} - onUpdateTemplates={({ alert_group_labels }) => { - const newCustom = [...alertGroupLabels.custom]; - newCustom[customLabelIndexToShowTemplateEditor].value.name = alert_group_labels; - - setAlertGroupLabels({ - ...alertGroupLabels, - custom: newCustom, - }); - - setCustomLabelIndexToShowTemplateEditor(undefined); - }} - /> - )} - {showTemplateEditor && ( - setShowTemplateEditor(false)} - onUpdateTemplates={({ alert_group_labels }) => { - setAlertGroupLabels({ - ...alertGroupLabels, - template: alert_group_labels, - }); - - setShowTemplateEditor(undefined); - }} - /> - )} - + )} + +
+ + + + +
+ + ); }); -interface CustomLabelsProps { - alertGroupLabels: AlertReceiveChannel['alert_group_labels']; - onChange: (value: AlertReceiveChannel['alert_group_labels']) => void; - onShowTemplateEditor: (index: number) => void; -} - -const CustomLabels = (props: CustomLabelsProps) => { - const { alertGroupLabels, onChange, onShowTemplateEditor } = props; - - const { labelsStore } = useStore(); - - const handlePlainLabelAdd = () => { - onChange({ - ...alertGroupLabels, - custom: [ - ...alertGroupLabels.custom, - { - key: { id: undefined, name: undefined }, - value: { id: undefined, name: undefined }, - }, - ], - }); - }; - const handleTemplatedLabelAdd = () => { - onChange({ - ...alertGroupLabels, - custom: [ - ...alertGroupLabels.custom, - { - key: { id: undefined, name: undefined }, - value: { id: null, name: undefined }, // id = null means it's a templated value - }, - ], - }); - }; - - const cachedOnLoadKeys = useCallback(() => { - let result = undefined; - return async (search?: string) => { - if (!result) { - try { - result = await labelsStore.loadKeys(); - } catch (error) { - openErrorNotification('There was an error processing your request. Please try again'); - } - } - - return result.filter((k) => k.name.toLowerCase().includes(search.toLowerCase())); - }; - }, []); - - const cachedOnLoadValuesForKey = useCallback(() => { - let result = undefined; - return async (key: string, search?: string) => { - if (!result) { - try { - const { values } = await labelsStore.loadValuesForKey(key, search); - result = values; - } catch (error) { - openErrorNotification('There was an error processing your request. Please try again'); - } - } - - return result.filter((k) => k.name.toLowerCase().includes(search.toLowerCase())); - }; - }, []); - - return ( - - - - - { - if (res?.response?.status === 409) { - openErrorNotification(`Duplicate values are not allowed`); - } else { - openErrorNotification('An error has occurred. Please try again'); - } - }} - renderValue={(option, index, renderValueDefault) => { - if (option.value.id === null) { - return ( - { - onShowTemplateEditor(index); - }} - /> - } - onChange={(e: ChangeEvent) => { - const newCustom = [...alertGroupLabels.custom]; - newCustom[index].value.name = e.currentTarget.value; - - onChange({ ...alertGroupLabels, custom: newCustom }); - }} - /> - ); - } else { - return renderValueDefault(option, index); - } - }} - onDataUpdate={(value) => { - onChange({ - ...alertGroupLabels, - custom: value, - }); - }} - /> - - - - - } - > - - - - ); -}; - export default IntegrationLabelsForm; diff --git a/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx b/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx index 98e98b71..7376ee96 100644 --- a/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx +++ b/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx @@ -5,7 +5,6 @@ import cn from 'classnames/bind'; import { debounce } from 'lodash-es'; import { observer } from 'mobx-react'; -import { templateForEdit } from 'components/AlertTemplates/AlertTemplatesForm.config'; import { TemplateForEdit } from 'components/AlertTemplates/CommonAlertTemplatesForm.config'; import CheatSheet from 'components/CheatSheet/CheatSheet'; import { @@ -39,7 +38,7 @@ interface IntegrationTemplateProps { templates: AlertTemplatesDTO[]; onHide: () => void; onUpdateTemplates: (values: any) => void; - onUpdateRoute?: (values: any, channelFilterId?: ChannelFilter['id']) => void; + onUpdateRoute: (values: any, channelFilterId?: ChannelFilter['id']) => void; } const IntegrationTemplate = observer((props: IntegrationTemplateProps) => { @@ -54,13 +53,11 @@ const IntegrationTemplate = observer((props: IntegrationTemplateProps) => { const [isRecentAlertGroupExisting, setIsRecentAlertGroupExisting] = useState(false); useEffect(() => { - if (templateForEdit[template.name]) { - const locationParams: any = { template: template.name }; - if (template.isRoute) { - locationParams.routeId = channelFilterId; - } - LocationHelper.update(locationParams, 'partial'); + const locationParams: any = { template: template.name }; + if (template.isRoute) { + locationParams.routeId = channelFilterId; } + LocationHelper.update(locationParams, 'partial'); }, []); useEffect(() => { diff --git a/grafana-plugin/src/containers/Labels/Labels.tsx b/grafana-plugin/src/containers/Labels/Labels.tsx index 60fc1324..21c4d46a 100644 --- a/grafana-plugin/src/containers/Labels/Labels.tsx +++ b/grafana-plugin/src/containers/Labels/Labels.tsx @@ -1,6 +1,6 @@ import React, { forwardRef, useCallback, useImperativeHandle, useState } from 'react'; -import { ServiceLabels, ServiceLabelsProps } from '@grafana/labels'; +import ServiceLabels, { ServiceLabelsProps } from '@grafana/labels'; import { Field } from '@grafana/ui'; import cn from 'classnames/bind'; import { isEmpty } from 'lodash-es'; diff --git a/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts b/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts index fd3ea806..ec366bf2 100644 --- a/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts +++ b/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts @@ -49,11 +49,7 @@ export interface AlertReceiveChannel { allow_delete: boolean; deleted?: boolean; labels: LabelKeyValue[]; - alert_group_labels: { - inheritable: Record; - custom: LabelKeyValue[]; - template: string; - }; + alert_group_labels: { inheritable: Record }; } export interface AlertReceiveChannelChoice { diff --git a/grafana-plugin/yarn.lock b/grafana-plugin/yarn.lock index 3f55fb20..2b225b09 100644 --- a/grafana-plugin/yarn.lock +++ b/grafana-plugin/yarn.lock @@ -1966,10 +1966,10 @@ "@opentelemetry/sdk-trace-web" "^1.8.0" "@opentelemetry/semantic-conventions" "^1.8.0" -"@grafana/labels@1.3.4": - version "1.3.4" - resolved "https://registry.yarnpkg.com/@grafana/labels/-/labels-1.3.4.tgz#8d9cdd215a80a1da1045d402c037be85d7efd6f5" - integrity sha512-YYCuLGvtrMz7KkbMc6qoNJQr6drDLo6mMI27LcqsTDMHCNO3uJWpzC1Q2Y9MIwctIuTFYhbgfLvIunEegCx6PQ== +"@grafana/labels@~1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@grafana/labels/-/labels-1.2.1.tgz#4113d584bf5cd826d011f957cb69c90bd0416ea8" + integrity sha512-Nlqqvjwh0MjWsqnfpYbKdYwByeKSmEpiit5mKd6Mnnbc5Hxb8ORIruMr40lTxxWLEnDfhENcAs6pvlBuIMG7tQ== dependencies: "@emotion/css" "^11.11.2" "@grafana/ui" "^10.0.0" From 9e889403f20b918f2e7296653201b9421158e0d2 Mon Sep 17 00:00:00 2001 From: Vadim Stepanov Date: Mon, 27 Nov 2023 17:53:54 +0000 Subject: [PATCH 17/18] Alert group payload labels (#3434) https://github.com/grafana/oncall/pull/3385 + handle null values --- ...nnel_alert_group_labels_custom_and_more.py | 23 ++ engine/apps/alerts/models/alert.py | 4 +- engine/apps/alerts/models/alert_group.py | 2 + .../alerts/models/alert_receive_channel.py | 32 +- .../api/serializers/alert_receive_channel.py | 201 ++++++++++- engine/apps/api/serializers/channel_filter.py | 2 +- .../api/tests/test_alert_receive_channel.py | 106 +++++- .../test_alert_receive_channel_template.py | 31 ++ engine/apps/labels/alert_group_labels.py | 161 +++++++++ engine/apps/labels/models.py | 12 +- engine/apps/labels/tests/test_alert_group.py | 82 ++++- engine/apps/labels/utils.py | 38 +- engine/apps/public_api/serializers/routes.py | 2 +- engine/common/api_helpers/mixins.py | 20 +- engine/common/api_helpers/utils.py | 11 + engine/conftest.py | 30 +- grafana-plugin/package.json | 2 +- grafana-plugin/playwright.config.ts | 2 +- .../LabelsTooltipBadge/LabelsTooltipBadge.tsx | 2 +- .../components/MonacoEditor/MonacoEditor.tsx | 1 + .../IntegrationLabelsForm.tsx | 333 +++++++++++++++--- .../IntegrationTemplate.tsx | 13 +- .../src/containers/Labels/Labels.tsx | 2 +- .../alert_receive_channel.types.ts | 6 +- grafana-plugin/yarn.lock | 8 +- 25 files changed, 965 insertions(+), 161 deletions(-) create mode 100644 engine/apps/alerts/migrations/0040_alertreceivechannel_alert_group_labels_custom_and_more.py create mode 100644 engine/apps/labels/alert_group_labels.py diff --git a/engine/apps/alerts/migrations/0040_alertreceivechannel_alert_group_labels_custom_and_more.py b/engine/apps/alerts/migrations/0040_alertreceivechannel_alert_group_labels_custom_and_more.py new file mode 100644 index 00000000..d35f9626 --- /dev/null +++ b/engine/apps/alerts/migrations/0040_alertreceivechannel_alert_group_labels_custom_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.7 on 2023-11-27 17:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('alerts', '0039_remove_alertreceivechannel_unique_integration_name'), + ] + + operations = [ + migrations.AddField( + model_name='alertreceivechannel', + name='alert_group_labels_custom', + field=models.JSONField(default=None, null=True), + ), + migrations.AddField( + model_name='alertreceivechannel', + name='alert_group_labels_template', + field=models.TextField(default=None, null=True), + ), + ] diff --git a/engine/apps/alerts/models/alert.py b/engine/apps/alerts/models/alert.py index 79f458db..203f208a 100644 --- a/engine/apps/alerts/models/alert.py +++ b/engine/apps/alerts/models/alert.py @@ -12,7 +12,7 @@ from django.db.models import JSONField from apps.alerts import tasks from apps.alerts.constants import TASK_DELAY_SECONDS from apps.alerts.incident_appearance.templaters import TemplateLoader -from apps.labels.utils import assign_labels +from apps.labels.alert_group_labels import assign_labels from common.jinja_templater import apply_jinja_template from common.jinja_templater.apply_jinja_template import JinjaTemplateError, JinjaTemplateWarning from common.public_primary_keys import generate_public_primary_key, increase_public_primary_key_length @@ -108,7 +108,7 @@ class Alert(models.Model): ) if group_created: - assign_labels(group, alert_receive_channel) + assign_labels(group, alert_receive_channel, raw_request_data) group.log_records.create(type=AlertGroupLogRecord.TYPE_REGISTERED) group.log_records.create(type=AlertGroupLogRecord.TYPE_ROUTE_ASSIGNED) diff --git a/engine/apps/alerts/models/alert_group.py b/engine/apps/alerts/models/alert_group.py index 6f6aec12..cbdb587a 100644 --- a/engine/apps/alerts/models/alert_group.py +++ b/engine/apps/alerts/models/alert_group.py @@ -43,6 +43,7 @@ if typing.TYPE_CHECKING: ResolutionNoteSlackMessage, ) from apps.base.models import UserNotificationPolicyLogRecord + from apps.labels.models import AlertGroupAssociatedLabel from apps.slack.models import SlackMessage logger = logging.getLogger(__name__) @@ -194,6 +195,7 @@ class AlertGroup(AlertGroupSlackRenderingMixin, EscalationSnapshotMixin, models. slack_log_message: typing.Optional["SlackMessage"] slack_messages: "RelatedManager['SlackMessage']" users: "RelatedManager['User']" + labels: "RelatedManager['AlertGroupAssociatedLabel']" objects: models.Manager["AlertGroup"] = AlertGroupQuerySet.as_manager() diff --git a/engine/apps/alerts/models/alert_receive_channel.py b/engine/apps/alerts/models/alert_receive_channel.py index fb465193..5ecc6a85 100644 --- a/engine/apps/alerts/models/alert_receive_channel.py +++ b/engine/apps/alerts/models/alert_receive_channel.py @@ -42,6 +42,7 @@ if typing.TYPE_CHECKING: from django.db.models.manager import RelatedManager from apps.alerts.models import AlertGroup, ChannelFilter + from apps.labels.models import AlertReceiveChannelAssociatedLabel from apps.user_management.models import Organization, Team logger = logging.getLogger(__name__) @@ -87,10 +88,6 @@ def number_to_smiles_translator(number): return "".join(reversed(smileset)) -class IntegrationAlertGroupLabels(typing.TypedDict): - inheritable: typing.Dict[str, bool] - - class AlertReceiveChannelQueryset(models.QuerySet): def delete(self): self.update(deleted_at=timezone.now()) @@ -123,6 +120,7 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): channel_filters: "RelatedManager['ChannelFilter']" organization: "Organization" team: typing.Optional["Team"] + labels: "RelatedManager['AlertReceiveChannelAssociatedLabel']" objects = AlertReceiveChannelManager() objects_with_maintenance = AlertReceiveChannelManagerWithMaintenance() @@ -206,6 +204,17 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): rate_limited_in_slack_at = models.DateTimeField(null=True, default=None) rate_limit_message_task_id = models.CharField(max_length=100, null=True, default=None) + AlertGroupCustomLabels = list[tuple[str, str | None, str | None]] | None + alert_group_labels_custom: AlertGroupCustomLabels = models.JSONField(null=True, default=None) + """ + Stores "custom labels" for alert group labels. Custom labels can be either "plain" or "templated". + For plain labels, the format is: [, , None] + For templated labels, the format is: [, None, ] + """ + + alert_group_labels_template: str | None = models.TextField(null=True, default=None) + """Stores a Jinja2 template for "advanced label templating" for alert group labels.""" + def __str__(self): short_name_with_emojis = emojize(self.short_name, language="alias") return f"{self.pk}: {short_name_with_emojis}" @@ -635,21 +644,6 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): result["team"] = "General" return result - @property - def alert_group_labels(self) -> IntegrationAlertGroupLabels: - """ - Alert group labels configuration for the integration used by AlertReceiveChannelSerializer. - See AlertReceiveChannelAssociatedLabel.inheritable for more details. - """ - return {"inheritable": {label.key_id: label.inheritable for label in self.labels.all()}} - - @alert_group_labels.setter - def alert_group_labels(self, value: IntegrationAlertGroupLabels) -> None: - """Setter for alert_group_labels used by AlertReceiveChannelSerializer""" - inheritable_key_ids = [key_id for key_id, inheritable in value["inheritable"].items() if inheritable] - self.labels.filter(key_id__in=inheritable_key_ids).update(inheritable=True) - self.labels.filter(~Q(key_id__in=inheritable_key_ids)).update(inheritable=False) - @receiver(post_save, sender=AlertReceiveChannel) def listen_for_alertreceivechannel_model_save( diff --git a/engine/apps/api/serializers/alert_receive_channel.py b/engine/apps/api/serializers/alert_receive_channel.py index ad9ddaf5..0aae34b6 100644 --- a/engine/apps/api/serializers/alert_receive_channel.py +++ b/engine/apps/api/serializers/alert_receive_channel.py @@ -4,6 +4,7 @@ from collections import OrderedDict from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError as DjangoValidationError +from django.db.models import Q from jinja2 import TemplateSyntaxError from rest_framework import serializers from rest_framework.exceptions import ValidationError @@ -14,30 +15,191 @@ from apps.alerts.models import AlertReceiveChannel from apps.alerts.models.channel_filter import ChannelFilter from apps.base.messaging import get_messaging_backends from apps.integrations.legacy_prefix import has_legacy_prefix +from apps.labels.models import LabelKeyCache, LabelValueCache +from apps.user_management.models import Organization from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField from common.api_helpers.exceptions import BadRequest from common.api_helpers.mixins import APPEARANCE_TEMPLATE_NAMES, EagerLoadingMixin from common.api_helpers.utils import CurrentTeamDefault -from common.jinja_templater import apply_jinja_template, jinja_template_env -from common.jinja_templater.apply_jinja_template import JinjaTemplateWarning +from common.jinja_templater import jinja_template_env from .integration_heartbeat import IntegrationHeartBeatSerializer from .labels import LabelsSerializerMixin -def valid_jinja_template_for_serializer_method_field(template): - for _, val in template.items(): - try: - apply_jinja_template(val, payload={}) - except JinjaTemplateWarning: - # Suppress warnings, template may be valid with payload - pass +class AlertGroupCustomLabelKey(typing.TypedDict): + id: str + name: str + + +class AlertGroupCustomLabelValue(typing.TypedDict): + id: str | None # None for templated labels, label value ID for plain labels + name: str # Jinja template for templated labels, label value name for plain labels + + +class AlertGroupCustomLabel(typing.TypedDict): + key: AlertGroupCustomLabelKey + value: AlertGroupCustomLabelValue + + +AlertGroupCustomLabels = list[AlertGroupCustomLabel] + + +class IntegrationAlertGroupLabels(typing.TypedDict): + inheritable: dict[str, bool] + custom: AlertGroupCustomLabels + template: str | None + + +class CustomLabelSerializer(serializers.Serializer): + """This serializer is consistent with apps.api.serializers.labels.LabelSerializer, but allows null for value ID.""" + + class KeySerializer(serializers.Serializer): + id = serializers.CharField() + name = serializers.CharField() + + class ValueSerializer(serializers.Serializer): + # ID is null for templated labels. For such labels, the "name" value is a Jinja2 template. + id = serializers.CharField(allow_null=True) + name = serializers.CharField() + + key = KeySerializer() + value = ValueSerializer() class IntegrationAlertGroupLabelsSerializer(serializers.Serializer): """Alert group labels configuration for the integration. See AlertReceiveChannel.alert_group_labels for details.""" inheritable = serializers.DictField(child=serializers.BooleanField()) + custom = CustomLabelSerializer(many=True) + template = serializers.CharField(allow_null=True) + + @staticmethod + def pop_alert_group_labels(validated_data: dict) -> IntegrationAlertGroupLabels | None: + """Get alert group labels from validated data.""" + + # the "alert_group_labels" field is optional, so either all 3 fields are present or none + if "inheritable" not in validated_data: + return None + + return { + "inheritable": validated_data.pop("inheritable"), + "custom": validated_data.pop("custom"), + "template": validated_data.pop("template"), + } + + @classmethod + def update( + cls, instance: AlertReceiveChannel, alert_group_labels: IntegrationAlertGroupLabels | None + ) -> AlertReceiveChannel: + if alert_group_labels is None: + return instance + + # update inheritable labels + inheritable_key_ids = [ + key_id for key_id, inheritable in alert_group_labels["inheritable"].items() if inheritable + ] + instance.labels.filter(key_id__in=inheritable_key_ids).update(inheritable=True) + instance.labels.filter(~Q(key_id__in=inheritable_key_ids)).update(inheritable=False) + + # update DB cache for custom labels + cls._create_custom_labels(instance.organization, alert_group_labels["custom"]) + # update custom labels + instance.alert_group_labels_custom = cls._custom_labels_to_internal_value(alert_group_labels["custom"]) + + # update template + instance.alert_group_labels_template = alert_group_labels["template"] + + instance.save(update_fields=["alert_group_labels_custom", "alert_group_labels_template"]) + return instance + + @staticmethod + def _create_custom_labels(organization: Organization, labels: AlertGroupCustomLabels) -> None: + """Create LabelKeyCache and LabelValueCache objects for custom labels.""" + + label_keys = [ + LabelKeyCache(id=label["key"]["id"], name=label["key"]["name"], organization=organization) + for label in labels + ] + + label_values = [ + LabelValueCache(id=label["value"]["id"], name=label["value"]["name"], key_id=label["key"]["id"]) + for label in labels + if label["value"]["id"] # don't create LabelValueCache objects for templated labels + ] + + LabelKeyCache.objects.bulk_create(label_keys, ignore_conflicts=True, batch_size=5000) + LabelValueCache.objects.bulk_create(label_values, ignore_conflicts=True, batch_size=5000) + + @classmethod + def to_representation(cls, instance: AlertReceiveChannel) -> IntegrationAlertGroupLabels: + """ + The API representation of alert group labels is very different from the underlying model. + + "inheritable" is based on AlertReceiveChannelAssociatedLabel.inheritable, a property of another model. + "custom" is based on AlertReceiveChannel.alert_group_labels_custom, a JSONField with a different schema. + "template" is based on AlertReceiveChannel.alert_group_labels_template, this one is straightforward. + """ + + return { + "inheritable": {label.key_id: label.inheritable for label in instance.labels.all()}, + "custom": cls._custom_labels_to_representation(instance.alert_group_labels_custom), + "template": instance.alert_group_labels_template, + } + + @staticmethod + def _custom_labels_to_internal_value( + custom_labels: AlertGroupCustomLabels, + ) -> AlertReceiveChannel.AlertGroupCustomLabels: + """Convert custom labels from API representation to the schema used by the JSONField on the model.""" + + return [ + [label["key"]["id"], label["value"]["id"], None if label["value"]["id"] else label["value"]["name"]] + for label in custom_labels + ] + + @staticmethod + def _custom_labels_to_representation( + custom_labels: AlertReceiveChannel.AlertGroupCustomLabels, + ) -> AlertGroupCustomLabels: + """ + Inverse of the _custom_labels_to_internal_value method above. + Fetches label names from DB cache, so the API response schema is consistent with other label endpoints. + """ + + from apps.labels.models import LabelKeyCache, LabelValueCache + + if custom_labels is None: + return [] + + # get up-to-date label key names + label_key_names = { + k.id: k.name + for k in LabelKeyCache.objects.filter(id__in=[label[0] for label in custom_labels]).only("id", "name") + } + + # get up-to-date label value names + label_value_names = { + v.id: v.name + for v in LabelValueCache.objects.filter(id__in=[label[1] for label in custom_labels if label[1]]).only( + "id", "name" + ) + } + + return [ + { + "key": { + "id": key_id, + "name": label_key_names[key_id], + }, + "value": { + "id": value_id if value_id else None, + "name": label_value_names[value_id] if value_id else typing.cast(str, template), + }, + } + for key_id, value_id, template in custom_labels + if key_id in label_key_names and (value_id in label_value_names or not value_id) + ] class AlertReceiveChannelSerializer( @@ -64,7 +226,7 @@ class AlertReceiveChannelSerializer( connected_escalations_chains_count = serializers.SerializerMethodField() inbound_email = serializers.CharField(required=False) is_legacy = serializers.SerializerMethodField() - alert_group_labels = IntegrationAlertGroupLabelsSerializer(required=False) + alert_group_labels = IntegrationAlertGroupLabelsSerializer(source="*", required=False) # integration heartbeat is in PREFETCH_RELATED not by mistake. # With using of select_related ORM builds strange join @@ -138,8 +300,10 @@ class AlertReceiveChannelSerializer( if _integration.slug == integration: is_able_to_autoresolve = _integration.is_able_to_autoresolve + # pop associated labels and alert group labels, so they are not passed to AlertReceiveChannel.create labels = validated_data.pop("labels", None) - alert_group_labels = validated_data.pop("alert_group_labels", None) + alert_group_labels = IntegrationAlertGroupLabelsSerializer.pop_alert_group_labels(validated_data) + try: instance = AlertReceiveChannel.create( **validated_data, @@ -150,17 +314,22 @@ class AlertReceiveChannelSerializer( except AlertReceiveChannel.DuplicateDirectPagingError: raise BadRequest(detail=AlertReceiveChannel.DuplicateDirectPagingError.DETAIL) - # Create label associations first, then update inheritable labels + # Create label associations first, then update alert group labels self.update_labels_association_if_needed(labels, instance, organization) - if alert_group_labels: - instance.alert_group_labels = alert_group_labels + instance = IntegrationAlertGroupLabelsSerializer.update(instance, alert_group_labels) return instance def update(self, instance, validated_data): + # update associated labels labels = validated_data.pop("labels", None) - organization = self.context["request"].auth.organization - self.update_labels_association_if_needed(labels, instance, organization) + self.update_labels_association_if_needed(labels, instance, self.context["request"].auth.organization) + + # update alert group labels + instance = IntegrationAlertGroupLabelsSerializer.update( + instance, IntegrationAlertGroupLabelsSerializer.pop_alert_group_labels(validated_data) + ) + try: return super().update(instance, validated_data) except AlertReceiveChannel.DuplicateDirectPagingError: diff --git a/engine/apps/api/serializers/channel_filter.py b/engine/apps/api/serializers/channel_filter.py index b9239d6d..7815a584 100644 --- a/engine/apps/api/serializers/channel_filter.py +++ b/engine/apps/api/serializers/channel_filter.py @@ -3,12 +3,12 @@ import typing from rest_framework import serializers from apps.alerts.models import AlertReceiveChannel, ChannelFilter, EscalationChain -from apps.api.serializers.alert_receive_channel import valid_jinja_template_for_serializer_method_field from apps.base.messaging import get_messaging_backend_from_id from apps.telegram.models import TelegramToOrganizationConnector from common.api_helpers.custom_fields import OrganizationFilteredPrimaryKeyRelatedField from common.api_helpers.exceptions import BadRequest from common.api_helpers.mixins import EagerLoadingMixin +from common.api_helpers.utils import valid_jinja_template_for_serializer_method_field from common.jinja_templater.apply_jinja_template import JinjaTemplateError from common.utils import is_regex_valid diff --git a/engine/apps/api/tests/test_alert_receive_channel.py b/engine/apps/api/tests/test_alert_receive_channel.py index 989171db..4855a87c 100644 --- a/engine/apps/api/tests/test_alert_receive_channel.py +++ b/engine/apps/api/tests/test_alert_receive_channel.py @@ -9,6 +9,7 @@ from rest_framework.test import APIClient from apps.alerts.models import AlertReceiveChannel, EscalationPolicy from apps.api.permissions import LegacyAccessControlRole +from apps.labels.models import LabelKeyCache, LabelValueCache @pytest.fixture() @@ -1383,23 +1384,49 @@ def test_update_alert_receive_channel_labels_duplicate_key( def test_alert_group_labels_get( make_organization_and_user_with_plugin_token, make_alert_receive_channel, + make_label_key_and_value, make_integration_label_association, make_user_auth_headers, ): organization, user, token = make_organization_and_user_with_plugin_token() alert_receive_channel = make_alert_receive_channel(organization) + label_key, label_value = make_label_key_and_value(organization) + label_key_1, _ = make_label_key_and_value(organization) client = APIClient() url = reverse("api-internal:alert_receive_channel-detail", kwargs={"pk": alert_receive_channel.public_primary_key}) response = client.get(url, **make_user_auth_headers(user, token)) assert response.status_code == status.HTTP_200_OK - assert response.json()["alert_group_labels"] == {"inheritable": {}} + assert response.json()["alert_group_labels"] == {"inheritable": {}, "custom": [], "template": None} label = make_integration_label_association(organization, alert_receive_channel) + + template = "{{ payload.labels | tojson }}" + alert_receive_channel.alert_group_labels_template = template + + alert_receive_channel.alert_group_labels_custom = [ + (label_key.id, label_value.id, None), + (label_key_1.id, None, "{{ payload.foo }}"), + ] + alert_receive_channel.save(update_fields=["alert_group_labels_custom", "alert_group_labels_template"]) + response = client.get(url, **make_user_auth_headers(user, token)) assert response.status_code == status.HTTP_200_OK - assert response.json()["alert_group_labels"] == {"inheritable": {label.key_id: True}} + assert response.json()["alert_group_labels"] == { + "inheritable": {label.key_id: True}, + "custom": [ + { + "key": {"id": label_key.id, "name": label_key.name}, + "value": {"id": label_value.id, "name": label_value.name}, + }, + { + "key": {"id": label_key_1.id, "name": label_key_1.name}, + "value": {"id": None, "name": "{{ payload.foo }}"}, + }, + ], + "template": template, + } @pytest.mark.django_db @@ -1413,14 +1440,75 @@ def test_alert_group_labels_put( alert_receive_channel = make_alert_receive_channel(organization) label_1 = make_integration_label_association(organization, alert_receive_channel) label_2 = make_integration_label_association(organization, alert_receive_channel, inheritable=False) + label_3 = make_integration_label_association(organization, alert_receive_channel, inheritable=False) + + custom = [ + # plain label + { + "key": {"id": label_2.key.id, "name": label_2.key.name}, + "value": {"id": label_2.value.id, "name": label_2.value.name}, + }, + # plain label not present in DB cache + { + "key": {"id": "hello", "name": "world"}, + "value": {"id": "foo", "name": "bar"}, + }, + # templated label + { + "key": {"id": label_3.key.id, "name": label_3.key.name}, + "value": {"id": None, "name": "{{ payload.foo }}"}, + }, + ] + template = "{{ payload.labels | tojson }}" # advanced template client = APIClient() url = reverse("api-internal:alert_receive_channel-detail", kwargs={"pk": alert_receive_channel.public_primary_key}) - data = {"alert_group_labels": {"inheritable": {label_1.key_id: False, label_2.key_id: True}}} + data = { + "alert_group_labels": { + "inheritable": {label_1.key_id: False, label_2.key_id: True, label_3.key_id: False}, + "custom": custom, + "template": template, + } + } response = client.put(url, data, format="json", **make_user_auth_headers(user, token)) assert response.status_code == status.HTTP_200_OK - assert response.json()["alert_group_labels"] == {"inheritable": {label_1.key_id: False, label_2.key_id: True}} + assert response.json()["alert_group_labels"] == { + "inheritable": {label_1.key_id: False, label_2.key_id: True, label_3.key_id: False}, + "custom": custom, + "template": template, + } + + alert_receive_channel.refresh_from_db() + assert alert_receive_channel.alert_group_labels_custom == [ + [label_2.key_id, label_2.value_id, None], + ["hello", "foo", None], + [label_3.key_id, None, "{{ payload.foo }}"], + ] + assert alert_receive_channel.alert_group_labels_template == template + + # check label keys & values are created + key = LabelKeyCache.objects.filter(id="hello", name="world", organization=organization).first() + assert key is not None + assert LabelValueCache.objects.filter(key=key, id="foo", name="bar").exists() + + +@pytest.mark.django_db +def test_alert_group_labels_put_none( + make_organization_and_user_with_plugin_token, + make_alert_receive_channel, + make_user_auth_headers, +): + organization, user, token = make_organization_and_user_with_plugin_token() + alert_receive_channel = make_alert_receive_channel(organization) + + client = APIClient() + url = reverse("api-internal:alert_receive_channel-detail", kwargs={"pk": alert_receive_channel.public_primary_key}) + response = client.put(url, {"verbal_name": "123"}, format="json", **make_user_auth_headers(user, token)) + + assert response.status_code == status.HTTP_200_OK + assert response.json()["verbal_name"] == "123" + assert response.json()["alert_group_labels"] == {"inheritable": {}, "custom": [], "template": None} @pytest.mark.django_db @@ -1428,7 +1516,11 @@ def test_alert_group_labels_post(alert_receive_channel_internal_api_setup, make_ user, token, _ = alert_receive_channel_internal_api_setup labels = [{"key": {"id": "test", "name": "test"}, "value": {"id": "123", "name": "123"}}] - alert_group_labels = {"inheritable": {"test": False}} + alert_group_labels = { + "inheritable": {"test": False}, + "custom": [{"key": {"id": "test", "name": "test"}, "value": {"id": "123", "name": "123"}}], + "template": "{{ payload.labels | tojson }}", + } data = { "integration": AlertReceiveChannel.INTEGRATION_GRAFANA, "team": None, @@ -1443,3 +1535,7 @@ def test_alert_group_labels_post(alert_receive_channel_internal_api_setup, make_ assert response.status_code == status.HTTP_201_CREATED assert response.json()["labels"] == labels assert response.json()["alert_group_labels"] == alert_group_labels + + alert_receive_channel = AlertReceiveChannel.objects.get(public_primary_key=response.json()["id"]) + assert alert_receive_channel.alert_group_labels_custom == [["test", "123", None]] + assert alert_receive_channel.alert_group_labels_template == "{{ payload.labels | tojson }}" diff --git a/engine/apps/api/tests/test_alert_receive_channel_template.py b/engine/apps/api/tests/test_alert_receive_channel_template.py index 111696cd..f494d776 100644 --- a/engine/apps/api/tests/test_alert_receive_channel_template.py +++ b/engine/apps/api/tests/test_alert_receive_channel_template.py @@ -337,6 +337,37 @@ def test_preview_alert_receive_channel_backend_templater( assert response.json() == {"preview": "title: alert!"} +@pytest.mark.django_db +def test_preview_alert_group_labels( + make_organization_and_user_with_plugin_token, + make_user_auth_headers, + make_alert_receive_channel, + make_channel_filter, + make_alert_group, + make_alert, +): + organization, user, token = make_organization_and_user_with_plugin_token() + alert_receive_channel = make_alert_receive_channel(organization) + default_channel_filter = make_channel_filter(alert_receive_channel, is_default=True) + alert_group = make_alert_group(alert_receive_channel, channel_filter=default_channel_filter) + make_alert(alert_group=alert_group, raw_request_data={"labels": {"1": "2"}}) + + client = APIClient() + url = reverse( + "api-internal:alert_receive_channel-preview-template", + kwargs={"pk": alert_receive_channel.public_primary_key}, + ) + + data = { + "template_body": "{{ payload.labels | tojson }}", + "template_name": "alert_group_labels", + } + response = client.post(url, format="json", data=data, **make_user_auth_headers(user, token)) + + assert response.status_code == status.HTTP_200_OK + assert response.json() == {"preview": '{"1": "2"}'} + + @pytest.mark.django_db def test_update_alert_receive_channel_templates( make_organization_and_user_with_plugin_token, diff --git a/engine/apps/labels/alert_group_labels.py b/engine/apps/labels/alert_group_labels.py new file mode 100644 index 00000000..68ef6ead --- /dev/null +++ b/engine/apps/labels/alert_group_labels.py @@ -0,0 +1,161 @@ +import json +import logging +import typing + +from apps.labels.utils import is_labels_feature_enabled +from common.jinja_templater import apply_jinja_template +from common.jinja_templater.apply_jinja_template import JinjaTemplateError, JinjaTemplateWarning + +if typing.TYPE_CHECKING: + from apps.alerts.models import AlertGroup, AlertReceiveChannel + + +logger = logging.getLogger(__name__) + + +# What can be used as a label key/value coming out from the template +LABEL_VALUE_TYPES = (str, int, float, bool) + + +def assign_labels( + alert_group: "AlertGroup", alert_receive_channel: "AlertReceiveChannel", raw_request_data: typing.Any +) -> None: + from apps.labels.models import AlertGroupAssociatedLabel + + if not is_labels_feature_enabled(alert_receive_channel.organization): + return + + # inherit labels from the integration + labels = { + label.key.name: label.value.name + for label in alert_receive_channel.labels.filter(inheritable=True).select_related("key", "value") + } + + # apply custom labels + labels.update(_custom_labels(alert_receive_channel, raw_request_data)) + + # apply template labels + labels.update(_template_labels(alert_receive_channel, raw_request_data)) + + # create associated labels + alert_group_labels = [ + AlertGroupAssociatedLabel( + alert_group=alert_group, + organization=alert_receive_channel.organization, + key_name=key, + value_name=value, + ) + for key, value in labels.items() + ] + # sort associated labels by key and value + alert_group_labels.sort(key=lambda label: (label.key_name, label.value_name)) + # bulk create associated labels + AlertGroupAssociatedLabel.objects.bulk_create(alert_group_labels) + + +def _custom_labels(alert_receive_channel: "AlertReceiveChannel", raw_request_data: typing.Any) -> dict[str, str]: + from apps.labels.models import MAX_VALUE_NAME_LENGTH, LabelKeyCache, LabelValueCache + + if alert_receive_channel.alert_group_labels_custom is None: + return {} + + # fetch up-to-date label key names + label_key_names = { + k.id: k.name + for k in LabelKeyCache.objects.filter( + id__in=[label[0] for label in alert_receive_channel.alert_group_labels_custom] + ).only("id", "name") + } + + # fetch up-to-date label value names + label_value_names = { + v.id: v.name + for v in LabelValueCache.objects.filter( + id__in=[label[1] for label in alert_receive_channel.alert_group_labels_custom if label[1]] + ).only("id", "name") + } + + rendered_labels = {} + for label in alert_receive_channel.alert_group_labels_custom: + key_id, value_id, template = label + + if key_id in label_key_names: + key = label_key_names[key_id] + else: + logger.warning("Label key cache not found. %s", key_id) + continue + + if value_id: + if value_id in label_value_names: + rendered_labels[key] = label_value_names[value_id] + else: + logger.warning("Label value cache not found. %s", value_id) + continue + else: + try: + rendered_labels[key] = apply_jinja_template(template, raw_request_data) + except (JinjaTemplateError, JinjaTemplateWarning) as e: + logger.warning("Failed to apply template. %s", e.fallback_message) + continue + + labels = {} + for key in rendered_labels: + value = rendered_labels[key] + + # check value length + if len(value) > MAX_VALUE_NAME_LENGTH: + logger.warning("Template result value is too long. %s", value) + continue + + labels[key] = value + + return labels + + +def _template_labels(alert_receive_channel: "AlertReceiveChannel", raw_request_data: typing.Any) -> dict[str, str]: + from apps.labels.models import MAX_KEY_NAME_LENGTH, MAX_VALUE_NAME_LENGTH + + if not alert_receive_channel.alert_group_labels_template: + return {} + + try: + rendered = apply_jinja_template(alert_receive_channel.alert_group_labels_template, raw_request_data) + except (JinjaTemplateError, JinjaTemplateWarning) as e: + logger.warning("Failed to apply template. %s", e.fallback_message) + return {} + + try: + rendered_labels = json.loads(rendered) + except (TypeError, json.JSONDecodeError): + logger.warning("Failed to parse template result. %s", rendered) + return {} + + if not isinstance(rendered_labels, dict): + logger.warning("Template result is not a dict. %s", rendered_labels) + return {} + + labels = {} + for key in rendered_labels: + value = rendered_labels[key] + + # check value type + if not isinstance(value, LABEL_VALUE_TYPES): + logger.warning("Template result value has invalid type. %s", value) + continue + + # convert value to string + value = str(value) + + # check key length + if len(key) > MAX_KEY_NAME_LENGTH: + logger.warning("Template result key is too long. %s", key) + continue + + # check value length + if len(value) > MAX_VALUE_NAME_LENGTH: + logger.warning("Template result value is too long. %s", value) + continue + + labels[key] = value + + return labels diff --git a/engine/apps/labels/models.py b/engine/apps/labels/models.py index 28947e75..14cd446c 100644 --- a/engine/apps/labels/models.py +++ b/engine/apps/labels/models.py @@ -10,9 +10,13 @@ if typing.TYPE_CHECKING: from apps.user_management.models import Organization +MAX_KEY_NAME_LENGTH = 200 +MAX_VALUE_NAME_LENGTH = 200 + + class LabelKeyCache(models.Model): id = models.CharField(primary_key=True, editable=False, max_length=36) - name = models.CharField(max_length=200) + name = models.CharField(max_length=MAX_KEY_NAME_LENGTH) organization = models.ForeignKey("user_management.Organization", on_delete=models.CASCADE) last_synced = models.DateTimeField(auto_now=True) @@ -23,7 +27,7 @@ class LabelKeyCache(models.Model): class LabelValueCache(models.Model): id = models.CharField(primary_key=True, editable=False, max_length=36) - name = models.CharField(max_length=200) + name = models.CharField(max_length=MAX_VALUE_NAME_LENGTH) key = models.ForeignKey("labels.LabelKeyCache", on_delete=models.CASCADE, related_name="values") last_synced = models.DateTimeField(auto_now=True) @@ -129,8 +133,8 @@ class AlertGroupAssociatedLabel(models.Model): "user_management.Organization", on_delete=models.CASCADE, related_name="alert_group_labels" ) - key_name = models.CharField(max_length=200) - value_name = models.CharField(max_length=200) + key_name = models.CharField(max_length=MAX_KEY_NAME_LENGTH) + value_name = models.CharField(max_length=MAX_VALUE_NAME_LENGTH) class Meta: constraints = [ diff --git a/engine/apps/labels/tests/test_alert_group.py b/engine/apps/labels/tests/test_alert_group.py index a5ac35ca..935e15af 100644 --- a/engine/apps/labels/tests/test_alert_group.py +++ b/engine/apps/labels/tests/test_alert_group.py @@ -3,9 +3,13 @@ from unittest import mock import pytest from apps.alerts.models import Alert +from apps.labels.models import MAX_KEY_NAME_LENGTH, MAX_VALUE_NAME_LENGTH + +TOO_LONG_KEY_NAME = "k" * (MAX_KEY_NAME_LENGTH + 1) +TOO_LONG_VALUE_NAME = "v" * (MAX_VALUE_NAME_LENGTH + 1) -@mock.patch("apps.labels.utils.is_labels_feature_enabled", return_value=False) +@mock.patch("apps.labels.alert_group_labels.is_labels_feature_enabled", return_value=False) @pytest.mark.django_db def test_assign_labels_feature_flag_disabled( _, make_organization, make_alert_receive_channel, make_integration_label_association @@ -28,11 +32,75 @@ def test_assign_labels_feature_flag_disabled( @pytest.mark.django_db -def test_assign_labels(make_organization, make_alert_receive_channel, make_integration_label_association): +def test_assign_labels( + make_organization, + make_alert_receive_channel, + make_label_key_and_value, + make_label_key, + make_integration_label_association, +): organization = make_organization() - alert_receive_channel = make_alert_receive_channel(organization) - label = make_integration_label_association(organization, alert_receive_channel) - make_integration_label_association(organization, alert_receive_channel, inheritable=False) + + # create label repo labels + label_key, label_value = make_label_key_and_value(organization, key_name="a", value_name="b") + label_key_1 = make_label_key(organization=organization, key_name="c") + label_key_2 = make_label_key(organization=organization) + label_key_3 = make_label_key(organization=organization) + + # create alert receive channel with all 3 types of labels + alert_receive_channel = make_alert_receive_channel( + organization, + alert_group_labels_custom=[ + [label_key.id, label_value.id, None], # plain label + ["nonexistent", label_value.id, None], # plain label with nonexistent key ID + [label_key_2.id, "nonexistent", None], # plain label with nonexistent value ID + [label_key_1.id, None, "{{ payload.c }}"], # templated label + [label_key_3.id, None, TOO_LONG_VALUE_NAME], # templated label too long + ], + alert_group_labels_template="{{ payload.advanced_template | tojson }}", + ) + make_integration_label_association(organization, alert_receive_channel, key_name="e", value_name="f") + + # create alert group + alert = Alert.create( + title="the title", + message="the message", + alert_receive_channel=alert_receive_channel, + raw_request_data={ + "c": "d", + "advanced_template": { + "g": 123, + "too_long": TOO_LONG_VALUE_NAME, + TOO_LONG_KEY_NAME: "too_long", + "invalid_type": {"test": "test"}, + }, + "extra": "hi", + }, + integration_unique_data={}, + image_url=None, + link_to_upstream_details=None, + ) + + # check alert group labels are assigned correctly, in the lexicographical order + assert [(label.key_name, label.value_name) for label in alert.group.labels.all()] == [ + ("a", "b"), + ("c", "d"), + ("e", "f"), + ("g", "123"), + ] + + +@pytest.mark.django_db +def test_assign_labels_custom_labels_none( + make_organization, + make_alert_receive_channel, + make_label_key_and_value, + make_label_key, + make_integration_label_association, +): + organization = make_organization() + alert_receive_channel = make_alert_receive_channel(organization, alert_group_labels_custom=None) + make_integration_label_association(organization, alert_receive_channel, key_name="a", value_name="b") alert = Alert.create( title="the title", @@ -44,6 +112,4 @@ def test_assign_labels(make_organization, make_alert_receive_channel, make_integ link_to_upstream_details=None, ) - assert alert.group.labels.count() == 1 - assert alert.group.labels.first().key_name == label.key.name - assert alert.group.labels.first().value_name == label.value.name + assert [(label.key_name, label.value_name) for label in alert.group.labels.all()] == [("a", "b")] diff --git a/engine/apps/labels/utils.py b/engine/apps/labels/utils.py index 46b0583a..bacf86c0 100644 --- a/engine/apps/labels/utils.py +++ b/engine/apps/labels/utils.py @@ -1,13 +1,16 @@ +import logging import typing from django.apps import apps # noqa: I251 from django.conf import settings if typing.TYPE_CHECKING: - from apps.alerts.models import AlertGroup, AlertReceiveChannel + from apps.alerts.models import AlertGroup from apps.labels.models import AssociatedLabel from apps.user_management.models import Organization +logger = logging.getLogger(__name__) + LABEL_OUTDATED_TIMEOUT_MINUTES = 30 ASSOCIATED_MODEL_NAME = "AssociatedLabel" @@ -54,35 +57,10 @@ def is_labels_feature_enabled(organization: "Organization") -> bool: ) -def assign_labels(alert_group: "AlertGroup", alert_receive_channel: "AlertReceiveChannel") -> None: - from apps.labels.models import AlertGroupAssociatedLabel - - if not is_labels_feature_enabled(alert_receive_channel.organization): - return - - # inherit labels from the integration - alert_group_labels = [ - AlertGroupAssociatedLabel( - alert_group=alert_group, - organization=alert_receive_channel.organization, - key_name=label.key.name, - value_name=label.value.name, - ) - for label in alert_receive_channel.labels.filter(inheritable=True).select_related("key", "value") - ] - AlertGroupAssociatedLabel.objects.bulk_create(alert_group_labels) +def get_label_verbal(obj: typing.Any) -> dict[str, str]: + return {label.key.name: label.value.name for label in obj.labels.all().select_related("key", "value")} -def get_label_verbal(labelable) -> typing.Dict[str, str]: - """ - label_verbal returns dict of labels' key and values names for the given object - """ - return {label.key.name: label.value.name for label in labelable.labels.all().select_related("key", "value")} - - -def get_alert_group_label_verbal(alert_group: "AlertGroup") -> typing.Dict[str, str]: - """ - get_alert_group_label_verbal returns dict of labels' key and values names for the given alert group. - It's different from get_label_verbal, because AlertGroupAssociated labels store key/value_name, not key/value_id - """ +def get_alert_group_label_verbal(alert_group: "AlertGroup") -> dict[str, str]: + """This is different from get_label_verbal because alert group labels store key/value names, not IDs""" return {label.key_name: label.value_name for label in alert_group.labels.all()} diff --git a/engine/apps/public_api/serializers/routes.py b/engine/apps/public_api/serializers/routes.py index 1907c9e9..bbe9ff55 100644 --- a/engine/apps/public_api/serializers/routes.py +++ b/engine/apps/public_api/serializers/routes.py @@ -1,10 +1,10 @@ from rest_framework import fields, serializers from apps.alerts.models import AlertReceiveChannel, ChannelFilter, EscalationChain -from apps.api.serializers.alert_receive_channel import valid_jinja_template_for_serializer_method_field from apps.base.messaging import get_messaging_backend_from_id, get_messaging_backends from common.api_helpers.custom_fields import OrganizationFilteredPrimaryKeyRelatedField from common.api_helpers.exceptions import BadRequest +from common.api_helpers.utils import valid_jinja_template_for_serializer_method_field from common.jinja_templater.apply_jinja_template import JinjaTemplateError from common.ordered_model.serializer import OrderedModelSerializer from common.utils import is_regex_valid diff --git a/engine/common/api_helpers/mixins.py b/engine/common/api_helpers/mixins.py index 4204095f..5f15f137 100644 --- a/engine/common/api_helpers/mixins.py +++ b/engine/common/api_helpers/mixins.py @@ -248,6 +248,7 @@ ACKNOWLEDGE_CONDITION = "acknowledge_condition" GROUPING_ID = "grouping_id" SOURCE_LINK = "source_link" ROUTE = "route" +ALERT_GROUP_LABELS = "alert_group_labels" NOTIFICATION_CHANNEL_TO_TEMPLATER_MAP = { SLACK: AlertSlackTemplater, @@ -264,9 +265,15 @@ for backend_id, backend in get_messaging_backends(): NOTIFICATION_CHANNEL_TO_TEMPLATER_MAP[backend.slug] = backend.get_templater_class() APPEARANCE_TEMPLATE_NAMES = [TITLE, MESSAGE, IMAGE_URL] -BEHAVIOUR_TEMPLATE_NAMES = [RESOLVE_CONDITION, ACKNOWLEDGE_CONDITION, GROUPING_ID, SOURCE_LINK] -ROUTE_TEMPLATE_NAMES = [ROUTE] -ALL_TEMPLATE_NAMES = APPEARANCE_TEMPLATE_NAMES + BEHAVIOUR_TEMPLATE_NAMES + ROUTE_TEMPLATE_NAMES +BEHAVIOUR_TEMPLATE_NAMES = [ + RESOLVE_CONDITION, + ACKNOWLEDGE_CONDITION, + GROUPING_ID, + SOURCE_LINK, + ROUTE, + ALERT_GROUP_LABELS, +] +ALL_TEMPLATE_NAMES = APPEARANCE_TEMPLATE_NAMES + BEHAVIOUR_TEMPLATE_NAMES class PreviewTemplateException(Exception): @@ -326,11 +333,6 @@ class PreviewTemplateMixin: templated_attr = apply_jinja_template(template_body, payload=alert_to_template.raw_request_data) except (JinjaTemplateError, JinjaTemplateWarning) as e: return Response({"preview": e.fallback_message}, status.HTTP_200_OK) - elif attr_name in ROUTE_TEMPLATE_NAMES: - try: - templated_attr = apply_jinja_template(template_body, payload=alert_to_template.raw_request_data) - except (JinjaTemplateError, JinjaTemplateWarning) as e: - return Response({"preview": e.fallback_message}, status.HTTP_200_OK) else: templated_attr = None response = {"preview": templated_attr} @@ -346,8 +348,6 @@ class PreviewTemplateMixin: destination = None if template_param.startswith(tuple(BEHAVIOUR_TEMPLATE_NAMES)): attr_name = template_param - if template_param.startswith(tuple(ROUTE_TEMPLATE_NAMES)): - attr_name = template_param elif template_param.startswith(tuple(NOTIFICATION_CHANNEL_OPTIONS)): for notification_channel in NOTIFICATION_CHANNEL_OPTIONS: if template_param.startswith(notification_channel): diff --git a/engine/common/api_helpers/utils.py b/engine/common/api_helpers/utils.py index 1233c4b0..0ec76200 100644 --- a/engine/common/api_helpers/utils.py +++ b/engine/common/api_helpers/utils.py @@ -14,6 +14,8 @@ from rest_framework.request import Request from apps.schedules.ical_utils import fetch_ical_file from common.api_helpers.exceptions import BadRequest +from common.jinja_templater import apply_jinja_template +from common.jinja_templater.apply_jinja_template import JinjaTemplateWarning from common.timezones import raise_exception_if_not_valid_timezone @@ -165,3 +167,12 @@ def check_phone_number_is_valid(phone_number): def serialize_datetime_as_utc_timestamp(dt: datetime.datetime) -> str: return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def valid_jinja_template_for_serializer_method_field(template): + for _, val in template.items(): + try: + apply_jinja_template(val, payload={}) + except JinjaTemplateWarning: + # Suppress warnings, template may be valid with payload + pass diff --git a/engine/conftest.py b/engine/conftest.py index 4f30e34a..1698befe 100644 --- a/engine/conftest.py +++ b/engine/conftest.py @@ -954,7 +954,13 @@ def webhook_preset_api_setup(): @pytest.fixture def make_label_key(): - def _make_label_key(organization, **kwargs): + def _make_label_key(organization, key_id=None, key_name=None, **kwargs): + if key_id is not None: + kwargs["id"] = key_id + + if key_name is not None: + kwargs["name"] = key_name + return LabelKeyFactory(organization=organization, **kwargs) return _make_label_key @@ -962,7 +968,13 @@ def make_label_key(): @pytest.fixture def make_label_value(): - def _make_label_value(key, **kwargs): + def _make_label_value(key, value_id=None, value_name=None, **kwargs): + if value_id is not None: + kwargs["id"] = value_id + + if value_name is not None: + kwargs["name"] = value_name + return LabelValueFactory(key=key, **kwargs) return _make_label_value @@ -970,9 +982,9 @@ def make_label_value(): @pytest.fixture def make_label_key_and_value(make_label_key, make_label_value): - def _make_label_key_and_value(organization): - key = make_label_key(organization=organization) - value = make_label_value(key=key) + def _make_label_key_and_value(organization, key_id=None, key_name=None, value_id=None, value_name=None): + key = make_label_key(organization=organization, key_id=key_id, key_name=key_name) + value = make_label_value(key=key, value_id=value_id, value_name=value_name) return key, value return _make_label_key_and_value @@ -980,8 +992,12 @@ def make_label_key_and_value(make_label_key, make_label_value): @pytest.fixture def make_integration_label_association(make_label_key_and_value): - def _make_integration_label_association(organization, alert_receive_channel, **kwargs): - key, value = make_label_key_and_value(organization) + def _make_integration_label_association( + organization, alert_receive_channel, key_id=None, key_name=None, value_id=None, value_name=None, **kwargs + ): + key, value = make_label_key_and_value( + organization, key_id=key_id, key_name=key_name, value_id=value_id, value_name=value_name + ) return AlertReceiveChannelAssociatedLabelFactory( alert_receive_channel=alert_receive_channel, organization=organization, key=key, value=value, **kwargs ) diff --git a/grafana-plugin/package.json b/grafana-plugin/package.json index 3b7bda98..0178bc57 100644 --- a/grafana-plugin/package.json +++ b/grafana-plugin/package.json @@ -115,7 +115,7 @@ "@grafana/data": "^9.2.4", "@grafana/faro-web-sdk": "^1.0.0-beta4", "@grafana/faro-web-tracing": "^1.0.0-beta4", - "@grafana/labels": "~1.2.1", + "@grafana/labels": "1.3.4", "@grafana/runtime": "9.3.0-beta1", "@grafana/ui": "^9.4.7", "@opentelemetry/api": "^1.3.0", diff --git a/grafana-plugin/playwright.config.ts b/grafana-plugin/playwright.config.ts index 689cd6cc..835639e5 100644 --- a/grafana-plugin/playwright.config.ts +++ b/grafana-plugin/playwright.config.ts @@ -1,4 +1,4 @@ -import { PlaywrightTestConfig, PlaywrightTestProject, defineConfig, devices } from '@playwright/test'; +import { PlaywrightTestProject, defineConfig, devices } from '@playwright/test'; import path from 'path'; /** diff --git a/grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx b/grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx index 5ec9d084..a2b9262d 100644 --- a/grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx +++ b/grafana-plugin/src/components/LabelsTooltipBadge/LabelsTooltipBadge.tsx @@ -22,7 +22,7 @@ const LabelsTooltipBadge: FC = ({ labels, onClick }) => {labels.map((label) => ( - + - - - - - + + +
+ + + + +
+ + + {customLabelIndexToShowTemplateEditor !== undefined && ( + setCustomLabelIndexToShowTemplateEditor(undefined)} + onUpdateTemplates={({ alert_group_labels }) => { + const newCustom = [...alertGroupLabels.custom]; + newCustom[customLabelIndexToShowTemplateEditor].value.name = alert_group_labels; + + setAlertGroupLabels({ + ...alertGroupLabels, + custom: newCustom, + }); + + setCustomLabelIndexToShowTemplateEditor(undefined); + }} + /> + )} + {showTemplateEditor && ( + setShowTemplateEditor(false)} + onUpdateTemplates={({ alert_group_labels }) => { + setAlertGroupLabels({ + ...alertGroupLabels, + template: alert_group_labels, + }); + + setShowTemplateEditor(undefined); + }} + /> + )} + ); }); +interface CustomLabelsProps { + alertGroupLabels: AlertReceiveChannel['alert_group_labels']; + onChange: (value: AlertReceiveChannel['alert_group_labels']) => void; + onShowTemplateEditor: (index: number) => void; +} + +const CustomLabels = (props: CustomLabelsProps) => { + const { alertGroupLabels, onChange, onShowTemplateEditor } = props; + + const { labelsStore } = useStore(); + + const handlePlainLabelAdd = () => { + onChange({ + ...alertGroupLabels, + custom: [ + ...alertGroupLabels.custom, + { + key: { id: undefined, name: undefined }, + value: { id: undefined, name: undefined }, + }, + ], + }); + }; + const handleTemplatedLabelAdd = () => { + onChange({ + ...alertGroupLabels, + custom: [ + ...alertGroupLabels.custom, + { + key: { id: undefined, name: undefined }, + value: { id: null, name: undefined }, // id = null means it's a templated value + }, + ], + }); + }; + + const cachedOnLoadKeys = useCallback(() => { + let result = undefined; + return async (search?: string) => { + if (!result) { + try { + result = await labelsStore.loadKeys(); + } catch (error) { + openErrorNotification('There was an error processing your request. Please try again'); + } + } + + return result.filter((k) => k.name.toLowerCase().includes(search.toLowerCase())); + }; + }, []); + + const cachedOnLoadValuesForKey = useCallback(() => { + let result = undefined; + return async (key: string, search?: string) => { + if (!result) { + try { + const { values } = await labelsStore.loadValuesForKey(key, search); + result = values; + } catch (error) { + openErrorNotification('There was an error processing your request. Please try again'); + } + } + + return result.filter((k) => k.name.toLowerCase().includes(search.toLowerCase())); + }; + }, []); + + return ( + + + + + { + if (res?.response?.status === 409) { + openErrorNotification(`Duplicate values are not allowed`); + } else { + openErrorNotification('An error has occurred. Please try again'); + } + }} + renderValue={(option, index, renderValueDefault) => { + if (option.value.id === null) { + return ( + { + onShowTemplateEditor(index); + }} + /> + } + onChange={(e: ChangeEvent) => { + const newCustom = [...alertGroupLabels.custom]; + newCustom[index].value.name = e.currentTarget.value; + + onChange({ ...alertGroupLabels, custom: newCustom }); + }} + /> + ); + } else { + return renderValueDefault(option, index); + } + }} + onDataUpdate={(value) => { + onChange({ + ...alertGroupLabels, + custom: value, + }); + }} + /> + + + + + } + > + + + + ); +}; + export default IntegrationLabelsForm; diff --git a/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx b/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx index 7376ee96..98e98b71 100644 --- a/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx +++ b/grafana-plugin/src/containers/IntegrationTemplate/IntegrationTemplate.tsx @@ -5,6 +5,7 @@ import cn from 'classnames/bind'; import { debounce } from 'lodash-es'; import { observer } from 'mobx-react'; +import { templateForEdit } from 'components/AlertTemplates/AlertTemplatesForm.config'; import { TemplateForEdit } from 'components/AlertTemplates/CommonAlertTemplatesForm.config'; import CheatSheet from 'components/CheatSheet/CheatSheet'; import { @@ -38,7 +39,7 @@ interface IntegrationTemplateProps { templates: AlertTemplatesDTO[]; onHide: () => void; onUpdateTemplates: (values: any) => void; - onUpdateRoute: (values: any, channelFilterId?: ChannelFilter['id']) => void; + onUpdateRoute?: (values: any, channelFilterId?: ChannelFilter['id']) => void; } const IntegrationTemplate = observer((props: IntegrationTemplateProps) => { @@ -53,11 +54,13 @@ const IntegrationTemplate = observer((props: IntegrationTemplateProps) => { const [isRecentAlertGroupExisting, setIsRecentAlertGroupExisting] = useState(false); useEffect(() => { - const locationParams: any = { template: template.name }; - if (template.isRoute) { - locationParams.routeId = channelFilterId; + if (templateForEdit[template.name]) { + const locationParams: any = { template: template.name }; + if (template.isRoute) { + locationParams.routeId = channelFilterId; + } + LocationHelper.update(locationParams, 'partial'); } - LocationHelper.update(locationParams, 'partial'); }, []); useEffect(() => { diff --git a/grafana-plugin/src/containers/Labels/Labels.tsx b/grafana-plugin/src/containers/Labels/Labels.tsx index 21c4d46a..60fc1324 100644 --- a/grafana-plugin/src/containers/Labels/Labels.tsx +++ b/grafana-plugin/src/containers/Labels/Labels.tsx @@ -1,6 +1,6 @@ import React, { forwardRef, useCallback, useImperativeHandle, useState } from 'react'; -import ServiceLabels, { ServiceLabelsProps } from '@grafana/labels'; +import { ServiceLabels, ServiceLabelsProps } from '@grafana/labels'; import { Field } from '@grafana/ui'; import cn from 'classnames/bind'; import { isEmpty } from 'lodash-es'; diff --git a/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts b/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts index ec366bf2..fd3ea806 100644 --- a/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts +++ b/grafana-plugin/src/models/alert_receive_channel/alert_receive_channel.types.ts @@ -49,7 +49,11 @@ export interface AlertReceiveChannel { allow_delete: boolean; deleted?: boolean; labels: LabelKeyValue[]; - alert_group_labels: { inheritable: Record }; + alert_group_labels: { + inheritable: Record; + custom: LabelKeyValue[]; + template: string; + }; } export interface AlertReceiveChannelChoice { diff --git a/grafana-plugin/yarn.lock b/grafana-plugin/yarn.lock index 2b225b09..3f55fb20 100644 --- a/grafana-plugin/yarn.lock +++ b/grafana-plugin/yarn.lock @@ -1966,10 +1966,10 @@ "@opentelemetry/sdk-trace-web" "^1.8.0" "@opentelemetry/semantic-conventions" "^1.8.0" -"@grafana/labels@~1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@grafana/labels/-/labels-1.2.1.tgz#4113d584bf5cd826d011f957cb69c90bd0416ea8" - integrity sha512-Nlqqvjwh0MjWsqnfpYbKdYwByeKSmEpiit5mKd6Mnnbc5Hxb8ORIruMr40lTxxWLEnDfhENcAs6pvlBuIMG7tQ== +"@grafana/labels@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@grafana/labels/-/labels-1.3.4.tgz#8d9cdd215a80a1da1045d402c037be85d7efd6f5" + integrity sha512-YYCuLGvtrMz7KkbMc6qoNJQr6drDLo6mMI27LcqsTDMHCNO3uJWpzC1Q2Y9MIwctIuTFYhbgfLvIunEegCx6PQ== dependencies: "@emotion/css" "^11.11.2" "@grafana/ui" "^10.0.0" From cb6a907b0309d2f7a8fa3d496d2594850cc38994 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 28 Nov 2023 09:53:50 +0800 Subject: [PATCH 18/18] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4938291..b98def39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## v1.3.62 (2023-11-28) + ### Added - Add ability to use Grafana Service Account Tokens for OnCall API (This is only enabled for resolution_notes