From 4f26ea3a689f1267e03f98bebcbb2ecd41938640 Mon Sep 17 00:00:00 2001 From: Matias Bordese Date: Fri, 14 Apr 2023 14:30:32 -0300 Subject: [PATCH 01/15] Fix responses context when no response (#1754) --- engine/apps/webhooks/models/webhook.py | 3 ++- engine/apps/webhooks/tests/test_trigger_webhook.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/engine/apps/webhooks/models/webhook.py b/engine/apps/webhooks/models/webhook.py index 6407b04c..11713eaa 100644 --- a/engine/apps/webhooks/models/webhook.py +++ b/engine/apps/webhooks/models/webhook.py @@ -274,4 +274,5 @@ class WebhookResponse(models.Model): content = models.TextField(null=True, default=None) def json(self): - return json.loads(self.content) + if self.content: + return json.loads(self.content) diff --git a/engine/apps/webhooks/tests/test_trigger_webhook.py b/engine/apps/webhooks/tests/test_trigger_webhook.py index 4fd1ee48..9380f560 100644 --- a/engine/apps/webhooks/tests/test_trigger_webhook.py +++ b/engine/apps/webhooks/tests/test_trigger_webhook.py @@ -297,6 +297,7 @@ def test_execute_webhook_using_responses_data( public_primary_key="response-1", ), trigger_type=Webhook.TRIGGER_FIRING, + status_code=200, content=json.dumps({"id": "third-party-id"}), ) make_webhook_response( @@ -306,8 +307,20 @@ def test_execute_webhook_using_responses_data( public_primary_key="response-2", ), trigger_type=Webhook.TRIGGER_ACKNOWLEDGE, + status_code=200, content=json.dumps({"id": "third-party-id", "status": "updated"}), ) + # webhook wasn't executed because of some error, there is no content or status_code + make_webhook_response( + alert_group=alert_group, + webhook=make_custom_webhook( + organization=organization, + public_primary_key="response-3", + ), + trigger_type=Webhook.TRIGGER_SILENCE, + content=None, + status_code=None, + ) mock_response = MockResponse() with patch("apps.webhooks.utils.socket.gethostbyname") as mock_gethostbyname: From c68fdf5681683552e07ec534768a2d10f3094cb4 Mon Sep 17 00:00:00 2001 From: Innokentii Konstantinov Date: Mon, 17 Apr 2023 15:16:18 +0800 Subject: [PATCH 02/15] Fix insight_logs exceptions (#1757) Most of the PR is just renaming ChatOpsType to ChatOpsPlug, core changes are linked below: - Fix insight_logs error writing unlink_backend error https://github.com/grafana/oncall/pull/1757/files#diff-7ae187be84e55ebac962bad0984f7569186cdc83c896132b2ebcbcbb31bbf5dd - Fix insight_logs error writing updated schedule with installed slack integration (https://github.com/grafana/oncall/pull/1757/files#diff-4037b7bbef9fc16d9b541beb3ed46f760916d7cd720847c3123adf7afb5ab4b4L690) --- engine/apps/api/views/telegram_channels.py | 4 ++-- engine/apps/api/views/user.py | 6 +++--- engine/apps/schedules/models/on_call_schedule.py | 2 +- engine/apps/slack/models/slack_team_identity.py | 4 ++-- engine/apps/slack/views.py | 4 ++-- engine/apps/social_auth/pipeline.py | 4 ++-- .../apps/telegram/models/connectors/channel.py | 4 ++-- .../apps/telegram/models/verification/channel.py | 6 +++--- .../telegram/models/verification/personal.py | 4 ++-- .../apps/user_management/models/organization.py | 4 ++-- engine/common/insight_log/__init__.py | 2 +- .../common/insight_log/chatops_insight_logs.py | 16 ++++++---------- 12 files changed, 28 insertions(+), 32 deletions(-) diff --git a/engine/apps/api/views/telegram_channels.py b/engine/apps/api/views/telegram_channels.py index 7fd9975d..c7bd6d50 100644 --- a/engine/apps/api/views/telegram_channels.py +++ b/engine/apps/api/views/telegram_channels.py @@ -8,7 +8,7 @@ from apps.api.permissions import RBACPermission from apps.api.serializers.telegram import TelegramToOrganizationConnectorSerializer from apps.auth_token.auth import PluginAuthentication from common.api_helpers.mixins import PublicPrimaryKeyMixin -from common.insight_log.chatops_insight_logs import ChatOpsEvent, ChatOpsType, write_chatops_insight_log +from common.insight_log.chatops_insight_logs import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log class TelegramChannelViewSet( @@ -47,7 +47,7 @@ class TelegramChannelViewSet( write_chatops_insight_log( author=user, event_name=ChatOpsEvent.CHANNEL_DISCONNECTED, - chatops_type=ChatOpsType.TELEGRAM, + chatops_type=ChatOpsTypePlug.TELEGRAM.value, channel_name=instance.channel_name, ) instance.delete() diff --git a/engine/apps/api/views/user.py b/engine/apps/api/views/user.py index b5ddcafa..ab9d05c0 100644 --- a/engine/apps/api/views/user.py +++ b/engine/apps/api/views/user.py @@ -49,7 +49,7 @@ from common.api_helpers.paginators import HundredPageSizePaginator from common.api_helpers.utils import create_engine_url from common.insight_log import ( ChatOpsEvent, - ChatOpsType, + ChatOpsTypePlug, EntityEvent, write_chatops_insight_log, write_resource_insight_log, @@ -417,7 +417,7 @@ class UserView( write_chatops_insight_log( author=request.user, event_name=ChatOpsEvent.USER_UNLINKED, - chatops_type=ChatOpsType.SLACK, + chatops_type=ChatOpsTypePlug.SLACK.value, linked_user=user.username, linked_user_id=user.public_primary_key, ) @@ -433,7 +433,7 @@ class UserView( write_chatops_insight_log( author=request.user, event_name=ChatOpsEvent.USER_UNLINKED, - chatops_type=ChatOpsType.TELEGRAM, + chatops_type=ChatOpsTypePlug.TELEGRAM.value, linked_user=user.username, linked_user_id=user.public_primary_key, ) diff --git a/engine/apps/schedules/models/on_call_schedule.py b/engine/apps/schedules/models/on_call_schedule.py index 1fde3070..8896811f 100644 --- a/engine/apps/schedules/models/on_call_schedule.py +++ b/engine/apps/schedules/models/on_call_schedule.py @@ -687,7 +687,7 @@ class OnCallSchedule(PolymorphicModel): result["notification_frequency"] = self.get_notify_oncall_shift_freq_display() result["current_shift_notification"] = self.mention_oncall_start result["next_shift_notification"] = self.mention_oncall_next - result["notify_empty_oncall"] = self.get_notify_empty_oncall_display + result["notify_empty_oncall"] = self.get_notify_empty_oncall_display() return result @property diff --git a/engine/apps/slack/models/slack_team_identity.py b/engine/apps/slack/models/slack_team_identity.py index 2c8bc4a8..969f097e 100644 --- a/engine/apps/slack/models/slack_team_identity.py +++ b/engine/apps/slack/models/slack_team_identity.py @@ -9,7 +9,7 @@ from apps.slack.constants import SLACK_INVALID_AUTH_RESPONSE, SLACK_WRONG_TEAM_N from apps.slack.slack_client import SlackClientWithErrorHandling from apps.slack.slack_client.exceptions import SlackAPIException, SlackAPITokenException from apps.user_management.models.user import User -from common.insight_log.chatops_insight_logs import ChatOpsEvent, ChatOpsType, write_chatops_insight_log +from common.insight_log.chatops_insight_logs import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log logger = logging.getLogger(__name__) @@ -65,7 +65,7 @@ class SlackTeamIdentity(models.Model): self.installed_via_granular_permissions = True self.save() write_chatops_insight_log( - author=user, event_name=ChatOpsEvent.WORKSPACE_CONNECTED, chatops_type=ChatOpsType.SLACK + author=user, event_name=ChatOpsEvent.WORKSPACE_CONNECTED, chatops_type=ChatOpsTypePlug.SLACK.value ) def get_cached_channels(self, search_term=None, slack_id=None): diff --git a/engine/apps/slack/views.py b/engine/apps/slack/views.py index a6659e07..829c7c49 100644 --- a/engine/apps/slack/views.py +++ b/engine/apps/slack/views.py @@ -53,7 +53,7 @@ from apps.slack.scenarios.slack_usergroup import STEPS_ROUTING as SLACK_USERGROU from apps.slack.slack_client import SlackClientWithErrorHandling from apps.slack.slack_client.exceptions import SlackAPIException, SlackAPITokenException from apps.slack.tasks import clean_slack_integration_leftovers, unpopulate_slack_user_identities -from common.insight_log import ChatOpsEvent, ChatOpsType, write_chatops_insight_log +from common.insight_log import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log from common.oncall_gateway import delete_slack_connector from .models import SlackMessage, SlackTeamIdentity, SlackUserIdentity @@ -555,7 +555,7 @@ class ResetSlackView(APIView): write_chatops_insight_log( author=request.user, event_name=ChatOpsEvent.WORKSPACE_DISCONNECTED, - chatops_type=ChatOpsType.SLACK, + chatops_type=ChatOpsTypePlug.SLACK.value, ) unpopulate_slack_user_identities(organization.pk, True) response = Response(status=200) diff --git a/engine/apps/social_auth/pipeline.py b/engine/apps/social_auth/pipeline.py index 4c53c9fd..96aa4bc0 100644 --- a/engine/apps/social_auth/pipeline.py +++ b/engine/apps/social_auth/pipeline.py @@ -14,7 +14,7 @@ from common.constants.slack_auth import ( SLACK_AUTH_SLACK_USER_ALREADY_CONNECTED_ERROR, SLACK_AUTH_WRONG_WORKSPACE_ERROR, ) -from common.insight_log import ChatOpsEvent, ChatOpsType, write_chatops_insight_log +from common.insight_log import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log from common.oncall_gateway import check_slack_installation_possible, create_slack_connector logger = logging.getLogger(__name__) @@ -74,7 +74,7 @@ def connect_user_to_slack(response, backend, strategy, user, organization, *args write_chatops_insight_log( author=user, event_name=ChatOpsEvent.USER_LINKED, - chatops_type=ChatOpsType.SLACK, + chatops_type=ChatOpsTypePlug.SLACK.value, linked_user=user.username, linked_user_id=user.public_primary_key, ) diff --git a/engine/apps/telegram/models/connectors/channel.py b/engine/apps/telegram/models/connectors/channel.py index 75190f6b..fc3c7794 100644 --- a/engine/apps/telegram/models/connectors/channel.py +++ b/engine/apps/telegram/models/connectors/channel.py @@ -10,7 +10,7 @@ from telegram import error from apps.alerts.models import AlertGroup from apps.telegram.client import TelegramClient from apps.telegram.models import TelegramMessage -from common.insight_log.chatops_insight_logs import ChatOpsEvent, ChatOpsType, write_chatops_insight_log +from common.insight_log.chatops_insight_logs import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log from common.public_primary_keys import generate_public_primary_key, increase_public_primary_key_length logger = logging.getLogger(__name__) @@ -102,7 +102,7 @@ class TelegramToOrganizationConnector(models.Model): write_chatops_insight_log( author=author, event_name=ChatOpsEvent.DEFAULT_CHANNEL_CHANGED, - chatops_type=ChatOpsType.TELEGRAM, + chatops_type=ChatOpsTypePlug.TELEGRAM.value, prev_channel=old_default_channel.channel_name if old_default_channel else None, new_channel=self.channel_name, ) diff --git a/engine/apps/telegram/models/verification/channel.py b/engine/apps/telegram/models/verification/channel.py index 1b24bb6e..f9e530ba 100644 --- a/engine/apps/telegram/models/verification/channel.py +++ b/engine/apps/telegram/models/verification/channel.py @@ -6,7 +6,7 @@ from django.db import models from django.utils import timezone from apps.telegram.models import TelegramToOrganizationConnector -from common.insight_log.chatops_insight_logs import ChatOpsEvent, ChatOpsType, write_chatops_insight_log +from common.insight_log.chatops_insight_logs import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log class TelegramChannelVerificationCode(models.Model): @@ -66,14 +66,14 @@ class TelegramChannelVerificationCode(models.Model): write_chatops_insight_log( author=code_instance.author, event_name=ChatOpsEvent.CHANNEL_CONNECTED, - chatops_type=ChatOpsType.TELEGRAM, + chatops_type=ChatOpsTypePlug.TELEGRAM.value, channel_name=channel_name, ) if not connector_exists: write_chatops_insight_log( author=code_instance.author, event_name=ChatOpsEvent.DEFAULT_CHANNEL_CHANGED, - chatops_type=ChatOpsType.TELEGRAM, + chatops_type=ChatOpsTypePlug.TELEGRAM.value, prev_channel=None, new_channel=channel_name, ) diff --git a/engine/apps/telegram/models/verification/personal.py b/engine/apps/telegram/models/verification/personal.py index 323b990a..e61664b2 100644 --- a/engine/apps/telegram/models/verification/personal.py +++ b/engine/apps/telegram/models/verification/personal.py @@ -6,7 +6,7 @@ from django.db import IntegrityError, models from django.utils import timezone from apps.telegram.models import TelegramToUserConnector -from common.insight_log import ChatOpsEvent, ChatOpsType, write_chatops_insight_log +from common.insight_log import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log class TelegramVerificationCode(models.Model): @@ -48,7 +48,7 @@ class TelegramVerificationCode(models.Model): write_chatops_insight_log( author=user, event_name=ChatOpsEvent.USER_LINKED, - chatops_type=ChatOpsType.TELEGRAM, + chatops_type=ChatOpsTypePlug.TELEGRAM.value, linked_user=user.username, linked_user_id=user.public_primary_key, ) diff --git a/engine/apps/user_management/models/organization.py b/engine/apps/user_management/models/organization.py index 73e3f9c3..151bfb34 100644 --- a/engine/apps/user_management/models/organization.py +++ b/engine/apps/user_management/models/organization.py @@ -14,7 +14,7 @@ from apps.alerts.models import MaintainableObject from apps.alerts.tasks import disable_maintenance from apps.slack.utils import post_message_to_channel from apps.user_management.subscription_strategy import FreePublicBetaSubscriptionStrategy -from common.insight_log import ChatOpsEvent, ChatOpsType, write_chatops_insight_log +from common.insight_log import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log from common.oncall_gateway import create_oncall_connector, delete_oncall_connector, delete_slack_connector from common.public_primary_keys import generate_public_primary_key, increase_public_primary_key_length @@ -302,7 +302,7 @@ class Organization(MaintainableObject): write_chatops_insight_log( author=user, event_name=ChatOpsEvent.DEFAULT_CHANNEL_CHANGED, - chatops_type=ChatOpsType.SLACK, + chatops_type=ChatOpsTypePlug.SLACK.value, prev_channel=old_channel_name, new_channel=channel_name, ) diff --git a/engine/common/insight_log/__init__.py b/engine/common/insight_log/__init__.py index 9bf46cf3..f71d6423 100644 --- a/engine/common/insight_log/__init__.py +++ b/engine/common/insight_log/__init__.py @@ -1,3 +1,3 @@ -from .chatops_insight_logs import ChatOpsEvent, ChatOpsType, write_chatops_insight_log # noqa +from .chatops_insight_logs import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log # noqa from .maintenance_insight_log import MaintenanceEvent, write_maintenance_insight_log # noqa from .resource_insight_logs import EntityEvent, write_resource_insight_log # noqa diff --git a/engine/common/insight_log/chatops_insight_logs.py b/engine/common/insight_log/chatops_insight_logs.py index 6bf6055e..75dfe148 100644 --- a/engine/common/insight_log/chatops_insight_logs.py +++ b/engine/common/insight_log/chatops_insight_logs.py @@ -18,17 +18,13 @@ class ChatOpsEvent(enum.Enum): DEFAULT_CHANNEL_CHANGED = "default_channel_changed" -class ChatOpsType(enum.Enum): - # Keep in sync with messaging backends' id. - # In perfect world backend_ids should be used intead of this enums - # It can be achieved when we move refactor slack and telegram to use the messaging_backend system. - SLACK = "SLACK" - MSTEAMS = "MSTEAMS" - TELEGRAM = "TELEGRAM" - MOBILE_APP = "MOBILE_APP" +class ChatOpsTypePlug(enum.Enum): + # ChatOpsTypePlug provides backend_id string for chatops integration not supporting messaging_backends. + SLACK = "slack" + TELEGRAM = "telegram" -def write_chatops_insight_log(author, event_name: ChatOpsEvent, chatops_type: ChatOpsType, **kwargs): +def write_chatops_insight_log(author, event_name: ChatOpsEvent, chatops_type: str, **kwargs): try: organization = author.organization @@ -37,7 +33,7 @@ def write_chatops_insight_log(author, event_name: ChatOpsEvent, chatops_type: Ch user_id = author.public_primary_key username = json.dumps(author.username) - log_line = f"tenant_id={tenant_id} author_id={user_id} author={username} action_type=chat_ops action_name={event_name.value} chat_ops_type={chatops_type.value}" # noqa + log_line = f"tenant_id={tenant_id} author_id={user_id} author={username} action_type=chat_ops action_name={event_name.value} chat_ops_type={chatops_type.lower()}" # noqa for k, v in kwargs.items(): log_line += f" {k}={json.dumps(v)}" From 75824dc9ad54fb90a97c2df8aadf7c8b1654bfca Mon Sep 17 00:00:00 2001 From: Joey Orlando Date: Mon, 17 Apr 2023 10:52:03 +0200 Subject: [PATCH 03/15] ignore .http file extensions (#1762) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0e47d8af..30a02411 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ venv .python-version .vscode +*.http .idea .DS_Store .env From 15e7baad88eb16f43ae4fa076022da721adf1b75 Mon Sep 17 00:00:00 2001 From: Innokentii Konstantinov Date: Mon, 17 Apr 2023 18:10:47 +0800 Subject: [PATCH 04/15] Insight logs format fixes (#1763) # What this PR does Minot fixes of insight logs formatting --- engine/common/insight_log/chatops_insight_logs.py | 4 ++-- engine/common/insight_log/insight_logs_enabled_check.py | 6 +++--- engine/common/insight_log/maintenance_insight_log.py | 4 ++-- engine/common/insight_log/resource_insight_logs.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/engine/common/insight_log/chatops_insight_logs.py b/engine/common/insight_log/chatops_insight_logs.py index 75dfe148..64808beb 100644 --- a/engine/common/insight_log/chatops_insight_logs.py +++ b/engine/common/insight_log/chatops_insight_logs.py @@ -9,8 +9,8 @@ logger = logging.getLogger(__name__) class ChatOpsEvent(enum.Enum): - WORKSPACE_CONNECTED = "started" - WORKSPACE_DISCONNECTED = "finished" + WORKSPACE_CONNECTED = "workspace_connected" + WORKSPACE_DISCONNECTED = "workspace_disconnected" CHANNEL_CONNECTED = "channel_connected" CHANNEL_DISCONNECTED = "channel_disconnected" USER_LINKED = "user_linked" diff --git a/engine/common/insight_log/insight_logs_enabled_check.py b/engine/common/insight_log/insight_logs_enabled_check.py index 0bf41933..67041bd9 100644 --- a/engine/common/insight_log/insight_logs_enabled_check.py +++ b/engine/common/insight_log/insight_logs_enabled_check.py @@ -8,8 +8,8 @@ logger = logging.getLogger(__name__) def is_insight_logs_enabled(organization): """ is_insight_logs_enabled checks if inside logs enabled for given organization. - Now it checks if oncall is deployed on same cluster that its grafana instance to be able to forward logs. - Or if it's Open Source :) + Now it checks if oncall is deployed on same cluster that its grafana instance to be able to forward logs + to Loki through logs-forwarder. """ logger.info( "is_insight_logs_enabled: " @@ -17,4 +17,4 @@ def is_insight_logs_enabled(organization): f"ONCALL_BACKEND_REGION={settings.ONCALL_BACKEND_REGION} " f"cluster_slug={organization.cluster_slug}" ) - return settings.IS_OPEN_SOURCE or settings.ONCALL_BACKEND_REGION == organization.cluster_slug + return not settings.IS_OPEN_SOURCE and settings.ONCALL_BACKEND_REGION == organization.cluster_slug diff --git a/engine/common/insight_log/maintenance_insight_log.py b/engine/common/insight_log/maintenance_insight_log.py index f1260351..d0426ebe 100644 --- a/engine/common/insight_log/maintenance_insight_log.py +++ b/engine/common/insight_log/maintenance_insight_log.py @@ -21,7 +21,7 @@ def write_maintenance_insight_log(instance, user, event: MaintenanceEvent): team = instance.get_team() entity_name = json.dumps(instance.insight_logs_verbal) entity_id = instance.public_primary_key - maintenance_mode = instance.get_maintenance_mode_display() + maintenance_mode = instance.get_maintenance_mode_display().lower() if is_insight_logs_enabled(organization): log_line = f"tenant_id={tenant_id} action_type=maintenance action_name={event.value} maintenance_mode={maintenance_mode} resource_id={entity_id} resource_name={entity_name}" # noqa @@ -32,7 +32,7 @@ def write_maintenance_insight_log(instance, user, event: MaintenanceEvent): if user: username = json.dumps(user.username) user_id = user.public_primary_key - log_line += f" user_id={user_id} username={username} " + log_line += f" author_id={user_id} author={username}" insight_logger.info(log_line) except Exception as e: logger.warning(f"insight_log.failed_to_write_maintenance_insight_log exception={e}") diff --git a/engine/common/insight_log/resource_insight_logs.py b/engine/common/insight_log/resource_insight_logs.py index 1b3a0ce9..7f5361fa 100644 --- a/engine/common/insight_log/resource_insight_logs.py +++ b/engine/common/insight_log/resource_insight_logs.py @@ -71,7 +71,7 @@ def write_resource_insight_log(instance: InsightLoggable, author, event: EntityE entity_id = instance.id entity_name = json.dumps(instance.insight_logs_verbal) metadata = instance.insight_logs_metadata - log_line = f"tenant_id={tenant_id} author_id={author_id} author={author} action_type=resource action={event.value} resource_type={entity_type} resource_id={entity_id} resource_name={entity_name}" # noqa + log_line = f"tenant_id={tenant_id} author_id={author_id} author={author} action_type=resource action_name={event.value} resource_type={entity_type} resource_id={entity_id} resource_name={entity_name}" # noqa for k, v in metadata.items(): log_line += f" {k}={json.dumps(v)}" if prev_state and new_state: @@ -82,7 +82,7 @@ def write_resource_insight_log(instance: InsightLoggable, author, event: EntityE log_line += f' new_state="{new_state}"' insight_logger.info(log_line) except Exception as e: - logger.warning(f"insight_log.failed_to_write_entity_insight_log exception={e}") + logger.warning(f"insight_log.failed_to_write_entity_insight_log exception={e} instance_id={instance.id}") def state_diff_finder(prev_state: dict, new_state: dict): From 0c42c2a86dc55fd62e5955188264980ed4e83538 Mon Sep 17 00:00:00 2001 From: Matthias Teich Date: Mon, 17 Apr 2023 12:22:05 +0200 Subject: [PATCH 05/15] Add option to use helm hook for migration job (#1386) # What this PR does This PR adds the option to use helm hooks for the database migration. ## Which issue(s) this PR fixes Currently oncall always shows as out-of-sync in argo-cd because the name changes on each hard refresh. When using a helm hook the job is executed on sync but does not show as out-of-sync ## Checklist - [ ] Tests updated - [ ] Documentation added - [x] `CHANGELOG.md` updated --------- Co-authored-by: Ildar Iskhakov --- CHANGELOG.md | 6 ++++++ helm/oncall/Chart.yaml | 4 ++-- helm/oncall/templates/engine/job-migrate.yaml | 6 ++++++ helm/oncall/values.yaml | 2 ++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed63e536..0aac851c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +- Helm chart: add the option to use a helm hook for the migration job ([1386](https://github.com/grafana/oncall/pull/1386)) + ## v1.2.11 (2023-04-14) ### Added diff --git a/helm/oncall/Chart.yaml b/helm/oncall/Chart.yaml index 5d65a126..575d8683 100644 --- a/helm/oncall/Chart.yaml +++ b/helm/oncall/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: oncall description: Developer-friendly incident response with brilliant Slack integration type: application -version: 1.2.8 -appVersion: v1.2.8 +version: 1.2.9 +appVersion: v1.2.9 dependencies: - name: cert-manager version: v1.8.0 diff --git a/helm/oncall/templates/engine/job-migrate.yaml b/helm/oncall/templates/engine/job-migrate.yaml index 2bb478fa..b92d2939 100644 --- a/helm/oncall/templates/engine/job-migrate.yaml +++ b/helm/oncall/templates/engine/job-migrate.yaml @@ -2,7 +2,13 @@ apiVersion: batch/v1 kind: Job metadata: + {{- if .Values.migrate.useHook }} + name: {{ printf "%s-migrate" (include "oncall.engine.fullname" .) }} + annotations: + "helm.sh/hook": post-install,post-upgrade + {{- else }} name: {{ printf "%s-migrate-%s" (include "oncall.engine.fullname" .) (now | date "2006-01-02-15-04-05") }} + {{- end }} labels: {{- include "oncall.engine.labels" . | nindent 4 }} spec: diff --git a/helm/oncall/values.yaml b/helm/oncall/values.yaml index 612e0c49..5bfb08bb 100644 --- a/helm/oncall/values.yaml +++ b/helm/oncall/values.yaml @@ -150,6 +150,8 @@ migrate: enabled: true # TTL can be unset by setting ttlSecondsAfterFinished: "" ttlSecondsAfterFinished: 20 + # use a helm hook to manage the migration job + useHook: false # Additional env variables to add to deployments env: {} From d99bb7b8bc24b4b4d3e5292cf3e39566fe353535 Mon Sep 17 00:00:00 2001 From: Vadim Stepanov Date: Mon, 17 Apr 2023 13:12:21 +0100 Subject: [PATCH 06/15] PD migrator: ignore 404s on delete + delete ONCALL_DEFAULT_CONTACT_METHOD (#1764) --- tools/pagerduty-migrator/README.md | 12 ++---------- tools/pagerduty-migrator/migrator/__main__.py | 4 ++-- tools/pagerduty-migrator/migrator/config.py | 3 --- .../pagerduty-migrator/migrator/oncall_api_client.py | 7 ++++++- tools/pagerduty-migrator/migrator/report.py | 4 ++-- .../migrator/resources/notification_rules.py | 9 ++------- 6 files changed, 14 insertions(+), 25 deletions(-) diff --git a/tools/pagerduty-migrator/README.md b/tools/pagerduty-migrator/README.md index 6a1b0a19..f0f38222 100644 --- a/tools/pagerduty-migrator/README.md +++ b/tools/pagerduty-migrator/README.md @@ -79,30 +79,22 @@ docker run --rm \ -e PAGERDUTY_API_TOKEN="" \ -e ONCALL_API_URL="" \ -e ONCALL_API_TOKEN="" \ --e ONCALL_DEFAULT_CONTACT_METHOD="sms" \ -e MODE="migrate" \ pd-oncall-migrator ``` -### Migrate unsupported user notification rules - -It's possible to specify a default contact method type for user notification rules that cannot be migrated as-is by -changing the `ONCALL_DEFAULT_CONTACT_METHOD` env variable. -Options are: `email`, `sms`, `phone_call`, `slack`, `telegram`, `mobile_app` (default is `email`). - ### Migrate unsupported integration types It's possible to migrate unsupported integration types to [Grafana OnCall incoming webhooks](https://grafana.com/docs/oncall/latest/integrations/available-integrations/configure-webhook/). -by changing UNSUPPORTED_INTEGRATION_TO_WEBHOOKS env variable: +To enable this feature, set env variable `UNSUPPORTED_INTEGRATION_TO_WEBHOOKS` to `true`: ```shell docker run --rm \ -e PAGERDUTY_API_TOKEN="" \ -e ONCALL_API_URL="" \ -e ONCALL_API_TOKEN="" \ --e ONCALL_DEFAULT_CONTACT_METHOD="sms" \ --e MODE="migrate" \ -e UNSUPPORTED_INTEGRATION_TO_WEBHOOKS="true" \ +-e MODE="migrate" \ pd-oncall-migrator ``` diff --git a/tools/pagerduty-migrator/migrator/__main__.py b/tools/pagerduty-migrator/migrator/__main__.py index 086273f0..8905f07c 100644 --- a/tools/pagerduty-migrator/migrator/__main__.py +++ b/tools/pagerduty-migrator/migrator/__main__.py @@ -98,7 +98,7 @@ def main() -> None: rulesets = None if EXPERIMENTAL_MIGRATE_EVENT_RULES: - print("▶ Fetching event rules (rulesets) ...") + print("▶ Fetching event rules (global rulesets)...") rulesets = session.list_all("rulesets") for ruleset in rulesets: rules = session.list_all(f"rulesets/{ruleset['id']}/rules") @@ -173,7 +173,7 @@ def main() -> None: print(TAB + format_integration(integration)) if rulesets is not None: - print("▶ Migrating event rules (rulesets) ...") + print("▶ Migrating event rules (global rulesets)...") for ruleset in rulesets: if not ruleset["flawed_escalation_policies"]: migrate_ruleset(ruleset, escalation_policies, services) diff --git a/tools/pagerduty-migrator/migrator/config.py b/tools/pagerduty-migrator/migrator/config.py index e9d65736..0bf60277 100644 --- a/tools/pagerduty-migrator/migrator/config.py +++ b/tools/pagerduty-migrator/migrator/config.py @@ -14,9 +14,6 @@ ONCALL_API_URL = urljoin( ) ONCALL_DELAY_OPTIONS = [1, 5, 15, 30, 60] -ONCALL_DEFAULT_CONTACT_METHOD = "notify_by_" + os.getenv( - "ONCALL_DEFAULT_CONTACT_METHOD", default="email" -) PAGERDUTY_TO_ONCALL_CONTACT_METHOD_MAP = { "sms_contact_method": "notify_by_sms", "phone_contact_method": "notify_by_phone_call", diff --git a/tools/pagerduty-migrator/migrator/oncall_api_client.py b/tools/pagerduty-migrator/migrator/oncall_api_client.py index 59056793..183253f3 100644 --- a/tools/pagerduty-migrator/migrator/oncall_api_client.py +++ b/tools/pagerduty-migrator/migrator/oncall_api_client.py @@ -73,7 +73,12 @@ def create(path: str, payload: dict) -> dict: def delete(path: str) -> None: - api_call("delete", path) + try: + api_call("delete", path) + except requests.exceptions.HTTPError as e: + # ignore 404s on delete so deleting resources manually while running the script doesn't break it + if e.response.status_code != 404: + raise def update(path: str, payload: dict) -> dict: diff --git a/tools/pagerduty-migrator/migrator/report.py b/tools/pagerduty-migrator/migrator/report.py index 69294d0a..86eabcf3 100644 --- a/tools/pagerduty-migrator/migrator/report.py +++ b/tools/pagerduty-migrator/migrator/report.py @@ -76,7 +76,7 @@ def format_integration(integration: dict) -> str: else: # check if integration not supported, but UNSUPPORTED_INTEGRATION_TO_WEBHOOKS set if integration.get("converted_to_webhook", False): - result = "{} {} – Webhook integration will be created, Grafana OnCall not support this type directly ".format( + result = "{} {} – cannot find appropriate Grafana OnCall integration type, integration will be migrated with type 'webhook'".format( WARNING_SIGN, result ) else: @@ -187,7 +187,7 @@ def format_ruleset(ruleset: dict) -> str: def ruleset_report(rulesets: list[dict]) -> str: - result = "Event rules (rulesets) report:" + result = "Event rules (global rulesets) report:" for ruleset in sorted( rulesets, diff --git a/tools/pagerduty-migrator/migrator/resources/notification_rules.py b/tools/pagerduty-migrator/migrator/resources/notification_rules.py index 7a26d71f..2ae78ff3 100644 --- a/tools/pagerduty-migrator/migrator/resources/notification_rules.py +++ b/tools/pagerduty-migrator/migrator/resources/notification_rules.py @@ -1,10 +1,7 @@ import copy from migrator import oncall_api_client -from migrator.config import ( - ONCALL_DEFAULT_CONTACT_METHOD, - PAGERDUTY_TO_ONCALL_CONTACT_METHOD_MAP, -) +from migrator.config import PAGERDUTY_TO_ONCALL_CONTACT_METHOD_MAP from migrator.utils import remove_duplicates, transform_wait_delay @@ -76,10 +73,8 @@ def transform_notification_rule( notification_rule: dict, delay: int, user_id: str ) -> list[dict]: contact_method_type = notification_rule["contact_method"]["type"] + oncall_type = PAGERDUTY_TO_ONCALL_CONTACT_METHOD_MAP[contact_method_type] - oncall_type = PAGERDUTY_TO_ONCALL_CONTACT_METHOD_MAP.get( - contact_method_type, ONCALL_DEFAULT_CONTACT_METHOD - ) notify_rule = {"user_id": user_id, "type": oncall_type, "important": False} if not delay: From e93347e75ba635cebdf4ceb0a656e51b779f1006 Mon Sep 17 00:00:00 2001 From: Joey Orlando Date: Mon, 17 Apr 2023 19:30:31 +0200 Subject: [PATCH 07/15] temporarily disable is_restricted column + migration (#1765) --- .../alerts/migrations/0012_auto_20230406_1010.py | 13 +++++++------ engine/apps/alerts/models/alert_group.py | 6 +++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/engine/apps/alerts/migrations/0012_auto_20230406_1010.py b/engine/apps/alerts/migrations/0012_auto_20230406_1010.py index ad6ab411..666b5c49 100644 --- a/engine/apps/alerts/migrations/0012_auto_20230406_1010.py +++ b/engine/apps/alerts/migrations/0012_auto_20230406_1010.py @@ -10,14 +10,15 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AddField( - model_name='alertgroup', - name='is_restricted', - field=models.BooleanField(default=False, null=True), - ), + # migrations.AddField( + # model_name='alertgroup', + # name='is_restricted', + # field=models.BooleanField(default=False, null=True), + # ), migrations.AlterField( model_name='alertgrouplogrecord', name='type', - field=models.IntegerField(choices=[(0, 'Acknowledged'), (1, 'Unacknowledged'), (2, 'Invite'), (3, 'Stop invitation'), (4, 'Re-invite'), (5, 'Escalation triggered'), (6, 'Invitation triggered'), (16, 'Escalation finished'), (7, 'Silenced'), (15, 'Unsilenced'), (8, 'Attached'), (9, 'Unattached'), (10, 'Custom button triggered'), (11, 'Unacknowledged by timeout'), (12, 'Failed attachment'), (13, 'Incident resolved'), (14, 'Incident unresolved'), (17, 'Escalation failed'), (18, 'Acknowledge reminder triggered'), (19, 'Wiped'), (20, 'Deleted'), (21, 'Incident registered'), (22, 'A route is assigned to the incident'), (23, 'Trigger direct paging escalation'), (24, 'Unpage a user'), (25, 'Restricted')]), + field=models.IntegerField(choices=[(0, 'Acknowledged'), (1, 'Unacknowledged'), (2, 'Invite'), (3, 'Stop invitation'), (4, 'Re-invite'), (5, 'Escalation triggered'), (6, 'Invitation triggered'), (16, 'Escalation finished'), (7, 'Silenced'), (15, 'Unsilenced'), (8, 'Attached'), (9, 'Unattached'), (10, 'Custom button triggered'), (11, 'Unacknowledged by timeout'), + (12, 'Failed attachment'), (13, 'Incident resolved'), (14, 'Incident unresolved'), (17, 'Escalation failed'), (18, 'Acknowledge reminder triggered'), (19, 'Wiped'), (20, 'Deleted'), (21, 'Incident registered'), (22, 'A route is assigned to the incident'), (23, 'Trigger direct paging escalation'), (24, 'Unpage a user'), (25, 'Restricted')]), ), ] diff --git a/engine/apps/alerts/models/alert_group.py b/engine/apps/alerts/models/alert_group.py index cdcba846..49c60498 100644 --- a/engine/apps/alerts/models/alert_group.py +++ b/engine/apps/alerts/models/alert_group.py @@ -352,7 +352,11 @@ class AlertGroup(AlertGroupSlackRenderingMixin, EscalationSnapshotMixin, models. # https://code.djangoproject.com/ticket/28545 is_open_for_grouping = models.BooleanField(default=None, null=True, blank=True) - is_restricted = models.BooleanField(default=False, null=True) + # is_restricted = models.BooleanField(default=False, null=True) + + @property + def is_restricted(self): + return False @staticmethod def get_silenced_state_filter(): From f825fdf1a3607cb75bd1568911e9cae6a83c02c0 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 18 Apr 2023 10:48:11 +0800 Subject: [PATCH 08/15] Send demo alert with dynamic payload and get demo payload example on private api (#1700) # What this PR does ## Which issue(s) this PR fixes ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required) --- CHANGELOG.md | 1 + .../alerts/models/alert_receive_channel.py | 29 ++++++++++------- engine/apps/alerts/models/channel_filter.py | 3 ++ .../tests/test_alert_receiver_channel.py | 31 ++++++++++++++++--- .../api/serializers/alert_receive_channel.py | 6 ++++ .../apps/api/views/alert_receive_channel.py | 14 +++++++-- engine/apps/api/views/channel_filter.py | 1 + engine/common/exceptions/exceptions.py | 2 +- 8 files changed, 69 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0aac851c..65048da4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Helm chart: add the option to use a helm hook for the migration job ([1386](https://github.com/grafana/oncall/pull/1386)) +- Send demo alert with dynamic payload and get demo payload example on private api ([1700](https://github.com/grafana/oncall/pull/1700)) ## v1.2.11 (2023-04-14) diff --git a/engine/apps/alerts/models/alert_receive_channel.py b/engine/apps/alerts/models/alert_receive_channel.py index 412193c3..80e6ccb5 100644 --- a/engine/apps/alerts/models/alert_receive_channel.py +++ b/engine/apps/alerts/models/alert_receive_channel.py @@ -501,19 +501,26 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): return getattr(heartbeat, self.INTEGRATIONS_TO_REVERSE_URL_MAP[self.integration], None) # Demo alerts - def send_demo_alert(self, force_route_id=None): + def send_demo_alert(self, force_route_id=None, payload=None): logger.info(f"send_demo_alert integration={self.pk} force_route_id={force_route_id}") + if payload is None: + payload = self.config.example_payload if self.is_demo_alert_enabled: if self.has_alertmanager_payload_structure: - for alert in self.config.example_payload.get("alerts", []): - create_alertmanager_alerts.apply_async( - [], - { - "alert_receive_channel_pk": self.pk, - "alert": alert, - "is_demo": True, - "force_route_id": force_route_id, - }, + if (alerts := payload.get("alerts", None)) and type(alerts) == list and len(alerts): + for alert in alerts: + create_alertmanager_alerts.apply_async( + [], + { + "alert_receive_channel_pk": self.pk, + "alert": alert, + "is_demo": True, + "force_route_id": force_route_id, + }, + ) + else: + raise UnableToSendDemoAlert( + "Unable to send demo alert as payload has no 'alerts' key, it is not array, or it is empty." ) else: create_alert.apply_async( @@ -525,7 +532,7 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): "link_to_upstream_details": None, "alert_receive_channel_pk": self.pk, "integration_unique_data": None, - "raw_request_data": self.config.example_payload, + "raw_request_data": payload, "is_demo": True, "force_route_id": force_route_id, }, diff --git a/engine/apps/alerts/models/channel_filter.py b/engine/apps/alerts/models/channel_filter.py index 7ea55abe..677f3a19 100644 --- a/engine/apps/alerts/models/channel_filter.py +++ b/engine/apps/alerts/models/channel_filter.py @@ -94,6 +94,8 @@ class ChannelFilter(OrderedModel): @classmethod def select_filter(cls, alert_receive_channel, raw_request_data, force_route_id=None): # Try to find force route first if force_route_id is given + # Force route was used to send demo alerts to specific route. + # It is deprecated and may be used by older versions of the plugins if force_route_id is not None: logger.info( f"start select_filter with force_route_id={force_route_id} alert_receive_channel={alert_receive_channel.pk}." @@ -164,6 +166,7 @@ class ChannelFilter(OrderedModel): raise Exception("Unknown filtering term") def send_demo_alert(self): + """Deprecated. May be used in the older versions of the plugin""" integration = self.alert_receive_channel integration.send_demo_alert(force_route_id=self.pk) diff --git a/engine/apps/alerts/tests/test_alert_receiver_channel.py b/engine/apps/alerts/tests/test_alert_receiver_channel.py index cdc204de..12aa63db 100644 --- a/engine/apps/alerts/tests/test_alert_receiver_channel.py +++ b/engine/apps/alerts/tests/test_alert_receiver_channel.py @@ -90,14 +90,25 @@ def test_get_default_template_attribute_fallback_to_web(make_organization, make_ @mock.patch("apps.integrations.tasks.create_alert.apply_async", return_value=None) @pytest.mark.django_db -def test_send_demo_alert(mocked_create_alert, make_organization, make_alert_receive_channel): +@pytest.mark.parametrize( + "payload", + [ + None, + {"foo": "bar"}, + ], +) +def test_send_demo_alert(mocked_create_alert, make_organization, make_alert_receive_channel, payload): organization = make_organization() alert_receive_channel = make_alert_receive_channel( organization, integration=AlertReceiveChannel.INTEGRATION_WEBHOOK ) - alert_receive_channel.send_demo_alert() + alert_receive_channel.send_demo_alert(payload=payload) assert mocked_create_alert.called assert mocked_create_alert.call_args.args[1]["is_demo"] + assert ( + mocked_create_alert.call_args.args[1]["raw_request_data"] == payload + or alert_receive_channel.config.example_payload + ) assert mocked_create_alert.call_args.args[1]["force_route_id"] is None @@ -111,14 +122,26 @@ def test_send_demo_alert(mocked_create_alert, make_organization, make_alert_rece AlertReceiveChannel.INTEGRATION_GRAFANA_ALERTING, ], ) +@pytest.mark.parametrize( + "payload", + [ + None, + {"alerts": [{"foo": "bar"}]}, + ], +) def test_send_demo_alert_alertmanager_payload_shape( - mocked_create_alert, make_organization, make_alert_receive_channel, integration + mocked_create_alert, make_organization, make_alert_receive_channel, integration, payload ): organization = make_organization() alert_receive_channel = make_alert_receive_channel(organization, integration=integration) - alert_receive_channel.send_demo_alert() + alert_receive_channel.send_demo_alert(payload=payload) assert mocked_create_alert.called assert mocked_create_alert.call_args.args[1]["is_demo"] + assert ( + mocked_create_alert.call_args.args[1]["alert"] == payload["alerts"][0] + if payload + else alert_receive_channel.config.example_payload["alerts"][0] + ) assert mocked_create_alert.call_args.args[1]["force_route_id"] is None diff --git a/engine/apps/api/serializers/alert_receive_channel.py b/engine/apps/api/serializers/alert_receive_channel.py index 7bd2ced4..64341317 100644 --- a/engine/apps/api/serializers/alert_receive_channel.py +++ b/engine/apps/api/serializers/alert_receive_channel.py @@ -50,6 +50,7 @@ class AlertReceiveChannelSerializer(EagerLoadingMixin, serializers.ModelSerializ maintenance_till = serializers.ReadOnlyField(source="till_maintenance_timestamp") heartbeat = serializers.SerializerMethodField() allow_delete = serializers.SerializerMethodField() + demo_alert_payload = serializers.SerializerMethodField() # integration heartbeat is in PREFETCH_RELATED not by mistake. # With using of select_related ORM builds strange join @@ -82,6 +83,7 @@ class AlertReceiveChannelSerializer(EagerLoadingMixin, serializers.ModelSerializ "heartbeat", "is_available_for_integration_heartbeat", "allow_delete", + "demo_alert_payload", ] read_only_fields = [ "created_at", @@ -92,6 +94,7 @@ class AlertReceiveChannelSerializer(EagerLoadingMixin, serializers.ModelSerializ "instructions", "demo_alert_enabled", "maintenance_mode", + "demo_alert_payload", ] extra_kwargs = {"integration": {"required": True}} @@ -153,6 +156,9 @@ class AlertReceiveChannelSerializer(EagerLoadingMixin, serializers.ModelSerializ def get_alert_groups_count(self, obj): return 0 + def get_demo_alert_payload(self, obj): + return obj.config.example_payload + class AlertReceiveChannelUpdateSerializer(AlertReceiveChannelSerializer): class Meta(AlertReceiveChannelSerializer.Meta): diff --git a/engine/apps/api/views/alert_receive_channel.py b/engine/apps/api/views/alert_receive_channel.py index 8675d95f..ff401793 100644 --- a/engine/apps/api/views/alert_receive_channel.py +++ b/engine/apps/api/views/alert_receive_channel.py @@ -149,9 +149,19 @@ class AlertReceiveChannelView( @action(detail=True, methods=["post"], throttle_classes=[DemoAlertThrottler]) def send_demo_alert(self, request, pk): - instance = AlertReceiveChannel.objects.get(public_primary_key=pk) + alert_receive_channel = AlertReceiveChannel.objects.get(public_primary_key=pk) + demo_alert_payload = request.data.get("demo_alert_payload", None) + + if not demo_alert_payload: + # If no payload provided, use the demo payload for backword compatibility + payload = alert_receive_channel.config.example_payload + else: + if type(demo_alert_payload) != dict: + raise BadRequest(detail="Payload for demo alert must be a valid json object") + payload = demo_alert_payload + try: - instance.send_demo_alert() + alert_receive_channel.send_demo_alert(payload=payload) except UnableToSendDemoAlert as e: raise BadRequest(detail=str(e)) return Response(status=status.HTTP_200_OK) diff --git a/engine/apps/api/views/channel_filter.py b/engine/apps/api/views/channel_filter.py index adfe534f..1ba3bb1e 100644 --- a/engine/apps/api/views/channel_filter.py +++ b/engine/apps/api/views/channel_filter.py @@ -137,6 +137,7 @@ class ChannelFilterView( @action(detail=True, methods=["post"], throttle_classes=[DemoAlertThrottler]) def send_demo_alert(self, request, pk): + """Deprecated action. May be used in the older version of the plugin.""" instance = ChannelFilter.objects.get(public_primary_key=pk) try: instance.send_demo_alert() diff --git a/engine/common/exceptions/exceptions.py b/engine/common/exceptions/exceptions.py index 9adf0b47..15f70520 100644 --- a/engine/common/exceptions/exceptions.py +++ b/engine/common/exceptions/exceptions.py @@ -1,6 +1,6 @@ class OperationCouldNotBePerformedError(Exception): """ - Indicates that operation could not be performed due to to application logic. + Indicates that operation could not be performed due to application logic. E.g. you can't ack resolved AlertGroup """ From 9d1949356144b932c8b5cf645727bc27d6825f79 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 18 Apr 2023 10:53:53 +0800 Subject: [PATCH 09/15] Added preview and migration API endpoints for route migration from regex into jinja2 (#1715) # What this PR does This PR adds new API endpoints for migrating routes from regex format to jinja2 format. The changes include the following: * `filtering_term_as_jinja2` field to GET `channels_filters` endpoint * A POST endpoint `channel_filters/ABCDEF123/convert_from_regex_to_jinja2/` for migrating routes to jinja2 format. These new endpoints will allow users to preview and migrate their existing regex routes to the more flexible and maintainable jinja2 format. Check the screenshot where this endpoints will be used Screenshot 2023-04-14 at 09 50 23 ## Which issue(s) this PR fixes ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required) --- CHANGELOG.md | 1 + engine/apps/api/serializers/channel_filter.py | 12 ++++ engine/apps/api/tests/test_channel_filter.py | 64 +++++++++++++++++++ engine/apps/api/views/channel_filter.py | 14 ++++ engine/common/jinja_templater/filters.py | 14 ++++ .../jinja_templater/jinja_template_env.py | 12 +++- .../common/tests/test_apply_jinja_template.py | 21 ++++++ 7 files changed, 137 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65048da4..198daae8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added preview and migration API endpoints for route migration from regex into jinja2 ([1715](https://github.com/grafana/oncall/pull/1715)) - Helm chart: add the option to use a helm hook for the migration job ([1386](https://github.com/grafana/oncall/pull/1386)) - Send demo alert with dynamic payload and get demo payload example on private api ([1700](https://github.com/grafana/oncall/pull/1700)) diff --git a/engine/apps/api/serializers/channel_filter.py b/engine/apps/api/serializers/channel_filter.py index 2347a78b..8987374f 100644 --- a/engine/apps/api/serializers/channel_filter.py +++ b/engine/apps/api/serializers/channel_filter.py @@ -26,6 +26,7 @@ class ChannelFilterSerializer(OrderedModelSerializerMixin, EagerLoadingMixin, se queryset=TelegramToOrganizationConnector.objects, filter_field="organization", allow_null=True, required=False ) order = serializers.IntegerField(required=False) + filtering_term_as_jinja2 = serializers.SerializerMethodField() SELECT_RELATED = ["escalation_chain", "alert_receive_channel"] @@ -45,6 +46,7 @@ class ChannelFilterSerializer(OrderedModelSerializerMixin, EagerLoadingMixin, se "notify_in_slack", "notify_in_telegram", "notification_backends", + "filtering_term_as_jinja2", ] read_only_fields = ["created_at", "is_default"] extra_kwargs = {"filtering_term": {"required": True, "allow_null": False}} @@ -107,6 +109,16 @@ class ChannelFilterSerializer(OrderedModelSerializerMixin, EagerLoadingMixin, se notification_backends = updated return notification_backends + def get_filtering_term_as_jinja2(self, obj): + """ + Returns the regex filtering term as a jinja2, for the preview before migration from regex to jinja2""" + if obj.filtering_term_type == ChannelFilter.FILTERING_TERM_TYPE_JINJA2: + return obj.filtering_term + elif obj.filtering_term_type == ChannelFilter.FILTERING_TERM_TYPE_REGEX: + # Four curly braces will result in two curly braces in the final string + # rf"..." is a raw f string, to keep original filtering_term + return rf'{{{{ payload | json_dumps | regex_search("{obj.filtering_term}") }}}}' + class ChannelFilterCreateSerializer(ChannelFilterSerializer): alert_receive_channel = OrganizationFilteredPrimaryKeyRelatedField(queryset=AlertReceiveChannel.objects) diff --git a/engine/apps/api/tests/test_channel_filter.py b/engine/apps/api/tests/test_channel_filter.py index fe02e97b..d22b7040 100644 --- a/engine/apps/api/tests/test_channel_filter.py +++ b/engine/apps/api/tests/test_channel_filter.py @@ -502,3 +502,67 @@ def test_channel_filter_update_invalid_notification_backends( assert response.status_code == status.HTTP_400_BAD_REQUEST assert response.json() == {"notification_backends": ["Invalid messaging backend"]} assert channel_filter.notification_backends is None + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "role,expected_status", + [ + (LegacyAccessControlRole.ADMIN, status.HTTP_200_OK), + (LegacyAccessControlRole.EDITOR, status.HTTP_403_FORBIDDEN), + (LegacyAccessControlRole.VIEWER, status.HTTP_403_FORBIDDEN), + ], +) +def test_channel_filter_convert_from_regex_to_jinja2( + make_organization_and_user_with_plugin_token, + make_alert_receive_channel, + make_channel_filter, + make_user_auth_headers, + role, + expected_status, +): + organization, user, token = make_organization_and_user_with_plugin_token(role) + alert_receive_channel = make_alert_receive_channel(organization) + + make_channel_filter(alert_receive_channel, is_default=True) + + # r"..." used to keep this string as raw string + regex_filtering_term = r"\".*\": \"This alert was sent by user for the demonstration purposes\"" + final_filtering_term = r'{{ payload | json_dumps | regex_search("\".*\": \"This alert was sent by user for the demonstration purposes\"") }}' + payload = {"description": "This alert was sent by user for the demonstration purposes"} + + regex_channel_filter = make_channel_filter( + alert_receive_channel, + filtering_term=regex_filtering_term, + is_default=False, + ) + # Check if the filtering term is a regex + assert regex_channel_filter.filtering_term_type == regex_channel_filter.FILTERING_TERM_TYPE_REGEX + # Check if the alert is matched to the channel filter (route) regex + assert bool(regex_channel_filter.is_satisfying(payload)) is True + + client = APIClient() + url = reverse("api-internal:channel_filter-detail", kwargs={"pk": regex_channel_filter.public_primary_key}) + + response = client.get(url, format="json", **make_user_auth_headers(user, token)) + + assert response.status_code == status.HTTP_200_OK + # Check if preview of the filtering term migration is correct + assert response.json()["filtering_term_as_jinja2"] == final_filtering_term + + url = reverse( + "api-internal:channel_filter-convert-from-regex-to-jinja2", + kwargs={"pk": regex_channel_filter.public_primary_key}, + ) + response = client.post(url, **make_user_auth_headers(user, token)) + # Only admins can convert from regex to jinja2 + assert response.status_code == expected_status + if expected_status == status.HTTP_200_OK: + regex_channel_filter.refresh_from_db() + # Regex is now converted to jinja2 + jinja2_channel_filter = regex_channel_filter + # Check if the filtering term is a jinja2, and if it is correct + assert jinja2_channel_filter.filtering_term_type == jinja2_channel_filter.FILTERING_TERM_TYPE_JINJA2 + assert jinja2_channel_filter.filtering_term == final_filtering_term + # Check if the same alert is matched to the channel filter (route) new jinja2 + assert bool(jinja2_channel_filter.is_satisfying(payload)) is True diff --git a/engine/apps/api/views/channel_filter.py b/engine/apps/api/views/channel_filter.py index 1ba3bb1e..2341c003 100644 --- a/engine/apps/api/views/channel_filter.py +++ b/engine/apps/api/views/channel_filter.py @@ -41,6 +41,7 @@ class ChannelFilterView( "destroy": [RBACPermission.Permissions.INTEGRATIONS_WRITE], "move_to_position": [RBACPermission.Permissions.INTEGRATIONS_WRITE], "send_demo_alert": [RBACPermission.Permissions.INTEGRATIONS_TEST], + "convert_from_regex_to_jinja2": [RBACPermission.Permissions.INTEGRATIONS_WRITE], } model = ChannelFilter @@ -144,3 +145,16 @@ class ChannelFilterView( except UnableToSendDemoAlert as e: raise BadRequest(detail=str(e)) return Response(status=status.HTTP_200_OK) + + @action(detail=True, methods=["post"]) + def convert_from_regex_to_jinja2(self, request, pk): + instance = self.get_queryset().get(public_primary_key=pk) + if not instance.filtering_term_type == ChannelFilter.FILTERING_TERM_TYPE_REGEX: + raise BadRequest(detail="Only regex filtering term type is supported") + + serializer_class = self.serializer_class + + instance.filtering_term = serializer_class(instance).get_filtering_term_as_jinja2(instance) + instance.filtering_term_type = ChannelFilter.FILTERING_TERM_TYPE_JINJA2 + instance.save() + return Response(status=status.HTTP_200_OK, data=serializer_class(instance).data) diff --git a/engine/common/jinja_templater/filters.py b/engine/common/jinja_templater/filters.py index 1264aa68..88b4797d 100644 --- a/engine/common/jinja_templater/filters.py +++ b/engine/common/jinja_templater/filters.py @@ -37,3 +37,17 @@ def regex_match(pattern, value): return bool(re.match(value, pattern)) except (ValueError, AttributeError, TypeError): return None + + +def regex_search(pattern, value): + try: + return bool(re.search(value, pattern)) + except (ValueError, AttributeError, TypeError): + return None + + +def json_dumps(value): + try: + return json.dumps(value) + except (ValueError, AttributeError, TypeError): + return None diff --git a/engine/common/jinja_templater/jinja_template_env.py b/engine/common/jinja_templater/jinja_template_env.py index 32ceda6b..f4d2d65a 100644 --- a/engine/common/jinja_templater/jinja_template_env.py +++ b/engine/common/jinja_templater/jinja_template_env.py @@ -3,7 +3,15 @@ from jinja2 import BaseLoader from jinja2.exceptions import SecurityError from jinja2.sandbox import SandboxedEnvironment -from .filters import datetimeformat, iso8601_to_time, regex_match, regex_replace, to_pretty_json +from .filters import ( + datetimeformat, + iso8601_to_time, + json_dumps, + regex_match, + regex_replace, + regex_search, + to_pretty_json, +) def raise_security_exception(name): @@ -19,3 +27,5 @@ jinja_template_env.globals["time"] = timezone.now jinja_template_env.globals["range"] = lambda *args: raise_security_exception("range") jinja_template_env.filters["regex_replace"] = regex_replace jinja_template_env.filters["regex_match"] = regex_match +jinja_template_env.filters["regex_search"] = regex_search +jinja_template_env.filters["json_dumps"] = json_dumps diff --git a/engine/common/tests/test_apply_jinja_template.py b/engine/common/tests/test_apply_jinja_template.py index 632f5026..aab3f488 100644 --- a/engine/common/tests/test_apply_jinja_template.py +++ b/engine/common/tests/test_apply_jinja_template.py @@ -14,6 +14,14 @@ def test_apply_jinja_template(): assert payload == result +def test_apply_jinja_template_json_dumps(): + payload = {"name": "test"} + + result = apply_jinja_template("{{ payload | json_dumps }}", payload) + expected = json.dumps(payload) + assert result == expected + + def test_apply_jinja_template_regex_match(): payload = {"name": "test"} @@ -26,6 +34,19 @@ def test_apply_jinja_template_regex_match(): apply_jinja_template("{{ payload.name | regex_match('*') }}", payload) +def test_apply_jinja_template_regex_search(): + payload = {"name": "test"} + + assert apply_jinja_template("{{ payload.name | regex_search('.*') }}", payload) == "True" + assert apply_jinja_template("{{ payload.name | regex_search('tes') }}", payload) == "True" + assert apply_jinja_template("{{ payload.name | regex_search('est') }}", payload) == "True" + assert apply_jinja_template("{{ payload.name | regex_search('test1') }}", payload) == "False" + + # Check that exception is raised when regex is invalid + with pytest.raises(JinjaTemplateError): + apply_jinja_template("{{ payload.name | regex_search('*') }}", payload) + + def test_apply_jinja_template_bad_syntax_error(): with pytest.raises(JinjaTemplateError): apply_jinja_template("{{%", payload={}) From 7dd726622a2d5dad15035e3998293969aaf33010 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 18 Apr 2023 11:31:49 +0800 Subject: [PATCH 10/15] =?UTF-8?q?Add=20endpoints=20to=20start=20and=20stop?= =?UTF-8?q?=20maintenance=20in=20alert=20receive=20channel=20=E2=80=A6=20(?= =?UTF-8?q?#1755)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …private api # What this PR does ## Which issue(s) this PR fixes ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required) --- CHANGELOG.md | 1 + .../api/tests/test_alert_receive_channel.py | 65 +++++++++++++++++++ .../apps/api/views/alert_receive_channel.py | 42 +++++++++++- engine/apps/api/views/maintenance.py | 6 ++ 4 files changed, 113 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 198daae8..5cdf60a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added preview and migration API endpoints for route migration from regex into jinja2 ([1715](https://github.com/grafana/oncall/pull/1715)) - Helm chart: add the option to use a helm hook for the migration job ([1386](https://github.com/grafana/oncall/pull/1386)) +- Add endpoints to start and stop maintenance in alert receive channel private api ([1755](https://github.com/grafana/oncall/pull/1755)) - Send demo alert with dynamic payload and get demo payload example on private api ([1700](https://github.com/grafana/oncall/pull/1700)) ## v1.2.11 (2023-04-14) diff --git a/engine/apps/api/tests/test_alert_receive_channel.py b/engine/apps/api/tests/test_alert_receive_channel.py index a1b47b60..53996c56 100644 --- a/engine/apps/api/tests/test_alert_receive_channel.py +++ b/engine/apps/api/tests/test_alert_receive_channel.py @@ -669,3 +669,68 @@ def test_get_alert_receive_channels_direct_paging_present_for_filters( # Check direct paging integration is in the response assert response.status_code == status.HTTP_200_OK assert response.json()[0]["value"] == alert_receive_channel.public_primary_key + + +@pytest.mark.django_db +def test_start_maintenance_integration( + make_user_auth_headers, + make_organization_and_user_with_plugin_token, + make_escalation_chain, + make_alert_receive_channel, +): + + organization, user, token = make_organization_and_user_with_plugin_token() + make_escalation_chain(organization) + alert_receive_channel = make_alert_receive_channel(organization) + + client = APIClient() + + url = reverse( + "api-internal:alert_receive_channel-start-maintenance", kwargs={"pk": alert_receive_channel.public_primary_key} + ) + + data = { + "mode": AlertReceiveChannel.MAINTENANCE, + "duration": AlertReceiveChannel.DURATION_ONE_HOUR.total_seconds(), + "type": "alert_receive_channel", + } + response = client.post(url, data=data, format="json", **make_user_auth_headers(user, token)) + + alert_receive_channel.refresh_from_db() + assert response.status_code == status.HTTP_200_OK + assert alert_receive_channel.maintenance_mode == AlertReceiveChannel.MAINTENANCE + assert alert_receive_channel.maintenance_duration == AlertReceiveChannel.DURATION_ONE_HOUR + assert alert_receive_channel.maintenance_uuid is not None + assert alert_receive_channel.maintenance_started_at is not None + assert alert_receive_channel.maintenance_author is not None + + +@pytest.mark.django_db +def test_stop_maintenance_integration( + mock_start_disable_maintenance_task, + make_user_auth_headers, + make_organization_and_user_with_plugin_token, + make_escalation_chain, + make_alert_receive_channel, +): + organization, user, token = make_organization_and_user_with_plugin_token() + make_escalation_chain(organization) + alert_receive_channel = make_alert_receive_channel(organization) + client = APIClient() + mode = AlertReceiveChannel.MAINTENANCE + duration = AlertReceiveChannel.DURATION_ONE_HOUR.seconds + alert_receive_channel.start_maintenance(mode, duration, user) + url = reverse( + "api-internal:alert_receive_channel-stop-maintenance", kwargs={"pk": alert_receive_channel.public_primary_key} + ) + data = { + "type": "alert_receive_channel", + } + response = client.post(url, data=data, format="json", **make_user_auth_headers(user, token)) + alert_receive_channel.refresh_from_db() + assert response.status_code == status.HTTP_200_OK + assert alert_receive_channel.maintenance_mode is None + assert alert_receive_channel.maintenance_duration is None + assert alert_receive_channel.maintenance_uuid is None + assert alert_receive_channel.maintenance_started_at is None + assert alert_receive_channel.maintenance_author is None diff --git a/engine/apps/api/views/alert_receive_channel.py b/engine/apps/api/views/alert_receive_channel.py index ff401793..5331dae3 100644 --- a/engine/apps/api/views/alert_receive_channel.py +++ b/engine/apps/api/views/alert_receive_channel.py @@ -9,6 +9,7 @@ from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet from apps.alerts.models import AlertReceiveChannel +from apps.alerts.models.maintainable_object import MaintainableObject from apps.api.permissions import RBACPermission from apps.api.serializers.alert_receive_channel import ( AlertReceiveChannelSerializer, @@ -26,7 +27,7 @@ from common.api_helpers.mixins import ( TeamFilteringMixin, UpdateSerializerMixin, ) -from common.exceptions import TeamCanNotBeChangedError, UnableToSendDemoAlert +from common.exceptions import MaintenanceCouldNotBeStartedError, TeamCanNotBeChangedError, UnableToSendDemoAlert from common.insight_log import EntityEvent, write_resource_insight_log @@ -95,6 +96,8 @@ class AlertReceiveChannelView( "destroy": [RBACPermission.Permissions.INTEGRATIONS_WRITE], "change_team": [RBACPermission.Permissions.INTEGRATIONS_WRITE], "filters": [RBACPermission.Permissions.INTEGRATIONS_READ], + "start_maintenance": [RBACPermission.Permissions.INTEGRATIONS_WRITE], + "stop_maintenance": [RBACPermission.Permissions.INTEGRATIONS_WRITE], } def create(self, request, *args, **kwargs): @@ -249,3 +252,40 @@ class AlertReceiveChannelView( filter_options = list(filter(lambda f: filter_name in f["name"], filter_options)) return Response(filter_options) + + @action(detail=True, methods=["post"]) + def start_maintenance(self, request, pk): + instance = self.get_queryset(eager=False).get(public_primary_key=pk) + + mode = request.data.get("mode", None) + duration = request.data.get("duration", None) + try: + mode = int(mode) + except (ValueError, TypeError): + raise BadRequest(detail={"mode": ["Invalid mode"]}) + if mode not in [MaintainableObject.DEBUG_MAINTENANCE, MaintainableObject.MAINTENANCE]: + raise BadRequest(detail={"mode": ["Unknown mode"]}) + try: + duration = int(duration) + except (ValueError, TypeError): + raise BadRequest(detail={"duration": ["Invalid duration"]}) + if duration not in MaintainableObject.maintenance_duration_options_in_seconds(): + raise BadRequest(detail={"mode": ["Unknown duration"]}) + + try: + instance.start_maintenance(mode, duration, request.user) + except MaintenanceCouldNotBeStartedError as e: + if type(instance) == AlertReceiveChannel: + detail = {"alert_receive_channel_id": ["Already on maintenance"]} + else: + detail = str(e) + raise BadRequest(detail=detail) + + return Response(status=status.HTTP_200_OK) + + @action(detail=True, methods=["post"]) + def stop_maintenance(self, request, pk): + instance = self.get_queryset(eager=False).get(public_primary_key=pk) + user = request.user + instance.force_disable_maintenance(user) + return Response(status=status.HTTP_200_OK) diff --git a/engine/apps/api/views/maintenance.py b/engine/apps/api/views/maintenance.py index 29042acb..c875d65c 100644 --- a/engine/apps/api/views/maintenance.py +++ b/engine/apps/api/views/maintenance.py @@ -39,6 +39,8 @@ class GetObjectMixin: class MaintenanceAPIView(APIView): + """Deprecated. Maintenance management is now performed on integrations page (alert_receive_channel/ endpoint))""" + authentication_classes = (PluginAuthentication,) permission_classes = (IsAuthenticated, RBACPermission) @@ -101,6 +103,8 @@ class MaintenanceAPIView(APIView): class MaintenanceStartAPIView(GetObjectMixin, APIView): + """Deprecated. Maintenance management is now performed on integrations page (alert_receive_channel/ endpoint))""" + authentication_classes = (PluginAuthentication,) permission_classes = (IsAuthenticated, RBACPermission) rbac_permissions = { @@ -137,6 +141,8 @@ class MaintenanceStartAPIView(GetObjectMixin, APIView): class MaintenanceStopAPIView(GetObjectMixin, APIView): + """Deprecated. Maintenance management is now performed on integrations page (alert_receive_channel/ endpoint))""" + authentication_classes = (PluginAuthentication,) permission_classes = (IsAuthenticated, RBACPermission) rbac_permissions = { From 5da5b8d430cf272d9ebf49a964af547c00ac6469 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 18 Apr 2023 11:57:40 +0800 Subject: [PATCH 11/15] =?UTF-8?q?Allow=20use=20of=20dynamic=20payloads=20i?= =?UTF-8?q?n=20alert=20receive=20channels=20preview=20templ=E2=80=A6=20(#1?= =?UTF-8?q?756)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ate in private api # What this PR does ## Which issue(s) this PR fixes ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required) --- CHANGELOG.md | 1 + .../api/tests/test_alert_receive_channel.py | 38 +++++++++++++++++++ engine/apps/api/views/alert_group.py | 2 +- .../apps/api/views/alert_receive_channel.py | 14 +++++-- engine/common/api_helpers/mixins.py | 20 +++++++--- 5 files changed, 65 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cdf60a4..a97214e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Helm chart: add the option to use a helm hook for the migration job ([1386](https://github.com/grafana/oncall/pull/1386)) - Add endpoints to start and stop maintenance in alert receive channel private api ([1755](https://github.com/grafana/oncall/pull/1755)) - Send demo alert with dynamic payload and get demo payload example on private api ([1700](https://github.com/grafana/oncall/pull/1700)) +- Allow use of dynamic payloads in alert receive channels preview template in private api ([1756](https://github.com/grafana/oncall/pull/1756)) ## v1.2.11 (2023-04-14) diff --git a/engine/apps/api/tests/test_alert_receive_channel.py b/engine/apps/api/tests/test_alert_receive_channel.py index 53996c56..9a487803 100644 --- a/engine/apps/api/tests/test_alert_receive_channel.py +++ b/engine/apps/api/tests/test_alert_receive_channel.py @@ -505,6 +505,44 @@ def test_alert_receive_channel_preview_template_require_notification_channel( assert response.status_code == status.HTTP_200_OK +@pytest.mark.django_db +@pytest.mark.parametrize("template_name", ["title", "message", "image_url"]) +@pytest.mark.parametrize("notification_channel", ["slack", "web", "telegram"]) +def test_alert_receive_channel_preview_template_dynamic_payload( + make_organization_and_user_with_plugin_token, + make_user_auth_headers, + make_alert_receive_channel, + template_name, + notification_channel, + make_alert_group, + make_alert, +): + organization, user, token = make_organization_and_user_with_plugin_token() + alert_receive_channel = make_alert_receive_channel(organization) + alert_group = make_alert_group(alert_receive_channel) + + make_alert(alert_group=alert_group, raw_request_data=alert_receive_channel.config.example_payload) + + client = APIClient() + url = reverse( + "api-internal:alert_receive_channel-preview-template", kwargs={"pk": alert_receive_channel.public_primary_key} + ) + + data = { + "template_body": "{{ payload.foo }}", + "template_name": f"{notification_channel}_{template_name}", + "payload": {"foo": "bar"}, + } + + response = client.post(url, data=data, format="json", **make_user_auth_headers(user, token)) + + assert response.status_code == status.HTTP_200_OK + if notification_channel == "web" and template_name == "message": + assert response.data["preview"] == "

bar

" + else: + assert response.data["preview"] == "bar" + + @pytest.mark.django_db @pytest.mark.parametrize( "role,expected_status", diff --git a/engine/apps/api/views/alert_group.py b/engine/apps/api/views/alert_group.py index a0a290be..96420041 100644 --- a/engine/apps/api/views/alert_group.py +++ b/engine/apps/api/views/alert_group.py @@ -657,5 +657,5 @@ class AlertGroupView( ) # This method is required for PreviewTemplateMixin - def get_alert_to_template(self): + def get_alert_to_template(self, payload=None): return self.get_object().alerts.first() diff --git a/engine/apps/api/views/alert_receive_channel.py b/engine/apps/api/views/alert_receive_channel.py index 5331dae3..ced06e18 100644 --- a/engine/apps/api/views/alert_receive_channel.py +++ b/engine/apps/api/views/alert_receive_channel.py @@ -8,7 +8,7 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet -from apps.alerts.models import AlertReceiveChannel +from apps.alerts.models import Alert, AlertGroup, AlertReceiveChannel from apps.alerts.models.maintainable_object import MaintainableObject from apps.api.permissions import RBACPermission from apps.api.serializers.alert_receive_channel import ( @@ -22,6 +22,7 @@ from common.api_helpers.exceptions import BadRequest from common.api_helpers.filters import ByTeamModelFieldFilterMixin, TeamModelMultipleChoiceFilter from common.api_helpers.mixins import ( FilterSerializerMixin, + PreviewTemplateException, PreviewTemplateMixin, PublicPrimaryKeyMixin, TeamFilteringMixin, @@ -228,9 +229,16 @@ class AlertReceiveChannelView( return Response(response) # This method is required for PreviewTemplateMixin - def get_alert_to_template(self): + def get_alert_to_template(self, payload=None): try: - return self.get_object().alert_groups.last().alerts.first() + if payload is None: + return self.get_object().alert_groups.last().alerts.first() + else: + if type(payload) != dict: + raise PreviewTemplateException("Payload must be a valid json object") + # Build Alert and AlertGroup objects to pass to templater without saving them to db + alert_group_to_template = AlertGroup(channel=self.get_object()) + return Alert(raw_request_data=payload, group=alert_group_to_template) except AttributeError: return None diff --git a/engine/common/api_helpers/mixins.py b/engine/common/api_helpers/mixins.py index 271a9555..9889e642 100644 --- a/engine/common/api_helpers/mixins.py +++ b/engine/common/api_helpers/mixins.py @@ -283,11 +283,16 @@ BEHAVIOUR_TEMPLATE_NAMES = [RESOLVE_CONDITION, ACKNOWLEDGE_CONDITION, GROUPING_I ALL_TEMPLATE_NAMES = APPEARANCE_TEMPLATE_NAMES + BEHAVIOUR_TEMPLATE_NAMES +class PreviewTemplateException(Exception): + pass + + class PreviewTemplateMixin: @action(methods=["post"], detail=True) def preview_template(self, request, pk): template_body = request.data.get("template_body", None) template_name = request.data.get("template_name", None) + payload = request.data.get("payload", None) if template_body is None or template_name is None: response = {"preview": None} @@ -295,18 +300,21 @@ class PreviewTemplateMixin: notification_channel, attr_name = self.parse_name_and_notification_channel(template_name) if attr_name is None: - raise BadRequest(detail={"template_name": "Attr name is required"}) + raise BadRequest(detail={"template_name": "Template name is missing"}) if attr_name not in ALL_TEMPLATE_NAMES: - raise BadRequest(detail={"template_name": "Unknown attr name"}) + raise BadRequest(detail={"template_name": "Unknown template name"}) if attr_name in APPEARANCE_TEMPLATE_NAMES: if notification_channel is None: raise BadRequest(detail={"notification_channel": "notification_channel is required"}) if notification_channel not in NOTIFICATION_CHANNEL_OPTIONS: raise BadRequest(detail={"notification_channel": "Unknown notification_channel"}) - alert_to_template = self.get_alert_to_template() - if alert_to_template is None: - raise BadRequest(detail="Alert to preview does not exist") + try: + alert_to_template = self.get_alert_to_template(payload=payload) + if alert_to_template is None: + raise BadRequest(detail="Alert to preview does not exist") + except PreviewTemplateException as e: + raise BadRequest(detail=str(e)) if attr_name in APPEARANCE_TEMPLATE_NAMES: @@ -337,7 +345,7 @@ class PreviewTemplateMixin: response = {"preview": templated_attr} return Response(response, status=status.HTTP_200_OK) - def get_alert_to_template(self): + def get_alert_to_template(self, payload=None): raise NotImplementedError @staticmethod From 61dced5bd91c3b91e77c81234908558a73740a2d Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 18 Apr 2023 12:44:51 +0800 Subject: [PATCH 12/15] =?UTF-8?q?Add=20is=5Fdefault=20fields=20to=20templa?= =?UTF-8?q?tes,=20remove=20WritableSerialiserMethodFi=E2=80=A6=20(#1759)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …eld, refactor fields # What this PR does ## Which issue(s) this PR fixes ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required) --- CHANGELOG.md | 1 + .../api/serializers/alert_receive_channel.py | 381 +++--------------- .../test_alert_receive_channel_template.py | 5 +- engine/common/api_helpers/custom_fields.py | 29 -- 4 files changed, 69 insertions(+), 347 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a97214e5..3dbd9fc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Helm chart: add the option to use a helm hook for the migration job ([1386](https://github.com/grafana/oncall/pull/1386)) - Add endpoints to start and stop maintenance in alert receive channel private api ([1755](https://github.com/grafana/oncall/pull/1755)) - Send demo alert with dynamic payload and get demo payload example on private api ([1700](https://github.com/grafana/oncall/pull/1700)) +- Add is_default fields to templates, remove WritableSerialiserMethodField ([1759](https://github.com/grafana/oncall/pull/1759)) - Allow use of dynamic payloads in alert receive channels preview template in private api ([1756](https://github.com/grafana/oncall/pull/1756)) ## v1.2.11 (2023-04-14) diff --git a/engine/apps/api/serializers/alert_receive_channel.py b/engine/apps/api/serializers/alert_receive_channel.py index 64341317..852c3229 100644 --- a/engine/apps/api/serializers/alert_receive_channel.py +++ b/engine/apps/api/serializers/alert_receive_channel.py @@ -1,22 +1,19 @@ from collections import OrderedDict -from collections.abc import Mapping from django.apps import apps from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError as DjangoValidationError from django.template.loader import render_to_string -from django.utils import timezone from jinja2 import TemplateSyntaxError from rest_framework import serializers from rest_framework.exceptions import ValidationError -from rest_framework.fields import SerializerMethodField, SkipField, get_error_detail, set_value -from rest_framework.settings import api_settings +from rest_framework.fields import SerializerMethodField, set_value from apps.alerts.grafana_alerting_sync_manager.grafana_alerting_sync import GrafanaAlertingSyncManager from apps.alerts.models import AlertReceiveChannel from apps.base.messaging import get_messaging_backends -from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField, WritableSerializerMethodField +from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField from common.api_helpers.exceptions import BadRequest from common.api_helpers.mixins import APPEARANCE_TEMPLATE_NAMES, EagerLoadingMixin from common.api_helpers.utils import CurrentTeamDefault @@ -197,96 +194,23 @@ class FilterAlertReceiveChannelSerializer(serializers.ModelSerializer): class AlertReceiveChannelTemplatesSerializer(EagerLoadingMixin, serializers.ModelSerializer): id = serializers.CharField(read_only=True, source="public_primary_key") - slack_title_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - slack_message_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - slack_image_url_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - web_title_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - web_message_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - web_image_url_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - sms_title_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - phone_call_title_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - telegram_title_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - telegram_message_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - telegram_image_url_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - source_link_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - grouping_id_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - acknowledge_condition_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) - resolve_condition_template = WritableSerializerMethodField( - allow_null=True, - deserializer_field=serializers.CharField(), - validators=[valid_jinja_template_for_serializer_method_field], - required=False, - ) + CORE_TEMPLATE_NAMES = [ + "slack_title_template", + "slack_message_template", + "slack_image_url_template", + "web_title_template", + "web_message_template", + "web_image_url_template", + "telegram_title_template", + "telegram_message_template", + "telegram_image_url_template", + "sms_title_template", + "phone_call_title_template", + "source_link_template", + "grouping_id_template", + "resolve_condition_template", + "acknowledge_condition_template", + ] payload_example = SerializerMethodField() @@ -295,207 +219,10 @@ class AlertReceiveChannelTemplatesSerializer(EagerLoadingMixin, serializers.Mode fields = [ "id", "verbal_name", - "slack_title_template", - "slack_message_template", - "slack_image_url_template", - "sms_title_template", - "phone_call_title_template", - "web_title_template", - "web_message_template", - "web_image_url_template", - "telegram_title_template", - "telegram_message_template", - "telegram_image_url_template", - "source_link_template", - "grouping_id_template", - "resolve_condition_template", "payload_example", - "acknowledge_condition_template", ] extra_kwargs = {"integration": {"required": True}} - # MethodFields are used instead of relevant properties because of properties hit db on each instance in queryset - - def get_slack_title_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_SLACK_TITLE_TEMPLATE[obj.integration] - return obj.slack_title_template or default_template - - def set_slack_title_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_SLACK_TITLE_TEMPLATE[self.instance.integration] - if default_template is None or default_template.strip() != value.strip(): - self.instance.slack_title_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.slack_title_template = None - - def get_slack_message_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_SLACK_MESSAGE_TEMPLATE[obj.integration] - return obj.slack_message_template or default_template - - def set_slack_message_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_SLACK_MESSAGE_TEMPLATE[self.instance.integration] - if default_template is None or default_template.strip() != value.strip(): - self.instance.slack_message_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.slack_message_template = None - - def get_slack_image_url_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_SLACK_IMAGE_URL_TEMPLATE[obj.integration] - return obj.slack_image_url_template or default_template - - def set_slack_image_url_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_SLACK_IMAGE_URL_TEMPLATE[ - self.instance.integration - ] - if default_template is None or default_template.strip() != value.strip(): - self.instance.slack_image_url_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.slack_image_url_template = None - - def get_sms_title_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_SMS_TITLE_TEMPLATE[obj.integration] - return obj.sms_title_template or default_template - - def set_sms_title_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_SMS_TITLE_TEMPLATE[self.instance.integration] - if default_template is None or default_template.strip() != value.strip(): - self.instance.sms_title_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.sms_title_template = None - - def get_phone_call_title_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_PHONE_CALL_TITLE_TEMPLATE[obj.integration] - return obj.phone_call_title_template or default_template - - def set_phone_call_title_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_PHONE_CALL_TITLE_TEMPLATE[ - self.instance.integration - ] - if default_template is None or default_template.strip() != value.strip(): - self.instance.phone_call_title_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.phone_call_title_template = None - - def get_web_title_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_WEB_TITLE_TEMPLATE[obj.integration] - return obj.web_title_template or default_template - - def set_web_title_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_WEB_TITLE_TEMPLATE[self.instance.integration] - if default_template is None or default_template.strip() != value.strip(): - self.instance.web_title_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.web_title_template = None - self.instance.web_templates_modified_at = timezone.now() - - def get_web_message_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_WEB_MESSAGE_TEMPLATE[obj.integration] - return obj.web_message_template or default_template - - def set_web_message_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_WEB_MESSAGE_TEMPLATE[self.instance.integration] - if default_template is None or default_template.strip() != value.strip(): - self.instance.web_message_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.web_message_template = None - self.instance.web_templates_modified_at = timezone.now() - - def get_web_image_url_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_WEB_IMAGE_URL_TEMPLATE[obj.integration] - return obj.web_image_url_template or default_template - - def set_web_image_url_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_WEB_IMAGE_URL_TEMPLATE[self.instance.integration] - if default_template is None or default_template.strip() != value.strip(): - self.instance.web_image_url_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.web_image_url_template = None - self.instance.web_templates_modified_at = timezone.now() - - def get_telegram_title_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_TELEGRAM_TITLE_TEMPLATE[obj.integration] - return obj.telegram_title_template or default_template - - def set_telegram_title_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_TELEGRAM_TITLE_TEMPLATE[self.instance.integration] - if default_template is None or default_template.strip() != value.strip(): - self.instance.telegram_title_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.telegram_title_template = None - - def get_telegram_message_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_TELEGRAM_MESSAGE_TEMPLATE[obj.integration] - return obj.telegram_message_template or default_template - - def set_telegram_message_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_TELEGRAM_MESSAGE_TEMPLATE[ - self.instance.integration - ] - if default_template is None or default_template.strip() != value.strip(): - self.instance.telegram_message_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.telegram_message_template = None - - def get_telegram_image_url_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_TELEGRAM_IMAGE_URL_TEMPLATE[obj.integration] - return obj.telegram_image_url_template or default_template - - def set_telegram_image_url_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_TELEGRAM_IMAGE_URL_TEMPLATE[ - self.instance.integration - ] - if default_template is None or default_template.strip() != value.strip(): - self.instance.telegram_image_url_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.telegram_image_url_template = None - - def get_source_link_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_SOURCE_LINK_TEMPLATE[obj.integration] - return obj.source_link_template or default_template - - def set_source_link_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_SOURCE_LINK_TEMPLATE[self.instance.integration] - if default_template is None or default_template.strip() != value.strip(): - self.instance.source_link_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.source_link_template = None - - def get_grouping_id_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_GROUPING_ID_TEMPLATE[obj.integration] - return obj.grouping_id_template or default_template - - def set_grouping_id_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_GROUPING_ID_TEMPLATE[self.instance.integration] - if default_template is None or default_template.strip() != value.strip(): - self.instance.grouping_id_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.grouping_id_template = None - - def get_acknowledge_condition_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_ACKNOWLEDGE_CONDITION_TEMPLATE[obj.integration] - return obj.acknowledge_condition_template or default_template - - def set_acknowledge_condition_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_ACKNOWLEDGE_CONDITION_TEMPLATE[ - self.instance.integration - ] - if default_template is None or default_template.strip() != value.strip(): - self.instance.acknowledge_condition_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.acknowledge_condition_template = None - - def get_resolve_condition_template(self, obj): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_RESOLVE_CONDITION_TEMPLATE[obj.integration] - return obj.resolve_condition_template or default_template - - def set_resolve_condition_template(self, value): - default_template = AlertReceiveChannel.INTEGRATION_TO_DEFAULT_RESOLVE_CONDITION_TEMPLATE[ - self.instance.integration - ] - if default_template is None or default_template.strip() != value.strip(): - self.instance.resolve_condition_template = value.strip() - elif default_template is not None and default_template.strip() == value.strip(): - self.instance.resolve_condition_template = None - def get_payload_example(self, obj): AlertGroup = apps.get_model("alerts", "AlertGroup") if "alert_group_id" in self.context["request"].query_params: @@ -517,33 +244,15 @@ class AlertReceiveChannelTemplatesSerializer(EagerLoadingMixin, serializers.Mode """ Dict of native values <- Dict of primitive datatypes. """ - if not isinstance(data, Mapping): - message = self.error_messages["invalid"].format(datatype=type(data).__name__) - raise ValidationError({api_settings.NON_FIELD_ERRORS_KEY: [message]}, code="invalid") + # First validate and save data from serializer fields + ret = super().to_internal_value(data) - ret = OrderedDict() + # Separately validate and save template fields we generate dynamically errors = OrderedDict() - fields = self._writable_fields - for field in fields: - validate_method = getattr(self, "validate_" + field.field_name, None) - primitive_value = field.get_value(data) - try: - validated_value = field.run_validation(primitive_value) - if validate_method is not None: - validated_value = validate_method(validated_value) - except ValidationError as exc: - errors[field.field_name] = exc.detail - except DjangoValidationError as exc: - errors[field.field_name] = get_error_detail(exc) - except SkipField: - pass - else: - # Line because of which method is overriden - if validated_value is None and isinstance(field, WritableSerializerMethodField): - set_value(ret, [field.field_name], validated_value) - else: - set_value(ret, field.source_attrs, validated_value) + # handle updates for core templates + core_template_errors = self._handle_core_template_updates(data, ret) + errors.update(core_template_errors) # handle updates for messaging backend templates messaging_backend_errors = self._handle_messaging_backend_updates(data, ret) @@ -551,7 +260,6 @@ class AlertReceiveChannelTemplatesSerializer(EagerLoadingMixin, serializers.Mode if errors: raise ValidationError(errors) - return ret def _handle_messaging_backend_updates(self, data, ret): @@ -586,10 +294,33 @@ class AlertReceiveChannelTemplatesSerializer(EagerLoadingMixin, serializers.Mode return errors + def _handle_core_template_updates(self, data, ret): + """Update core templates if needed.""" + errors = {} + + core_template_names = self.CORE_TEMPLATE_NAMES + + for field_name in core_template_names: + value = data.get(field_name) + validator = jinja_template_env.from_string + if value is not None: + try: + if value: + validator(value) + except TemplateSyntaxError: + errors[field_name] = "invalid template" + except DjangoValidationError: + errors[field_name] = "invalid URL" + set_value(ret, [field_name], value) + return errors + def to_representation(self, obj): ret = super().to_representation(obj) ret = self._get_templates_to_show(ret) + core_templates = self._get_core_templates(obj) + ret.update(core_templates) + # include messaging backend templates additional_templates = self._get_messaging_backend_templates(obj) ret.update(additional_templates) @@ -627,10 +358,26 @@ class AlertReceiveChannelTemplatesSerializer(EagerLoadingMixin, serializers.Mode continue for field in backend.template_fields: value = None + is_default = False if obj.messaging_backends_templates: value = obj.messaging_backends_templates.get(backend_id, {}).get(field) if not value: value = obj.get_default_template_attribute(backend_id, field) + is_default = True field_name = f"{backend.slug}_{field}_template" templates[field_name] = value + templates[f"{field_name}_is_default"] = is_default return templates + + def _get_core_templates(self, obj): + core_templates = {} + + core_template_names = self.CORE_TEMPLATE_NAMES + for template_name in core_template_names: + template_value = getattr(obj, template_name) + defaults = getattr(obj, f"INTEGRATION_TO_DEFAULT_{template_name.upper()}", {}) + default_template_value = defaults.get(obj.integration) + core_templates[template_name] = template_value or default_template_value + core_templates[f"{template_name}_is_default"] = not bool(template_value) + + return core_templates diff --git a/engine/apps/api/tests/test_alert_receive_channel_template.py b/engine/apps/api/tests/test_alert_receive_channel_template.py index 1b2a1d6a..e00f75b9 100644 --- a/engine/apps/api/tests/test_alert_receive_channel_template.py +++ b/engine/apps/api/tests/test_alert_receive_channel_template.py @@ -376,4 +376,7 @@ def test_update_alert_receive_channel_templates( # check if updated templates are applied updated_templates_data = response.json() for template_name, prev_template_value in existing_templates_data.items(): - assert updated_templates_data[template_name] == template_update_func(prev_template_value) + if template_name.endswith("_is_default"): + assert updated_templates_data[template_name] is False + else: + assert updated_templates_data[template_name] == template_update_func(prev_template_value) diff --git a/engine/common/api_helpers/custom_fields.py b/engine/common/api_helpers/custom_fields.py index d7114b67..28bc9e30 100644 --- a/engine/common/api_helpers/custom_fields.py +++ b/engine/common/api_helpers/custom_fields.py @@ -101,35 +101,6 @@ class UsersFilteredByOrganizationField(serializers.Field): return queryset.filter(organization=request.user.organization, public_primary_key__in=data).distinct() -class WritableSerializerMethodField(serializers.SerializerMethodField): - """ - Please, NEVER use this field. - It was a mistake to create this one due to necessity to dig deep in drf to fix bugs there. - This field is a workaround to allow to write into SerializerMethodField. - """ - - def __init__(self, method_name=None, **kwargs): - self.method_name = method_name - self.setter_method_name = kwargs.pop("setter_method_name", None) - self.deserializer_field = kwargs.pop("deserializer_field") - - kwargs["source"] = "*" - super(serializers.SerializerMethodField, self).__init__(**kwargs) - - def bind(self, field_name, parent): - retval = super().bind(field_name, parent) - if not self.setter_method_name: - self.setter_method_name = f"set_{field_name}" - - return retval - - def to_internal_value(self, data): - value = self.deserializer_field.to_internal_value(data) - method = getattr(self.parent, self.setter_method_name) - method(value) - return {self.method_name: value} - - class CustomTimeField(fields.TimeField): def to_representation(self, value): result = super().to_representation(value) From cee0fdccd7fbc0b943337d47a4edc8e9a8943934 Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 18 Apr 2023 12:55:55 +0800 Subject: [PATCH 13/15] Add new field description_short to private and public api (#1698) # What this PR does Required for new Integrations page Screenshot 2023-04-04 at 20 32 03 ## Which issue(s) this PR fixes ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required) --------- Co-authored-by: Joey Orlando --- CHANGELOG.md | 3 +- ...2_alertreceivechannel_description_short.py | 18 +++++ .../migrations/0013_merge_20230418_0336.py | 14 ++++ .../alerts/models/alert_receive_channel.py | 1 + .../api/serializers/alert_receive_channel.py | 2 + .../public_api/serializers/integrations.py | 2 + .../public_api/tests/test_integrations.py | 72 ++++++++++++++++++- 7 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 engine/apps/alerts/migrations/0012_alertreceivechannel_description_short.py create mode 100644 engine/apps/alerts/migrations/0013_merge_20230418_0336.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dbd9fc5..859c8846 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add new field description_short to private api ([#1698](https://github.com/grafana/oncall/pull/1698)) - Added preview and migration API endpoints for route migration from regex into jinja2 ([1715](https://github.com/grafana/oncall/pull/1715)) - Helm chart: add the option to use a helm hook for the migration job ([1386](https://github.com/grafana/oncall/pull/1386)) - Add endpoints to start and stop maintenance in alert receive channel private api ([1755](https://github.com/grafana/oncall/pull/1755)) @@ -52,7 +53,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Save selected teams filter in local storage ([1611](https://github.com/grafana/oncall/issues/1611)) +- Save selected teams filter in local storage ([#1611](https://github.com/grafana/oncall/issues/1611)) ### Changed diff --git a/engine/apps/alerts/migrations/0012_alertreceivechannel_description_short.py b/engine/apps/alerts/migrations/0012_alertreceivechannel_description_short.py new file mode 100644 index 00000000..4e35755a --- /dev/null +++ b/engine/apps/alerts/migrations/0012_alertreceivechannel_description_short.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.18 on 2023-04-17 01:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('alerts', '0011_auto_20230329_1617'), + ] + + operations = [ + migrations.AddField( + model_name='alertreceivechannel', + name='description_short', + field=models.CharField(default=None, max_length=250, null=True), + ), + ] diff --git a/engine/apps/alerts/migrations/0013_merge_20230418_0336.py b/engine/apps/alerts/migrations/0013_merge_20230418_0336.py new file mode 100644 index 00000000..0b6a15df --- /dev/null +++ b/engine/apps/alerts/migrations/0013_merge_20230418_0336.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.18 on 2023-04-18 03:36 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('alerts', '0012_alertreceivechannel_description_short'), + ('alerts', '0012_auto_20230406_1010'), + ] + + operations = [ + ] diff --git a/engine/apps/alerts/models/alert_receive_channel.py b/engine/apps/alerts/models/alert_receive_channel.py index 80e6ccb5..e26e762a 100644 --- a/engine/apps/alerts/models/alert_receive_channel.py +++ b/engine/apps/alerts/models/alert_receive_channel.py @@ -145,6 +145,7 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): smile_code = models.TextField(default=":slightly_smiling_face:") verbal_name = models.CharField(max_length=150, null=True, default=None) + description_short = models.CharField(max_length=250, null=True, default=None) integration_slack_channel_id = models.CharField(max_length=150, null=True, default=None) diff --git a/engine/apps/api/serializers/alert_receive_channel.py b/engine/apps/api/serializers/alert_receive_channel.py index 852c3229..a84fb5fe 100644 --- a/engine/apps/api/serializers/alert_receive_channel.py +++ b/engine/apps/api/serializers/alert_receive_channel.py @@ -47,6 +47,7 @@ class AlertReceiveChannelSerializer(EagerLoadingMixin, serializers.ModelSerializ maintenance_till = serializers.ReadOnlyField(source="till_maintenance_timestamp") heartbeat = serializers.SerializerMethodField() allow_delete = serializers.SerializerMethodField() + description_short = serializers.CharField(max_length=250, required=False) demo_alert_payload = serializers.SerializerMethodField() # integration heartbeat is in PREFETCH_RELATED not by mistake. @@ -60,6 +61,7 @@ class AlertReceiveChannelSerializer(EagerLoadingMixin, serializers.ModelSerializ fields = [ "id", "description", + "description_short", "integration", "smile_code", "verbal_name", diff --git a/engine/apps/public_api/serializers/integrations.py b/engine/apps/public_api/serializers/integrations.py index 2c42da5d..26af0fe8 100644 --- a/engine/apps/public_api/serializers/integrations.py +++ b/engine/apps/public_api/serializers/integrations.py @@ -80,6 +80,7 @@ class IntegrationSerializer(EagerLoadingMixin, serializers.ModelSerializer, Main templates = serializers.DictField(required=False) default_route = serializers.DictField(required=False) heartbeat = serializers.SerializerMethodField() + description_short = serializers.CharField(max_length=250, required=False) PREFETCH_RELATED = ["channel_filters"] SELECT_RELATED = ["organization", "integration_heartbeat"] @@ -89,6 +90,7 @@ class IntegrationSerializer(EagerLoadingMixin, serializers.ModelSerializer, Main fields = MaintainableObjectSerializerMixin.Meta.fields + [ "id", "name", + "description_short", "team_id", "link", "type", diff --git a/engine/apps/public_api/tests/test_integrations.py b/engine/apps/public_api/tests/test_integrations.py index d508e15a..6bbea4ae 100644 --- a/engine/apps/public_api/tests/test_integrations.py +++ b/engine/apps/public_api/tests/test_integrations.py @@ -17,7 +17,7 @@ def test_get_list_integrations( make_integration_heartbeat, ): organization, user, token = make_organization_and_user_with_token() - integration = make_alert_receive_channel(organization, verbal_name="grafana") + integration = make_alert_receive_channel(organization, verbal_name="grafana", description_short="Some description") default_channel_filter = make_channel_filter(integration, is_default=True) make_integration_heartbeat(integration) @@ -31,6 +31,7 @@ def test_get_list_integrations( "id": integration.public_primary_key, "team_id": None, "name": "grafana", + "description_short": "Some description", "link": integration.integration_url, "type": "grafana", "default_route": { @@ -162,6 +163,7 @@ def test_update_integration_template( "id": integration.public_primary_key, "team_id": None, "name": "grafana", + "description_short": None, "link": integration.integration_url, "type": "grafana", "default_route": { @@ -223,6 +225,7 @@ def test_update_integration_template_messaging_backend( "id": integration.public_primary_key, "team_id": None, "name": "grafana", + "description_short": None, "link": integration.integration_url, "type": "grafana", "default_route": { @@ -300,6 +303,7 @@ def test_update_resolve_signal_template( "id": integration.public_primary_key, "team_id": None, "name": "grafana", + "description_short": None, "link": integration.integration_url, "type": "grafana", "default_route": { @@ -409,6 +413,7 @@ def test_update_sms_template_with_empty_dict( "id": integration.public_primary_key, "team_id": None, "name": "grafana", + "description_short": None, "link": integration.integration_url, "type": "grafana", "default_route": { @@ -470,6 +475,69 @@ def test_update_integration_name( "id": integration.public_primary_key, "team_id": None, "name": "grafana_updated", + "description_short": None, + "link": integration.integration_url, + "type": "grafana", + "default_route": { + "escalation_chain_id": None, + "id": default_channel_filter.public_primary_key, + "slack": {"channel_id": None, "enabled": True}, + "telegram": {"id": None, "enabled": False}, + TEST_MESSAGING_BACKEND_FIELD: {"id": None, "enabled": False}, + }, + "heartbeat": { + "link": f"{integration.integration_url}heartbeat/", + }, + "templates": { + "grouping_key": None, + "resolve_signal": None, + "acknowledge_signal": None, + "source_link": None, + "slack": {"title": None, "message": None, "image_url": None}, + "web": {"title": None, "message": None, "image_url": None}, + "sms": { + "title": None, + }, + "phone_call": { + "title": None, + }, + "telegram": { + "title": None, + "message": None, + "image_url": None, + }, + TEST_MESSAGING_BACKEND_FIELD: { + "title": None, + "message": None, + "image_url": None, + }, + }, + "maintenance_mode": None, + "maintenance_started_at": None, + "maintenance_end_at": None, + } + url = reverse("api-public:integrations-detail", args=[integration.public_primary_key]) + response = client.put(url, data=data_for_update, format="json", HTTP_AUTHORIZATION=f"{token}") + assert response.status_code == status.HTTP_200_OK + assert response.data == expected_response + + +@pytest.mark.django_db +def test_update_integration_name_and_description_short( + make_organization_and_user_with_token, make_alert_receive_channel, make_channel_filter, make_integration_heartbeat +): + organization, user, token = make_organization_and_user_with_token() + integration = make_alert_receive_channel(organization, verbal_name="grafana", description_short="Some description") + default_channel_filter = make_channel_filter(integration, is_default=True) + make_integration_heartbeat(integration) + + client = APIClient() + data_for_update = {"name": "grafana_updated"} + expected_response = { + "id": integration.public_primary_key, + "team_id": None, + "name": "grafana_updated", + "description_short": "Some description", "link": integration.integration_url, "type": "grafana", "default_route": { @@ -534,6 +602,7 @@ def test_set_default_template( "id": integration.public_primary_key, "team_id": None, "name": "grafana", + "description_short": None, "link": integration.integration_url, "type": "grafana", "default_route": { @@ -601,6 +670,7 @@ def test_set_default_messaging_backend_template( "id": integration.public_primary_key, "team_id": None, "name": "grafana", + "description_short": None, "link": integration.integration_url, "type": "grafana", "default_route": { From 5f9e79d50f1f384a1bde9383b661be14616738e9 Mon Sep 17 00:00:00 2001 From: Joey Orlando Date: Tue, 18 Apr 2023 12:02:56 +0200 Subject: [PATCH 14/15] move alert_group.is_restricted to alert_receive_channel.restricted_at (#1770) --- CHANGELOG.md | 6 +++++- .../migrations/0012_auto_20230406_1010.py | 5 ----- .../0014_alertreceivechannel_restricted_at.py | 18 ++++++++++++++++++ engine/apps/alerts/models/alert_group.py | 7 ++++--- .../alerts/models/alert_receive_channel.py | 2 ++ 5 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 engine/apps/alerts/migrations/0014_alertreceivechannel_restricted_at.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 859c8846..35579982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## v1.2.12 (2023-04-18) + +### Changed + +- Move `alerts_alertgroup.is_restricted` column to `alerts_alertreceivechannel.restricted_at` by @joeyorlando ([#1770](https://github.com/grafana/oncall/pull/1770)) ### Added diff --git a/engine/apps/alerts/migrations/0012_auto_20230406_1010.py b/engine/apps/alerts/migrations/0012_auto_20230406_1010.py index 666b5c49..59136798 100644 --- a/engine/apps/alerts/migrations/0012_auto_20230406_1010.py +++ b/engine/apps/alerts/migrations/0012_auto_20230406_1010.py @@ -10,11 +10,6 @@ class Migration(migrations.Migration): ] operations = [ - # migrations.AddField( - # model_name='alertgroup', - # name='is_restricted', - # field=models.BooleanField(default=False, null=True), - # ), migrations.AlterField( model_name='alertgrouplogrecord', name='type', diff --git a/engine/apps/alerts/migrations/0014_alertreceivechannel_restricted_at.py b/engine/apps/alerts/migrations/0014_alertreceivechannel_restricted_at.py new file mode 100644 index 00000000..e7b2e1ec --- /dev/null +++ b/engine/apps/alerts/migrations/0014_alertreceivechannel_restricted_at.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.18 on 2023-04-18 05:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('alerts', '0013_merge_20230418_0336'), + ] + + operations = [ + migrations.AddField( + model_name='alertreceivechannel', + name='restricted_at', + field=models.DateTimeField(default=None, null=True), + ), + ] diff --git a/engine/apps/alerts/models/alert_group.py b/engine/apps/alerts/models/alert_group.py index 49c60498..4d71eb0a 100644 --- a/engine/apps/alerts/models/alert_group.py +++ b/engine/apps/alerts/models/alert_group.py @@ -352,11 +352,12 @@ class AlertGroup(AlertGroupSlackRenderingMixin, EscalationSnapshotMixin, models. # https://code.djangoproject.com/ticket/28545 is_open_for_grouping = models.BooleanField(default=None, null=True, blank=True) - # is_restricted = models.BooleanField(default=False, null=True) - @property def is_restricted(self): - return False + integration_restricted_at = self.channel.restricted_at + if integration_restricted_at is None: + return False + return self.started_at >= integration_restricted_at @staticmethod def get_silenced_state_filter(): diff --git a/engine/apps/alerts/models/alert_receive_channel.py b/engine/apps/alerts/models/alert_receive_channel.py index e26e762a..94db5d00 100644 --- a/engine/apps/alerts/models/alert_receive_channel.py +++ b/engine/apps/alerts/models/alert_receive_channel.py @@ -187,6 +187,8 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject): rate_limited_in_slack_at = models.DateTimeField(null=True, default=None) rate_limit_message_task_id = models.CharField(max_length=100, null=True, default=None) + restricted_at = models.DateTimeField(null=True, default=None) + class Meta: constraints = [ models.UniqueConstraint( From 16d64485f314321c6be6282da362183058d7b65e Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 18 Apr 2023 18:55:53 +0800 Subject: [PATCH 15/15] Fix templates bug (#1776) # What this PR does ## Which issue(s) this PR fixes ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required) --- engine/apps/api/serializers/alert_receive_channel.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/engine/apps/api/serializers/alert_receive_channel.py b/engine/apps/api/serializers/alert_receive_channel.py index a84fb5fe..c7de4f7c 100644 --- a/engine/apps/api/serializers/alert_receive_channel.py +++ b/engine/apps/api/serializers/alert_receive_channel.py @@ -156,7 +156,12 @@ class AlertReceiveChannelSerializer(EagerLoadingMixin, serializers.ModelSerializ return 0 def get_demo_alert_payload(self, obj): - return obj.config.example_payload + if obj.is_demo_alert_enabled: + try: + return obj.config.example_payload + except AttributeError: + return "{}" + return None class AlertReceiveChannelUpdateSerializer(AlertReceiveChannelSerializer):