From d6467e9cb7dcf227478a4b0d2109a17915cfff5c Mon Sep 17 00:00:00 2001 From: Matias Bordese Date: Tue, 20 Feb 2024 10:02:23 -0300 Subject: [PATCH] Add escalation step to notify all members from a team (#3908) Based on https://github.com/grafana/oncall/pull/3477 --------- Co-authored-by: xssfox --- CHANGELOG.md | 1 + .../escalation-chains-and-routes/index.md | 1 + .../escalation_policies.md | 32 ++++++++- .../escalation_snapshot_mixin.py | 2 + .../serializers/escalation_policy_snapshot.py | 1 + .../escalation_policy_snapshot.py | 54 +++++++++++++++ ...npolicy_notify_to_team_members_and_more.py | 25 +++++++ .../alerts/models/alert_group_log_record.py | 8 ++- .../apps/alerts/models/escalation_policy.py | 35 +++++++++- .../tests/test_escalation_policy_snapshot.py | 54 ++++++++++++++- .../alerts/tests/test_escalation_snapshot.py | 3 + .../tests/test_escalation_snapshot_mixin.py | 60 ++++++++++++++++ .../apps/api/serializers/escalation_policy.py | 13 +++- .../apps/api/tests/test_escalation_policy.py | 42 ++++++++++++ .../serializers/escalation_policies.py | 22 +++++- .../tests/test_escalation_policies.py | 68 +++++++++++++++++++ engine/settings/celery_task_routes.py | 1 + .../components/Policy/EscalationPolicy.tsx | 30 ++++++++ .../escalation_policy.types.ts | 2 + 19 files changed, 448 insertions(+), 6 deletions(-) create mode 100644 engine/apps/alerts/migrations/0045_escalationpolicy_notify_to_team_members_and_more.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ae7c232..44479387 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update OnCall Insights dashboard @Ferril ([#3875](https://github.com/grafana/oncall/pull/3875)) - Do not delete webhook if its team is deleted @mderynck ([#3873](https://github.com/grafana/oncall/pull/3873)) - Update user details internal API perms ([#3900](https://github.com/grafana/oncall/pull/3900)) +- Add escalation to notify entire Grafana team @xssfox ([#3477](https://github.com/grafana/oncall/pull/3477)) ## v1.3.105 (2024-02-13) diff --git a/docs/sources/configure/escalation-chains-and-routes/index.md b/docs/sources/configure/escalation-chains-and-routes/index.md index 7d4287a2..da799119 100644 --- a/docs/sources/configure/escalation-chains-and-routes/index.md +++ b/docs/sources/configure/escalation-chains-and-routes/index.md @@ -80,6 +80,7 @@ need a larger time interval, use multiple wait steps in a row. * `Notify users` - send a notification to a user or a group of users. * `Notify users from on-call schedule` - send a notification to a user or a group of users from an on-call schedule. +* `Notify all users from a team` - send a notification to all users in a team. * `Resolve incident automatically` - resolve the alert group right now with status `Resolved automatically`. * `Notify whole slack channel` - send a notification to the users in the slack channel. diff --git a/docs/sources/oncall-api-reference/escalation_policies.md b/docs/sources/oncall-api-reference/escalation_policies.md index ce0fdd66..12ce328a 100644 --- a/docs/sources/oncall-api-reference/escalation_policies.md +++ b/docs/sources/oncall-api-reference/escalation_policies.md @@ -35,7 +35,7 @@ The above command returns JSON structured in the following way: | `escalation_chain_id` | Yes | Each escalation policy is assigned to a specific escalation chain. | | `position` | Optional | Escalation policies execute one after another starting from `position=0`. `Position=-1` will put the escalation policy to the end of the list. A new escalation policy created with a position of an existing escalation policy will move the old one (and all following) down in the list. | | `type` | Yes | One of: `wait`, `notify_persons`, `notify_person_next_each_time`, `notify_on_call_from_schedule`, `notify_user_group`, `trigger_action`, `resolve`, `notify_whole_channel`, `notify_if_time_from_to`. | -| `important` | Optional | Default is `false`. Will assign "important" to personal notification rules if `true`. This can be used to distinguish alerts on which you want to be notified immediately by phone. Applicable for types `notify_persons`, `notify_on_call_from_schedule`, and `notify_user_group`. | +| `important` | Optional | Default is `false`. Will assign "important" to personal notification rules if `true`. This can be used to distinguish alerts on which you want to be notified immediately by phone. Applicable for types `notify_persons`, `notify_team_members`, `notify_on_call_from_schedule`, and `notify_user_group`. | | `duration` | If type = `wait` | The duration, in seconds, when type `wait` is chosen. Valid values are: `60`, `300`, `900`, `1800`, `3600`. | | `action_to_trigger` | If type = `trigger_action` | ID of a webhook. | | `group_to_notify` | If type = `notify_user_group` | ID of a `User Group`. | @@ -44,6 +44,7 @@ The above command returns JSON structured in the following way: | `notify_on_call _from_schedule` | If type = `notify_on_call_from_schedule` | ID of a Schedule. | | `notify_if_time_from` | If type = `notify_if_time_from_to` | UTC time represents the beginning of the time period, for example `09:00:00Z`. | | `notify_if_time_to` | If type = `notify_if_time_from_to` | UTC time represents the end of the time period, for example `18:00:00Z`. | +| `team_to_notify` | If type = `notify_team_members` | ID of a team. | **HTTP request** @@ -70,6 +71,35 @@ The above command returns JSON structured in the following way: } ``` +# Update an escalation policy + +```shell +curl "{{API_URL}}/api/v1/escalation_policies/E3GA6SJETWWJS/" \ + --request PUT \ + --header "Authorization: meowmeowmeow" \ + --header "Content-Type: application/json" \ + --data '{ + "type": "wait", + "duration": 300, + }' +``` + +The above command returns JSON structured in the following way: + +```json +{ + "id": "E3GA6SJETWWJS", + "escalation_chain_id": "F5JU6KJET33FE", + "position": 0, + "type": "wait", + "duration": 300 +} +``` + +**HTTP request** + +`PUT {{API_URL}}/api/v1/on_call_shifts//` + **HTTP request** `GET {{API_URL}}/api/v1/escalation_policies//` diff --git a/engine/apps/alerts/escalation_snapshot/escalation_snapshot_mixin.py b/engine/apps/alerts/escalation_snapshot/escalation_snapshot_mixin.py index 8483ef55..8cf137b3 100644 --- a/engine/apps/alerts/escalation_snapshot/escalation_snapshot_mixin.py +++ b/engine/apps/alerts/escalation_snapshot/escalation_snapshot_mixin.py @@ -67,6 +67,7 @@ class EscalationSnapshotMixin: 'wait_delay': None, 'notify_schedule': None, 'notify_to_group': None, + 'notify_to_team_members': None, 'passed_last_time': None, 'escalation_counter': 0, 'last_notified_user': None, @@ -84,6 +85,7 @@ class EscalationSnapshotMixin: 'wait_delay': '00:05:00', 'notify_schedule': None, 'notify_to_group': None, + 'notify_to_team_members': None, 'passed_last_time': None, 'escalation_counter': 0, 'last_notified_user': None, diff --git a/engine/apps/alerts/escalation_snapshot/serializers/escalation_policy_snapshot.py b/engine/apps/alerts/escalation_snapshot/serializers/escalation_policy_snapshot.py index 6d648af2..b676d6be 100644 --- a/engine/apps/alerts/escalation_snapshot/serializers/escalation_policy_snapshot.py +++ b/engine/apps/alerts/escalation_snapshot/serializers/escalation_policy_snapshot.py @@ -83,6 +83,7 @@ class EscalationPolicySnapshotSerializer(serializers.ModelSerializer): "custom_webhook", "notify_schedule", "notify_to_group", + "notify_to_team_members", "escalation_counter", "passed_last_time", "pause_escalation", diff --git a/engine/apps/alerts/escalation_snapshot/snapshot_classes/escalation_policy_snapshot.py b/engine/apps/alerts/escalation_snapshot/snapshot_classes/escalation_policy_snapshot.py index 69809dc0..10a0c4c9 100644 --- a/engine/apps/alerts/escalation_snapshot/snapshot_classes/escalation_policy_snapshot.py +++ b/engine/apps/alerts/escalation_snapshot/snapshot_classes/escalation_policy_snapshot.py @@ -41,6 +41,7 @@ class EscalationPolicySnapshot: "custom_webhook", "notify_schedule", "notify_to_group", + "notify_to_team_members", "escalation_counter", "passed_last_time", "pause_escalation", @@ -72,6 +73,7 @@ class EscalationPolicySnapshot: escalation_counter, passed_last_time, pause_escalation, + notify_to_team_members=None, ): self.id = id self.order = order @@ -87,6 +89,7 @@ class EscalationPolicySnapshot: self.custom_webhook = custom_webhook self.notify_schedule = notify_schedule self.notify_to_group = notify_to_group + self.notify_to_team_members = notify_to_team_members self.escalation_counter = escalation_counter # used for STEP_REPEAT_ESCALATION_N_TIMES self.passed_last_time = passed_last_time # used for building escalation plan self.pause_escalation = pause_escalation # used for STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW @@ -124,6 +127,8 @@ class EscalationPolicySnapshot: EscalationPolicy.STEP_FINAL_RESOLVE: self._escalation_step_resolve, EscalationPolicy.STEP_NOTIFY_GROUP: self._escalation_step_notify_user_group, EscalationPolicy.STEP_NOTIFY_GROUP_IMPORTANT: self._escalation_step_notify_user_group, + EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS: self._escalation_step_notify_team_members, + EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT: self._escalation_step_notify_team_members, EscalationPolicy.STEP_NOTIFY_SCHEDULE: self._escalation_step_notify_on_call_schedule, EscalationPolicy.STEP_NOTIFY_SCHEDULE_IMPORTANT: self._escalation_step_notify_on_call_schedule, EscalationPolicy.STEP_TRIGGER_CUSTOM_BUTTON: self._escalation_step_trigger_custom_button, @@ -358,6 +363,55 @@ class EscalationPolicySnapshot: tasks.append(notify_group) self._execute_tasks(tasks) + def _escalation_step_notify_team_members(self, alert_group: "AlertGroup", reason: str) -> None: + tasks = [] + + if self.notify_to_team_members is None: + log_record = AlertGroupLogRecord( + type=AlertGroupLogRecord.TYPE_ESCALATION_FAILED, + alert_group=alert_group, + reason=reason, + escalation_policy=self.escalation_policy, + escalation_error_code=AlertGroupLogRecord.ERROR_ESCALATION_NOTIFY_TEAM_MEMBERS_STEP_IS_NOT_CONFIGURED, + escalation_policy_step=self.step, + ) + log_record.save() + else: + log_record = AlertGroupLogRecord( + type=AlertGroupLogRecord.TYPE_ESCALATION_TRIGGERED, + alert_group=alert_group, + reason=reason, + escalation_policy=self.escalation_policy, + escalation_policy_step=self.step, + step_specific_info={"team": self.notify_to_team_members.name}, + ) + log_record.save() + self.notify_to_users_queue = self.notify_to_team_members.users.all() + reason = "user belongs to team {}".format(self.notify_to_team_members.name) + for notify_to_user in self.notify_to_users_queue: + notify_task = notify_user_task.signature( + ( + notify_to_user.pk, + alert_group.pk, + ), + { + "reason": reason, + "important": self.step == EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT, + }, + immutable=True, + ) + tasks.append(notify_task) + AlertGroupLogRecord.objects.create( + type=AlertGroupLogRecord.TYPE_ESCALATION_TRIGGERED, + author=notify_to_user, + alert_group=alert_group, + reason=reason, + escalation_policy=self.escalation_policy, + escalation_policy_step=self.step, + ) + + self._execute_tasks(tasks) + def _escalation_step_notify_if_time(self, alert_group: "AlertGroup", _reason: str) -> StepExecutionResultData: eta = None diff --git a/engine/apps/alerts/migrations/0045_escalationpolicy_notify_to_team_members_and_more.py b/engine/apps/alerts/migrations/0045_escalationpolicy_notify_to_team_members_and_more.py new file mode 100644 index 00000000..b0ec6d63 --- /dev/null +++ b/engine/apps/alerts/migrations/0045_escalationpolicy_notify_to_team_members_and_more.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.10 on 2024-02-16 17:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('user_management', '0020_organization_is_grafana_labels_enabled'), + ('alerts', '0044_alertreceivechannel_alertmanager_v2_backup_templates_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='escalationpolicy', + name='notify_to_team_members', + field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='escalation_policies', to='user_management.team'), + ), + migrations.AlterField( + model_name='escalationpolicy', + name='step', + field=models.IntegerField(choices=[(0, 'Wait'), (1, 'Notify User'), (2, 'Notify Whole Channel'), (3, 'Repeat Escalation (5 times max)'), (4, 'Resolve'), (5, 'Notify Group'), (6, 'Notify Schedule'), (7, 'Notify User (Important)'), (8, 'Notify Group (Important)'), (9, 'Notify Schedule (Important)'), (10, 'Trigger Outgoing Webhook'), (11, 'Notify User (next each time)'), (12, 'Continue escalation only if time is from'), (13, 'Notify multiple Users'), (14, 'Notify multiple Users (Important)'), (15, 'Continue escalation if >X alerts per Y minutes'), (16, 'Trigger Webhook'), (17, 'Notify all users in a Team'), (18, 'Notify all users in a Team (Important)')], default=None, null=True), + ), + ] diff --git a/engine/apps/alerts/models/alert_group_log_record.py b/engine/apps/alerts/models/alert_group_log_record.py index 3187ebb7..64d6876a 100644 --- a/engine/apps/alerts/models/alert_group_log_record.py +++ b/engine/apps/alerts/models/alert_group_log_record.py @@ -159,7 +159,8 @@ class AlertGroupLogRecord(models.Model): ERROR_ESCALATION_NOTIFY_IN_SLACK, ERROR_ESCALATION_NOTIFY_IF_NUM_ALERTS_IN_WINDOW_STEP_IS_NOT_CONFIGURED, ERROR_ESCALATION_TRIGGER_CUSTOM_WEBHOOK_ERROR, - ) = range(18) + ERROR_ESCALATION_NOTIFY_TEAM_MEMBERS_STEP_IS_NOT_CONFIGURED, + ) = range(19) type = models.IntegerField(choices=TYPE_CHOICES) @@ -519,6 +520,11 @@ class AlertGroupLogRecord(models.Model): result += 'skipped escalation step "Notify Schedule" because it is not configured' elif self.escalation_error_code == AlertGroupLogRecord.ERROR_ESCALATION_NOTIFY_GROUP_STEP_IS_NOT_CONFIGURED: result += 'skipped escalation step "Notify Group" because it is not configured' + elif ( + self.escalation_error_code + == AlertGroupLogRecord.ERROR_ESCALATION_NOTIFY_TEAM_MEMBERS_STEP_IS_NOT_CONFIGURED + ): + result += 'skipped escalation step "Notify Team Members" because it is not configured' elif ( self.escalation_error_code == AlertGroupLogRecord.ERROR_ESCALATION_TRIGGER_CUSTOM_BUTTON_STEP_IS_NOT_CONFIGURED diff --git a/engine/apps/alerts/models/escalation_policy.py b/engine/apps/alerts/models/escalation_policy.py index a7f58839..7a9005aa 100644 --- a/engine/apps/alerts/models/escalation_policy.py +++ b/engine/apps/alerts/models/escalation_policy.py @@ -45,7 +45,9 @@ class EscalationPolicy(OrderedModel): STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT, STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW, STEP_TRIGGER_CUSTOM_WEBHOOK, - ) = range(17) + STEP_NOTIFY_TEAM_MEMBERS, + STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT, + ) = range(19) # Must be the same order as previous STEP_CHOICES = ( @@ -66,6 +68,8 @@ class EscalationPolicy(OrderedModel): (STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT, "Notify multiple Users (Important)"), (STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW, "Continue escalation if >X alerts per Y minutes"), (STEP_TRIGGER_CUSTOM_WEBHOOK, "Trigger Webhook"), + (STEP_NOTIFY_TEAM_MEMBERS, "Notify all users in a Team"), + (STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT, "Notify all users in a Team (Important)"), ) # Ordered step choices available for internal api. @@ -74,6 +78,7 @@ class EscalationPolicy(OrderedModel): # Common STEP_WAIT, STEP_NOTIFY_MULTIPLE_USERS, + STEP_NOTIFY_TEAM_MEMBERS, STEP_NOTIFY_SCHEDULE, STEP_FINAL_RESOLVE, # Slack @@ -100,6 +105,8 @@ class EscalationPolicy(OrderedModel): STEP_NOTIFY_USERS_QUEUE, STEP_NOTIFY_IF_TIME, STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW, + STEP_NOTIFY_TEAM_MEMBERS, + STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT, STEP_NOTIFY_MULTIPLE_USERS, STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT, STEP_TRIGGER_CUSTOM_BUTTON, @@ -113,6 +120,10 @@ class EscalationPolicy(OrderedModel): # Common steps STEP_WAIT: ("Wait {{wait_delay}} minute(s)", "Wait"), STEP_NOTIFY_MULTIPLE_USERS: ("Start {{importance}} notification for {{users}}", "Notify users"), + STEP_NOTIFY_TEAM_MEMBERS: ( + "Start {{importance}} notification for {{team}} team members", + "Notify all team members", + ), STEP_NOTIFY_SCHEDULE: ( "Start {{importance}} notification for schedule {{schedule}}", "Notify users from on-call schedule", @@ -157,11 +168,13 @@ class EscalationPolicy(OrderedModel): STEP_NOTIFY_GROUP: STEP_NOTIFY_GROUP_IMPORTANT, STEP_NOTIFY_SCHEDULE: STEP_NOTIFY_SCHEDULE_IMPORTANT, STEP_NOTIFY_MULTIPLE_USERS: STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT, + STEP_NOTIFY_TEAM_MEMBERS: STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT, } IMPORTANT_TO_DEFAULT_STEP_MAPPING = { STEP_NOTIFY_GROUP_IMPORTANT: STEP_NOTIFY_GROUP, STEP_NOTIFY_SCHEDULE_IMPORTANT: STEP_NOTIFY_SCHEDULE, STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT: STEP_NOTIFY_MULTIPLE_USERS, + STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT: STEP_NOTIFY_TEAM_MEMBERS, } # Default steps are just usual version of important steps. E.g. notify group - notify group important @@ -169,12 +182,14 @@ class EscalationPolicy(OrderedModel): STEP_NOTIFY_GROUP, STEP_NOTIFY_SCHEDULE, STEP_NOTIFY_MULTIPLE_USERS, + STEP_NOTIFY_TEAM_MEMBERS, } IMPORTANT_STEPS_SET = { STEP_NOTIFY_GROUP_IMPORTANT, STEP_NOTIFY_SCHEDULE_IMPORTANT, STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT, + STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT, } SLACK_INTEGRATION_REQUIRED_STEPS = [ @@ -187,6 +202,7 @@ class EscalationPolicy(OrderedModel): STEP_WAIT, STEP_NOTIFY_SCHEDULE, STEP_NOTIFY_MULTIPLE_USERS, + STEP_NOTIFY_TEAM_MEMBERS, STEP_NOTIFY_USERS_QUEUE, STEP_NOTIFY_GROUP, STEP_FINAL_RESOLVE, @@ -213,6 +229,8 @@ class EscalationPolicy(OrderedModel): STEP_NOTIFY_USERS_QUEUE: "notify_person_next_each_time", STEP_NOTIFY_MULTIPLE_USERS: "notify_persons", STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT: "notify_persons", + STEP_NOTIFY_TEAM_MEMBERS: "notify_team_members", + STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT: "notify_team_members", STEP_NOTIFY_IF_TIME: "notify_if_time_from_to", STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW: "notify_if_num_alerts_in_window", STEP_REPEAT_ESCALATION_N_TIMES: "repeat_escalation", @@ -244,6 +262,14 @@ class EscalationPolicy(OrderedModel): step = models.IntegerField(choices=STEP_CHOICES, default=None, null=True) + notify_to_team_members = models.ForeignKey( + "user_management.Team", + on_delete=models.SET_NULL, + related_name="escalation_policies", + default=None, + null=True, + ) + notify_to_group = models.ForeignKey( "slack.SlackUserGroup", on_delete=models.SET_NULL, @@ -368,6 +394,13 @@ class EscalationPolicy(OrderedModel): if self.notify_to_group: result["user_group"] = self.notify_to_group.name result["user_group_id"] = self.notify_to_group.public_primary_key + elif self.step in [ + EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS, + EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT, + ]: + if self.notify_to_team_members: + result["team"] = self.notify_to_team_members.name + result["team_id"] = self.notify_to_team_members.public_primary_key elif self.step in [EscalationPolicy.STEP_NOTIFY_SCHEDULE, EscalationPolicy.STEP_NOTIFY_SCHEDULE_IMPORTANT]: if self.notify_schedule: result["on-call_schedule"] = self.notify_schedule.insight_logs_verbal diff --git a/engine/apps/alerts/tests/test_escalation_policy_snapshot.py b/engine/apps/alerts/tests/test_escalation_policy_snapshot.py index 97a11b25..e1c74216 100644 --- a/engine/apps/alerts/tests/test_escalation_policy_snapshot.py +++ b/engine/apps/alerts/tests/test_escalation_policy_snapshot.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import call, patch import pytest from django.utils import timezone @@ -636,3 +636,55 @@ def test_escalation_step_with_deleted_user( deserialized_escalation_snapshot = EscalationPolicySnapshotSerializer().to_internal_value(raw_snapshot) assert deserialized_escalation_snapshot["notify_to_users_queue"] == [user] + + +@patch("apps.alerts.escalation_snapshot.snapshot_classes.EscalationPolicySnapshot._execute_tasks", return_value=None) +@pytest.mark.django_db +@pytest.mark.parametrize( + "step", + (EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS, EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT), +) +def test_notify_team_members( + mocked_execute_tasks, escalation_step_test_setup, make_user, make_team, make_escalation_policy, step +): + organization, user, _, channel_filter, alert_group, reason = escalation_step_test_setup + user_1 = make_user(organization=organization) + user_2 = make_user(organization=organization) + team_1 = make_team(organization=organization) + team_1.users.add(user_1) + team_1.users.add(user_2) + + notify_team_members_step = make_escalation_policy( + escalation_chain=channel_filter.escalation_chain, + escalation_policy_step=step, + notify_to_team_members=team_1, + ) + escalation_policy_snapshot = get_escalation_policy_snapshot_from_model(notify_team_members_step) + expected_eta = timezone.now() + timezone.timedelta(seconds=NEXT_ESCALATION_DELAY) + with patch( + "apps.alerts.escalation_snapshot.snapshot_classes.escalation_policy_snapshot.notify_user_task" + ) as mock_execute: + result = escalation_policy_snapshot.execute(alert_group, reason) + + expected_result = EscalationPolicySnapshot.StepExecutionResultData( + eta=result.eta, + stop_escalation=False, + pause_escalation=False, + start_from_beginning=False, + ) + assert expected_eta + timezone.timedelta(seconds=15) > result.eta > expected_eta - timezone.timedelta(seconds=15) + assert result == expected_result + assert notify_team_members_step.log_records.filter(type=AlertGroupLogRecord.TYPE_ESCALATION_TRIGGERED).exists() + assert list(escalation_policy_snapshot.notify_to_users_queue) == list(team_1.users.all()) + assert mocked_execute_tasks.called + expected_kwargs = { + "reason": f"user belongs to team {team_1.name}", + "important": step == EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT, + } + assert mock_execute.signature.call_args_list[0] == call( + (user_1.pk, alert_group.pk), expected_kwargs, immutable=True + ) + assert mock_execute.signature.call_args_list[1] == call( + (user_2.pk, alert_group.pk), expected_kwargs, immutable=True + ) + assert mock_execute.signature.call_count == 2 diff --git a/engine/apps/alerts/tests/test_escalation_snapshot.py b/engine/apps/alerts/tests/test_escalation_snapshot.py index 2bfbf2f8..96d62694 100644 --- a/engine/apps/alerts/tests/test_escalation_snapshot.py +++ b/engine/apps/alerts/tests/test_escalation_snapshot.py @@ -40,6 +40,7 @@ def test_raw_escalation_snapshot(escalation_snapshot_test_setup): "last_notified_user": None, "notify_schedule": None, "notify_to_group": None, + "notify_to_team_members": None, "from_time": None, "to_time": None, "num_alerts_in_window": None, @@ -59,6 +60,7 @@ def test_raw_escalation_snapshot(escalation_snapshot_test_setup): "last_notified_user": None, "notify_schedule": None, "notify_to_group": None, + "notify_to_team_members": None, "from_time": None, "to_time": None, "num_alerts_in_window": None, @@ -78,6 +80,7 @@ def test_raw_escalation_snapshot(escalation_snapshot_test_setup): "last_notified_user": None, "notify_schedule": None, "notify_to_group": None, + "notify_to_team_members": None, "from_time": notify_if_time_step.from_time.isoformat(), "to_time": notify_if_time_step.to_time.isoformat(), "num_alerts_in_window": None, diff --git a/engine/apps/alerts/tests/test_escalation_snapshot_mixin.py b/engine/apps/alerts/tests/test_escalation_snapshot_mixin.py index f29ec590..676fefad 100644 --- a/engine/apps/alerts/tests/test_escalation_snapshot_mixin.py +++ b/engine/apps/alerts/tests/test_escalation_snapshot_mixin.py @@ -499,6 +499,66 @@ def test_deserialize_escalation_snapshot( assert deserialized_escalation_snapshot.stop_escalation is False +@pytest.mark.django_db +def test_deserialize_escalation_snapshot_missing_notify_to_team_members( + make_organization_and_user, + make_alert_receive_channel, + make_channel_filter, + make_escalation_chain, + make_escalation_policy, + make_alert_group, +): + organization, _ = make_organization_and_user() + alert_receive_channel = make_alert_receive_channel(organization) + escalation_chain = make_escalation_chain(organization=organization) + channel_filter = make_channel_filter(alert_receive_channel, escalation_chain=escalation_chain) + make_escalation_policy( + escalation_chain=channel_filter.escalation_chain, + escalation_policy_step=EscalationPolicy.STEP_WAIT, + wait_delay=EscalationPolicy.FIFTEEN_MINUTES, + ) + + alert_group = make_alert_group(alert_receive_channel, channel_filter=channel_filter) + alert_group.raw_escalation_snapshot = alert_group.build_raw_escalation_snapshot() + del alert_group.raw_escalation_snapshot["escalation_policies_snapshots"][0]["notify_to_team_members"] + + deserialized_escalation_snapshot = alert_group._deserialize_escalation_snapshot(alert_group.raw_escalation_snapshot) + assert deserialized_escalation_snapshot.escalation_policies_snapshots[0].notify_to_team_members is None + + +@patch("apps.alerts.models.alert_group.AlertGroup.slack_channel_id", new_callable=PropertyMock) +@pytest.mark.django_db +def test_deserialize_escalation_snapshot_notify_to_team_members( + mock_alert_group_slack_channel_id, + make_organization_and_user, + make_alert_receive_channel, + make_channel_filter, + make_escalation_chain, + make_escalation_policy, + make_alert_group, + make_team, +): + mock_alert_group_slack_channel_id.return_value = MOCK_SLACK_CHANNEL_ID + + organization, _ = make_organization_and_user() + alert_receive_channel = make_alert_receive_channel(organization) + escalation_chain = make_escalation_chain(organization=organization) + channel_filter = make_channel_filter(alert_receive_channel, escalation_chain=escalation_chain) + team = make_team(organization) + make_escalation_policy( + escalation_chain=channel_filter.escalation_chain, + escalation_policy_step=EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS, + notify_to_team_members=team, + ) + + alert_group = make_alert_group(alert_receive_channel, channel_filter=channel_filter) + alert_group.raw_escalation_snapshot = alert_group.build_raw_escalation_snapshot() + alert_group.raw_escalation_snapshot["escalation_policies_snapshots"][0]["notify_to_team_members"] + + deserialized_escalation_snapshot = alert_group._deserialize_escalation_snapshot(alert_group.raw_escalation_snapshot) + assert deserialized_escalation_snapshot.escalation_policies_snapshots[0].notify_to_team_members.id == team.id + + @pytest.mark.django_db def test_escalation_chain_exists( make_organization_and_user, diff --git a/engine/apps/api/serializers/escalation_policy.py b/engine/apps/api/serializers/escalation_policy.py index ee8fdf4d..ace5d708 100644 --- a/engine/apps/api/serializers/escalation_policy.py +++ b/engine/apps/api/serializers/escalation_policy.py @@ -6,7 +6,7 @@ from rest_framework import serializers from apps.alerts.models import CustomButton, EscalationChain, EscalationPolicy from apps.schedules.models import OnCallSchedule from apps.slack.models import SlackUserGroup -from apps.user_management.models import User +from apps.user_management.models import Team, User from apps.webhooks.models import Webhook from common.api_helpers.custom_fields import ( OrganizationFilteredPrimaryKeyRelatedField, @@ -18,6 +18,7 @@ WAIT_DELAY = "wait_delay" NOTIFY_SCHEDULE = "notify_schedule" NOTIFY_TO_USERS_QUEUE = "notify_to_users_queue" NOTIFY_GROUP = "notify_to_group" +NOTIFY_TEAM_MEMBERS = "notify_to_team_members" FROM_TIME = "from_time" TO_TIME = "to_time" NUM_ALERTS_IN_WINDOW = "num_alerts_in_window" @@ -31,6 +32,7 @@ STEP_TYPE_TO_RELATED_FIELD_MAP = { EscalationPolicy.STEP_NOTIFY_USERS_QUEUE: [NOTIFY_TO_USERS_QUEUE], EscalationPolicy.STEP_NOTIFY_MULTIPLE_USERS: [NOTIFY_TO_USERS_QUEUE], EscalationPolicy.STEP_NOTIFY_GROUP: [NOTIFY_GROUP], + EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS: [NOTIFY_TEAM_MEMBERS], EscalationPolicy.STEP_NOTIFY_IF_TIME: [FROM_TIME, TO_TIME], EscalationPolicy.STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW: [NUM_ALERTS_IN_WINDOW, NUM_MINUTES_IN_WINDOW], EscalationPolicy.STEP_TRIGGER_CUSTOM_BUTTON: [CUSTOM_BUTTON_TRIGGER], @@ -62,6 +64,11 @@ class EscalationPolicySerializer(EagerLoadingMixin, serializers.ModelSerializer) required=False, allow_null=True, ) + notify_to_team_members = OrganizationFilteredPrimaryKeyRelatedField( + queryset=Team.objects, + required=False, + allow_null=True, + ) notify_to_group = OrganizationFilteredPrimaryKeyRelatedField( queryset=SlackUserGroup.objects, required=False, @@ -98,6 +105,7 @@ class EscalationPolicySerializer(EagerLoadingMixin, serializers.ModelSerializer) "custom_webhook", "notify_schedule", "notify_to_group", + "notify_to_team_members", "important", ] @@ -105,6 +113,7 @@ class EscalationPolicySerializer(EagerLoadingMixin, serializers.ModelSerializer) "escalation_chain", "notify_schedule", "notify_to_group", + "notify_to_team_members", "custom_button_trigger", "custom_webhook", ] @@ -115,6 +124,7 @@ class EscalationPolicySerializer(EagerLoadingMixin, serializers.ModelSerializer) WAIT_DELAY, NOTIFY_SCHEDULE, NOTIFY_TO_USERS_QUEUE, + NOTIFY_TEAM_MEMBERS, NOTIFY_GROUP, FROM_TIME, TO_TIME, @@ -224,6 +234,7 @@ class EscalationPolicyUpdateSerializer(EscalationPolicySerializer): NOTIFY_SCHEDULE, NOTIFY_TO_USERS_QUEUE, NOTIFY_GROUP, + NOTIFY_TEAM_MEMBERS, FROM_TIME, TO_TIME, NUM_ALERTS_IN_WINDOW, diff --git a/engine/apps/api/tests/test_escalation_policy.py b/engine/apps/api/tests/test_escalation_policy.py index 30609c41..73601d7f 100644 --- a/engine/apps/api/tests/test_escalation_policy.py +++ b/engine/apps/api/tests/test_escalation_policy.py @@ -101,6 +101,43 @@ def test_update_notify_multiple_users_step(escalation_policy_internal_api_setup, ) +@pytest.mark.django_db +def test_manage_escalation_policy_notify_team(escalation_policy_internal_api_setup, make_team, make_user_auth_headers): + token, escalation_chain, _, user, _ = escalation_policy_internal_api_setup + client = APIClient() + url = reverse("api-internal:escalation_policy-list") + + team = make_team(organization=user.organization) + data = { + "step": EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS, + "escalation_chain": escalation_chain.public_primary_key, + "notify_to_team_members": team.public_primary_key, + } + + max_order = EscalationPolicy.objects.filter(escalation_chain=escalation_chain).aggregate(maxorder=Max("order"))[ + "maxorder" + ] + + response = client.post(url, data, format="json", **make_user_auth_headers(user, token)) + assert response.status_code == status.HTTP_201_CREATED + assert response.data["notify_to_team_members"] == team.public_primary_key + escalation_policy = EscalationPolicy.objects.get(public_primary_key=response.data["id"]) + assert escalation_policy.order == max_order + 1 + assert escalation_policy.notify_to_team_members == team + + # update team in policy + url = reverse("api-internal:escalation_policy-detail", kwargs={"pk": escalation_policy.public_primary_key}) + another_team = make_team(organization=user.organization) + data = { + "step": EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS, + "notify_to_team_members": another_team.public_primary_key, + } + response = client.put(url, data, format="json", **make_user_auth_headers(user, token)) + assert response.status_code == status.HTTP_200_OK + assert response.json()["step"] == EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS + assert response.json()["notify_to_team_members"] == another_team.public_primary_key + + @pytest.mark.django_db def test_move_to_position(escalation_policy_internal_api_setup, make_user_auth_headers): token, _, escalation_policy, user, _ = escalation_policy_internal_api_setup @@ -713,6 +750,7 @@ def test_escalation_policy_update_drop_non_step_type_related_data( "notify_schedule", "notify_to_users_queue", "notify_to_group", + "notify_to_team_members", "from_time", "to_time", "custom_button_trigger", @@ -770,6 +808,7 @@ def test_escalation_policy_switch_importance( "custom_webhook": None, "notify_schedule": None, "notify_to_group": None, + "notify_to_team_members": None, "important": True, "wait_delay": None, } @@ -827,6 +866,7 @@ def test_escalation_policy_filter_by_user( "custom_webhook": None, "notify_schedule": None, "notify_to_group": None, + "notify_to_team_members": None, "important": False, }, { @@ -844,6 +884,7 @@ def test_escalation_policy_filter_by_user( "custom_webhook": None, "notify_schedule": None, "notify_to_group": None, + "notify_to_team_members": None, "important": False, }, ] @@ -909,6 +950,7 @@ def test_escalation_policy_filter_by_slack_channel( "custom_webhook": None, "notify_schedule": None, "notify_to_group": None, + "notify_to_team_members": None, "important": False, }, ] diff --git a/engine/apps/public_api/serializers/escalation_policies.py b/engine/apps/public_api/serializers/escalation_policies.py index a100d6fb..d32d0ebb 100644 --- a/engine/apps/public_api/serializers/escalation_policies.py +++ b/engine/apps/public_api/serializers/escalation_policies.py @@ -7,7 +7,7 @@ from rest_framework import fields, serializers from apps.alerts.models import EscalationChain, EscalationPolicy from apps.schedules.models import OnCallSchedule from apps.slack.models import SlackUserGroup -from apps.user_management.models import User +from apps.user_management.models import Team, User from apps.webhooks.models import Webhook from common.api_helpers.custom_fields import ( OrganizationFilteredPrimaryKeyRelatedField, @@ -57,6 +57,11 @@ class EscalationPolicySerializer(EagerLoadingMixin, OrderedModelSerializer): required=False, source="notify_to_users_queue", ) + team_to_notify = OrganizationFilteredPrimaryKeyRelatedField( + queryset=Team.objects, + required=False, + source="notify_to_team_members", + ) persons_to_notify_next_each_time = UsersFilteredByOrganizationField( queryset=User.objects, required=False, @@ -95,6 +100,7 @@ class EscalationPolicySerializer(EagerLoadingMixin, OrderedModelSerializer): "duration", "important", "persons_to_notify", + "team_to_notify", "persons_to_notify_next_each_time", "notify_on_call_from_schedule", "group_to_notify", @@ -154,6 +160,7 @@ class EscalationPolicySerializer(EagerLoadingMixin, OrderedModelSerializer): def _get_field_to_represent(self, step, result): fields_to_remove = [ "duration", + "team_to_notify", "persons_to_notify", "persons_to_notify_next_each_time", "notify_on_call_from_schedule", @@ -174,6 +181,11 @@ class EscalationPolicySerializer(EagerLoadingMixin, OrderedModelSerializer): EscalationPolicy.STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT, ]: fields_to_remove.remove("persons_to_notify") + elif step in [ + EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS, + EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT, + ]: + fields_to_remove.remove("team_to_notify") elif step == EscalationPolicy.STEP_NOTIFY_USERS_QUEUE: fields_to_remove.remove("persons_to_notify_next_each_time") elif step in [EscalationPolicy.STEP_NOTIFY_GROUP, EscalationPolicy.STEP_NOTIFY_GROUP_IMPORTANT]: @@ -203,6 +215,7 @@ class EscalationPolicySerializer(EagerLoadingMixin, OrderedModelSerializer): "wait_delay", "notify_schedule", "notify_to_group", + "notify_to_team_members", "custom_button_trigger", "custom_webhook", "from_time", @@ -229,6 +242,8 @@ class EscalationPolicySerializer(EagerLoadingMixin, OrderedModelSerializer): validated_data_fields_to_remove.remove("notify_to_users_queue") elif step in [EscalationPolicy.STEP_NOTIFY_GROUP, EscalationPolicy.STEP_NOTIFY_GROUP_IMPORTANT]: validated_data_fields_to_remove.remove("notify_to_group") + elif step in [EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS, EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT]: + validated_data_fields_to_remove.remove("notify_to_team_members") elif step == EscalationPolicy.STEP_TRIGGER_CUSTOM_BUTTON: validated_data_fields_to_remove.remove("custom_button_trigger") elif step == EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK: @@ -280,6 +295,11 @@ class EscalationPolicyUpdateSerializer(EscalationPolicySerializer): EscalationPolicy.STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT, ]: instance.notify_to_users_queue.clear() + if step not in [ + EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS, + EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT, + ]: + instance.notify_to_team_members = None if step not in [EscalationPolicy.STEP_NOTIFY_GROUP, EscalationPolicy.STEP_NOTIFY_GROUP_IMPORTANT]: instance.notify_to_group = None if step != EscalationPolicy.STEP_TRIGGER_CUSTOM_BUTTON: diff --git a/engine/apps/public_api/tests/test_escalation_policies.py b/engine/apps/public_api/tests/test_escalation_policies.py index a04a2440..078d0d15 100644 --- a/engine/apps/public_api/tests/test_escalation_policies.py +++ b/engine/apps/public_api/tests/test_escalation_policies.py @@ -456,3 +456,71 @@ def test_update_escalation_policy_from_and_to_time( assert response.data == serializer.data else: assert response.json()[field][0] == "Time has wrong format. Use one of these formats instead: hh:mm:ssZ." + + +@pytest.mark.django_db +def test_create_escalation_policy_using_notify_team_members( + make_organization_and_user_with_token, + make_team, + escalation_policies_setup, +): + organization, user, token = make_organization_and_user_with_token() + escalation_chain, _, _ = escalation_policies_setup(organization, user) + team = make_team(organization) + + data_for_create = { + "escalation_chain_id": escalation_chain.public_primary_key, + "type": "notify_team_members", + "position": 0, + "notify_to_team_members": team.team_id, + } + + client = APIClient() + url = reverse("api-public:escalation_policies-list") + response = client.post(url, data=data_for_create, format="json", HTTP_AUTHORIZATION=token) + + assert response.status_code == status.HTTP_201_CREATED + + escalation_policy = EscalationPolicy.objects.get(public_primary_key=response.data["id"]) + serializer = EscalationPolicySerializer(escalation_policy) + assert response.data == serializer.data + + # update to important + data_to_change = {"important": True} + url = reverse("api-public:escalation_policies-detail", kwargs={"pk": escalation_policy.public_primary_key}) + response = client.put(url, data=data_to_change, format="json", HTTP_AUTHORIZATION=token) + + assert response.status_code == status.HTTP_200_OK + escalation_policy.refresh_from_db() + serializer = EscalationPolicySerializer(escalation_policy) + assert response.data == serializer.data + # step is migrated + assert escalation_policy.step == EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT + + +@pytest.mark.django_db +def test_update_escalation_policy_using_notify_team_members( + make_organization_and_user_with_token, + make_team, + escalation_policies_setup, +): + organization, user, token = make_organization_and_user_with_token() + escalation_chain, _, _ = escalation_policies_setup(organization, user) + team = make_team(organization) + + data_for_create = { + "escalation_chain_id": escalation_chain.public_primary_key, + "type": "notify_team_members", + "position": 0, + "notify_to_team_members": team.team_id, + } + + client = APIClient() + url = reverse("api-public:escalation_policies-list") + response = client.post(url, data=data_for_create, format="json", HTTP_AUTHORIZATION=token) + + assert response.status_code == status.HTTP_201_CREATED + + escalation_policy = EscalationPolicy.objects.get(public_primary_key=response.data["id"]) + serializer = EscalationPolicySerializer(escalation_policy) + assert response.data == serializer.data diff --git a/engine/settings/celery_task_routes.py b/engine/settings/celery_task_routes.py index d77ee25c..14b70d90 100644 --- a/engine/settings/celery_task_routes.py +++ b/engine/settings/celery_task_routes.py @@ -98,6 +98,7 @@ CELERY_TASK_ROUTES = { "apps.alerts.tasks.maintenance.disable_maintenance": {"queue": "critical"}, "apps.alerts.tasks.notify_all.notify_all_task": {"queue": "critical"}, "apps.alerts.tasks.notify_group.notify_group_task": {"queue": "critical"}, + "apps.alerts.tasks.notify_team_members.notify_team_members_task": {"queue": "critical"}, "apps.alerts.tasks.notify_ical_schedule_shift.notify_ical_schedule_shift": {"queue": "critical"}, "apps.alerts.tasks.notify_user.notify_user_task": {"queue": "critical"}, "apps.alerts.tasks.notify_user.perform_notification": {"queue": "critical"}, diff --git a/grafana-plugin/src/components/Policy/EscalationPolicy.tsx b/grafana-plugin/src/components/Policy/EscalationPolicy.tsx index 7b6e3481..5f743d60 100644 --- a/grafana-plugin/src/components/Policy/EscalationPolicy.tsx +++ b/grafana-plugin/src/components/Policy/EscalationPolicy.tsx @@ -20,6 +20,7 @@ import { EscalationPolicy as EscalationPolicyType, EscalationPolicyOption, } from 'models/escalation_policy/escalation_policy.types'; +import { GrafanaTeam } from 'models/grafana_team/grafana_team.types'; import { OutgoingWebhook } from 'models/outgoing_webhook/outgoing_webhook.types'; import { Schedule } from 'models/schedule/schedule.types'; import { User } from 'models/user/user.types'; @@ -109,6 +110,8 @@ class _EscalationPolicy extends React.Component { return this.renderWaitDelays(); case 'slack_user_group': return this.renderNotifyUserGroup(); + case 'team': + return this.renderNotifyTeam(); case 'schedule': return this.renderNotifySchedule(); case 'custom_webhook': @@ -420,6 +423,33 @@ class _EscalationPolicy extends React.Component { ); } + renderNotifyTeam() { + const { + data, + isDisabled, + store: { grafanaTeamStore }, + } = this.props; + const { notify_to_team_members } = data; + + return ( + + + disabled={isDisabled} + items={grafanaTeamStore.items} + fetchItemsFn={grafanaTeamStore.updateItems} + getSearchResult={grafanaTeamStore.getSearchResult} + displayField="name" + valueField="id" + placeholder="Select Team" + className={cx('select', 'control')} + value={notify_to_team_members} + onChange={this.getOnChangeHandler('notify_to_team_members')} + width={'auto'} + /> + + ); + } + getOnSelectChangeHandler = (field: string) => { return (option: SelectableValue) => { const { data, onChange = () => {} } = this.props; diff --git a/grafana-plugin/src/models/escalation_policy/escalation_policy.types.ts b/grafana-plugin/src/models/escalation_policy/escalation_policy.types.ts index 93c8a9d0..bf8729ea 100644 --- a/grafana-plugin/src/models/escalation_policy/escalation_policy.types.ts +++ b/grafana-plugin/src/models/escalation_policy/escalation_policy.types.ts @@ -1,5 +1,6 @@ import { Channel } from 'models/channel/channel'; import { EscalationChain } from 'models/escalation_chain/escalation_chain.types'; +import { GrafanaTeam } from 'models/grafana_team/grafana_team.types'; import { OutgoingWebhook } from 'models/outgoing_webhook/outgoing_webhook.types'; import { Schedule } from 'models/schedule/schedule.types'; import { User } from 'models/user/user.types'; @@ -19,6 +20,7 @@ export interface EscalationPolicy { notify_to_channel: Channel['id'] | null; custom_webhook: OutgoingWebhook['id'] | null; notify_to_group: UserGroup['id'] | null; + notify_to_team_members: GrafanaTeam['id'] | null; notify_schedule: Schedule['id'] | null; important: boolean | null; num_alerts_in_window: number;