Add escalation chain option when creating a direct page alert group (#1143)

Also changes the default integration used when creating an alert group
for a direct page to a custom manual integration to avoid
conflicts/unexpected behaviors with existing manual alerts.

engine/apps/alerts/models/alert.py

@@ -79,6 +79,7 @@ class Alert(models.Model):
         raw_request_data,
         enable_autoresolve=True,
         is_demo=False,
+        channel_filter=None,
         force_route_id=None,
     ):
         ChannelFilter = apps.get_model("alerts", "ChannelFilter")
@@ -87,9 +88,10 @@ class Alert(models.Model):
         AlertGroupLogRecord = apps.get_model("alerts", "AlertGroupLogRecord")
 
         group_data = Alert.render_group_data(alert_receive_channel, raw_request_data, is_demo)
-        channel_filter = ChannelFilter.select_filter(
-            alert_receive_channel, raw_request_data, title, message, force_route_id
-        )
+        if channel_filter is None:
+            channel_filter = ChannelFilter.select_filter(
+                alert_receive_channel, raw_request_data, title, message, force_route_id
+            )
 
         group, group_created = AlertGroup.all_objects.get_or_create_grouping(
             channel=alert_receive_channel,

engine/apps/alerts/paging.py

@@ -3,7 +3,15 @@ from typing import Any
 from django.db import transaction
 from django.db.models import Q
 
-from apps.alerts.models import Alert, AlertGroup, AlertGroupLogRecord, AlertReceiveChannel, UserHasNotification
+from apps.alerts.models import (
+    Alert,
+    AlertGroup,
+    AlertGroupLogRecord,
+    AlertReceiveChannel,
+    ChannelFilter,
+    EscalationChain,
+    UserHasNotification,
+)
 from apps.alerts.tasks.notify_user import notify_user_task
 from apps.schedules.ical_utils import list_users_to_notify_from_ical
 from apps.schedules.models import OnCallSchedule
@@ -17,18 +25,43 @@ UserNotifications = list[tuple[User, bool]]
 ScheduleNotifications = list[tuple[OnCallSchedule, bool]]
 
 
-def _trigger_alert(organization: Organization, team: Team, title: str, message: str, from_user: User) -> AlertGroup:
+def _trigger_alert(
+    organization: Organization,
+    team: Team,
+    title: str,
+    message: str,
+    from_user: User,
+    escalation_chain: EscalationChain = None,
+) -> AlertGroup:
     """Trigger manual integration alert from params."""
     alert_receive_channel = AlertReceiveChannel.get_or_create_manual_integration(
         organization=organization,
         team=team,
-        integration=AlertReceiveChannel.INTEGRATION_MANUAL,
+        integration=AlertReceiveChannel.INTEGRATION_DIRECT_PAGING,
         deleted_at=None,
         defaults={
             "author": from_user,
-            "verbal_name": f"Manual alert groups ({team.name if team else 'General'} team)",
+            "verbal_name": f"Direct paging ({team.name if team else 'General'} team)",
         },
     )
+    if alert_receive_channel.default_channel_filter is None:
+        ChannelFilter.objects.create(
+            alert_receive_channel=alert_receive_channel,
+            notify_in_slack=True,
+            is_default=True,
+        )
+
+    channel_filter = None
+    if escalation_chain is not None:
+        channel_filter, _ = ChannelFilter.objects.get_or_create(
+            alert_receive_channel=alert_receive_channel,
+            escalation_chain=escalation_chain,
+            is_default=False,
+            defaults={
+                "filtering_term": f"escalate to {escalation_chain.name}",
+                "notify_in_slack": True,
+            },
+        )
 
     permalink = None
     if not title:
@@ -49,6 +82,7 @@ def _trigger_alert(organization: Organization, team: Team, title: str, message:
         integration_unique_data={"created_by": from_user.username},
         image_url=None,
         link_to_upstream_details=None,
+        channel_filter=channel_filter,
     )
     return alert.group
 
@@ -103,6 +137,7 @@ def direct_paging(
     message: str = None,
     users: UserNotifications = None,
     schedules: ScheduleNotifications = None,
+    escalation_chain: EscalationChain = None,
     alert_group: AlertGroup = None,
 ) -> None:
     """Trigger escalation targeting given users/schedules.
@@ -111,7 +146,7 @@ def direct_paging(
     Otherwise, create a new alert using given title and message.
 
     """
-    if not users and not schedules:
+    if not users and not schedules and not escalation_chain:
         return
 
     if users is None:
@@ -120,9 +155,12 @@ def direct_paging(
     if schedules is None:
         schedules = []
 
+    if escalation_chain is not None and alert_group is not None:
+        raise ValueError("Cannot change an existing alert group escalation chain")
+
     # create alert group if needed
     if alert_group is None:
-        alert_group = _trigger_alert(organization, team, title, message, from_user)
+        alert_group = _trigger_alert(organization, team, title, message, from_user, escalation_chain=escalation_chain)
 
     # get on call users, add log entry for each schedule
     for (s, important) in schedules:

engine/apps/alerts/tests/test_alert.py

@@ -0,0 +1,43 @@
+import pytest
+
+from apps.alerts.models import Alert
+
+
+@pytest.mark.django_db
+def test_alert_create_default_channel_filter(make_organization, make_alert_receive_channel, make_channel_filter):
+    organization = make_organization()
+    alert_receive_channel = make_alert_receive_channel(organization)
+    channel_filter = make_channel_filter(alert_receive_channel, is_default=True)
+
+    alert = Alert.create(
+        title="the title",
+        message="the message",
+        alert_receive_channel=alert_receive_channel,
+        raw_request_data={},
+        integration_unique_data={},
+        image_url=None,
+        link_to_upstream_details=None,
+    )
+
+    assert alert.group.channel_filter == channel_filter
+
+
+@pytest.mark.django_db
+def test_alert_create_custom_channel_filter(make_organization, make_alert_receive_channel, make_channel_filter):
+    organization = make_organization()
+    alert_receive_channel = make_alert_receive_channel(organization)
+    make_channel_filter(alert_receive_channel, is_default=True)
+    other_channel_filter = make_channel_filter(alert_receive_channel)
+
+    alert = Alert.create(
+        title="the title",
+        message="the message",
+        alert_receive_channel=alert_receive_channel,
+        raw_request_data={},
+        integration_unique_data={},
+        image_url=None,
+        link_to_upstream_details=None,
+        channel_filter=other_channel_filter,
+    )
+
+    assert alert.group.channel_filter == other_channel_filter

engine/apps/alerts/tests/test_paging.py

@@ -244,6 +244,41 @@ def test_direct_paging_reusing_alert_group(
     assert notify_task.apply_async.called_with((user.pk, ag.pk), {"important": False})
 
 
+@pytest.mark.django_db
+def test_direct_paging_reusing_alert_group_custom_chain_raises(
+    make_organization, make_user_for_organization, make_alert_receive_channel, make_alert_group, make_escalation_chain
+):
+    organization = make_organization()
+    from_user = make_user_for_organization(organization)
+    alert_receive_channel = make_alert_receive_channel(organization=organization)
+    alert_group = make_alert_group(alert_receive_channel=alert_receive_channel)
+    custom_chain = make_escalation_chain(organization)
+
+    with pytest.raises(ValueError):
+        direct_paging(organization, None, from_user, alert_group=alert_group, escalation_chain=custom_chain)
+
+
+@pytest.mark.django_db
+def test_direct_paging_custom_chain(
+    make_organization, make_user_for_organization, make_alert_receive_channel, make_alert_group, make_escalation_chain
+):
+    organization = make_organization()
+    from_user = make_user_for_organization(organization)
+    custom_chain = make_escalation_chain(organization)
+
+    direct_paging(organization, None, from_user, escalation_chain=custom_chain)
+
+    # alert group created
+    alert_groups = AlertGroup.all_objects.all()
+    assert alert_groups.count() == 1
+    ag = alert_groups.get()
+    channel_filter = ag.channel_filter_with_respect_to_escalation_snapshot
+    assert channel_filter is not None
+    assert not channel_filter.is_default
+    assert channel_filter.notify_in_slack
+    assert ag.escalation_chain_with_respect_to_escalation_snapshot == custom_chain
+
+
 @pytest.mark.django_db
 def test_unpage_user_not_exists(
     make_organization, make_user_for_organization, make_alert_receive_channel, make_alert_group

engine/apps/integrations/templates/html/integration_direct_paging.html

@@ -0,0 +1,2 @@
+
+<p>You can create a direct page alert group from the web UI</p>

engine/config_integrations/direct_paging.py

@@ -0,0 +1,56 @@
+# Main
+enabled = True
+title = "Direct paging"
+slug = "direct_paging"
+short_description = None
+description = None
+is_displayed_on_web = False
+is_featured = False
+is_able_to_autoresolve = False
+is_demo_alert_enabled = False
+
+description = None
+
+# Default templates
+slack_title = """\
+*<{{ grafana_oncall_link }}|#{{ grafana_oncall_incident_id }} {{ payload.oncall.title }}>* via {{ integration_name }}
+{% if source_link %}
+ (*<{{ source_link }}|source>*)
+{% endif %}
+"""
+
+slack_message = """{{ payload.oncall.message }}
+
+created by {{ payload.oncall.author_username }}
+"""
+
+slack_image_url = None
+
+web_title = "{{ payload.oncall.title }}"
+
+web_message = """{{ payload.oncall.message }}
+{% if source_link %}
+<{{ source_link }} | Link to the original message >
+{% endif %}
+created by {{ payload.oncall.author_username }}
+"""
+
+web_image_url = slack_image_url
+
+sms_title = web_title
+
+phone_call_title = sms_title
+
+telegram_title = sms_title
+
+telegram_message = slack_message
+
+telegram_image_url = slack_image_url
+
+source_link = "{{ payload.oncall.permalink }}"
+
+grouping_id = """{{ payload }}"""
+
+resolve_condition = None
+
+acknowledge_condition = None

engine/settings/base.py

@@ -617,6 +617,7 @@ INSTALLED_ONCALL_INTEGRATIONS = [
     "config_integrations.manual",
     "config_integrations.slack_channel",
     "config_integrations.zabbix",
+    "config_integrations.direct_paging",
 ]
 
 if OSS_INSTALLATION: