Add escalation chain support for new webhooks (#1654)

Allow setting a webhook as escalation chain policy step.
This commit is contained in:
Matias Bordese 2023-04-05 09:03:55 -03:00 committed by GitHub
parent 617c7bf724
commit 2a89374adf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 466 additions and 56 deletions

View file

@ -4,6 +4,7 @@ from apps.alerts.models.custom_button import CustomButton
from apps.alerts.models.escalation_policy import EscalationPolicy
from apps.schedules.models import OnCallSchedule
from apps.user_management.models import User
from apps.webhooks.models import Webhook
class PrimaryKeyRelatedFieldWithNoneValue(serializers.PrimaryKeyRelatedField):
@ -58,6 +59,7 @@ class EscalationPolicySnapshotSerializer(serializers.ModelSerializer):
escalation_counter = serializers.IntegerField(default=0)
passed_last_time = serializers.DateTimeField(allow_null=True, default=None)
custom_button_trigger = PrimaryKeyRelatedFieldWithNoneValue(allow_null=True, queryset=CustomButton.objects)
custom_webhook = PrimaryKeyRelatedFieldWithNoneValue(allow_null=True, queryset=Webhook.objects, default=None)
notify_schedule = PrimaryKeyRelatedFieldWithNoneValue(allow_null=True, queryset=OnCallSchedule.objects)
num_alerts_in_window = serializers.IntegerField(allow_null=True, default=None)
num_minutes_in_window = serializers.IntegerField(allow_null=True, default=None)
@ -77,6 +79,7 @@ class EscalationPolicySnapshotSerializer(serializers.ModelSerializer):
"num_alerts_in_window",
"num_minutes_in_window",
"custom_button_trigger",
"custom_webhook",
"notify_schedule",
"notify_to_group",
"escalation_counter",

View file

@ -10,6 +10,7 @@ from apps.alerts.models.alert_group_log_record import AlertGroupLogRecord
from apps.alerts.models.escalation_policy import EscalationPolicy
from apps.alerts.tasks import (
custom_button_result,
custom_webhook_result,
notify_all_task,
notify_group_task,
notify_user_task,
@ -32,6 +33,7 @@ class EscalationPolicySnapshot:
"num_alerts_in_window",
"num_minutes_in_window",
"custom_button_trigger",
"custom_webhook",
"notify_schedule",
"notify_to_group",
"escalation_counter",
@ -57,6 +59,7 @@ class EscalationPolicySnapshot:
num_alerts_in_window,
num_minutes_in_window,
custom_button_trigger,
custom_webhook,
notify_schedule,
notify_to_group,
escalation_counter,
@ -74,6 +77,7 @@ class EscalationPolicySnapshot:
self.num_alerts_in_window = num_alerts_in_window
self.num_minutes_in_window = num_minutes_in_window
self.custom_button_trigger = custom_button_trigger
self.custom_webhook = custom_webhook
self.notify_schedule = notify_schedule
self.notify_to_group = notify_to_group
self.escalation_counter = escalation_counter # used for STEP_REPEAT_ESCALATION_N_TIMES
@ -116,6 +120,7 @@ class EscalationPolicySnapshot:
EscalationPolicy.STEP_NOTIFY_SCHEDULE: self._escalation_step_notify_on_call_schedule,
EscalationPolicy.STEP_NOTIFY_SCHEDULE_IMPORTANT: self._escalation_step_notify_on_call_schedule,
EscalationPolicy.STEP_TRIGGER_CUSTOM_BUTTON: self._escalation_step_trigger_custom_button,
EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK: self._escalation_step_trigger_custom_webhook,
EscalationPolicy.STEP_NOTIFY_USERS_QUEUE: self._escalation_step_notify_users_queue,
EscalationPolicy.STEP_NOTIFY_IF_TIME: self._escalation_step_notify_if_time,
EscalationPolicy.STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW: self._escalation_step_notify_if_num_alerts_in_time_window,
@ -431,6 +436,29 @@ class EscalationPolicySnapshot:
log_record.save()
self._execute_tasks(tasks)
def _escalation_step_trigger_custom_webhook(self, alert_group, **kwargs) -> None:
tasks = []
webhook = self.custom_webhook
if webhook is not None:
custom_webhook_task = custom_webhook_result.signature(
(webhook.pk, alert_group.pk),
{
"escalation_policy_pk": self.id,
},
immutable=True,
)
tasks.append(custom_webhook_task)
else:
log_record = AlertGroupLogRecord(
type=AlertGroupLogRecord.TYPE_ESCALATION_FAILED,
alert_group=alert_group,
escalation_policy=self.escalation_policy,
escalation_error_code=AlertGroupLogRecord.ERROR_ESCALATION_TRIGGER_CUSTOM_BUTTON_STEP_IS_NOT_CONFIGURED,
escalation_policy_step=self.step,
)
log_record.save()
self._execute_tasks(tasks)
def _escalation_step_repeat_escalation_n_times(self, alert_group, **kwargs) -> StepExecutionResultData:
if self.escalation_counter < EscalationPolicy.MAX_TIMES_REPEAT:
log_record = AlertGroupLogRecord(

View file

@ -0,0 +1,25 @@
# Generated by Django 3.2.18 on 2023-03-29 16:17
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('webhooks', '0002_auto_20230320_1604'),
('alerts', '0010_channelfilter_filtering_term_type'),
]
operations = [
migrations.AddField(
model_name='escalationpolicy',
name='custom_webhook',
field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='escalation_policies', to='webhooks.webhook'),
),
migrations.AlterField(
model_name='escalationpolicy',
name='step',
field=models.IntegerField(choices=[(0, 'Wait'), (1, 'Notify User'), (2, 'Notify Whole Channel'), (3, 'Repeat Escalation (5 times max)'), (4, 'Resolve'), (5, 'Notify Group'), (6, 'Notify Schedule'), (7, 'Notify User (Important)'), (8, 'Notify Group (Important)'), (9, 'Notify Schedule (Important)'), (10, 'Trigger Outgoing Webhook'), (11, 'Notify User (next each time)'), (12, 'Continue escalation only if time is from'), (13, 'Notify multiple Users'), (14, 'Notify multiple Users (Important)'), (15, 'Continue escalation if >X alerts per Y minutes'), (16, 'Trigger Webhook')], default=None, null=True),
),
]

View file

@ -135,7 +135,8 @@ class AlertGroupLogRecord(models.Model):
ERROR_ESCALATION_TRIGGER_CUSTOM_BUTTON_STEP_IS_NOT_CONFIGURED,
ERROR_ESCALATION_NOTIFY_IN_SLACK,
ERROR_ESCALATION_NOTIFY_IF_NUM_ALERTS_IN_WINDOW_STEP_IS_NOT_CONFIGURED,
) = range(17)
ERROR_ESCALATION_TRIGGER_CUSTOM_WEBHOOK_ERROR,
) = range(18)
type = models.IntegerField(choices=TYPE_CHOICES)
@ -436,18 +437,18 @@ class AlertGroupLogRecord(models.Model):
f"{f' by {author_name}' if author_name else ''}"
)
elif self.type == AlertGroupLogRecord.TYPE_CUSTOM_BUTTON_TRIGGERED:
webhook_name = ""
trigger = None
if step_specific_info is not None:
custom_button_name = step_specific_info.get("custom_button_name")
custom_button_name = f"`{custom_button_name}`" or ""
webhook_name = step_specific_info.get("webhook_name") or step_specific_info.get("custom_button_name")
trigger = step_specific_info.get("trigger")
elif self.custom_button is not None:
custom_button_name = f"`{self.custom_button.name}`"
webhook_name = f"`{self.custom_button.name}`"
if trigger is None and self.author:
trigger = f"{author_name}"
else:
custom_button_name = ""
result += f"outgoing webhook {custom_button_name} triggered by "
if self.author:
result += f"{author_name}"
else:
result += "escalation chain"
trigger = trigger or "escalation chain"
result += f"outgoing webhook `{webhook_name}` triggered by {trigger}"
elif self.type == AlertGroupLogRecord.TYPE_FAILED_ATTACHMENT:
if self.alert_group.slack_message is not None:
result += (
@ -491,6 +492,14 @@ class AlertGroupLogRecord(models.Model):
== AlertGroupLogRecord.ERROR_ESCALATION_TRIGGER_CUSTOM_BUTTON_STEP_IS_NOT_CONFIGURED
):
result += 'skipped escalation step "Trigger Outgoing Webhook" because it is not configured'
elif self.escalation_error_code == AlertGroupLogRecord.ERROR_ESCALATION_TRIGGER_CUSTOM_WEBHOOK_ERROR:
webhook_name = trigger = ""
if step_specific_info is not None:
webhook_name = step_specific_info.get("webhook_name", "")
trigger = step_specific_info.get("trigger", "")
result += f"skipped {trigger} outgoing webhook `{webhook_name}`"
if self.reason:
result += f": {self.reason}"
elif self.escalation_error_code == AlertGroupLogRecord.ERROR_ESCALATION_NOTIFY_IF_TIME_IS_NOT_CONFIGURED:
result += 'skipped escalation step "Continue escalation if time" because it is not configured'
elif (

View file

@ -44,7 +44,8 @@ class EscalationPolicy(OrderedModel):
STEP_NOTIFY_MULTIPLE_USERS,
STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT,
STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW,
) = range(16)
STEP_TRIGGER_CUSTOM_WEBHOOK,
) = range(17)
# Must be the same order as previous
STEP_CHOICES = (
@ -64,6 +65,7 @@ class EscalationPolicy(OrderedModel):
(STEP_NOTIFY_MULTIPLE_USERS, "Notify multiple Users"),
(STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT, "Notify multiple Users (Important)"),
(STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW, "Continue escalation if >X alerts per Y minutes"),
(STEP_TRIGGER_CUSTOM_WEBHOOK, "Trigger Webhook"),
)
# Ordered step choices available for internal api.
@ -79,6 +81,7 @@ class EscalationPolicy(OrderedModel):
STEP_NOTIFY_GROUP,
# Other
STEP_TRIGGER_CUSTOM_BUTTON,
STEP_TRIGGER_CUSTOM_WEBHOOK,
STEP_NOTIFY_USERS_QUEUE,
STEP_NOTIFY_IF_TIME,
STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW,
@ -100,6 +103,7 @@ class EscalationPolicy(OrderedModel):
STEP_NOTIFY_MULTIPLE_USERS,
STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT,
STEP_TRIGGER_CUSTOM_BUTTON,
STEP_TRIGGER_CUSTOM_WEBHOOK,
STEP_REPEAT_ESCALATION_N_TIMES,
]
@ -122,6 +126,7 @@ class EscalationPolicy(OrderedModel):
),
# Other
STEP_TRIGGER_CUSTOM_BUTTON: ("Trigger outgoing webhook {{custom_action}}", "Trigger outgoing webhook"),
STEP_TRIGGER_CUSTOM_WEBHOOK: ("Trigger webhook {{custom_webhook}}", "Trigger webhook"),
STEP_NOTIFY_USERS_QUEUE: ("Round robin notification for {{users}}", "Notify users one by one (round-robin)"),
STEP_NOTIFY_IF_TIME: (
"Continue escalation if current time is in {{timerange}} ",
@ -142,6 +147,7 @@ class EscalationPolicy(OrderedModel):
STEP_FINAL_NOTIFYALL,
STEP_FINAL_RESOLVE,
STEP_TRIGGER_CUSTOM_BUTTON,
STEP_TRIGGER_CUSTOM_WEBHOOK,
STEP_NOTIFY_USERS_QUEUE,
STEP_NOTIFY_IF_TIME,
STEP_REPEAT_ESCALATION_N_TIMES,
@ -186,6 +192,7 @@ class EscalationPolicy(OrderedModel):
STEP_FINAL_RESOLVE,
STEP_FINAL_NOTIFYALL,
STEP_TRIGGER_CUSTOM_BUTTON,
STEP_TRIGGER_CUSTOM_WEBHOOK,
STEP_NOTIFY_IF_TIME,
STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW,
STEP_REPEAT_ESCALATION_N_TIMES,
@ -202,6 +209,7 @@ class EscalationPolicy(OrderedModel):
STEP_NOTIFY_SCHEDULE: "notify_on_call_from_schedule",
STEP_NOTIFY_SCHEDULE_IMPORTANT: "notify_on_call_from_schedule",
STEP_TRIGGER_CUSTOM_BUTTON: "trigger_action",
STEP_TRIGGER_CUSTOM_WEBHOOK: "trigger_webhook",
STEP_NOTIFY_USERS_QUEUE: "notify_person_next_each_time",
STEP_NOTIFY_MULTIPLE_USERS: "notify_persons",
STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT: "notify_persons",
@ -256,6 +264,14 @@ class EscalationPolicy(OrderedModel):
null=True,
)
custom_webhook = models.ForeignKey(
"webhooks.Webhook",
on_delete=models.CASCADE,
related_name="escalation_policies",
default=None,
null=True,
)
ONE_MINUTE = timezone.timedelta(minutes=1)
FIVE_MINUTES = timezone.timedelta(minutes=5)
FIFTEEN_MINUTES = timezone.timedelta(minutes=15)
@ -350,6 +366,10 @@ class EscalationPolicy(OrderedModel):
if self.custom_button_trigger:
result["outgoing_webhook"] = self.custom_button_trigger.insight_logs_verbal
result["outgoing_webhook_id"] = self.custom_button_trigger.public_primary_key
elif self.step == EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK:
if self.custom_button_trigger:
result["outgoing_webhook"] = self.custom_webhook.insight_logs_verbal
result["outgoing_webhook_id"] = self.custom_webhook.public_primary_key
elif self.step in [
EscalationPolicy.STEP_NOTIFY_USERS_QUEUE,
EscalationPolicy.STEP_NOTIFY_MULTIPLE_USERS,

View file

@ -8,6 +8,7 @@ from .check_escalation_finished import check_escalation_finished_task # noqa: F
from .create_contact_points_for_datasource import create_contact_points_for_datasource # noqa: F401
from .create_contact_points_for_datasource import schedule_create_contact_points_for_datasource # noqa: F401
from .custom_button_result import custom_button_result # noqa: F401
from .custom_webhook_result import custom_webhook_result # noqa: F401
from .delete_alert_group import delete_alert_group # noqa: F401
from .distribute_alert import distribute_alert # noqa: F401
from .escalate_alert_group import escalate_alert_group # noqa: F401

View file

@ -0,0 +1,16 @@
import logging
from django.conf import settings
from common.custom_celery_tasks import shared_dedicated_queue_retry_task
logger = logging.getLogger(__name__)
@shared_dedicated_queue_retry_task(
autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else None
)
def custom_webhook_result(webhook_pk, alert_group_pk, escalation_policy_pk=None):
from apps.webhooks.tasks import execute_webhook
execute_webhook.apply_async((webhook_pk, alert_group_pk, None, escalation_policy_pk))

View file

@ -0,0 +1,17 @@
from unittest.mock import call, patch
import pytest
from apps.alerts.tasks import custom_webhook_result
@pytest.mark.django_db
def test_custom_webhook_result_executes_webhook():
webhook_id = 42
alert_group_id = 13
escalation_policy_id = 11
with patch("apps.webhooks.tasks.trigger_webhook.execute_webhook.apply_async") as mock_execute:
custom_webhook_result(webhook_id, alert_group_id, escalation_policy_id)
assert mock_execute.call_args == call((webhook_id, alert_group_id, None, escalation_policy_id))

View file

@ -442,6 +442,37 @@ def test_escalation_step_trigger_custom_button(
assert mocked_execute_tasks.called
@patch("apps.alerts.escalation_snapshot.snapshot_classes.EscalationPolicySnapshot._execute_tasks", return_value=None)
@pytest.mark.django_db
def test_escalation_step_trigger_custom_webhook(
mocked_execute_tasks,
escalation_step_test_setup,
make_custom_webhook,
make_escalation_policy,
):
organization, _, _, channel_filter, alert_group, reason = escalation_step_test_setup
custom_webhook = make_custom_webhook(organization=organization)
trigger_custom_webhook_step = make_escalation_policy(
escalation_chain=channel_filter.escalation_chain,
escalation_policy_step=EscalationPolicy.STEP_TRIGGER_CUSTOM_BUTTON,
custom_webhook=custom_webhook,
)
escalation_policy_snapshot = get_escalation_policy_snapshot_from_model(trigger_custom_webhook_step)
expected_eta = timezone.now() + timezone.timedelta(seconds=NEXT_ESCALATION_DELAY)
result = escalation_policy_snapshot.execute(alert_group, reason)
expected_result = EscalationPolicySnapshot.StepExecutionResultData(
eta=result.eta,
stop_escalation=False,
pause_escalation=False,
start_from_beginning=False,
)
assert expected_eta + timezone.timedelta(seconds=15) > result.eta > expected_eta - timezone.timedelta(seconds=15)
assert result == expected_result
assert mocked_execute_tasks.called
@patch("apps.alerts.escalation_snapshot.snapshot_classes.EscalationPolicySnapshot._execute_tasks", return_value=None)
@pytest.mark.django_db
def test_escalation_step_repeat_escalation_n_times(

View file

@ -45,6 +45,7 @@ def test_raw_escalation_snapshot(escalation_snapshot_test_setup):
"num_alerts_in_window": None,
"num_minutes_in_window": None,
"custom_button_trigger": None,
"custom_webhook": None,
"escalation_counter": 0,
"passed_last_time": None,
"pause_escalation": False,
@ -63,6 +64,7 @@ def test_raw_escalation_snapshot(escalation_snapshot_test_setup):
"num_alerts_in_window": None,
"num_minutes_in_window": None,
"custom_button_trigger": None,
"custom_webhook": None,
"escalation_counter": 0,
"passed_last_time": None,
"pause_escalation": False,
@ -81,6 +83,7 @@ def test_raw_escalation_snapshot(escalation_snapshot_test_setup):
"num_alerts_in_window": None,
"num_minutes_in_window": None,
"custom_button_trigger": None,
"custom_webhook": None,
"escalation_counter": 0,
"passed_last_time": None,
"pause_escalation": False,

View file

@ -7,6 +7,7 @@ from apps.alerts.models import CustomButton, EscalationChain, EscalationPolicy
from apps.schedules.models import OnCallSchedule
from apps.slack.models import SlackUserGroup
from apps.user_management.models import User
from apps.webhooks.models import Webhook
from common.api_helpers.custom_fields import (
OrganizationFilteredPrimaryKeyRelatedField,
UsersFilteredByOrganizationField,
@ -22,6 +23,7 @@ TO_TIME = "to_time"
NUM_ALERTS_IN_WINDOW = "num_alerts_in_window"
NUM_MINUTES_IN_WINDOW = "num_minutes_in_window"
CUSTOM_BUTTON_TRIGGER = "custom_button_trigger"
CUSTOM_WEBHOOK_TRIGGER = "custom_webhook"
STEP_TYPE_TO_RELATED_FIELD_MAP = {
EscalationPolicy.STEP_WAIT: [WAIT_DELAY],
@ -32,6 +34,7 @@ STEP_TYPE_TO_RELATED_FIELD_MAP = {
EscalationPolicy.STEP_NOTIFY_IF_TIME: [FROM_TIME, TO_TIME],
EscalationPolicy.STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW: [NUM_ALERTS_IN_WINDOW, NUM_MINUTES_IN_WINDOW],
EscalationPolicy.STEP_TRIGGER_CUSTOM_BUTTON: [CUSTOM_BUTTON_TRIGGER],
EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK: [CUSTOM_WEBHOOK_TRIGGER],
}
@ -71,6 +74,12 @@ class EscalationPolicySerializer(EagerLoadingMixin, serializers.ModelSerializer)
allow_null=True,
filter_field="organization",
)
custom_webhook = OrganizationFilteredPrimaryKeyRelatedField(
queryset=Webhook.objects,
required=False,
allow_null=True,
filter_field="organization",
)
class Meta:
model = EscalationPolicy
@ -87,13 +96,20 @@ class EscalationPolicySerializer(EagerLoadingMixin, serializers.ModelSerializer)
"num_minutes_in_window",
"slack_integration_required",
"custom_button_trigger",
"custom_webhook",
"notify_schedule",
"notify_to_group",
"important",
]
read_only_fields = ("order",)
SELECT_RELATED = ["escalation_chain", "notify_schedule", "notify_to_group", "custom_button_trigger"]
SELECT_RELATED = [
"escalation_chain",
"notify_schedule",
"notify_to_group",
"custom_button_trigger",
"custom_webhook",
]
PREFETCH_RELATED = ["notify_to_users_queue"]
def validate(self, data):
@ -108,6 +124,7 @@ class EscalationPolicySerializer(EagerLoadingMixin, serializers.ModelSerializer)
NUM_ALERTS_IN_WINDOW,
NUM_MINUTES_IN_WINDOW,
CUSTOM_BUTTON_TRIGGER,
CUSTOM_WEBHOOK_TRIGGER,
]
step = data.get("step")
@ -216,6 +233,7 @@ class EscalationPolicyUpdateSerializer(EscalationPolicySerializer):
NUM_ALERTS_IN_WINDOW,
NUM_MINUTES_IN_WINDOW,
CUSTOM_BUTTON_TRIGGER,
CUSTOM_WEBHOOK_TRIGGER,
]
for f in STEP_TYPE_TO_RELATED_FIELD_MAP.get(step, []):

View file

@ -59,9 +59,7 @@ class WebhookSerializer(serializers.ModelSerializer):
"last_response_log",
]
extra_kwargs = {
"authorization_header": {"write_only": True},
"name": {"required": True, "allow_null": False, "allow_blank": False},
"password": {"write_only": True},
"url": {"required": True, "allow_null": False, "allow_blank": False},
}
@ -113,6 +111,6 @@ class WebhookSerializer(serializers.ModelSerializer):
def get_trigger_type_name(self, obj):
trigger_type_name = ""
if obj.trigger_type:
if obj.trigger_type is not None:
trigger_type_name = Webhook.TRIGGER_TYPES[int(obj.trigger_type)][1]
return trigger_type_name

View file

@ -55,6 +55,33 @@ def test_create_escalation_policy(escalation_policy_internal_api_setup, make_use
assert response.data["order"] == max_order + 1
@pytest.mark.django_db
def test_create_escalation_policy_webhook(
escalation_policy_internal_api_setup, make_custom_webhook, make_user_auth_headers
):
token, escalation_chain, _, user, _ = escalation_policy_internal_api_setup
client = APIClient()
url = reverse("api-internal:escalation_policy-list")
webhook = make_custom_webhook(organization=user.organization)
data = {
"step": EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK,
"escalation_chain": escalation_chain.public_primary_key,
"custom_webhook": webhook.public_primary_key,
}
max_order = EscalationPolicy.objects.filter(escalation_chain=escalation_chain).aggregate(maxorder=Max("order"))[
"maxorder"
]
response = client.post(url, data, format="json", **make_user_auth_headers(user, token))
assert response.status_code == status.HTTP_201_CREATED
assert response.data["order"] == max_order + 1
assert response.data["custom_webhook"] == webhook.public_primary_key
escalation_policy = EscalationPolicy.objects.get(public_primary_key=response.data["id"])
assert escalation_policy.custom_webhook == webhook
@pytest.mark.django_db
def test_update_notify_multiple_users_step(escalation_policy_internal_api_setup, make_user_auth_headers):
token, _, escalation_policy, first_user, second_user = escalation_policy_internal_api_setup
@ -624,6 +651,7 @@ def test_escalation_policy_can_not_create_with_non_step_type_related_data(
(EscalationPolicy.STEP_NOTIFY_IF_TIME, ["from_time", "to_time"]),
(EscalationPolicy.STEP_NOTIFY_MULTIPLE_USERS, ["notify_to_users_queue"]),
(EscalationPolicy.STEP_TRIGGER_CUSTOM_BUTTON, ["custom_button_trigger"]),
(EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK, ["custom_webhook"]),
],
)
def test_escalation_policy_update_drop_non_step_type_related_data(
@ -662,6 +690,7 @@ def test_escalation_policy_update_drop_non_step_type_related_data(
"from_time",
"to_time",
"custom_button_trigger",
"custom_webhook",
]
for f in related_fields:
fields_to_check.remove(f)
@ -713,6 +742,7 @@ def test_escalation_policy_switch_importance(
"num_minutes_in_window": None,
"slack_integration_required": escalation_policy.slack_integration_required,
"custom_button_trigger": None,
"custom_webhook": None,
"notify_schedule": None,
"notify_to_group": None,
"important": True,
@ -770,6 +800,7 @@ def test_escalation_policy_filter_by_user(
"num_minutes_in_window": None,
"slack_integration_required": False,
"custom_button_trigger": None,
"custom_webhook": None,
"notify_schedule": None,
"notify_to_group": None,
"important": False,
@ -787,6 +818,7 @@ def test_escalation_policy_filter_by_user(
"num_minutes_in_window": None,
"slack_integration_required": False,
"custom_button_trigger": None,
"custom_webhook": None,
"notify_schedule": None,
"notify_to_group": None,
"important": False,
@ -849,6 +881,7 @@ def test_escalation_policy_filter_by_slack_channel(
"num_minutes_in_window": None,
"slack_integration_required": False,
"custom_button_trigger": None,
"custom_webhook": None,
"notify_schedule": None,
"notify_to_group": None,
"important": False,
@ -864,3 +897,25 @@ def test_escalation_policy_filter_by_slack_channel(
assert response.status_code == status.HTTP_200_OK
assert response.json() == expected_payload
@pytest.mark.django_db
@pytest.mark.parametrize("enabled", [True, False])
def test_escalation_policy_escalation_options_webhooks(
make_organization_and_user_with_plugin_token,
make_user_auth_headers,
enabled,
):
_, user, token = make_organization_and_user_with_plugin_token()
client = APIClient()
url = reverse("api-internal:escalation_policy-escalation-options")
with patch("apps.api.views.escalation_policy.is_webhooks_enabled_for_organization", return_value=enabled):
response = client.get(url, format="json", **make_user_auth_headers(user, token))
returned_options = [option["value"] for option in response.json()]
if enabled:
assert EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK in returned_options
else:
assert EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK not in returned_options

View file

@ -44,6 +44,8 @@ def test_get_list_webhooks(webhook_internal_api_setup, make_user_auth_headers):
"url": "https://github.com/",
"data": '{"name": "{{ alert_payload }}"}',
"username": "Chris Vanstras",
"password": "qwerty",
"authorization_header": "auth_token",
"forward_all": False,
"headers": None,
"http_method": "POST",
@ -81,6 +83,8 @@ def test_get_detail_webhook(webhook_internal_api_setup, make_user_auth_headers):
"url": "https://github.com/",
"data": '{"name": "{{ alert_payload }}"}',
"username": "Chris Vanstras",
"password": "qwerty",
"authorization_header": "auth_token",
"forward_all": False,
"headers": None,
"http_method": "POST",
@ -123,6 +127,8 @@ def test_create_webhook(mocked_check_webhooks_2_enabled, webhook_internal_api_se
"id": webhook.public_primary_key,
"data": None,
"username": None,
"password": None,
"authorization_header": None,
"forward_all": True,
"headers": None,
"http_method": "POST",
@ -177,6 +183,8 @@ def test_create_valid_templated_field(
expected_response = data | {
"id": webhook.public_primary_key,
"username": None,
"password": None,
"authorization_header": None,
"forward_all": True,
"headers": None,
"data": None,

View file

@ -14,6 +14,7 @@ from apps.api.serializers.escalation_policy import (
EscalationPolicyUpdateSerializer,
)
from apps.auth_token.auth import PluginAuthentication
from apps.webhooks.utils import is_webhooks_enabled_for_organization
from common.api_helpers.exceptions import BadRequest
from common.api_helpers.mixins import (
CreateSerializerMixin,
@ -140,6 +141,10 @@ class EscalationPolicyView(
def escalation_options(self, request):
choices = []
for step in EscalationPolicy.INTERNAL_API_STEPS:
if step == EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK and not is_webhooks_enabled_for_organization(
self.request.auth.organization.pk
):
continue
verbal = EscalationPolicy.INTERNAL_API_STEPS_TO_VERBAL_MAP[step]
can_change_importance = (
step in EscalationPolicy.IMPORTANT_STEPS_SET or step in EscalationPolicy.DEFAULT_STEPS_SET

View file

@ -1,4 +1,3 @@
from django.apps import apps
from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
from django_filters import rest_framework as filters
from rest_framework.decorators import action
@ -12,6 +11,7 @@ from apps.api.permissions import RBACPermission
from apps.api.serializers.webhook import WebhookSerializer
from apps.auth_token.auth import PluginAuthentication
from apps.webhooks.models import Webhook
from apps.webhooks.utils import is_webhooks_enabled_for_organization
from common.api_helpers.filters import ByTeamModelFieldFilterMixin, ModelFieldFilterMixin, TeamModelMultipleChoiceFilter
from common.api_helpers.mixins import PublicPrimaryKeyMixin, TeamFilteringMixin
@ -85,16 +85,7 @@ class WebhooksView(TeamFilteringMixin, PublicPrimaryKeyMixin, ModelViewSet):
instance.delete()
def check_webhooks_2_enabled(self):
DynamicSetting = apps.get_model("base", "DynamicSetting")
enabled_webhooks_2_orgs = DynamicSetting.objects.get_or_create(
name="enabled_webhooks_2_orgs",
defaults={
"json_value": {
"org_ids": [],
}
},
)[0]
if self.request.auth.organization.pk not in enabled_webhooks_2_orgs.json_value["org_ids"]:
if not is_webhooks_enabled_for_organization(self.request.auth.organization.pk):
raise PermissionDenied("Webhooks 2 not enabled for organization. Permission denied.")
@action(methods=["get"], detail=False)

View file

@ -6,7 +6,7 @@ from celery.utils.log import get_task_logger
from django.apps import apps
from django.conf import settings
from apps.alerts.models import AlertGroup
from apps.alerts.models import AlertGroup, AlertGroupLogRecord, EscalationPolicy
from apps.user_management.models import User
from apps.webhooks.models import Webhook, WebhookResponse
from apps.webhooks.utils import (
@ -29,6 +29,7 @@ TRIGGER_TYPE_TO_LABEL = {
Webhook.TRIGGER_SILENCE: "silence",
Webhook.TRIGGER_UNSILENCE: "unsilence",
Webhook.TRIGGER_UNRESOLVE: "unresolve",
Webhook.TRIGGER_ESCALATION_STEP: "escalation",
}
@ -40,18 +41,14 @@ def send_webhook_event(trigger_type, alert_group_id, team_id=None, organization_
webhooks_qs = Webhooks.objects.filter(trigger_type=trigger_type, organization_id=organization_id, team_id=team_id)
for webhook in webhooks_qs:
execute_webhook.apply_async((webhook.pk, alert_group_id, user_id))
execute_webhook.apply_async((webhook.pk, alert_group_id, user_id, None))
def _isoformat_date(date_value):
return date_value.isoformat() if date_value else None
def _build_payload(trigger_type, alert_group, user_id):
user = None
if user_id is not None:
user = User.objects.filter(pk=user_id).first()
def _build_payload(trigger_type, alert_group, user):
event = {
"type": TRIGGER_TYPE_TO_LABEL[trigger_type],
}
@ -83,7 +80,7 @@ def _build_payload(trigger_type, alert_group, user_id):
@shared_dedicated_queue_retry_task(
autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else None
)
def execute_webhook(webhook_pk, alert_group_id, user_id):
def execute_webhook(webhook_pk, alert_group_id, user_id, escalation_policy_id):
Webhooks = apps.get_model("webhooks", "Webhook")
try:
webhook = Webhooks.objects.get(pk=webhook_pk)
@ -96,7 +93,11 @@ def execute_webhook(webhook_pk, alert_group_id, user_id):
except AlertGroup.DoesNotExist:
return
data = _build_payload(webhook.trigger_type, alert_group, user_id)
user = None
if user_id is not None:
user = User.objects.filter(pk=user_id).first()
data = _build_payload(webhook.trigger_type, alert_group, user)
status = {
"url": None,
"request_trigger": None,
@ -107,7 +108,7 @@ def execute_webhook(webhook_pk, alert_group_id, user_id):
"webhook": webhook,
}
exception = None
exception = error = None
try:
triggered, status["request_trigger"] = webhook.check_trigger(data)
if triggered:
@ -128,15 +129,15 @@ def execute_webhook(webhook_pk, alert_group_id, user_id):
# do not add a log entry if the webhook is not triggered
return
except InvalidWebhookUrl as e:
status["url"] = e.message
status["url"] = error = e.message
except InvalidWebhookTrigger as e:
status["request_trigger"] = e.message
status["request_trigger"] = error = e.message
except InvalidWebhookHeaders as e:
status["request_headers"] = e.message
status["request_headers"] = error = e.message
except InvalidWebhookData as e:
status["request_data"] = e.message
status["request_data"] = error = e.message
except Exception as e:
status["content"] = str(e)
status["content"] = error = str(e)
exception = e
# create response entry
@ -146,5 +147,35 @@ def execute_webhook(webhook_pk, alert_group_id, user_id):
**status,
)
escalation_policy = step = None
if escalation_policy_id:
escalation_policy = EscalationPolicy.objects.filter(pk=escalation_policy_id).first()
step = EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK
# create log record
error_code = None
# reuse existing webhooks record type (TODO: rename after migration)
log_type = AlertGroupLogRecord.TYPE_CUSTOM_BUTTON_TRIGGERED
reason = str(status["status_code"])
if error is not None:
log_type = AlertGroupLogRecord.TYPE_ESCALATION_FAILED
error_code = AlertGroupLogRecord.ERROR_ESCALATION_TRIGGER_CUSTOM_WEBHOOK_ERROR
reason = error
AlertGroupLogRecord.objects.create(
type=log_type,
alert_group=alert_group,
author=user,
reason=reason,
step_specific_info={
"webhook_name": webhook.name,
"webhook_id": webhook.public_primary_key,
"trigger": TRIGGER_TYPE_TO_LABEL[webhook.trigger_type],
},
escalation_policy=escalation_policy,
escalation_policy_step=step,
escalation_error_code=error_code,
)
if exception:
raise exception

View file

@ -4,6 +4,7 @@ from unittest.mock import call, patch
import pytest
from django.utils import timezone
from apps.alerts.models import AlertGroupLogRecord, EscalationPolicy
from apps.public_api.serializers import IncidentSerializer
from apps.webhooks.models import Webhook
from apps.webhooks.tasks import execute_webhook, send_webhook_event
@ -42,7 +43,7 @@ def test_send_webhook_event_filters(
for trigger_type, _ in Webhook.TRIGGER_TYPES:
with patch("apps.webhooks.tasks.trigger_webhook.execute_webhook.apply_async") as mock_execute:
send_webhook_event(trigger_type, alert_group.pk, organization_id=organization.pk)
assert mock_execute.call_args == call((webhooks[trigger_type].pk, alert_group.pk, None))
assert mock_execute.call_args == call((webhooks[trigger_type].pk, alert_group.pk, None, None))
# other team
alert_receive_channel = make_alert_receive_channel(organization, team=other_team)
@ -51,14 +52,14 @@ def test_send_webhook_event_filters(
send_webhook_event(
Webhook.TRIGGER_ACKNOWLEDGE, alert_group.pk, organization_id=organization.pk, team_id=other_team.pk
)
assert mock_execute.call_args == call((other_team_webhook.pk, alert_group.pk, None))
assert mock_execute.call_args == call((other_team_webhook.pk, alert_group.pk, None, None))
# other org
alert_receive_channel = make_alert_receive_channel(other_organization)
alert_group = make_alert_group(alert_receive_channel)
with patch("apps.webhooks.tasks.trigger_webhook.execute_webhook.apply_async") as mock_execute:
send_webhook_event(Webhook.TRIGGER_NEW, alert_group.pk, organization_id=other_organization.pk)
assert mock_execute.call_args == call((other_org_webhook.pk, alert_group.pk, None))
assert mock_execute.call_args == call((other_org_webhook.pk, alert_group.pk, None, None))
@pytest.mark.django_db
@ -89,7 +90,7 @@ def test_execute_webhook_ok(
mock_gethostbyname.return_value = "8.8.8.8"
with patch("apps.webhooks.models.webhook.requests") as mock_requests:
mock_requests.post.return_value = mock_response
execute_webhook(webhook.pk, alert_group.pk, user.pk)
execute_webhook(webhook.pk, alert_group.pk, user.pk, None)
assert mock_requests.post.called
expected_call = call(
@ -106,6 +107,75 @@ def test_execute_webhook_ok(
assert log.request_data == json.dumps({"value": alert_group.public_primary_key})
assert log.request_headers == json.dumps({"some-header": alert_group.public_primary_key})
assert log.url == "https://something/{}/".format(alert_group.public_primary_key)
# check log record
log_record = alert_group.log_records.last()
assert log_record.type == AlertGroupLogRecord.TYPE_CUSTOM_BUTTON_TRIGGERED
expected_info = {
"trigger": "acknowledge",
"webhook_id": webhook.public_primary_key,
"webhook_name": webhook.name,
}
assert log_record.step_specific_info == expected_info
assert log_record.escalation_policy is None
assert log_record.escalation_policy_step is None
assert log_record.rendered_log_line_action() == f"outgoing webhook `{webhook.name}` triggered by acknowledge"
@pytest.mark.django_db
def test_execute_webhook_via_escalation_ok(
make_organization,
make_user_for_organization,
make_alert_receive_channel,
make_alert_group,
make_custom_webhook,
make_escalation_chain,
make_escalation_policy,
):
organization = make_organization()
user = make_user_for_organization(organization)
alert_receive_channel = make_alert_receive_channel(organization)
alert_group = make_alert_group(
alert_receive_channel, acknowledged_at=timezone.now(), acknowledged=True, acknowledged_by=user.pk
)
webhook = make_custom_webhook(
organization=organization,
url="https://something/{{ alert_group_id }}/",
http_method="POST",
trigger_type=Webhook.TRIGGER_ESCALATION_STEP,
trigger_template="{{{{ alert_group.integration_id == '{}' }}}}".format(
alert_receive_channel.public_primary_key
),
headers='{"some-header": "{{ alert_group_id }}"}',
data='{"value": "{{ alert_group_id }}"}',
forward_all=False,
)
escalation_chain = make_escalation_chain(organization)
escalation_policy = make_escalation_policy(
escalation_chain=escalation_chain,
escalation_policy_step=EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK,
custom_webhook=webhook,
)
mock_response = MockResponse()
with patch("apps.webhooks.utils.socket.gethostbyname") as mock_gethostbyname:
mock_gethostbyname.return_value = "8.8.8.8"
with patch("apps.webhooks.models.webhook.requests") as mock_requests:
mock_requests.post.return_value = mock_response
execute_webhook(webhook.pk, alert_group.pk, user.pk, escalation_policy.pk)
assert mock_requests.post.called
# check log record
log_record = alert_group.log_records.last()
assert log_record.type == AlertGroupLogRecord.TYPE_CUSTOM_BUTTON_TRIGGERED
expected_info = {
"trigger": "escalation",
"webhook_id": webhook.public_primary_key,
"webhook_name": webhook.name,
}
assert log_record.step_specific_info == expected_info
assert log_record.escalation_policy == escalation_policy
assert log_record.escalation_policy_step == EscalationPolicy.STEP_TRIGGER_CUSTOM_WEBHOOK
assert log_record.rendered_log_line_action() == f"outgoing webhook `{webhook.name}` triggered by escalation"
@pytest.mark.django_db
@ -131,7 +201,7 @@ def test_execute_webhook_ok_forward_all(
mock_gethostbyname.return_value = "8.8.8.8"
with patch("apps.webhooks.models.webhook.requests") as mock_requests:
mock_requests.post.return_value = mock_response
execute_webhook(webhook.pk, alert_group.pk, user.pk)
execute_webhook(webhook.pk, alert_group.pk, user.pk, None)
assert mock_requests.post.called
expected_data = {
@ -197,7 +267,7 @@ def test_execute_webhook_using_responses_data(
mock_gethostbyname.return_value = "8.8.8.8"
with patch("apps.webhooks.models.webhook.requests") as mock_requests:
mock_requests.post.return_value = mock_response
execute_webhook(webhook.pk, alert_group.pk, user.pk)
execute_webhook(webhook.pk, alert_group.pk, user.pk, None)
assert mock_requests.post.called
expected_data = {"value": "updated"}
@ -232,7 +302,7 @@ def test_execute_webhook_trigger_false(
)
with patch("apps.webhooks.models.webhook.requests") as mock_requests:
execute_webhook(webhook.pk, alert_group.pk, None)
execute_webhook(webhook.pk, alert_group.pk, None, None)
assert not mock_requests.post.called
# check no logs
@ -293,7 +363,7 @@ def test_execute_webhook_errors(
# make it a valid URL when resolving name
mock_gethostbyname.return_value = "8.8.8.8"
with patch("apps.webhooks.models.webhook.requests") as mock_requests:
execute_webhook(webhook.pk, alert_group.pk, None)
execute_webhook(webhook.pk, alert_group.pk, None, None)
assert not mock_requests.post.called
log = webhook.responses.all()[0]
@ -301,3 +371,16 @@ def test_execute_webhook_errors(
assert log.content is None
error = getattr(log, log_field_name)
assert error == expected_error
# check log record
log_record = alert_group.log_records.last()
assert log_record.type == AlertGroupLogRecord.ERROR_ESCALATION_TRIGGER_CUSTOM_WEBHOOK_ERROR
expected_info = {
"trigger": "resolve",
"webhook_id": webhook.public_primary_key,
"webhook_name": webhook.name,
}
assert log_record.step_specific_info == expected_info
assert log_record.reason == expected_error
assert (
log_record.rendered_log_line_action() == f"skipped resolve outgoing webhook `{webhook.name}`: {expected_error}"
)

View file

@ -4,6 +4,7 @@ import re
import socket
from urllib.parse import urlparse
from django.apps import apps
from django.conf import settings
from apps.base.utils import live_settings
@ -132,3 +133,16 @@ def serialize_event(event, alert_group, user, responses=None):
data["responses"] = responses
return data
def is_webhooks_enabled_for_organization(organization_id):
DynamicSetting = apps.get_model("base", "DynamicSetting")
enabled_webhooks_orgs = DynamicSetting.objects.get_or_create(
name="enabled_webhooks_2_orgs",
defaults={
"json_value": {
"org_ids": [],
}
},
)[0]
return organization_id in enabled_webhooks_orgs.json_value["org_ids"]

View file

@ -22,6 +22,7 @@ import {
} from 'models/escalation_policy/escalation_policy.types';
import { GrafanaTeamStore } from 'models/grafana_team/grafana_team';
import { OutgoingWebhookStore } from 'models/outgoing_webhook/outgoing_webhook';
import { OutgoingWebhook2Store } from 'models/outgoing_webhook_2/outgoing_webhook_2';
import { ScheduleStore } from 'models/schedule/schedule';
import { WaitDelay } from 'models/wait_delay';
import { SelectOption } from 'state/types';
@ -47,6 +48,7 @@ export interface EscalationPolicyProps {
isSlackInstalled: boolean;
teamStore: GrafanaTeamStore;
outgoingWebhookStore: OutgoingWebhookStore;
outgoingWebhook2Store: OutgoingWebhook2Store;
scheduleStore: ScheduleStore;
}
@ -99,6 +101,8 @@ export class EscalationPolicy extends React.Component<EscalationPolicyProps, any
return this._renderNotifySchedule();
case 'custom_action':
return this._renderTriggerCustomAction();
case 'custom_webhook':
return this._renderTriggerCustomWebhook();
case 'num_alerts_in_window':
return this.renderNumAlertsInWindow();
case 'num_minutes_in_window':
@ -387,6 +391,40 @@ export class EscalationPolicy extends React.Component<EscalationPolicyProps, any
);
}
private _renderTriggerCustomWebhook() {
const { data, teamStore, outgoingWebhook2Store } = this.props;
const { custom_webhook } = data;
return (
<WithPermissionControlTooltip
key="custom-webhook"
disableByPaywall
userAction={UserActions.EscalationChainsWrite}
>
<GSelect
showSearch
modelName="outgoingWebhook2Store"
displayField="name"
valueField="id"
placeholder="Select Webhook"
className={cx('select', 'control')}
value={custom_webhook}
onChange={this._getOnChangeHandler('custom_webhook')}
getOptionLabel={(item: SelectableValue) => {
const team = teamStore.items[outgoingWebhook2Store.items[item.value].team];
return (
<>
<Text>{item.label} </Text>
<TeamName team={team} size="small" />
</>
);
}}
width={'auto'}
/>
</WithPermissionControlTooltip>
);
}
_getOnSelectChangeHandler = (field: string) => {
return (option: SelectableValue) => {
const { data, onChange = () => {} } = this.props;

View file

@ -90,6 +90,7 @@ const EscalationChainSteps = observer((props: EscalationChainStepsProps) => {
teamStore={store.grafanaTeamStore}
scheduleStore={store.scheduleStore}
outgoingWebhookStore={store.outgoingWebhookStore}
outgoingWebhook2Store={store.outgoingWebhook2Store}
/>
);
})

View file

@ -19,6 +19,7 @@ export interface EscalationPolicy {
to_time: string | null;
notify_to_channel: Channel['id'] | null;
custom_button_trigger: ActionDTO['id'] | null;
custom_webhook: ActionDTO['id'] | null;
notify_to_group: UserGroup['id'] | null;
notify_schedule: Schedule['id'] | null;
important: boolean | null;

View file

@ -1,6 +1,10 @@
export const getApiPathByPage = (page: string) => {
return (
{ outgoing_webhooks: 'custom_buttons', incidents: 'alertgroups', integrations: 'alert_receive_channels' }[page] ||
page
{
outgoing_webhooks: 'custom_buttons',
outgoing_webhooks_2: 'webhooks',
incidents: 'alertgroups',
integrations: 'alert_receive_channels',
}[page] || page
);
};

View file

@ -1,7 +1,6 @@
import { action, observable } from 'mobx';
import BaseStore from 'models/base_store';
import { OutgoingWebhook } from 'models/outgoing_webhook/outgoing_webhook.types';
import { makeRequest } from 'network';
import { RootStore } from 'state';
@ -14,6 +13,9 @@ export class OutgoingWebhook2Store extends BaseStore {
@observable.shallow
searchResult: { [key: string]: Array<OutgoingWebhook2['id']> } = {};
@observable
incidentFilters: any;
constructor(rootStore: RootStore) {
super(rootStore);
@ -45,7 +47,6 @@ export class OutgoingWebhook2Store extends BaseStore {
@action
async updateItem(id: OutgoingWebhook2['id'], fromOrganization = false) {
const response = await this.getById(id, false, fromOrganization);
this.items = {
...this.items,
[id]: response,
@ -75,10 +76,17 @@ export class OutgoingWebhook2Store extends BaseStore {
this.searchResult = {
...this.searchResult,
[key]: results.map((item: OutgoingWebhook) => item.id),
[key]: results.map((item: OutgoingWebhook2) => item.id),
};
}
@action
async updateOutgoingWebhooks2Filters(params: any) {
this.incidentFilters = params;
this.updateItems();
}
getSearchResult(query = '') {
if (!this.searchResult[query]) {
return undefined;

View file

@ -1,3 +1,5 @@
import { GrafanaTeam } from 'models/grafana_team/grafana_team.types';
export interface OutgoingWebhook2 {
authorization_header: string;
data: string;
@ -7,7 +9,7 @@ export interface OutgoingWebhook2 {
last_run: string;
name: string;
password: string;
team: null;
team: GrafanaTeam['id'];
trigger_type: number;
trigger_type_name: string;
url: string;

View file

@ -163,7 +163,7 @@ export const Root = observer((props: AppRootProps) => {
<OutgoingWebhooks query={query} />
</Route>
<Route path={getRoutesForPage('outgoing_webhooks_2')} exact>
<OutgoingWebhooks2 />
<OutgoingWebhooks2 query={query} />
</Route>
<Route path={getRoutesForPage('maintenance')} exact>
<Maintenance query={query} />