From f04430568fa3a7630a766712c0131b25b1c6d24f Mon Sep 17 00:00:00 2001 From: Ildar Iskhakov Date: Tue, 23 May 2023 09:56:33 +0800 Subject: [PATCH] Refactor alertmanager templates (#1944) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What this PR does This PR changes default templates for alertmanager: Web: Screenshot 2023-05-17 at 6 20 06 PM Slack: Screenshot 2023-05-17 at 6 17 30 PM Telegram: Screenshot 2023-05-17 at 6 16 46 PM MS teams: ![Uploading Screenshot 2023-05-23 at 9.46.21 AM.png…]() ## Which issue(s) this PR fixes ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required) --- engine/config_integrations/alertmanager.py | 294 +++++++++++++-------- 1 file changed, 186 insertions(+), 108 deletions(-) diff --git a/engine/config_integrations/alertmanager.py b/engine/config_integrations/alertmanager.py index 8a2f8464..579174de 100644 --- a/engine/config_integrations/alertmanager.py +++ b/engine/config_integrations/alertmanager.py @@ -20,10 +20,67 @@ Alerts from Grafana Alertmanager are automatically routed to this integration.
Creating contact points and routes for other alertmanagers... {% endif %}""" -# Default templates +# Web +web_title = """{{- payload.get("labels", {}).get("alertname", "No title (check Title Template)") -}}""" +web_message = """\ +{%- set annotations = payload.annotations.copy() -%} +{%- set labels = payload.labels.copy() -%} + +{%- if "summary" in annotations %} +{{ annotations.summary }} +{%- set _ = annotations.pop('summary') -%} +{%- endif %} + +{%- if "message" in annotations %} +{{ annotations.message }} +{%- set _ = annotations.pop('message') -%} +{%- endif %} + +{% set severity = labels.severity | default("Unknown") -%} +{%- set severity_emoji = {"critical": ":rotating_light:", "warning": ":warning:" }[severity] | default(":question:") -%} +Severity: {{ severity }} {{ severity_emoji }} + +{%- set status = payload.status | default("Unknown") %} +{%- set status_emoji = {"firing": ":fire:", "resolved": ":white_check_mark:"}[status] | default(":warning:") %} +Status: {{ status }} {{ status_emoji }} (on the source) + +{% if "runbook_url" in annotations -%} +[:book: Runbook:link:]({{ annotations.runbook_url }}) +{%- set _ = annotations.pop('runbook_url') -%} +{%- endif %} + +{%- if "runbook_url_internal" in annotations -%} +[:closed_book: Runbook (internal):link:]({{ annotations.runbook_url_internal }}) +{%- set _ = annotations.pop('runbook_url_internal') -%} +{%- endif %} + +:label: Labels: +{%- for k, v in payload["labels"].items() %} +- {{ k }}: {{ v }} +{%- endfor %} + +{% if annotations | length > 0 -%} +:pushpin: Other annotations: +{%- for k, v in annotations.items() %} +- {{ k }}: {{ v }} +{%- endfor %} +{% endif %} +""" # noqa: W291 + +web_image_url = None + +# Behaviour +source_link = "{{ payload.generatorURL }}" + +grouping_id = "{{ payload.labels }}" + +resolve_condition = """{{ payload.status == "resolved" }}""" + +acknowledge_condition = None + +# Slack slack_title = """\ -{# Usually title is located in payload.labels.alertname #} -{% set title = payload.get("labels", {}).get("alertname", "No title (check Web Title Template)") %} +{% set title = payload.get("labels", {}).get("alertname", "No title (check Title Template)") %} {# Combine the title from different built-in variables into slack-formatted url #} *<{{ grafana_oncall_link }}|#{{ grafana_oncall_incident_id }} {{ title }}>* via {{ integration_name }} {% if source_link %} @@ -31,74 +88,131 @@ slack_title = """\ {%- endif %} """ -slack_message = """\ -{{- payload.message }} -{%- if "status" in payload -%} -*Status*: {{ payload.status }} -{% endif -%} -*Labels:* {% for k, v in payload["labels"].items() %} -{{ k }}: {{ v }}{% endfor %} -*Annotations:* -{%- for k, v in payload.get("annotations", {}).items() %} -{#- render annotation as slack markdown url if it starts with http #} -{{ k }}: {% if v.startswith("http") %} <{{v}}|here> {% else %} {{v}} {% endif -%} -{% endfor %} -""" # noqa: W291 +# default slack message template is identical to web message template, except urls +# It can be based on web message template (see example), but it can affect existing templates +# slack_message = """ +# {% set mkdwn_link_regex = "\[([\w\s\d:]+)\]\((https?:\/\/[\w\d./?=#]+)\)" %} +# {{ web_message +# | regex_replace(mkdwn_link_regex, "<\\2|\\1>") +# }} +# """ +slack_message = """\ +{%- set annotations = payload.annotations.copy() -%} +{%- set labels = payload.labels.copy() -%} + +{%- if "summary" in annotations %} +{{ annotations.summary }} +{%- set _ = annotations.pop('summary') -%} +{%- endif %} + +{%- if "message" in annotations %} +{{ annotations.message }} +{%- set _ = annotations.pop('message') -%} +{%- endif %} + +{# Optionally set oncall_slack_user_group to slack user group in the following format "@users-oncall" #} +{%- set oncall_slack_user_group = None -%} +{%- if oncall_slack_user_group %} +Heads up {{ oncall_slack_user_group }} +{%- endif %} + +{% set severity = labels.severity | default("Unknown") -%} +{%- set severity_emoji = {"critical": ":rotating_light:", "warning": ":warning:" }[severity] | default(":question:") -%} +Severity: {{ severity }} {{ severity_emoji }} + +{%- set status = payload.status | default("Unknown") %} +{%- set status_emoji = {"firing": ":fire:", "resolved": ":white_check_mark:"}[status] | default(":warning:") %} +Status: {{ status }} {{ status_emoji }} (on the source) + +{% if "runbook_url" in annotations -%} +<{{ annotations.runbook_url }}|:book: Runbook:link:> +{%- set _ = annotations.pop('runbook_url') -%} +{%- endif %} + +{%- if "runbook_url_internal" in annotations -%} +<{{ annotations.runbook_url_internal }}|:closed_book: Runbook (internal):link:> +{%- set _ = annotations.pop('runbook_url_internal') -%} +{%- endif %} + +:label: Labels: +{%- for k, v in payload["labels"].items() %} +- {{ k }}: {{ v }} +{%- endfor %} + +{% if annotations | length > 0 -%} +:pushpin: Other annotations: +{%- for k, v in annotations.items() %} +- {{ k }}: {{ v }} +{%- endfor %} +{% endif %} +""" # noqa: W291 slack_image_url = None -web_title = """\ -{# Usually title is located in payload.labels.alertname #} -{{- payload.get("labels", {}).get("alertname", "No title (check Web Title Template)") }} -""" +# SMS +sms_title = web_title -web_message = """\ -{{- payload.message }} -{%- if "status" in payload %} -**Status**: {{ payload.status }} -{% endif -%} -**Labels:** {% for k, v in payload["labels"].items() %} -*{{ k }}*: {{ v }}{% endfor %} -**Annotations:** -{%- for k, v in payload.get("annotations", {}).items() %} -{#- render annotation as markdown url if it starts with http #} -*{{ k }}*: {% if v.startswith("http") %} [here]({{v}}){% else %} {{v}} {% endif -%} -{% endfor %} +# Phone +phone_call_title = web_title + +# Telegram +telegram_title = web_title + +# default telegram message template is identical to web message template, except urls +# It can be based on web message template (see example), but it can affect existing templates +# telegram_message = """ +# {% set mkdwn_link_regex = "\[([\w\s\d:]+)\]\((https?:\/\/[\w\d./?=#]+)\)" %} +# {{ web_message +# | regex_replace(mkdwn_link_regex, "\\1") +# }} +# """ +telegram_message = """\ +{%- set annotations = payload.annotations.copy() -%} +{%- set labels = payload.labels.copy() -%} + +{%- if "summary" in annotations %} +{{ annotations.summary }} +{%- set _ = annotations.pop('summary') -%} +{%- endif %} + +{%- if "message" in annotations %} +{{ annotations.message }} +{%- set _ = annotations.pop('message') -%} +{%- endif %} + +{% set severity = labels.severity | default("Unknown") -%} +{%- set severity_emoji = {"critical": ":rotating_light:", "warning": ":warning:" }[severity] | default(":question:") -%} +Severity: {{ severity }} {{ severity_emoji }} + +{%- set status = payload.status | default("Unknown") %} +{%- set status_emoji = {"firing": ":fire:", "resolved": ":white_check_mark:"}[status] | default(":warning:") %} +Status: {{ status }} {{ status_emoji }} (on the source) + +{% if "runbook_url" in annotations -%} +:book: Runbook:link: +{%- set _ = annotations.pop('runbook_url') -%} +{%- endif %} + +{%- if "runbook_url_internal" in annotations -%} +:closed_book: Runbook (internal):link: +{%- set _ = annotations.pop('runbook_url_internal') -%} +{%- endif %} + +:label: Labels: +{%- for k, v in payload["labels"].items() %} +- {{ k }}: {{ v }} +{%- endfor %} + +{% if annotations | length > 0 -%} +:pushpin: Other annotations: +{%- for k, v in annotations.items() %} +- {{ k }}: {{ v }} +{%- endfor %} +{% endif %} """ # noqa: W291 - -web_image_url = slack_image_url - -sms_title = '{{ payload.get("labels", {}).get("alertname", "Title undefined") }}' -phone_call_title = sms_title - -telegram_title = sms_title - -telegram_message = """\ -{{- payload.messsage }} -{%- if "status" in payload -%} -Status: {{ payload.status }} -{% endif -%} -Labels: {% for k, v in payload["labels"].items() %} -{{ k }}: {{ v }}{% endfor %} -Annotations: -{%- for k, v in payload.get("annotations", {}).items() %} -{#- render annotation as markdown url if it starts with http #} -{{ k }}: {{ v }} -{% endfor %}""" # noqa: W291 - -telegram_image_url = slack_image_url - -source_link = "{{ payload.generatorURL }}" - -grouping_id = "{{ payload.labels }}" - -resolve_condition = """\ -{{ payload.get("status", "") == "resolved" }} -""" - -acknowledge_condition = None +telegram_image_url = None tests = { "payload": { @@ -131,36 +245,12 @@ tests = { "+-+kube_job_status_succeeded%7Bjob%3D%22kube-state-metrics%22%7D+%3E+0&g0.tab=1" "|source>*)" ), - "message": ( - "*Status*: firing\n" - "*Labels:* \n" - "job: kube-state-metrics\n" - "instance: 10.143.139.7:8443\n" - "job_name: email-tracking-perform-initialization-1.0.50\n" - "severity: warning\n" - "alertname: KubeJobCompletion\n" - "namespace: default\n" - "prometheus: monitoring/k8s\n" - "*Annotations:*\n" - "message: Job default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete. \n" - "runbook_url: " - ), + "message": "\nJob default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete.\n\n\n\nSeverity: warning :warning:\nStatus: firing :fire: (on the source)\n\n\n\n:label: Labels:\n- job: kube-state-metrics\n- instance: 10.143.139.7:8443\n- job_name: email-tracking-perform-initialization-1.0.50\n- severity: warning\n- alertname: KubeJobCompletion\n- namespace: default\n- prometheus: monitoring/k8s\n\n", # noqa "image_url": None, }, "web": { "title": "KubeJobCompletion", - "message": """

Status: firing
-Labels:
-job: kube-state-metrics
-instance: 10.143.139.7:8443
-job_name: email-tracking-perform-initialization-1.0.50
-severity: warning
-alertname: KubeJobCompletion
-namespace: default
-prometheus: monitoring/k8s
-Annotations:
-message: Job default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete.
-runbook_url: here

""", # noqa + "message": '

Job default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete.

\n

Severity: warning ⚠️
\nStatus: firing 🔥 (on the source)

\n

📖 Runbook🔗

\n

🏷️ Labels:

\n
    \n
  • job: kube-state-metrics
    • \n
  • instance: 10.143.139.7:8443
    • \n
  • job_name: email-tracking-perform-initialization-1.0.50
    • \n
  • severity: warning
    • \n
  • alertname: KubeJobCompletion
    • \n
  • namespace: default
    • \n
  • prometheus: monitoring/k8s
    • \n
', # noqa "image_url": None, }, "sms": { @@ -171,20 +261,7 @@ tests = { }, "telegram": { "title": "KubeJobCompletion", - "message": ( - "Status: firing\n" - "Labels: \n" - "job: kube-state-metrics\n" - "instance: 10.143.139.7:8443\n" - "job_name: email-tracking-perform-initialization-1.0.50\n" - "severity: warning\n" - "alertname: KubeJobCompletion\n" - "namespace: default\n" - "prometheus: monitoring/k8s\n" - "Annotations:\n" - "message: Job default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete.\n\n" - "runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion\n" - ), + "message": "\nJob default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete.\n\nSeverity: warning ⚠️\nStatus: firing 🔥 (on the source)\n\n📖 Runbook🔗\n\n🏷️ Labels:\n- job: kube-state-metrics\n- instance: 10.143.139.7:8443\n- job_name: email-tracking-perform-initialization-1.0.50\n- severity: warning\n- alertname: KubeJobCompletion\n- namespace: default\n- prometheus: monitoring/k8s\n\n", # noqa "image_url": None, }, } @@ -196,11 +273,12 @@ example_payload = { "alerts": [ { "status": "firing", - "labels": { - "alertname": "TestAlert", - "region": "eu-1", + "labels": {"alertname": "TestAlert", "region": "eu-1", "severity": "critical"}, + "annotations": { + "message": "This is test alert", + "description": "This alert was sent by user for the demonstration purposes", + "runbook_url": "https://grafana.com/", }, - "annotations": {"description": "This alert was sent by user for the demonstration purposes"}, "startsAt": "2018-12-25T15:47:47.377363608Z", "endsAt": "0001-01-01T00:00:00Z", "generatorURL": "",