centralcloud/oncall-engine
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor alertmanager templates (#1944)

Browse source 
# What this PR does

This PR changes default templates for alertmanager:

Web:
<img width="460" alt="Screenshot 2023-05-17 at 6 20 06 PM"
src="https://github.com/grafana/oncall/assets/2262529/1c63af70-6636-4d4a-bd9f-853d6e1e51e7">

Slack:
<img width="595" alt="Screenshot 2023-05-17 at 6 17 30 PM"
src="https://github.com/grafana/oncall/assets/2262529/893758ff-999b-40d0-a5c1-db12f2c1e534">

Telegram:
<img width="503" alt="Screenshot 2023-05-17 at 6 16 46 PM"
src="https://github.com/grafana/oncall/assets/2262529/70906c7d-a2e5-4d45-b973-f491eb9a70d7">


MS teams:
![Uploading Screenshot 2023-05-23 at 9.46.21 AM.png…]()

## Which issue(s) this PR fixes

## Checklist

- [ ] Unit, integration, and e2e (if applicable) tests updated
- [ ] Documentation added (or `pr:no public docs` PR label added if not
required)
- [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not
required)
This commit is contained in:
Ildar Iskhakov 2023-05-23 09:56:33 +08:00 • committed by GitHub
parent 53d34164ef
commit f04430568f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 186 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

294
engine/config_integrations/alertmanager.py
View file

@ -20,10 +20,67 @@ Alerts from Grafana Alertmanager are automatically routed to this integration.
<br>Creating contact points and routes for other alertmanagers...
{% endif %}"""
# Default templates
# Web
web_title = """{{- payload.get("labels", {}).get("alertname", "No title (check Title Template)") -}}"""
web_message = """\
{%- set annotations = payload.annotations.copy() -%}
{%- set labels = payload.labels.copy() -%}
{%- if "summary" in annotations %}
{{ annotations.summary }}
{%- set _ = annotations.pop('summary') -%}
{%- endif %}
{%- if "message" in annotations %}
{{ annotations.message }}
{%- set _ = annotations.pop('message') -%}
{%- endif %}
{% set severity = labels.severity | default("Unknown") -%}
{%- set severity_emoji = {"critical": ":rotating_light:", "warning": ":warning:" }[severity] | default(":question:") -%}
Severity: {{ severity }} {{ severity_emoji }}
{%- set status = payload.status | default("Unknown") %}
{%- set status_emoji = {"firing": ":fire:", "resolved": ":white_check_mark:"}[status] | default(":warning:") %}
Status: {{ status }} {{ status_emoji }} (on the source)
{% if "runbook_url" in annotations -%}
[:book: Runbook:link:]({{ annotations.runbook_url }})
{%- set _ = annotations.pop('runbook_url') -%}
{%- endif %}
{%- if "runbook_url_internal" in annotations -%}
[:closed_book: Runbook (internal):link:]({{ annotations.runbook_url_internal }})
{%- set _ = annotations.pop('runbook_url_internal') -%}
{%- endif %}
:label: Labels:
{%- for k, v in payload["labels"].items() %}
- {{ k }}: {{ v }}
{%- endfor %}
{% if annotations | length > 0 -%}
:pushpin: Other annotations:
{%- for k, v in annotations.items() %}
- {{ k }}: {{ v }}
{%- endfor %}
{% endif %}
""" # noqa: W291
web_image_url = None
# Behaviour
source_link = "{{ payload.generatorURL }}"
grouping_id = "{{ payload.labels }}"
resolve_condition = """{{ payload.status == "resolved" }}"""
acknowledge_condition = None
# Slack
slack_title = """\
{# Usually title is located in payload.labels.alertname #}
{% set title = payload.get("labels", {}).get("alertname", "No title (check Web Title Template)") %}
{% set title = payload.get("labels", {}).get("alertname", "No title (check Title Template)") %}
{# Combine the title from different built-in variables into slack-formatted url #}
*<{{ grafana_oncall_link }}|#{{ grafana_oncall_incident_id }} {{ title }}>* via {{ integration_name }}
{% if source_link %}
@ -31,74 +88,131 @@ slack_title = """\
{%- endif %}
"""
slack_message = """\
{{- payload.message }}
{%- if "status" in payload -%}
*Status*: {{ payload.status }}
{% endif -%}
*Labels:* {% for k, v in payload["labels"].items() %}
{{ k }}: {{ v }}{% endfor %}
*Annotations:*
{%- for k, v in payload.get("annotations", {}).items() %}
{#- render annotation as slack markdown url if it starts with http #}
{{ k }}: {% if v.startswith("http") %} <{{v}}|here> {% else %} {{v}} {% endif -%}
{% endfor %}
""" # noqa: W291
# default slack message template is identical to web message template, except urls
# It can be based on web message template (see example), but it can affect existing templates
# slack_message = """
# {% set mkdwn_link_regex = "\[([\w\s\d:]+)\]\((https?:\/\/[\w\d./?=#]+)\)" %}
# {{ web_message
# | regex_replace(mkdwn_link_regex, "<\\2|\\1>")
# }}
# """
slack_message = """\
{%- set annotations = payload.annotations.copy() -%}
{%- set labels = payload.labels.copy() -%}
{%- if "summary" in annotations %}
{{ annotations.summary }}
{%- set _ = annotations.pop('summary') -%}
{%- endif %}
{%- if "message" in annotations %}
{{ annotations.message }}
{%- set _ = annotations.pop('message') -%}
{%- endif %}
{# Optionally set oncall_slack_user_group to slack user group in the following format "@users-oncall" #}
{%- set oncall_slack_user_group = None -%}
{%- if oncall_slack_user_group %}
Heads up {{ oncall_slack_user_group }}
{%- endif %}
{% set severity = labels.severity | default("Unknown") -%}
{%- set severity_emoji = {"critical": ":rotating_light:", "warning": ":warning:" }[severity] | default(":question:") -%}
Severity: {{ severity }} {{ severity_emoji }}
{%- set status = payload.status | default("Unknown") %}
{%- set status_emoji = {"firing": ":fire:", "resolved": ":white_check_mark:"}[status] | default(":warning:") %}
Status: {{ status }} {{ status_emoji }} (on the source)
{% if "runbook_url" in annotations -%}
<{{ annotations.runbook_url }}|:book: Runbook:link:>
{%- set _ = annotations.pop('runbook_url') -%}
{%- endif %}
{%- if "runbook_url_internal" in annotations -%}
<{{ annotations.runbook_url_internal }}|:closed_book: Runbook (internal):link:>
{%- set _ = annotations.pop('runbook_url_internal') -%}
{%- endif %}
:label: Labels:
{%- for k, v in payload["labels"].items() %}
- {{ k }}: {{ v }}
{%- endfor %}
{% if annotations | length > 0 -%}
:pushpin: Other annotations:
{%- for k, v in annotations.items() %}
- {{ k }}: {{ v }}
{%- endfor %}
{% endif %}
""" # noqa: W291
slack_image_url = None
web_title = """\
{# Usually title is located in payload.labels.alertname #}
{{- payload.get("labels", {}).get("alertname", "No title (check Web Title Template)") }}
"""
# SMS
sms_title = web_title
web_message = """\
{{- payload.message }}
{%- if "status" in payload %}
**Status**: {{ payload.status }}
{% endif -%}
**Labels:** {% for k, v in payload["labels"].items() %}
*{{ k }}*: {{ v }}{% endfor %}
**Annotations:**
{%- for k, v in payload.get("annotations", {}).items() %}
{#- render annotation as markdown url if it starts with http #}
*{{ k }}*: {% if v.startswith("http") %} [here]({{v}}){% else %} {{v}} {% endif -%}
{% endfor %}
# Phone
phone_call_title = web_title
# Telegram
telegram_title = web_title
# default telegram message template is identical to web message template, except urls
# It can be based on web message template (see example), but it can affect existing templates
# telegram_message = """
# {% set mkdwn_link_regex = "\[([\w\s\d:]+)\]\((https?:\/\/[\w\d./?=#]+)\)" %}
# {{ web_message
# | regex_replace(mkdwn_link_regex, "<a href='\\2'>\\1</a>")
# }}
# """
telegram_message = """\
{%- set annotations = payload.annotations.copy() -%}
{%- set labels = payload.labels.copy() -%}
{%- if "summary" in annotations %}
{{ annotations.summary }}
{%- set _ = annotations.pop('summary') -%}
{%- endif %}
{%- if "message" in annotations %}
{{ annotations.message }}
{%- set _ = annotations.pop('message') -%}
{%- endif %}
{% set severity = labels.severity | default("Unknown") -%}
{%- set severity_emoji = {"critical": ":rotating_light:", "warning": ":warning:" }[severity] | default(":question:") -%}
Severity: {{ severity }} {{ severity_emoji }}
{%- set status = payload.status | default("Unknown") %}
{%- set status_emoji = {"firing": ":fire:", "resolved": ":white_check_mark:"}[status] | default(":warning:") %}
Status: {{ status }} {{ status_emoji }} (on the source)
{% if "runbook_url" in annotations -%}
<a href='{{ annotations.runbook_url }}'>:book: Runbook:link:</a>
{%- set _ = annotations.pop('runbook_url') -%}
{%- endif %}
{%- if "runbook_url_internal" in annotations -%}
<a href='{{ annotations.runbook_url_internal }}'>:closed_book: Runbook (internal):link:</a>
{%- set _ = annotations.pop('runbook_url_internal') -%}
{%- endif %}
:label: Labels:
{%- for k, v in payload["labels"].items() %}
- {{ k }}: {{ v }}
{%- endfor %}
{% if annotations | length > 0 -%}
:pushpin: Other annotations:
{%- for k, v in annotations.items() %}
- {{ k }}: {{ v }}
{%- endfor %}
{% endif %}
""" # noqa: W291
web_image_url = slack_image_url
sms_title = '{{ payload.get("labels", {}).get("alertname", "Title undefined") }}'
phone_call_title = sms_title
telegram_title = sms_title
telegram_message = """\
{{- payload.messsage }}
{%- if "status" in payload -%}
<b>Status</b>: {{ payload.status }}
{% endif -%}
<b>Labels:</b> {% for k, v in payload["labels"].items() %}
{{ k }}: {{ v }}{% endfor %}
<b>Annotations:</b>
{%- for k, v in payload.get("annotations", {}).items() %}
{#- render annotation as markdown url if it starts with http #}
{{ k }}: {{ v }}
{% endfor %}""" # noqa: W291
telegram_image_url = slack_image_url
source_link = "{{ payload.generatorURL }}"
grouping_id = "{{ payload.labels }}"
resolve_condition = """\
{{ payload.get("status", "") == "resolved" }}
"""
acknowledge_condition = None
telegram_image_url = None
tests = {
"payload": {
@ -131,36 +245,12 @@ tests = {
"+-+kube_job_status_succeeded%7Bjob%3D%22kube-state-metrics%22%7D+%3E+0&g0.tab=1"
"|source>*)"
),
"message": (
"*Status*: firing\n"
"*Labels:* \n"
"job: kube-state-metrics\n"
"instance: 10.143.139.7:8443\n"
"job_name: email-tracking-perform-initialization-1.0.50\n"
"severity: warning\n"
"alertname: KubeJobCompletion\n"
"namespace: default\n"
"prometheus: monitoring/k8s\n"
"*Annotations:*\n"
"message: Job default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete. \n"
"runbook_url: <https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion|here> "
),
"message": "\nJob default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete.\n\n\n\nSeverity: warning :warning:\nStatus: firing :fire: (on the source)\n\n<https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion|:book: Runbook:link:>\n\n:label: Labels:\n- job: kube-state-metrics\n- instance: 10.143.139.7:8443\n- job_name: email-tracking-perform-initialization-1.0.50\n- severity: warning\n- alertname: KubeJobCompletion\n- namespace: default\n- prometheus: monitoring/k8s\n\n", # noqa
"image_url": None,
},
"web": {
"title": "KubeJobCompletion",
"message": """<p><strong>Status</strong>: firing <br/>
<strong>Labels:</strong> <br/>
<em>job</em>: kube-state-metrics <br/>
<em>instance</em>: 10.143.139.7:8443 <br/>
<em>job_name</em>: email-tracking-perform-initialization-1.0.50 <br/>
<em>severity</em>: warning <br/>
<em>alertname</em>: KubeJobCompletion <br/>
<em>namespace</em>: default <br/>
<em>prometheus</em>: monitoring/k8s <br/>
<strong>Annotations:</strong> <br/>
<em>message</em>: Job default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete. <br/>
<em>runbook_url</em>: <a href="https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion" rel="nofollow noopener" target="_blank">here</a></p>""", # noqa
"message": '<p>Job default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete. </p>\n<p>Severity: warning ⚠️ <br/>\nStatus: firing 🔥 (on the source) </p>\n<p><a href="https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion" rel="nofollow noopener" target="_blank">📖 Runbook🔗</a> </p>\n<p>🏷️ Labels: </p>\n<ul>\n<li>job: kube-state-metrics </li>\n<li>instance: 10.143.139.7:8443 </li>\n<li>job_name: email-tracking-perform-initialization-1.0.50 </li>\n<li>severity: warning </li>\n<li>alertname: KubeJobCompletion </li>\n<li>namespace: default </li>\n<li>prometheus: monitoring/k8s </li>\n</ul>', # noqa
"image_url": None,
},
"sms": {
@ -171,20 +261,7 @@ tests = {
},
"telegram": {
"title": "KubeJobCompletion",
"message": (
"<b>Status</b>: firing\n"
"<b>Labels:</b> \n"
"job: kube-state-metrics\n"
"instance: 10.143.139.7:8443\n"
"job_name: email-tracking-perform-initialization-1.0.50\n"
"severity: warning\n"
"alertname: KubeJobCompletion\n"
"namespace: default\n"
"prometheus: monitoring/k8s\n"
"<b>Annotations:</b>\n"
"message: Job default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete.\n\n"
"runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion\n"
),
"message": "\nJob default/email-tracking-perform-initialization-1.0.50 is taking more than one hour to complete.\n\nSeverity: warning ⚠️\nStatus: firing 🔥 (on the source)\n\n<a href='https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion'>📖 Runbook🔗</a>\n\n🏷️ Labels:\n- job: kube-state-metrics\n- instance: 10.143.139.7:8443\n- job_name: email-tracking-perform-initialization-1.0.50\n- severity: warning\n- alertname: KubeJobCompletion\n- namespace: default\n- prometheus: monitoring/k8s\n\n", # noqa
"image_url": None,
},
}
@ -196,11 +273,12 @@ example_payload = {
"alerts": [
{
"status": "firing",
"labels": {
"alertname": "TestAlert",
"region": "eu-1",
"labels": {"alertname": "TestAlert", "region": "eu-1", "severity": "critical"},
"annotations": {
"message": "This is test alert",
"description": "This alert was sent by user for the demonstration purposes",
"runbook_url": "https://grafana.com/",
},
"annotations": {"description": "This alert was sent by user for the demonstration purposes"},
"startsAt": "2018-12-25T15:47:47.377363608Z",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",