Merge pull request #4985 from grafana/dev

v1.9.22

.github/actions/install-frontend-dependencies/action.yml

@@ -1,5 +1,10 @@
 name: Install frontend dependencies
 description: Setup node/pnpm + install frontend dependencies
+inputs:
+  oncall-directory:
+    description: "Relative path to oncall directory"
+    required: false
+    default: "."
 runs:
   using: composite
   steps:
@@ -7,12 +12,22 @@ runs:
       uses: pnpm/action-setup@v4
       with:
         version: 9.1.4
+    - name: Determine grafana-plugin directory location
+      id: grafana-plugin-directory
+      shell: bash
+      run: echo "grafana-plugin-directory=${{ inputs.oncall-directory }}/grafana-plugin" >> $GITHUB_OUTPUT
+    - name: Determine pnpm-lock.yaml location
+      id: pnpm-lock-location
+      shell: bash
+      # yamllint disable rule:line-length
+      run: echo "pnpm-lock-location=${{ steps.grafana-plugin-directory.outputs.grafana-plugin-directory }}/pnpm-lock.yaml" >> $GITHUB_OUTPUT
+      # yamllint enable rule:line-length
     - uses: actions/setup-node@v4
       with:
         node-version: 20.15.1
         cache: pnpm
-        cache-dependency-path: grafana-plugin/pnpm-lock.yaml
+        cache-dependency-path: ${{ steps.pnpm-lock-location.outputs.pnpm-lock-location }}
     - name: Install frontend dependencies
       shell: bash
-      working-directory: grafana-plugin
+      working-directory: ${{ steps.grafana-plugin-directory.outputs.grafana-plugin-directory }}
       run: pnpm install --frozen-lockfile --prefer-offline

docs/sources/configure/jinja2-templating/advanced-templates/index.md

@@ -95,6 +95,8 @@ Grafana OnCall enhances Jinja with additional functions:
 - `regex_replace`: Performs a regex find and replace
 - `regex_match`: Performs a regex match, returns `True` or `False`
   - Usage example: `{{ payload.ruleName | regex_match(".*") }}`
+- `regex_search`: Performs a regex search, returns `True` or `False`
+  - Usage example: `{{ payload.message | regex_search("Severity: (High|Critical)") }}`
 - `b64decode`: Performs a base64 string decode
   - Usage example: `{{ payload.data | b64decode }}`
 - `parse_json`:Parses a JSON string to an object

engine/apps/email/tasks.py

@@ -21,7 +21,7 @@ def get_from_email(user):
         return live_settings.EMAIL_FROM_ADDRESS
 
     if settings.LICENSE == settings.CLOUD_LICENSE_NAME:
-        return "oncall@{}.grafana.net".format(user.organization.stack_slug)
+        return "oncall@{}.{}".format(user.organization.stack_slug, settings.EMAIL_FROM_DOMAIN)
 
     return live_settings.EMAIL_HOST_USER
 

engine/common/jinja_templater/filters.py

@@ -1,11 +1,13 @@
 import base64
 import json
-import re
 from datetime import datetime
 
+import regex
 from django.utils.dateparse import parse_datetime
 from pytz import timezone
 
+REGEX_TIMEOUT = 2
+
 
 def datetimeparse(value, format="%H:%M / %d-%m-%Y"):
     try:
@@ -52,22 +54,22 @@ def json_dumps(value):
 
 def regex_replace(value, find, replace):
     try:
-        return re.sub(find, replace, value)
-    except (ValueError, AttributeError, TypeError):
+        return regex.sub(find, replace, value, timeout=REGEX_TIMEOUT)
+    except (ValueError, AttributeError, TypeError, TimeoutError):
         return None
 
 
 def regex_match(pattern, value):
     try:
-        return bool(re.match(value, pattern))
-    except (ValueError, AttributeError, TypeError):
+        return bool(regex.match(value, pattern, timeout=REGEX_TIMEOUT))
+    except (ValueError, AttributeError, TypeError, TimeoutError):
         return None
 
 
 def regex_search(pattern, value):
     try:
-        return bool(re.search(value, pattern))
-    except (ValueError, AttributeError, TypeError):
+        return bool(regex.search(value, pattern, timeout=REGEX_TIMEOUT))
+    except (ValueError, AttributeError, TypeError, TimeoutError):
         return None
 
 

engine/common/tests/test_apply_jinja_template.py

@@ -15,6 +15,38 @@ from common.jinja_templater.apply_jinja_template import (
     templated_value_is_truthy,
 )
 
+EMAIL_SAMPLE_PAYLOAD = {
+    "subject": "[Reminder] Review GKE getServerConfig API permission changes",
+    "message": "Hello Google Kubernetes Customer,\r\n"
+    "\r\n"
+    "We’re writing to remind you that starting October 22, 2024, "
+    "the  \r\n"
+    "getServerConfig API for Google Kubernetes Engine (GKE) will "
+    "enforce  \r\n"
+    "Identity and Access Management (IAM) container.clusters.list "
+    "checks. This  \r\n"
+    "change follows a series of security improvements as IAM  \r\n"
+    "container.clusters.list permissions are being enforced across "
+    "the  \r\n"
+    "getServerConfig API.\r\n"
+    "\r\n"
+    "We’ve provided additional information below to guide you through "
+    "this  \r\n"
+    "change.\r\n"
+    "\r\n"
+    "What you need to know\r\n"
+    "\r\n"
+    "The current implementation doesn’t apply a specific permissions "
+    "check via  \r\n"
+    "getServerConfig API. After this change goes into effect for the "
+    "Google  \r\n"
+    "Kubernetes Engine API getServerConfig, only authorized users with "
+    "the  \r\n"
+    "container.clusters.list permissions will be able to call the  \r\n"
+    "GetServerConfig.\r\n",
+    "sender": "someone@somewhere.dev",
+}
+
 
 def test_apply_jinja_template():
     payload = {"name": "test"}
@@ -127,25 +159,49 @@ def test_apply_jinja_template_json_dumps():
     assert result == expected
 
 
+@pytest.mark.filterwarnings("ignore:::jinja2.*")  # ignore regex escape sequence warning
 def test_apply_jinja_template_regex_match():
-    payload = {"name": "test"}
+    payload = {
+        "name": "test",
+        "message": json.dumps(EMAIL_SAMPLE_PAYLOAD),
+    }
 
     assert apply_jinja_template("{{ payload.name | regex_match('.*') }}", payload) == "True"
     assert apply_jinja_template("{{ payload.name | regex_match('tes') }}", payload) == "True"
     assert apply_jinja_template("{{ payload.name | regex_match('test1') }}", payload) == "False"
+    # check for timeouts
+    with patch("common.jinja_templater.filters.REGEX_TIMEOUT", 1):
+        assert (
+            apply_jinja_template(
+                "{{ payload.message | regex_match('(.|\\s)+Severity(.|\\s){2}High(.|\\s)+') }}", payload
+            )
+            == "False"
+        )
 
     # Check that exception is raised when regex is invalid
     with pytest.raises(JinjaTemplateError):
         apply_jinja_template("{{ payload.name | regex_match('*') }}", payload)
 
 
+@pytest.mark.filterwarnings("ignore:::jinja2.*")  # ignore regex escape sequence warning
 def test_apply_jinja_template_regex_search():
-    payload = {"name": "test"}
+    payload = {
+        "name": "test",
+        "message": json.dumps(EMAIL_SAMPLE_PAYLOAD),
+    }
 
     assert apply_jinja_template("{{ payload.name | regex_search('.*') }}", payload) == "True"
     assert apply_jinja_template("{{ payload.name | regex_search('tes') }}", payload) == "True"
     assert apply_jinja_template("{{ payload.name | regex_search('est') }}", payload) == "True"
     assert apply_jinja_template("{{ payload.name | regex_search('test1') }}", payload) == "False"
+    # check for timeouts
+    with patch("common.jinja_templater.filters.REGEX_TIMEOUT", 1):
+        assert (
+            apply_jinja_template(
+                "{{ payload.message | regex_search('(.|\\s)+Severity(.|\\s){2}High(.|\\s)+') }}", payload
+            )
+            == "False"
+        )
 
     # Check that exception is raised when regex is invalid
     with pytest.raises(JinjaTemplateError):

engine/pyproject.toml

@@ -71,6 +71,7 @@ module = [
     "polymorphic.*",
     "pyroscope.*",
     "ratelimit.*",
+    "regex.*",
     "recurring_ical_events.*",
     "rest_polymorphic.*",
     "slackclient.*",

engine/requirements.in

@@ -1,7 +1,7 @@
 babel==2.12.1
 beautifulsoup4==4.12.2
 celery[redis]==5.3.1
-cryptography==42.0.8
+cryptography==43.0.1
 django==4.2.15
 django-add-default-value==0.10.0
 django-amazon-ses==4.0.1
@@ -52,7 +52,7 @@ PyMySQL==1.1.1
 python-telegram-bot==13.13
 recurring-ical-events==2.1.0
 redis==5.0.1
-regex==2021.11.2
+regex==2024.7.24
 requests==2.32.3
 slack-export-viewer==1.1.4
 slack_sdk==3.21.3

engine/requirements.txt

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile requirements.in -o requirements.txt
+#    uv pip compile engine/requirements.in -o engine/requirements.txt
 amqp==5.2.0
     # via kombu
 appdirs==1.4.4
@@ -15,9 +15,9 @@ attrs==23.2.0
 autopep8==2.0.4
     # via django-silk
 babel==2.12.1
-    # via -r requirements.in
+    # via -r engine/requirements.in
 beautifulsoup4==4.12.2
-    # via -r requirements.in
+    # via -r engine/requirements.in
 billiard==4.2.0
     # via celery
 blinker==1.7.0
@@ -35,7 +35,7 @@ cachetools==4.2.2
     #   google-auth
     #   python-telegram-bot
 celery==5.3.1
-    # via -r requirements.in
+    # via -r engine/requirements.in
 certifi==2024.7.4
     # via
     #   python-telegram-bot
@@ -60,9 +60,9 @@ click-plugins==1.1.1
     # via celery
 click-repl==0.3.0
     # via celery
-cryptography==42.0.8
+cryptography==43.0.1
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   django-mirage-field
     #   pyopenssl
     #   social-auth-core
@@ -77,7 +77,7 @@ deprecated==1.2.14
     #   opentelemetry-semantic-conventions
 django==4.2.15
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   django-add-default-value
     #   django-amazon-ses
     #   django-anymail
@@ -98,62 +98,62 @@ django==4.2.15
     #   fcm-django
     #   social-auth-app-django
 django-add-default-value==0.10.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-amazon-ses==4.0.1
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-anymail==11.1
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-cors-headers==3.7.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-dbconn-retry==0.1.7
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-debug-toolbar==4.1.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-deprecate-fields==0.1.1
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-filter==2.4.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-ipware==4.0.2
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-log-request-id==1.6.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-migration-linter==4.1.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-mirage-field==1.3.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-mysql==4.6.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-polymorphic==3.1.0
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   django-rest-polymorphic
 django-ratelimit==2.0.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-redis==5.4.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-rest-polymorphic==0.1.10
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-silk==5.0.3
-    # via -r requirements.in
+    # via -r engine/requirements.in
 django-sns-view==0.1.2
-    # via -r requirements.in
+    # via -r engine/requirements.in
 djangorestframework==3.15.2
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   django-rest-polymorphic
     #   drf-spectacular
 drf-spectacular==0.26.5
-    # via -r requirements.in
+    # via -r engine/requirements.in
 emoji==2.4.0
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   slack-export-viewer
 factory-boy==2.12.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 faker==23.1.0
     # via factory-boy
 fcm-django @ https://github.com/grafana/fcm-django/archive/refs/tags/v1.0.12r1.tar.gz#sha256=7ec7cd9d353fc9edf19a4acd4fa14090a31d83d02ac986c5e5e081dea29f564f
-    # via -r requirements.in
+    # via -r engine/requirements.in
 firebase-admin==5.4.0
     # via fcm-django
 flask==3.0.2
@@ -167,7 +167,7 @@ google-api-core==2.17.0
     #   google-cloud-storage
 google-api-python-client==2.122.0
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   firebase-admin
 google-auth==2.27.0
     # via
@@ -179,10 +179,10 @@ google-auth==2.27.0
     #   google-cloud-storage
 google-auth-httplib2==0.2.0
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   google-api-python-client
 google-auth-oauthlib==1.2.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 google-cloud-core==2.4.1
     # via
     #   google-cloud-firestore
@@ -206,28 +206,28 @@ gprof2dot==2022.7.29
     # via django-silk
 grpcio==1.64.1
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   google-api-core
     #   grpcio-status
     #   opentelemetry-exporter-otlp-proto-grpc
 grpcio-status==1.57.0
     # via google-api-core
 hiredis==2.2.3
-    # via -r requirements.in
+    # via -r engine/requirements.in
 httplib2==0.22.0
     # via
     #   google-api-python-client
     #   google-auth-httplib2
 humanize==4.10.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 icalendar==5.0.10
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   recurring-ical-events
     #   x-wr-timezone
 idna==3.7
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   requests
 importlib-metadata==6.11.0
     # via opentelemetry-api
@@ -248,12 +248,12 @@ jsonschema-specifications==2023.12.1
 kombu==5.3.5
     # via celery
 lxml==5.2.2
-    # via -r requirements.in
+    # via -r engine/requirements.in
 markdown==3.5.2
     # via pymdown-extensions
 markdown2==2.4.10
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   slack-export-viewer
 markupsafe==2.1.5
     # via
@@ -267,7 +267,7 @@ oauthlib==3.2.2
     #   social-auth-core
 opentelemetry-api==1.26.0
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-instrumentation
     #   opentelemetry-instrumentation-django
@@ -279,7 +279,7 @@ opentelemetry-api==1.26.0
 opentelemetry-exporter-otlp-proto-common==1.26.0
     # via opentelemetry-exporter-otlp-proto-grpc
 opentelemetry-exporter-otlp-proto-grpc==1.26.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 opentelemetry-instrumentation==0.47b0
     # via
     #   opentelemetry-instrumentation-django
@@ -287,14 +287,14 @@ opentelemetry-instrumentation==0.47b0
     #   opentelemetry-instrumentation-requests
     #   opentelemetry-instrumentation-wsgi
 opentelemetry-instrumentation-django==0.47b0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 opentelemetry-instrumentation-logging==0.47b0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 opentelemetry-instrumentation-requests==0.47b0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 opentelemetry-instrumentation-wsgi==0.47b0
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   opentelemetry-instrumentation-django
 opentelemetry-proto==1.26.0
     # via
@@ -302,7 +302,7 @@ opentelemetry-proto==1.26.0
     #   opentelemetry-exporter-otlp-proto-grpc
 opentelemetry-sdk==1.26.0
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   opentelemetry-exporter-otlp-proto-grpc
 opentelemetry-semantic-conventions==0.47b0
     # via
@@ -318,9 +318,9 @@ opentelemetry-util-http==0.47b0
 pem==23.1.0
     # via django-sns-view
 phonenumbers==8.10.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 prometheus-client==0.16.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 prompt-toolkit==3.0.43
     # via click-repl
 proto-plus==1.23.0
@@ -334,9 +334,9 @@ protobuf==4.25.2
     #   opentelemetry-proto
     #   proto-plus
 psutil==5.9.4
-    # via -r requirements.in
+    # via -r engine/requirements.in
 psycopg2==2.9.3
-    # via -r requirements.in
+    # via -r engine/requirements.in
 pyasn1==0.5.1
     # via
     #   pyasn1-modules
@@ -352,10 +352,10 @@ pyjwt==2.8.0
     #   social-auth-core
     #   twilio
 pymdown-extensions==10.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 pymysql==1.1.1
-    # via -r requirements.in
-pyopenssl==24.1.0
+    # via -r engine/requirements.in
+pyopenssl==24.2.1
     # via django-sns-view
 pyparsing==3.1.1
     # via httplib2
@@ -367,7 +367,7 @@ python-dateutil==2.8.2
     #   icalendar
     #   recurring-ical-events
 python-telegram-bot==13.13
-    # via -r requirements.in
+    # via -r engine/requirements.in
 python3-openid==3.2.0
     # via social-auth-core
 pytz==2024.1
@@ -383,21 +383,21 @@ pyyaml==6.0.1
     #   drf-spectacular
     #   pymdown-extensions
 recurring-ical-events==2.1.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 redis==5.0.1
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   celery
     #   django-redis
 referencing==0.33.0
     # via
     #   jsonschema
     #   jsonschema-specifications
-regex==2021.11.2
-    # via -r requirements.in
+regex==2024.7.24
+    # via -r engine/requirements.in
 requests==2.32.3
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   cachecontrol
     #   django-anymail
     #   django-sns-view
@@ -429,11 +429,11 @@ six==1.16.0
     #   python-dateutil
     #   twilio
 slack-export-viewer==1.1.4
-    # via -r requirements.in
+    # via -r engine/requirements.in
 slack-sdk==3.21.3
-    # via -r requirements.in
+    # via -r engine/requirements.in
 social-auth-app-django==5.4.1
-    # via -r requirements.in
+    # via -r engine/requirements.in
 social-auth-core==4.5.2
     # via social-auth-app-django
 soupsieve==2.5
@@ -450,7 +450,7 @@ tornado==6.4.1
 tqdm==4.66.3
     # via django-mirage-field
 twilio==6.37.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 typing-extensions==4.9.0
     # via opentelemetry-sdk
 tzdata==2024.1
@@ -463,12 +463,12 @@ uritemplate==4.1.1
     #   google-api-python-client
 urllib3==1.26.19
     # via
-    #   -r requirements.in
+    #   -r engine/requirements.in
     #   botocore
     #   django-anymail
     #   requests
 uwsgi==2.0.26
-    # via -r requirements.in
+    # via -r engine/requirements.in
 vine==5.1.0
     # via
     #   amqp
@@ -479,7 +479,7 @@ wcwidth==0.2.13
 werkzeug==3.0.3
     # via flask
 whitenoise==5.3.0
-    # via -r requirements.in
+    # via -r engine/requirements.in
 wrapt==1.16.0
     # via
     #   deprecated

engine/settings/base.py

@@ -844,6 +844,7 @@ EMAIL_PORT = getenv_integer("EMAIL_PORT", 587)
 EMAIL_USE_TLS = getenv_boolean("EMAIL_USE_TLS", True)
 EMAIL_USE_SSL = getenv_boolean("EMAIL_USE_SSL", False)
 EMAIL_FROM_ADDRESS = os.getenv("EMAIL_FROM_ADDRESS")
+EMAIL_FROM_DOMAIN = os.getenv("EMAIL_FROM_DOMAIN", "grafana.net")
 EMAIL_NOTIFICATIONS_LIMIT = getenv_integer("EMAIL_NOTIFICATIONS_LIMIT", 200)
 
 EMAIL_BACKEND_INTERNAL_ID = 8

grafana-plugin/src/components/CheatSheet/CheatSheet.config.ts

@@ -21,7 +21,7 @@ export const groupingTemplateCheatSheet: CheatSheetInterface = {
       name: 'Additional variables and functions',
       listItems: [
         { listItemName: 'time(), datetimeformat, iso8601_to_time' },
-        { listItemName: 'regex_replace, regex_match' },
+        { listItemName: 'regex_replace, regex_match, regex_search' },
       ],
     },
     {
@@ -86,7 +86,7 @@ export const genericTemplateCheatSheet: CheatSheetInterface = {
         { listItemName: 'payload, grafana_oncall_link, grafana_oncall_incident_id, integration_name, source_link' },
         { listItemName: 'time(), datetimeformat, datetimeformat_as_timezone, datetimeparse, iso8601_to_time' },
         { listItemName: 'to_pretty_json' },
-        { listItemName: 'regex_replace, regex_match' },
+        { listItemName: 'regex_replace, regex_match, regex_search' },
         { listItemName: 'b64decode' },
       ],
     },
@@ -143,7 +143,7 @@ export const slackMessageTemplateCheatSheet: CheatSheetInterface = {
         { listItemName: 'payload, grafana_oncall_link, grafana_oncall_incident_id, integration_name, source_link' },
         { listItemName: 'time(), datetimeformat, iso8601_to_time' },
         { listItemName: 'to_pretty_json' },
-        { listItemName: 'regex_replace, regex_match' },
+        { listItemName: 'regex_replace, regex_match, regex_search' },
       ],
     },
     {

grafana-plugin/src/plugin.json

@@ -616,6 +616,10 @@
         "action": "plugins.app:access",
         "scope": "plugins:id:grafana-labels-app"
       },
+      {
+        "action": "plugins.app:access",
+        "scope": "plugins:id:grafana-ml-app"
+      },
       {
         "action": "plugins:write",
         "scope": "plugins:id:grafana-oncall-app"