From e16064c6bafea233df9979e27d38b8f9f3d382e9 Mon Sep 17 00:00:00 2001 From: Joey Orlando Date: Thu, 22 Sep 2022 15:17:17 +0200 Subject: [PATCH] #539 - add slack permalink to alert group public API response (#543) * add .python-version to .gitignore * add .nvmrc to frontend Also update DEVELOPER.md to mention optionally using nvm * update DEVELOPER.md to reflect running successfully locally * markdown autoformatter styling changes * add slack permalink to alertgroup public api http response * update changelog * address PR comments - rename permalink to permalinks in alert group public api seralizer - add permalinks property to AlertGroup model - update public api alert groups test - update alertgroups public documentation to include permalinks property * add default DEBUG = True in dev.py settings --- .gitignore | 1 + CHANGELOG.md | 31 +++++++ DEVELOPER.md | 92 ++++++++++++------- .../oncall-api-reference/alertgroups.md | 5 +- engine/apps/alerts/models/alert_group.py | 12 +++ .../apps/public_api/serializers/incidents.py | 3 +- .../apps/public_api/tests/test_incidents.py | 3 + engine/settings/dev.py | 2 + grafana-plugin/.nvmrc | 1 + 9 files changed, 113 insertions(+), 37 deletions(-) create mode 100644 grafana-plugin/.nvmrc diff --git a/.gitignore b/.gitignore index b00b88a2..cadd75d3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ */db.sqlite3 *.pyc venv +.python-version .env .env_hobby .vscode diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ce5d9b7..c13f348c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,71 +1,95 @@ # Change Log +## v1.0.37 (2022-09-21) + +- Add `permalinks` property to `AlertGroup` public API response schema + ## v1.0.36 (2022-09-12) + - Alpha web schedules frontend/backend updates - Bug fixes ## v1.0.35 (2022-09-07) + - Bug fixes ## v1.0.34 (2022-09-06) + - Fix schedule notification spam ## v1.0.33 (2022-09-06) + - Add raw alert view - Add GitHub star button for OSS installations - Restore alert group search functionality - Bug fixes ## v1.0.32 (2022-09-01) + - Bug fixes ## v1.0.31 (2022-09-01) + - Bump celery version - Fix oss to cloud connection ## v1.0.30 (2022-08-31) + - Bug fix: check user notification policy before access ## v1.0.29 (2022-08-31) + - Add arm64 docker image ## v1.0.28 (2022-08-31) + - Bug fixes ## v1.0.27 (2022-08-30) + - Bug fixes ## v1.0.26 (2022-08-26) + - Insight log's format fixes - Remove UserNotificationPolicy auto-recreating ## v1.0.25 (2022-08-24) + - Bug fixes ## v1.0.24 (2022-08-24) + - Insight logs - Default DATA_UPLOAD_MAX_MEMORY_SIZE to 1mb ## v1.0.23 (2022-08-23) + - Bug fixes ## v1.0.22 (2022-08-16) + - Make STATIC_URL configurable from environment variable ## v1.0.21 (2022-08-12) + - Bug fixes ## v1.0.19 (2022-08-10) + - Bug fixes ## v1.0.15 (2022-08-03) + - Bug fixes ## v1.0.13 (2022-07-27) + - Optimize alert group list view - Fix a bug related to Twilio setup ## v1.0.12 (2022-07-26) + - Update push-notifications dependency - Rework how absolute URLs are built - Fix to show maintenance windows per team @@ -73,15 +97,18 @@ - Internal api to get a schedule final events ## v1.0.10 (2022-07-22) + - Speed-up of alert group web caching - Internal api for OnCall shifts ## v1.0.9 (2022-07-21) + - Frontend bug fixes & improvements - Support regex_replace() in templates - Bring back alert group caching and list view ## v1.0.7 (2022-07-18) + - Backend & frontend bug fixes - Deployment improvements - Reshape webhook payload for outgoing webhooks @@ -89,18 +116,22 @@ - Improve alert group list load speeds and simplify caching system ## v1.0.6 (2022-07-12) + - Manual Incidents enabled for teams - Fix phone notifications for OSS - Public API improvements ## v1.0.5 (2022-07-06) + - Bump Django to 3.2.14 - Fix PagerDuty iCal parsing ## 1.0.4 (2022-06-28) + - Allow Telegram DMs without channel connection. ## 1.0.3 (2022-06-27) + - Fix users public api endpoint. Now it returns users with all roles. - Fix redundant notifications about gaps in schedules. - Frontend fixes. diff --git a/DEVELOPER.md b/DEVELOPER.md index 6612a8e7..da536813 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -1,22 +1,22 @@ -* [Developer quickstart](#developer-quickstart) - * [Code style](#code-style) - * [Backend setup](#backend-setup) - * [Frontend setup](#frontend-setup) - * [Slack application setup](#slack-application-setup) - * [Update drone build](#update-drone-build) -* [Troubleshooting](#troubleshooting) - * [ld: library not found for -lssl](#ld-library-not-found-for--lssl) - * [Could not build wheels for cryptography which use PEP 517 and cannot be installed directly](#could-not-build-wheels-for-cryptography-which-use-pep-517-and-cannot-be-installed-directly) - * [django.db.utils.OperationalError: (1366, "Incorrect string value ...")](#djangodbutilsoperationalerror-1366-incorrect-string-value-) - * [Empty queryset when filtering against datetime field](#empty-queryset-when-filtering-against-datetime-field) -* [Hints](#hints) - * [Building the all-in-one docker container](#building-the-all-in-one-docker-container) - * [Running Grafana with plugin (frontend) folder mounted for dev purposes](#running-grafana-with-plugin-frontend-folder-mounted-for-dev-purposes) - * [How to recreate the local database](#recreating-the-local-database) - * [Running tests locally](#running-tests-locally) -* [IDE Specific Instructions](#ide-specific-instructions) - * [PyCharm](#pycharm) - +- [Developer quickstart](#developer-quickstart) + - [Code style](#code-style) + - [Backend setup](#backend-setup) + - [Frontend setup](#frontend-setup) + - [Slack application setup](#slack-application-setup) + - [Update drone build](#update-drone-build) +- [Troubleshooting](#troubleshooting) + - [ld: library not found for -lssl](#ld-library-not-found-for--lssl) + - [Could not build wheels for cryptography which use PEP 517 and cannot be installed directly](#could-not-build-wheels-for-cryptography-which-use-pep-517-and-cannot-be-installed-directly) + - [django.db.utils.OperationalError: (1366, "Incorrect string value ...")](#djangodbutilsoperationalerror-1366-incorrect-string-value-) + - [Empty queryset when filtering against datetime field](#empty-queryset-when-filtering-against-datetime-field) +- [Hints](#hints) + - [Building the all-in-one docker container](#building-the-all-in-one-docker-container) + - [Running Grafana with plugin (frontend) folder mounted for dev purposes](#running-grafana-with-plugin-frontend-folder-mounted-for-dev-purposes) + - [How to recreate the local database](#recreating-the-local-database) + - [Running tests locally](#running-tests-locally) +- [IDE Specific Instructions](#ide-specific-instructions) + - [PyCharm](#pycharm) + ## Developer quickstart ### Code style @@ -29,13 +29,23 @@ ### Backend setup 1. Start stateful services (RabbitMQ, Redis, Grafana with mounted plugin folder) + ```bash docker-compose -f docker-compose-developer.yml up -d ``` NOTE: to use a PostgreSQL db backend, use the `docker-compose-developer-pg.yml` file instead. -2. Prepare a python environment: +2. `postgres` is a dependency on some of our Python dependencies (notably `psycopg2` ([docs](https://www.psycopg.org/docs/install.html#prerequisites))). To install this on Mac you can simply run: + +```bash +brew install postgresql@14 +``` + +For non Mac installation please visit [here](https://www.postgresql.org/download/) for more information on how to install. + +3. Prepare a python environment: + ```bash # Create and activate the virtual environment python3.9 -m venv venv && source venv/bin/activate @@ -67,8 +77,8 @@ python manage.py migrate python manage.py createsuperuser ``` +4. Launch the backend: -3. Launch the backend: ```bash # Http server: python manage.py runserver 0.0.0.0:8080 @@ -80,14 +90,14 @@ python manage.py start_celery celery -A engine beat -l info ``` -4. All set! Check out internal API endpoints at http://localhost:8080/. - +5. All set! Check out internal API endpoints at http://localhost:8000/. ### Frontend setup -1. Make sure you have [NodeJS v.14+ < 17](https://nodejs.org/) and [yarn](https://yarnpkg.com/) installed. +1. Make sure you have [NodeJS v.14+ < 17](https://nodejs.org/) and [yarn](https://yarnpkg.com/) installed. **Note**: If you are using [`nvm`](https://github.com/nvm-sh/nvm) feel free to simply run `cd grafana-plugin && nvm install` to install the proper Node version. 2. Install the dependencies with `yarn` and launch the frontend server (on port `3000` by default) + ```bash cd grafana-plugin yarn install @@ -96,19 +106,21 @@ yarn watch ``` 3. Ensure /grafana-plugin/provisioning has no grafana-plugin.yml - 4. Generate an invitation token: + ```bash cd engine; python manage.py issue_invite_for_the_frontend --override ``` + ... or use output of all-in-one docker container described in the README.md. 5. Open Grafana in the browser http://localhost:3000 (login: oncall, password: oncall) notice OnCall Plugin is not enabled, navigate to Configuration->Plugins and click Grafana OnCall 6. Some configuration fields will appear be available. Fill them out and click Initialize OnCall + ``` -OnCall API URL: +OnCall API URL: http://host.docker.internal:8080 Invitation Token (Single use token to connect Grafana instance): @@ -120,6 +132,7 @@ http://localhost:3000 NOTE: you may not have `host.docker.internal` available, in that case you can get the host IP from inside the container by running: + ```bash /sbin/ip route|awk '/default/ { print $3 }' @@ -133,13 +146,14 @@ extra_hosts: For Slack app configuration check our docs: https://grafana.com/docs/grafana-cloud/oncall/open-source/#slack-setup - ### Update drone build -The .drone.yml build file must be signed when changes are made to it. Follow these steps: + +The .drone.yml build file must be signed when changes are made to it. Follow these steps: If you have not installed drone CLI follow [these instructions](https://docs.drone.io/cli/install/) To sign the .drone.yml file: + ```bash export DRONE_SERVER=https://drone.grafana.net @@ -154,6 +168,7 @@ drone sign --save grafana/oncall .drone.yml ### ld: library not found for -lssl **Problem:** + ``` pip install -r requirements.txt ... @@ -162,6 +177,7 @@ pip install -r requirements.txt error: command 'gcc' failed with exit status 1 ... ``` + **Solution:** ``` @@ -174,6 +190,7 @@ pip install -r requirements.txt Happens on Apple Silicon **Problem:** + ``` build/temp.macosx-12-arm64-3.9/_openssl.c:575:10: fatal error: 'openssl/opensslv.h' file not found #include @@ -183,7 +200,9 @@ Happens on Apple Silicon ---------------------------------------- ERROR: Failed building wheel for cryptography ``` + **Solution:** + ``` LDFLAGS="-L$(brew --prefix openssl@1.1)/lib" CFLAGS="-I$(brew --prefix openssl@1.1)/include" pip install `cat requirements.txt | grep cryptography` ``` @@ -191,6 +210,7 @@ LDFLAGS="-L$(brew --prefix openssl@1.1)/lib" CFLAGS="-I$(brew --prefix openssl@1 ### django.db.utils.OperationalError: (1366, "Incorrect string value ...") **Problem:** + ``` django.db.utils.OperationalError: (1366, "Incorrect string value: '\\xF0\\x9F\\x98\\x8A\\xF0\\x9F...' for column 'cached_name' at row 1") ``` @@ -198,15 +218,15 @@ django.db.utils.OperationalError: (1366, "Incorrect string value: '\\xF0\\x9F\\x **Solution:** Recreate the database with the correct encoding. - - ### Grafana OnCall plugin does not show up in plugin list - + +### Grafana OnCall plugin does not show up in plugin list + **Problem:** I've run `yarn watch` in `grafana_plugin` but I do not see Grafana OnCall in the list of plugins - + **Solution:** If it is the first time you have run `yarn watch` and it was run after starting Grafana in docker-compose; Grafana will not have detected a plugin to fix: `docker-compose -f developer-docker-compose.yml restart grafana` - + ## Hints: ### Building the all-in-one docker container @@ -219,9 +239,11 @@ docker build -t grafana/oncall-all-in-one -f Dockerfile.all-in-one . ### Running Grafana with plugin (frontend) folder mounted for dev purposes Do it only after you built frontend at least once! Also developer-docker-compose.yml has similar Grafana included. + ```bash docker run --rm -it -p 3000:3000 -v "$(pwd)"/grafana-plugin:/var/lib/grafana/plugins/grafana-plugin -e GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=grafana-oncall-app --name=grafana grafana/grafana:8.3.2 ``` + Credentials: admin/admin ### Running tests locally @@ -239,10 +261,10 @@ pip install pytest.xdist pytest -n4 ``` - ## IDE Specific Instructions ### PyCharm + 1. Create venv and copy .env file ```bash python3.9 -m venv venv @@ -252,7 +274,7 @@ pytest -n4 3. Settings → Project OnCall - In Python Interpreter click the gear and create a new Virtualenv from existing environment selecting the venv created in Step 1. - In Project Structure make sure the project root is the content root and add /engine to Sources -4. Under Settings → Languages & Frameworks → Django +4. Under Settings → Languages & Frameworks → Django - Enable Django support - Set Django project root to /engine - Set Settings to settings/dev.py diff --git a/docs/sources/oncall-api-reference/alertgroups.md b/docs/sources/oncall-api-reference/alertgroups.md index 59b27ead..8b673b64 100644 --- a/docs/sources/oncall-api-reference/alertgroups.md +++ b/docs/sources/oncall-api-reference/alertgroups.md @@ -33,7 +33,10 @@ The above command returns JSON structured in the following way: "created_at": "2020-05-19T12:37:01.430444Z", "resolved_at": "2020-05-19T13:37:01.429805Z", "acknowledged_at": null, - "title": "Memory above 90% threshold" + "title": "Memory above 90% threshold", + "permalinks": { + "slack": null + } } ] } diff --git a/engine/apps/alerts/models/alert_group.py b/engine/apps/alerts/models/alert_group.py index db079c16..5ef9b453 100644 --- a/engine/apps/alerts/models/alert_group.py +++ b/engine/apps/alerts/models/alert_group.py @@ -1,4 +1,5 @@ import logging +import typing from collections import namedtuple from typing import Optional from urllib.parse import urljoin @@ -45,6 +46,10 @@ def generate_public_primary_key_for_alert_group(): return new_public_primary_key +class Permalinks(typing.TypedDict): + slack: str + + class AlertGroupQuerySet(models.QuerySet): def create(self, **kwargs): organization = kwargs["channel"].organization @@ -400,6 +405,13 @@ class AlertGroup(AlertGroupSlackRenderingMixin, EscalationSnapshotMixin, models. if self.slack_message is not None: return self.slack_message.permalink + @property + def permalinks(self) -> Permalinks: + # TODO: refactor 'permalink' property (maybe 'slack_permalink'?) once we add the next permalink + return { + "slack": self.permalink, + } + @property def web_link(self): return urljoin(self.channel.organization.web_link, f"?page=incident&id={self.public_primary_key}") diff --git a/engine/apps/public_api/serializers/incidents.py b/engine/apps/public_api/serializers/incidents.py index 1d3bc174..5010f88f 100644 --- a/engine/apps/public_api/serializers/incidents.py +++ b/engine/apps/public_api/serializers/incidents.py @@ -14,7 +14,7 @@ class IncidentSerializer(EagerLoadingMixin, serializers.ModelSerializer): title = serializers.SerializerMethodField() state = serializers.SerializerMethodField() - SELECT_RELATED = ["channel", "channel_filter"] + SELECT_RELATED = ["channel", "channel_filter", "slack_message"] PREFETCH_RELATED = ["alerts"] class Meta: @@ -29,6 +29,7 @@ class IncidentSerializer(EagerLoadingMixin, serializers.ModelSerializer): "resolved_at", "acknowledged_at", "title", + "permalinks", ] def get_alerts_count(self, obj): diff --git a/engine/apps/public_api/tests/test_incidents.py b/engine/apps/public_api/tests/test_incidents.py index ea1198a0..360a5e5e 100644 --- a/engine/apps/public_api/tests/test_incidents.py +++ b/engine/apps/public_api/tests/test_incidents.py @@ -38,6 +38,9 @@ def construct_expected_response_from_incidents(incidents): "resolved_at": resolved_at, "acknowledged_at": acknowledged_at, "title": None, + "permalinks": { + "slack": None, + }, } ) expected_response = {"count": incidents.count(), "next": None, "previous": None, "results": results} diff --git a/engine/settings/dev.py b/engine/settings/dev.py index 87a82d27..fb7ddc3e 100644 --- a/engine/settings/dev.py +++ b/engine/settings/dev.py @@ -9,6 +9,8 @@ if DB_BACKEND == "mysql": # noqa pymysql.install_as_MySQLdb() +DEBUG = True + DATABASES = { "default": { "ENGINE": "django.db.backends.{}".format(DB_BACKEND), # noqa diff --git a/grafana-plugin/.nvmrc b/grafana-plugin/.nvmrc new file mode 100644 index 00000000..62df50f1 --- /dev/null +++ b/grafana-plugin/.nvmrc @@ -0,0 +1 @@ +14.17.0