Add latest alert to public api alert groups endpoint (#5059)

# What this PR does

Added last alert information 
and optimized the API call so it makes 10x less queries by:
* prefetching chatops messages (based on @vadimkerr 's
https://github.com/grafana/oncall/pull/4738)
* using `enrich` from private api

Previously: 
<img width="1102" alt="Screenshot 2024-09-24 at 4 47 00 PM"
src="https://github.com/user-attachments/assets/84edb78e-257a-49cd-bc94-083dd8d043d7">
Now:
<img width="1066" alt="Screenshot 2024-09-24 at 4 44 56 PM"
src="https://github.com/user-attachments/assets/e7dfcc40-dce6-4a0d-9677-910aab2b4f17">



## Which issue(s) this PR closes

Related to [issue link here]

<!--
*Note*: If you want the issue to be auto-closed once the PR is merged,
change "Related to" to "Closes" in the line above.
If you have more than one GitHub issue that this PR closes, be sure to
preface
each issue link with a [closing
keyword](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/using-keywords-in-issues-and-pull-requests#linking-a-pull-request-to-an-issue).
This ensures that the issue(s) are auto-closed once the PR has been
merged.
-->

## Checklist

- [ ] Unit, integration, and e2e (if applicable) tests updated
- [ ] Documentation added (or `pr:no public docs` PR label added if not
required)
- [ ] Added the relevant release notes label (see labels prefixed w/
`release:`). These labels dictate how your PR will
    show up in the autogenerated release notes.

---------

Co-authored-by: Vadim Stepanov <vadimkerr@gmail.com>

docs/sources/oncall-api-reference/alertgroups.md

@@ -46,6 +46,31 @@ The above command returns JSON structured in the following way:
         "telegram": "https://t.me/c/5354/1234?thread=1234"
       },
       "silenced_at": "2020-05-19T13:37:01.429805Z",
+      "last_alert": {
+        "id": "AA74DN7T4JQB6",
+        "alert_group_id": "I68T24C13IFW1",
+        "created_at": "2020-05-11T20:08:43Z",
+        "payload": {
+          "state": "alerting",
+          "title": "[Alerting] Test notification",
+          "ruleId": 0,
+          "message": "Someone is testing the alert notification within Grafana.",
+          "ruleUrl": "{{API_URL}}/",
+          "ruleName": "Test notification",
+          "evalMatches": [
+            {
+              "tags": null,
+              "value": 100,
+              "metric": "High value"
+            },
+            {
+              "tags": null,
+              "value": 200,
+              "metric": "Higher Value"
+            }
+          ]
+        }
+      },
     }
   ],
   "current_page_number": 1,

engine/apps/api/serializers/alert_group.py

@@ -2,7 +2,6 @@ import datetime
 import logging
 import typing
 
-from django.conf import settings
 from django.core.cache import cache
 from django.db.models import Prefetch
 from django.utils import timezone
@@ -133,26 +132,23 @@ class AlertGroupListSerializer(
 
     labels = AlertGroupLabelSerializer(many=True, read_only=True)
 
-    PREFETCH_RELATED: list[str | Prefetch] = [
+    PREFETCH_RELATED = [
         "dependent_alert_groups",
         "log_records__author",
         "labels",
+        Prefetch(
+            "slack_messages",
+            queryset=SlackMessage.objects.select_related("_slack_team_identity").order_by("created_at")[:1],
+            to_attr="prefetched_slack_messages",
+        ),
+        Prefetch(
+            "telegram_messages",
+            queryset=TelegramMessage.objects.filter(
+                chat_id__startswith="-", message_type=TelegramMessage.ALERT_GROUP_MESSAGE
+            ).order_by("id")[:1],
+            to_attr="prefetched_telegram_messages",
+        ),
     ]
-    if settings.ALERT_GROUP_LIST_TRY_PREFETCH:
-        PREFETCH_RELATED += [
-            Prefetch(
-                "slack_messages",
-                queryset=SlackMessage.objects.select_related("_slack_team_identity").order_by("created_at")[:1],
-                to_attr="prefetched_slack_messages",
-            ),
-            Prefetch(
-                "telegram_messages",
-                queryset=TelegramMessage.objects.filter(
-                    chat_id__startswith="-", message_type=TelegramMessage.ALERT_GROUP_MESSAGE
-                ).order_by("id")[:1],
-                to_attr="prefetched_telegram_messages",
-            ),
-        ]
 
     SELECT_RELATED = [
         "channel__organization",

engine/apps/api/views/alert_group.py

@@ -1,9 +1,8 @@
-import typing
 from datetime import timedelta
 
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
-from django.db.models import Count, Max, Q
+from django.db.models import Q
 from django.utils import timezone
 from django_filters import rest_framework as filters
 from drf_spectacular.utils import extend_schema, inline_serializer
@@ -15,7 +14,7 @@ from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 
 from apps.alerts.constants import ActionSource
-from apps.alerts.models import Alert, AlertGroup, AlertReceiveChannel, EscalationChain, ResolutionNote
+from apps.alerts.models import AlertGroup, AlertReceiveChannel, EscalationChain, ResolutionNote
 from apps.alerts.paging import unpage_user
 from apps.alerts.tasks import delete_alert_group, send_update_resolution_note_signal
 from apps.alerts.utils import is_declare_incident_step_enabled
@@ -37,7 +36,12 @@ from common.api_helpers.filters import (
     ModelFieldFilterMixin,
     MultipleChoiceCharFilter,
 )
-from common.api_helpers.mixins import PreviewTemplateMixin, PublicPrimaryKeyMixin, TeamFilteringMixin
+from common.api_helpers.mixins import (
+    AlertGroupEnrichingMixin,
+    PreviewTemplateMixin,
+    PublicPrimaryKeyMixin,
+    TeamFilteringMixin,
+)
 from common.api_helpers.paginators import AlertGroupCursorPaginator
 
 
@@ -257,6 +261,7 @@ class AlertGroupSearchFilter(SearchFilter):
 
 
 class AlertGroupView(
+    AlertGroupEnrichingMixin,
     PreviewTemplateMixin,
     AlertGroupTeamFilteringMixin,
     PublicPrimaryKeyMixin[AlertGroup],
@@ -356,19 +361,8 @@ class AlertGroupView(
                 labels__value_name=value,
             )
 
-        queryset = queryset.only("id")
-
         return queryset
 
-    def paginate_queryset(self, queryset):
-        """
-        All SQL joins (select_related and prefetch_related) will be performed AFTER pagination, so it only joins tables
-        for 25 alert groups, not the whole table.
-        """
-        alert_groups = super().paginate_queryset(queryset)
-        alert_groups = self.enrich(alert_groups)
-        return alert_groups
-
     def get_object(self):
         obj = super().get_object()
         obj = self.enrich([obj])[0]
@@ -434,48 +428,6 @@ class AlertGroupView(
         """
         return super().retrieve(request, *args, **kwargs)
 
-    def enrich(self, alert_groups: typing.List[AlertGroup]) -> typing.List[AlertGroup]:
-        """
-        This method performs select_related and prefetch_related (using setup_eager_loading) as well as in-memory joins
-        to add additional info like alert_count and last_alert for every alert group efficiently.
-        We need the last_alert because it's used by AlertGroupWebRenderer.
-        """
-
-        # enrich alert groups with select_related and prefetch_related
-        alert_group_pks = [alert_group.pk for alert_group in alert_groups]
-        queryset = AlertGroup.objects.filter(pk__in=alert_group_pks).order_by("-started_at")
-
-        queryset = self.get_serializer_class().setup_eager_loading(queryset)
-        alert_groups = list(queryset)
-
-        # get info on alerts count and last alert ID for every alert group
-        alerts_info = (
-            Alert.objects.values("group_id")
-            .filter(group_id__in=alert_group_pks)
-            .annotate(alerts_count=Count("group_id"), last_alert_id=Max("id"))
-        )
-        alerts_info_map = {info["group_id"]: info for info in alerts_info}
-
-        # fetch last alerts for every alert group
-        last_alert_ids = [info["last_alert_id"] for info in alerts_info_map.values()]
-        last_alerts = Alert.objects.filter(pk__in=last_alert_ids)
-        for alert in last_alerts:
-            # link group back to alert
-            alert.group = [alert_group for alert_group in alert_groups if alert_group.pk == alert.group_id][0]
-            alerts_info_map[alert.group_id].update({"last_alert": alert})
-
-        # add additional "alerts_count" and "last_alert" fields to every alert group
-        for alert_group in alert_groups:
-            try:
-                alert_group.last_alert = alerts_info_map[alert_group.pk]["last_alert"]
-                alert_group.alerts_count = alerts_info_map[alert_group.pk]["alerts_count"]
-            except KeyError:
-                # alert group has no alerts
-                alert_group.last_alert = None
-                alert_group.alerts_count = 0
-
-        return alert_groups
-
     def destroy(self, request, *args, **kwargs):
         instance = self.get_object()
         delete_alert_group.apply_async((instance.pk, request.user.pk))

engine/apps/public_api/serializers/alert_groups.py

@@ -1,7 +1,11 @@
+from django.db.models import Prefetch
 from rest_framework import serializers
 
 from apps.alerts.models import AlertGroup
 from apps.api.serializers.alert_group import AlertGroupLabelSerializer
+from apps.public_api.serializers.alerts import AlertSerializer
+from apps.slack.models import SlackMessage
+from apps.telegram.models import TelegramMessage
 from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField, UserIdField
 from common.api_helpers.mixins import EagerLoadingMixin
 
@@ -18,9 +22,31 @@ class AlertGroupSerializer(EagerLoadingMixin, serializers.ModelSerializer):
     acknowledged_by = UserIdField(read_only=True, source="acknowledged_by_user")
     resolved_by = UserIdField(read_only=True, source="resolved_by_user")
     labels = AlertGroupLabelSerializer(many=True, read_only=True)
+    last_alert = serializers.SerializerMethodField()
 
-    SELECT_RELATED = ["channel", "channel_filter", "slack_message", "channel__organization", "channel__team"]
-    PREFETCH_RELATED = ["labels"]
+    SELECT_RELATED = [
+        "channel",
+        "channel_filter",
+        "channel__organization",
+        "channel__team",
+        "acknowledged_by_user",
+        "resolved_by_user",
+    ]
+    PREFETCH_RELATED = [
+        "labels",
+        Prefetch(
+            "slack_messages",
+            queryset=SlackMessage.objects.select_related("_slack_team_identity").order_by("created_at")[:1],
+            to_attr="prefetched_slack_messages",
+        ),
+        Prefetch(
+            "telegram_messages",
+            queryset=TelegramMessage.objects.filter(
+                chat_id__startswith="-", message_type=TelegramMessage.ALERT_GROUP_MESSAGE
+            ).order_by("id")[:1],
+            to_attr="prefetched_telegram_messages",
+        ),
+    ]
 
     class Meta:
         model = AlertGroup
@@ -40,14 +66,12 @@ class AlertGroupSerializer(EagerLoadingMixin, serializers.ModelSerializer):
             "title",
             "permalinks",
             "silenced_at",
+            "last_alert",
         ]
 
     def get_title(self, obj):
         return obj.web_title_cache
 
-    def get_alerts_count(self, obj):
-        return obj.alerts.count()
-
     def get_state(self, obj):
         return obj.state
 
@@ -56,3 +80,20 @@ class AlertGroupSerializer(EagerLoadingMixin, serializers.ModelSerializer):
             return obj.channel_filter.public_primary_key
         else:
             return None
+
+    def get_last_alert(self, obj):
+        if hasattr(obj, "last_alert"):  # could be set by AlertGroupEnrichingMixin.enrich
+            last_alert = obj.last_alert
+        else:
+            last_alert = obj.alerts.order_by("-created_at").first()
+
+        if last_alert is None:
+            return None
+
+        return AlertSerializer(last_alert).data
+
+    def get_alerts_count(self, obj):
+        if hasattr(obj, "alerts_count"):  # could be set by AlertGroupEnrichingMixin.enrich
+            return obj.alerts_count
+
+        return obj.alerts.count()

engine/apps/public_api/tests/test_alert_groups.py

@@ -71,6 +71,12 @@ def construct_expected_response_from_alert_groups(alert_groups):
                     "web": alert_group.web_link,
                 },
                 "silenced_at": silenced_at,
+                "last_alert": {
+                    "id": alert_group.alerts.last().public_primary_key,
+                    "alert_group_id": alert_group.public_primary_key,
+                    "created_at": alert_group.alerts.last().created_at.isoformat().replace("+00:00", "Z"),
+                    "payload": alert_group.channel.config.example_payload,
+                },
             }
         )
     return {
@@ -110,7 +116,7 @@ def alert_group_public_api_setup(
 
     make_alert(alert_group=grafana_alert_group_default_route, raw_request_data=grafana.config.example_payload)
     make_alert(alert_group=grafana_alert_group_non_default_route, raw_request_data=grafana.config.example_payload)
-    make_alert(alert_group=formatted_webhook_alert_group, raw_request_data=grafana.config.example_payload)
+    make_alert(alert_group=formatted_webhook_alert_group, raw_request_data=formatted_webhook.config.example_payload)
 
     integrations = grafana, formatted_webhook
     alert_groups = (

engine/apps/public_api/tests/test_escalation.py

@@ -73,6 +73,12 @@ def test_escalation_new_alert_group(
             "web": f"a/grafana-oncall-app/alert-groups/{ag.public_primary_key}",
         },
         "silenced_at": None,
+        "last_alert": {
+            "id": ag.alerts.last().public_primary_key,
+            "alert_group_id": ag.public_primary_key,
+            "created_at": ag.alerts.last().created_at.isoformat().replace("+00:00", "Z"),
+            "payload": ag.alerts.last().raw_request_data,
+        },
     }
 
     alert = ag.alerts.get()

engine/apps/public_api/views/alert_groups.py

@@ -8,7 +8,7 @@ from rest_framework.response import Response
 from rest_framework.viewsets import GenericViewSet
 
 from apps.alerts.constants import ActionSource
-from apps.alerts.models import AlertGroup
+from apps.alerts.models import AlertGroup, AlertReceiveChannel
 from apps.alerts.tasks import delete_alert_group, wipe
 from apps.api.label_filtering import parse_label_query
 from apps.auth_token.auth import ApiTokenAuthentication
@@ -23,7 +23,7 @@ from common.api_helpers.filters import (
     DateRangeFilterMixin,
     get_team_queryset,
 )
-from common.api_helpers.mixins import RateLimitHeadersMixin
+from common.api_helpers.mixins import AlertGroupEnrichingMixin, RateLimitHeadersMixin
 from common.api_helpers.paginators import FiftyPageSizePaginator
 
 
@@ -49,7 +49,12 @@ class AlertGroupFilters(ByTeamModelFieldFilterMixin, DateRangeFilterMixin, filte
 
 
 class AlertGroupView(
-    RateLimitHeadersMixin, mixins.ListModelMixin, mixins.RetrieveModelMixin, mixins.DestroyModelMixin, GenericViewSet
+    AlertGroupEnrichingMixin,
+    RateLimitHeadersMixin,
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.DestroyModelMixin,
+    GenericViewSet,
 ):
     authentication_classes = (ApiTokenAuthentication,)
     permission_classes = (IsAuthenticated,)
@@ -64,18 +69,23 @@ class AlertGroupView(
     filterset_class = AlertGroupFilters
 
     def get_queryset(self):
+        # no select_related or prefetch_related is used at this point, it will be done on paginate_queryset.
+
         route_id = self.request.query_params.get("route_id", None)
         integration_id = self.request.query_params.get("integration_id", None)
         state = self.request.query_params.get("state", None)
 
-        queryset = AlertGroup.objects.filter(
-            channel__organization=self.request.auth.organization,
-        ).order_by("-started_at")
+        alert_receive_channels_qs = AlertReceiveChannel.objects_with_deleted.filter(
+            organization_id=self.request.auth.organization.id
+        )
+        if integration_id:
+            alert_receive_channels_qs = alert_receive_channels_qs.filter(public_primary_key=integration_id)
+
+        alert_receive_channels_ids = list(alert_receive_channels_qs.values_list("id", flat=True))
+        queryset = AlertGroup.objects.filter(channel__in=alert_receive_channels_ids).order_by("-started_at")
 
         if route_id:
             queryset = queryset.filter(channel_filter__public_primary_key=route_id)
-        if integration_id:
-            queryset = queryset.filter(channel__public_primary_key=integration_id)
         if state:
             choices = dict(AlertGroup.STATUS_CHOICES)
             try:
@@ -112,9 +122,11 @@ class AlertGroupView(
         public_primary_key = self.kwargs["pk"]
 
         try:
-            return AlertGroup.objects.filter(
+            obj = AlertGroup.objects.filter(
                 channel__organization=self.request.auth.organization,
             ).get(public_primary_key=public_primary_key)
+            obj = self.enrich([obj])[0]
+            return obj
         except AlertGroup.DoesNotExist:
             raise NotFound
 

engine/common/api_helpers/mixins.py

@@ -4,7 +4,7 @@ import typing
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
-from django.db.models import Q
+from django.db.models import Count, Max, Q
 from django.utils.functional import cached_property
 from drf_spectacular.utils import extend_schema, inline_serializer
 from rest_framework import serializers, status
@@ -21,6 +21,7 @@ from apps.alerts.incident_appearance.templaters import (
     AlertWebTemplater,
     TemplateLoader,
 )
+from apps.alerts.models import Alert, AlertGroup
 from apps.base.messaging import get_messaging_backends
 from common.api_helpers.exceptions import BadRequest
 from common.jinja_templater import apply_jinja_template
@@ -411,3 +412,56 @@ class GrafanaHeadersMixin:
         else:
             instance_context = None
         return instance_context
+
+
+class AlertGroupEnrichingMixin:
+    def paginate_queryset(self, queryset):
+        """
+        All SQL joins (select_related and prefetch_related) will be performed AFTER pagination, so it only joins tables
+        for one page of alert groups, not the whole table.
+        """
+        alert_groups = super().paginate_queryset(queryset.only("id"))
+        alert_groups = self.enrich(alert_groups)
+        return alert_groups
+
+    def enrich(self, alert_groups: typing.List[AlertGroup]) -> typing.List[AlertGroup]:
+        """
+        This method performs select_related and prefetch_related (using setup_eager_loading) as well as in-memory joins
+        to add additional info like alert_count and last_alert for every alert group efficiently.
+        We need the last_alert because it's used by AlertGroupWebRenderer.
+        """
+
+        # enrich alert groups with select_related and prefetch_related
+        alert_group_pks = [alert_group.pk for alert_group in alert_groups]
+        queryset = AlertGroup.objects.filter(pk__in=alert_group_pks).order_by("-started_at")
+
+        queryset = self.get_serializer_class().setup_eager_loading(queryset)
+        alert_groups = list(queryset)
+
+        # get info on alerts count and last alert ID for every alert group
+        alerts_info = (
+            Alert.objects.values("group_id")
+            .filter(group_id__in=alert_group_pks)
+            .annotate(alerts_count=Count("group_id"), last_alert_id=Max("id"))
+        )
+        alerts_info_map = {info["group_id"]: info for info in alerts_info}
+
+        # fetch last alerts for every alert group
+        last_alert_ids = [info["last_alert_id"] for info in alerts_info_map.values()]
+        last_alerts = Alert.objects.filter(pk__in=last_alert_ids)
+        for alert in last_alerts:
+            # link group back to alert
+            alert.group = [alert_group for alert_group in alert_groups if alert_group.pk == alert.group_id][0]
+            alerts_info_map[alert.group_id].update({"last_alert": alert})
+
+        # add additional "alerts_count" and "last_alert" fields to every alert group
+        for alert_group in alert_groups:
+            try:
+                alert_group.last_alert = alerts_info_map[alert_group.pk]["last_alert"]
+                alert_group.alerts_count = alerts_info_map[alert_group.pk]["alerts_count"]
+            except KeyError:
+                # alert group has no alerts
+                alert_group.last_alert = None
+                alert_group.alerts_count = 0
+
+        return alert_groups

engine/settings/base.py

@@ -207,7 +207,6 @@ DJANGO_MYSQL_REWRITE_QUERIES = True
 ALERT_GROUPS_DISABLE_PREFER_ORDERING_INDEX = DATABASE_TYPE == DatabaseTypes.MYSQL and getenv_boolean(
     "ALERT_GROUPS_DISABLE_PREFER_ORDERING_INDEX", default=False
 )
-ALERT_GROUP_LIST_TRY_PREFETCH = getenv_boolean("ALERT_GROUP_LIST_TRY_PREFETCH", default=False)
 
 # Redis
 REDIS_USERNAME = os.getenv("REDIS_USERNAME", "")