WIP: Direct paging improvements (#3064)

# What this PR does * Create Direct Paging integration (with default route) when team is created with bulk_update * Create notification policies when user is created with bulk_update * If user notification policies are empty change it to Email * Minor markup and wording improvements * Add grafana queue to helm chart * Remove disabled commands for redis helm chart * Improve Dockerfile caching ## Which issue(s) this PR fixes ## Checklist - [ ] Unit, integration, and e2e (if applicable) tests updated - [ ] Documentation added (or `pr:no public docs` PR label added if not required) - [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required)
2023-09-28 11:57:49 +08:00 · 2023-09-28 11:57:49 +08:00 · 51014735aa
commit 51014735aa
parent 9126f214eb
29 changed files with 317 additions and 185 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -0,0 +1,6 @@
+# Ignore everything
+*
+
+# Allow directories
+!/engine
+!/grafana-plugin
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -28,6 +28,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix shifts for current user internal endpoint to return the right shift PK ([#3036](https://github.com/grafana/oncall/pull/3036))
 - Handle Slack ratelimit on alert group deletion by @vadimkerr ([#3038](https://github.com/grafana/oncall/pull/3038))

+### Added
+
+- Create Direct Paging integration by default for every team, create default E-Mail notification policy for every user ([#3064](https://github.com/grafana/oncall/pull/3064))
+
 ## v1.3.37 (2023-09-12)

 ### Added
--- a/136
+++ b/136
@ -1,82 +1,116 @@
-running_under_parent_tiltfile = os.getenv('TILT_PARENT', 'false') == 'true'
+running_under_parent_tiltfile = os.getenv("TILT_PARENT", "false") == "true"
 # The user/pass that you will login to Grafana with
-grafana_admin_user_pass = os.getenv('GRAFANA_ADMIN_USER_PASS', 'oncall')
+grafana_admin_user_pass = os.getenv("GRAFANA_ADMIN_USER_PASS", "oncall")
 # HELM_PREFIX must be "oncall-dev" as it is hardcoded in dev/helm-local.yml
-HELM_PREFIX="oncall-dev"
+HELM_PREFIX = "oncall-dev"
 # Use docker registery generated by ctlptl (dev/kind-config.yaml)
-DOCKER_REGISTRY="localhost:63628/"
+DOCKER_REGISTRY = "localhost:63628/"

 if not running_under_parent_tiltfile:
    # Load the custom Grafana extensions
-    v1alpha1.extension_repo(name='grafana-tilt-extensions', 
-       ref='main',
-       url='https://github.com/grafana/tilt-extensions')
-v1alpha1.extension(name='grafana', repo_name='grafana-tilt-extensions', repo_path='grafana')
+    v1alpha1.extension_repo(
+        name="grafana-tilt-extensions",
+        ref="main",
+        url="https://github.com/grafana/tilt-extensions",
+    )
+v1alpha1.extension(
+    name="grafana", repo_name="grafana-tilt-extensions", repo_path="grafana"
+)

-load('ext://grafana', 'grafana')
-load('ext://configmap', 'configmap_create')
+load("ext://grafana", "grafana")
+load("ext://configmap", "configmap_create")
+load("ext://docker_build_sub", "docker_build_sub")

 # Tell ops-devenv/Tiltifle where our plugin.json file lives
-plugin_file = os.path.abspath('grafana-plugin/src/plugin.json')
+plugin_file = os.path.abspath("grafana-plugin/src/plugin.json")
+
+
 def plugin_json():
    return plugin_file

+
 allow_k8s_contexts(["kind-kind"])

-docker_build(
-     		 "localhost:63628/oncall/engine:dev", 
-     		 "./engine", 
-     		 target = 'prod',
-		     live_update=[
-			 	sync('./engine/', '/etc/app'),
-				run('cd /etc/app && pip install -r requirements.txt',
-					trigger='./engine/requirements.txt'),
-			 ]
+local_resource("download-cache", cmd="docker pull grafana/oncall:latest; docker tag grafana/oncall localhost:63628/grafana/oncall:latest")
+
+# Build the image including frontend folder for pytest
+docker_build_sub(
+    "localhost:63628/oncall/engine:dev",
+    context="./engine",
+    cache_from="localhost:63628/grafana/oncall:latest",
+    # only=["./engine", "./grafana-plugin"],
+    ignore=["./grafana-plugin/test-results/", "./grafana-plugin/dist/", "./grafana-plugin/e2e-tests/"],
+    child_context=".",
+    target="dev",
+    extra_cmds=["ADD ./grafana-plugin/src/plugin.json /etc/grafana-plugin/src/plugin.json"],
+    live_update=[
+        sync("./engine/", "/etc/app"),
+        run(
+            "cd /etc/app && pip install -r requirements.txt",
+            trigger="./engine/requirements.txt",
+        ),
+    ],
 )

 # Build the plugin in the background
-local_resource('build-ui',
-				labels=['OnCallUI'],
-				cmd='cd grafana-plugin && yarn install  && yarn build:dev',
-				serve_cmd='cd grafana-plugin && ONCALL_API_URL=http://oncall-dev-engine:8080 yarn watch',
-				allow_parallel=True)
+local_resource(
+    "build-ui",
+    labels=["OnCallUI"],
+    cmd="cd grafana-plugin && yarn install && yarn build:dev",
+    serve_cmd="cd grafana-plugin && ONCALL_API_URL=http://oncall-dev-engine:8080 yarn watch",
+    allow_parallel=True,
+)

-yaml = helm(
-  'helm/oncall',
-  name=HELM_PREFIX,
-  values=['./dev/helm-local.yml'])
+yaml = helm("helm/oncall", name=HELM_PREFIX, values=["./dev/helm-local.yml"])

 k8s_yaml(yaml)

 # Generate and load the grafana deploy yaml
-configmap_create('grafana-oncall-app-provisioning',
-				  namespace='default',
-				  from_file='dev/grafana/provisioning/plugins/grafana-oncall-app-provisioning.yaml')
+configmap_create(
+    "grafana-oncall-app-provisioning",
+    namespace="default",
+    from_file="dev/grafana/provisioning/plugins/grafana-oncall-app-provisioning.yaml",
+)

-k8s_resource(objects=['grafana-oncall-app-provisioning:configmap'],
-    		new_name='grafana-oncall-app-provisioning-configmap',
-			resource_deps = ['build-ui', 'engine'],
-			labels=['Grafana'])
+k8s_resource(
+    objects=["grafana-oncall-app-provisioning:configmap"],
+    new_name="grafana-oncall-app-provisioning-configmap",
+    resource_deps=["build-ui", "engine"],
+    labels=["Grafana"],
+)

 # Use separate grafana helm chart
 if not running_under_parent_tiltfile:
-	grafana(context='grafana-plugin',
-			plugin_files = ['grafana-plugin/src/plugin.json'],
-			namespace='default',
-			deps = ['grafana-oncall-app-provisioning-configmap', 'build-ui', 'engine'],
-			extra_env={
-				'GF_SECURITY_ADMIN_PASSWORD': 'oncall',
-				'GF_SECURITY_ADMIN_USER': 'oncall',
-                'GF_AUTH_ANONYMOUS_ENABLED': 'false',
-		    },
-			)
+    grafana(
+        context="grafana-plugin",
+        plugin_files=["grafana-plugin/src/plugin.json"],
+        namespace="default",
+        deps=["grafana-oncall-app-provisioning-configmap", "build-ui", "engine"],
+        extra_env={
+            "GF_SECURITY_ADMIN_PASSWORD": "oncall",
+            "GF_SECURITY_ADMIN_USER": "oncall",
+            "GF_AUTH_ANONYMOUS_ENABLED": "false",
+        },
+    )
+
+k8s_resource(
+    workload="celery",
+    resource_deps=["mariadb", "redis-master"],
+    labels=["OnCallBackend"],
+)
+k8s_resource(
+    workload="engine",
+    port_forwards=8080,
+    resource_deps=["mariadb", "redis-master"],
+    labels=["OnCallBackend"],
+)
+k8s_resource(workload="redis-master", labels=["OnCallDeps"])
+k8s_resource(workload="mariadb", labels=["OnCallDeps"])

-k8s_resource(workload='celery', resource_deps=['mariadb', 'redis-master'], labels=['OnCallBackend'])
-k8s_resource(workload='engine', port_forwards=8080, resource_deps=['mariadb', 'redis-master'], labels=['OnCallBackend'])
-k8s_resource(workload='redis-master', labels=['OnCallDeps'])
-k8s_resource(workload='mariadb', labels=['OnCallDeps'])

 # name all tilt resources after the k8s object namespace + name
 def resource_name(id):
-  return id.name.replace(HELM_PREFIX + '-', '')
+    return id.name.replace(HELM_PREFIX + "-", "")
+
+
 workload_to_resource_function(resource_name)
--- a/dev/helm-local.yml
+++ b/dev/helm-local.yml
@ -16,6 +16,8 @@ redis:
    tag: 7.0.5
  auth:
    password: oncallpassword
+  master:
+    disableCommands: []
 rabbitmq:
  enabled: false
 oncall:
--- a/engine/Dockerfile
+++ b/engine/Dockerfile
@ -17,10 +17,7 @@ RUN apk add bash \

 WORKDIR /etc/app
 COPY ./requirements.txt ./
-COPY ./pip/cache ./pip/cache
-RUN if uname -m | grep -q "aarch64" ; then pip install pip/cache/grpcio-1.57.0-cp311-cp311-linux_aarch64.whl ; else echo "skip"  ; fi
-RUN pip install --upgrade pip
-RUN pip install --upgrade setuptools wheel
+COPY ./pip/cache /root/.cache/pip/wheels/
 RUN pip install -r requirements.txt

 # we intentionally have two COPY commands, this is to have the requirements.txt in a separate build step
--- a/engine/apps/alerts/incident_log_builder/incident_log_builder.py
+++ b/engine/apps/alerts/incident_log_builder/incident_log_builder.py
@ -668,7 +668,7 @@ class IncidentLogBuilder:
                    # last passed step order + 1
                    notification_policy_order = last_user_log.notification_policy.order + 1

-        notification_policies = UserNotificationPolicy.objects.filter(user=user_to_notify, important=important)
+        notification_policies = user_to_notify.get_or_create_notification_policies(important=important)

        for notification_policy in notification_policies:
            future_notification = notification_policy.order >= notification_policy_order
--- a/engine/apps/alerts/models/alert_receive_channel.py
+++ b/engine/apps/alerts/models/alert_receive_channel.py
@ -232,8 +232,9 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject):

    @classmethod
    def create(cls, **kwargs):
+        organization = kwargs["organization"]
        with transaction.atomic():
-            other_channels = cls.objects_with_deleted.select_for_update().filter(organization=kwargs["organization"])
+            other_channels = cls.objects_with_deleted.select_for_update().filter(organization=organization)
            channel = cls(**kwargs)
            smile_code = number_to_smiles_translator(other_channels.count())
            verbal_name = (
--- a/engine/apps/alerts/tasks/notify_group.py
+++ b/engine/apps/alerts/tasks/notify_group.py
@ -76,10 +76,8 @@ def notify_group_task(alert_group_pk, escalation_policy_snapshot_order=None):
            if not user.is_notification_allowed:
                continue

-            notification_policies = UserNotificationPolicy.objects.filter(
-                user=user,
-                important=escalation_policy_step == EscalationPolicy.STEP_NOTIFY_GROUP_IMPORTANT,
-            )
+            important = escalation_policy_step == EscalationPolicy.STEP_NOTIFY_GROUP_IMPORTANT
+            notification_policies = user.get_or_create_notification_policies(important=important)

            if notification_policies:
                usergroup_notification_plan += "\n_{} (".format(
--- a/engine/apps/alerts/tasks/notify_user.py
+++ b/engine/apps/alerts/tasks/notify_user.py
@ -69,7 +69,7 @@ def notify_user_task(
        user_has_notification = UserHasNotification.objects.filter(pk=user_has_notification.pk).select_for_update()[0]

        if previous_notification_policy_pk is None:
-            notification_policy = UserNotificationPolicy.objects.filter(user=user, important=important).first()
+            notification_policy = user.get_or_create_notification_policies(important=important).first()
            if notification_policy is None:
                task_logger.info(
                    f"notify_user_task: Failed to notify. No notification policies. user_id={user_pk} alert_group_id={alert_group_pk} important={important}"
--- a/engine/apps/api/views/user_notification_policy.py
+++ b/engine/apps/api/views/user_notification_policy.py
@ -67,15 +67,13 @@ class UserNotificationPolicyView(UpdateSerializerMixin, OrderedModelViewSet):
        except ValueError:
            raise BadRequest(detail="Invalid user param")
        if user_id is None or user_id == self.request.user.public_primary_key:
-            queryset = self.model.objects.filter(user=self.request.user, important=important)
+            target_user = self.request.user
        else:
            try:
                target_user = User.objects.get(public_primary_key=user_id)
            except User.DoesNotExist:
                raise BadRequest(detail="User does not exist")
-
-            queryset = self.model.objects.filter(user=target_user, important=important)
-
+        queryset = target_user.get_or_create_notification_policies(important=important)
        return self.serializer_class.setup_eager_loading(queryset)

    def get_object(self):
--- a/engine/apps/base/models/user_notification_policy.py
+++ b/engine/apps/base/models/user_notification_policy.py
@ -71,17 +71,7 @@ class UserNotificationPolicyQuerySet(models.QuerySet):
        if user.notification_policies.filter(important=False).exists():
            return

-        model = self.model
-        policies_to_create = (
-            model(
-                user=user,
-                step=model.Step.NOTIFY,
-                notify_by=NotificationChannelOptions.DEFAULT_NOTIFICATION_CHANNEL,
-                order=0,
-            ),
-            model(user=user, step=model.Step.WAIT, wait_delay=datetime.timedelta(minutes=15), order=1),
-            model(user=user, step=model.Step.NOTIFY, notify_by=model.NotificationChannel.PHONE_CALL, order=2),
-        )
+        policies_to_create = user.default_notification_policies_defaults

        try:
            super().bulk_create(policies_to_create)
@ -92,16 +82,7 @@ class UserNotificationPolicyQuerySet(models.QuerySet):
        if user.notification_policies.filter(important=True).exists():
            return

-        model = self.model
-        policies_to_create = (
-            model(
-                user=user,
-                step=model.Step.NOTIFY,
-                notify_by=model.NotificationChannel.PHONE_CALL,
-                important=True,
-                order=0,
-            ),
-        )
+        policies_to_create = user.important_notification_policies_defaults

        try:
            super().bulk_create(policies_to_create)
--- a/engine/apps/user_management/models/team.py
+++ b/engine/apps/user_management/models/team.py
@ -2,9 +2,10 @@ import typing

 from django.conf import settings
 from django.core.validators import MinLengthValidator
-from django.db import models
+from django.db import models, transaction

-from apps.metrics_exporter.helpers import metrics_bulk_update_team_label_cache
+from apps.alerts.models import AlertReceiveChannel, ChannelFilter
+from apps.metrics_exporter.helpers import metrics_add_integration_to_cache, metrics_bulk_update_team_label_cache
 from apps.metrics_exporter.metrics_cache_manager import MetricsCacheManager
 from common.public_primary_keys import generate_public_primary_key, increase_public_primary_key_length

@ -51,7 +52,44 @@ class TeamManager(models.Manager["Team"]):
            for team in grafana_teams.values()
            if team["id"] not in existing_team_ids
        )
-        organization.teams.bulk_create(teams_to_create, batch_size=5000)
+
+        with transaction.atomic():
+            organization.teams.bulk_create(teams_to_create, batch_size=5000)
+            # Retrieve primary keys for the newly created users
+            #
+            # If the model’s primary key is an AutoField, the primary key attribute can only be retrieved
+            # on certain databases (currently PostgreSQL, MariaDB 10.5+, and SQLite 3.35+).
+            # On other databases, it will not be set.
+            # https://docs.djangoproject.com/en/4.1/ref/models/querysets/#django.db.models.query.QuerySet.bulk_create
+            created_teams = organization.teams.exclude(team_id__in=existing_team_ids)
+            direct_paging_integrations_to_create = []
+            for team in created_teams:
+                alert_receive_channel = AlertReceiveChannel(
+                    organization=organization,
+                    team=team,
+                    integration=AlertReceiveChannel.INTEGRATION_DIRECT_PAGING,
+                    verbal_name=f"Direct paging ({team.name if team else 'No'} team)",
+                )
+                direct_paging_integrations_to_create.append(alert_receive_channel)
+            AlertReceiveChannel.objects.bulk_create(direct_paging_integrations_to_create, batch_size=5000)
+            created_direct_paging_integrations = AlertReceiveChannel.objects.filter(
+                organization=organization,
+                integration=AlertReceiveChannel.INTEGRATION_DIRECT_PAGING,
+            ).exclude(team__team_id__in=existing_team_ids)
+            default_channel_filters_to_create = []
+            for integration in created_direct_paging_integrations:
+                channel_filter = ChannelFilter(
+                    alert_receive_channel=integration,
+                    filtering_term=None,
+                    is_default=True,
+                    order=0,
+                )
+                default_channel_filters_to_create.append(channel_filter)
+            ChannelFilter.objects.bulk_create(default_channel_filters_to_create, batch_size=5000)
+
+            # Add direct paging integrations to metrics cache
+            for integration in direct_paging_integrations_to_create:
+                metrics_add_integration_to_cache(integration)

        # delete excess teams
        team_ids_to_delete = existing_team_ids - grafana_teams.keys()
--- a/engine/apps/user_management/models/user.py
+++ b/engine/apps/user_management/models/user.py
@ -7,7 +7,7 @@ from urllib.parse import urljoin
 import pytz
 from django.conf import settings
 from django.core.validators import MinLengthValidator
-from django.db import models
+from django.db import models, transaction
 from django.db.models import Q
 from django.db.models.signals import post_save
 from django.dispatch import receiver
@ -75,6 +75,8 @@ class UserManager(models.Manager["User"]):

    @staticmethod
    def sync_for_organization(organization, api_users: list[dict]):
+        from apps.base.models import UserNotificationPolicy
+
        grafana_users = {user["userId"]: user for user in api_users}
        existing_user_ids = set(organization.users.all().values_list("user_id", flat=True))

@ -93,7 +95,22 @@ class UserManager(models.Manager["User"]):
            for user in grafana_users.values()
            if user["userId"] not in existing_user_ids
        )
-        organization.users.bulk_create(users_to_create, batch_size=5000)
+
+        with transaction.atomic():
+            organization.users.bulk_create(users_to_create, batch_size=5000)
+            # Retrieve primary keys for the newly created users
+            #
+            # If the model’s primary key is an AutoField, the primary key attribute can only be retrieved
+            # on certain databases (currently PostgreSQL, MariaDB 10.5+, and SQLite 3.35+).
+            # On other databases, it will not be set.
+            # https://docs.djangoproject.com/en/4.1/ref/models/querysets/#django.db.models.query.QuerySet.bulk_create
+            created_users = organization.users.exclude(user_id__in=existing_user_ids)
+
+            policies_to_create = ()
+            for user in created_users:
+                policies_to_create = policies_to_create + user.default_notification_policies_defaults
+                policies_to_create = policies_to_create + user.important_notification_policies_defaults
+            UserNotificationPolicy.objects.bulk_create(policies_to_create, batch_size=5000)

        # delete excess users
        user_ids_to_delete = existing_user_ids - grafana_users.keys()
@ -384,6 +401,44 @@ class User(models.Model):
            return PermissionsRegexQuery(permissions__regex=r".*{0}.*".format(permission.value))
        return RoleInQuery(role__lte=permission.fallback_role.value)

+    def get_or_create_notification_policies(self, important=False):
+        if not self.notification_policies.filter(important=important).exists():
+            if important:
+                self.notification_policies.create_important_policies_for_user(self)
+            else:
+                self.notification_policies.create_default_policies_for_user(self)
+        notification_policies = self.notification_policies.filter(important=important)
+        return notification_policies
+
+    @property
+    def default_notification_policies_defaults(self):
+        from apps.base.models import UserNotificationPolicy
+
+        print(self)
+
+        return (
+            UserNotificationPolicy(
+                user=self,
+                step=UserNotificationPolicy.Step.NOTIFY,
+                notify_by=settings.EMAIL_BACKEND_INTERNAL_ID,
+                order=0,
+            ),
+        )
+
+    @property
+    def important_notification_policies_defaults(self):
+        from apps.base.models import UserNotificationPolicy
+
+        return (
+            UserNotificationPolicy(
+                user=self,
+                step=UserNotificationPolicy.Step.NOTIFY,
+                notify_by=settings.EMAIL_BACKEND_INTERNAL_ID,
+                important=True,
+                order=0,
+            ),
+        )
+

 # TODO: check whether this signal can be moved to save method of the model
@receiver(post_save, sender=User)
--- a/engine/apps/user_management/tests/test_sync.py
+++ b/engine/apps/user_management/tests/test_sync.py
@ -4,6 +4,7 @@ import pytest
 from django.conf import settings
 from django.test import override_settings

+from apps.alerts.models import AlertReceiveChannel
 from apps.grafana_plugin.helpers.client import GcomAPIClient, GrafanaAPIClient
 from apps.user_management.models import Team, User
 from apps.user_management.sync import check_grafana_incident_is_enabled, cleanup_organization, sync_organization
@ -48,6 +49,18 @@ def test_sync_users_for_organization(make_organization, make_user_for_organizati
    assert created_user.name == api_users[1]["name"]
    assert created_user.avatar_full_url == "https://test.test/test/1234"

+    assert created_user.notification_policies.filter(important=False).count() == 1
+    assert (
+        created_user.notification_policies.filter(important=False).first().notify_by
+        == settings.EMAIL_BACKEND_INTERNAL_ID
+    )
+
+    assert created_user.notification_policies.filter(important=True).count() == 1
+    assert (
+        created_user.notification_policies.filter(important=True).first().notify_by
+        == settings.EMAIL_BACKEND_INTERNAL_ID
+    )
+

@pytest.mark.django_db
 def test_sync_teams_for_organization(make_organization, make_team):
@ -77,6 +90,15 @@ def test_sync_teams_for_organization(make_organization, make_team):
    assert created_team.team_id == api_teams[1]["id"]
    assert created_team.name == api_teams[1]["name"]

+    direct_paging_integration = AlertReceiveChannel.objects.get(
+        organization=organization,
+        integration=AlertReceiveChannel.INTEGRATION_DIRECT_PAGING,
+        team=created_team,
+    )
+    assert direct_paging_integration.channel_filters.count() == 1
+    assert direct_paging_integration.channel_filters.first().order == 0
+    assert direct_paging_integration.channel_filters.first().is_default
+

@pytest.mark.django_db
 def test_sync_users_for_team(make_organization, make_user_for_organization, make_team):
--- a/engine/requirements.txt
+++ b/engine/requirements.txt
@ -58,3 +58,4 @@ prometheus_client==0.16.0
 lxml==4.9.2
 babel==2.12.1
 drf-spectacular==0.26.2
+grpcio==1.57.0
--- a/engine/settings/base.py
+++ b/engine/settings/base.py
@ -696,8 +696,9 @@ EMAIL_USE_TLS = getenv_boolean("EMAIL_USE_TLS", True)
 EMAIL_FROM_ADDRESS = os.getenv("EMAIL_FROM_ADDRESS")
 EMAIL_NOTIFICATIONS_LIMIT = getenv_integer("EMAIL_NOTIFICATIONS_LIMIT", 200)

+EMAIL_BACKEND_INTERNAL_ID = 8
 if FEATURE_EMAIL_INTEGRATION_ENABLED:
-    EXTRA_MESSAGING_BACKENDS += [("apps.email.backend.EmailBackend", 8)]
+    EXTRA_MESSAGING_BACKENDS += [("apps.email.backend.EmailBackend", EMAIL_BACKEND_INTERNAL_ID)]

 # Inbound email settings
 INBOUND_EMAIL_ESP = os.getenv("INBOUND_EMAIL_ESP")
--- a/grafana-plugin/.dockerignore
+++ b/grafana-plugin/.dockerignore
@ -1,3 +1,5 @@
 node_modules
 frontend_enterprise
 .DS_Store
+test-results
+playwright-report
--- a/grafana-plugin/e2e-tests/utils/integrations.ts
+++ b/grafana-plugin/e2e-tests/utils/integrations.ts
@ -40,7 +40,7 @@ export const assignEscalationChainToIntegration = async (page: Page, escalationC
    selectType: 'grafanaSelect',
    placeholderText: 'Select Escalation Chain',
    value: escalationChainName,
-    startingLocator: page.getByTestId('integration-block-item'),
+    startingLocator: page.getByTestId('escalation-chain-select'),
  });
 };

--- a/grafana-plugin/src/components/Integrations/IntegrationBlockItem.module.scss
+++ b/grafana-plugin/src/components/Integrations/IntegrationBlockItem.module.scss
@ -3,7 +3,6 @@
  flex-direction: row;
  margin-bottom: 4px;
  max-width: 100%;
-  margin-left: 16px;

  &__content {
    width: 100%;
@ -11,9 +10,4 @@
    padding-bottom: 12px;
  }

-  &__leftDelimitator {
-    border-left: var(--border-medium);
-    border-left-width: 3px;
-    margin-right: 16px;
-  }
 }
--- a/grafana-plugin/src/components/Integrations/IntegrationBlockItem.tsx
+++ b/grafana-plugin/src/components/Integrations/IntegrationBlockItem.tsx
@ -13,7 +13,6 @@ interface IntegrationBlockItemProps {
 const IntegrationBlockItem: React.FC<IntegrationBlockItemProps> = (props) => {
  return (
    <div className={cx('blockItem')} data-testid="integration-block-item">
-      <div className={cx('blockItem__leftDelimitator')} />
      <div className={cx('blockItem__content')}>{props.children}</div>
    </div>
  );
--- a/grafana-plugin/src/components/Policy/EscalationPolicy.tsx
+++ b/grafana-plugin/src/components/Policy/EscalationPolicy.tsx
@ -73,9 +73,11 @@ export class EscalationPolicy extends React.Component<EscalationPolicyProps, any
        textColor={isDisabled ? getVar('--tag-text-success') : undefined}
        backgroundColor={backgroundColor}
      >
-        <WithPermissionControlTooltip disableByPaywall userAction={UserActions.EscalationChainsWrite}>
-          <DragHandle />
-        </WithPermissionControlTooltip>
+        {!isDisabled && (
+          <WithPermissionControlTooltip disableByPaywall userAction={UserActions.EscalationChainsWrite}>
+            <DragHandle />
+          </WithPermissionControlTooltip>
+        )}
        {escalationOption &&
          reactStringReplace(escalationOption.display_name, /\{\{([^}]+)\}\}/g, this.replacePlaceholder)}
        {this._renderNote()}
--- a/grafana-plugin/src/components/Policy/NotificationPolicy.tsx
+++ b/grafana-plugin/src/components/Policy/NotificationPolicy.tsx
@ -44,34 +44,39 @@ export interface NotificationPolicyProps {
  number: number;
  userAction: UserAction;
  store: RootStore;
+  isDisabled: boolean;
 }

 export class NotificationPolicy extends React.Component<NotificationPolicyProps, any> {
  render() {
-    const { data, notificationChoices, number, color, userAction } = this.props;
+    const { data, notificationChoices, number, color, userAction, isDisabled } = this.props;
    const { id, step } = data;

    return (
      <Timeline.Item className={cx('root')} number={number} backgroundColor={color}>
        <div className={cx('step')}>
-          <WithPermissionControlTooltip disableByPaywall userAction={userAction}>
-            <DragHandle />
-          </WithPermissionControlTooltip>
+          {!isDisabled && (
+            <WithPermissionControlTooltip disableByPaywall userAction={userAction}>
+              <DragHandle />
+            </WithPermissionControlTooltip>
+          )}
          <WithPermissionControlTooltip disableByPaywall userAction={userAction}>
            <Select
              className={cx('select', 'control')}
              onChange={this._getOnChangeHandler('step')}
              value={step}
              options={notificationChoices.map((option: any) => ({ label: option.display_name, value: option.value }))}
+              disabled={isDisabled}
            />
          </WithPermissionControlTooltip>
-          {this._renderControls()}
+          {this._renderControls(isDisabled)}
          <WithPermissionControlTooltip userAction={userAction}>
            <IconButton
              className={cx('control')}
              name="trash-alt"
              onClick={this._getDeleteClickHandler(id)}
              variant="secondary"
+              disabled={isDisabled}
            />
          </WithPermissionControlTooltip>
          {this._renderNote()}
@ -80,16 +85,16 @@ export class NotificationPolicy extends React.Component<NotificationPolicyProps,
    );
  }

-  _renderControls() {
+  _renderControls(disabled: boolean) {
    const { data } = this.props;
    const { step } = data;

    switch (step) {
      case 0:
-        return <>{this._renderWaitDelays()}</>;
+        return <>{this._renderWaitDelays(disabled)}</>;

      case 1:
-        return <>{this._renderNotifyBy()}</>;
+        return <>{this._renderNotifyBy(disabled)}</>;

      default:
        return null;
@ -165,7 +170,7 @@ export class NotificationPolicy extends React.Component<NotificationPolicyProps,
    );
  }

-  private _renderWaitDelays() {
+  private _renderWaitDelays(disabled: boolean) {
    const { data, waitDelays = [], userAction } = this.props;
    const { wait_delay } = data;

@ -177,6 +182,7 @@ export class NotificationPolicy extends React.Component<NotificationPolicyProps,
          className={cx('select', 'control')}
          // @ts-ignore
          value={wait_delay}
+          disabled={disabled}
          onChange={this._getOnChangeHandler('wait_delay')}
          options={waitDelays.map((waitDelay: WaitDelay) => ({
            label: waitDelay.display_name,
@ -187,7 +193,7 @@ export class NotificationPolicy extends React.Component<NotificationPolicyProps,
    );
  }

-  private _renderNotifyBy() {
+  private _renderNotifyBy(disabled: boolean) {
    const { data, notifyByOptions = [], userAction } = this.props;
    const { notify_by } = data;

@ -199,6 +205,7 @@ export class NotificationPolicy extends React.Component<NotificationPolicyProps,
          className={cx('select', 'control')}
          // @ts-ignore
          value={notify_by}
+          disabled={disabled}
          onChange={this._getOnChangeHandler('notify_by')}
          options={notifyByOptions.map((notifyByOption: NotifyBy) => ({
            label: notifyByOption.display_name,
--- a/grafana-plugin/src/containers/IntegrationContainers/ExpandedIntegrationRouteDisplay/ExpandedIntegrationRouteDisplay.tsx
+++ b/grafana-plugin/src/containers/IntegrationContainers/ExpandedIntegrationRouteDisplay/ExpandedIntegrationRouteDisplay.tsx
@ -18,7 +18,6 @@ import CopyToClipboard from 'react-copy-to-clipboard';

 import HamburgerMenu from 'components/HamburgerMenu/HamburgerMenu';
 import IntegrationBlock from 'components/Integrations/IntegrationBlock';
-import IntegrationBlockItem from 'components/Integrations/IntegrationBlockItem';
 import MonacoEditor from 'components/MonacoEditor/MonacoEditor';
 import { MONACO_READONLY_CONFIG } from 'components/MonacoEditor/MonacoEditor.config';
 import PluginLink from 'components/PluginLink/PluginLink';
@ -102,7 +101,6 @@ const ExpandedIntegrationRouteDisplay: React.FC<ExpandedIntegrationRouteDisplayP
    }, []);

    const channelFilter = alertReceiveChannelStore.channelFilters[channelFilterId];
-    const channelFiltersTotal = Object.keys(alertReceiveChannelStore.channelFilters);
    if (!channelFilter) {
      return null;
    }
@ -149,20 +147,18 @@ const ExpandedIntegrationRouteDisplay: React.FC<ExpandedIntegrationRouteDisplayP
            </HorizontalGroup>
          }
          content={
-            <VerticalGroup spacing="xs">
-              {routeIndex !== channelFiltersTotal.length - 1 && (
-                <IntegrationBlockItem>
-                  <VerticalGroup>
-                    <Text type="secondary">
-                      If the Routing Template is True, group alerts with the Grouping Template, send them to messengers,
-                      and trigger the escalation chain.
-                    </Text>
-                  </VerticalGroup>
-                </IntegrationBlockItem>
-              )}
-              {/* Show Routing Template only for If/Else Routes, not for Default */}
-              {!isDefault && (
-                <IntegrationBlockItem>
+            <VerticalGroup>
+              {isDefault ? (
+                <Text type="secondary">
+                  All unmatched alerts are directed to this route, grouped using the Grouping Template, sent to
+                  messengers, and trigger the escalation chain
+                </Text>
+              ) : (
+                <VerticalGroup>
+                  <Text type="secondary">
+                    If the Routing Template is True, group alerts with the Grouping Template, send them to messengers,
+                    and trigger the escalation chain.
+                  </Text>
                  <HorizontalGroup spacing="xs">
                    <InlineLabel
                      width={20}
@ -187,21 +183,19 @@ const ExpandedIntegrationRouteDisplay: React.FC<ExpandedIntegrationRouteDisplayP
                      onClick={() => handleEditRoutingTemplate(channelFilter, channelFilterId)}
                    />
                  </HorizontalGroup>
-                </IntegrationBlockItem>
+                </VerticalGroup>
              )}

              {IntegrationHelper.hasChatopsInstalled(store) && (
-                <IntegrationBlockItem>
-                  <VerticalGroup spacing="md">
-                    <Text type="primary">Publish to ChatOps</Text>
-                    <ChatOpsConnectors channelFilterId={channelFilterId} showLineNumber={false} />
-                  </VerticalGroup>
-                </IntegrationBlockItem>
+                <VerticalGroup spacing="md">
+                  <Text type="primary">Publish to ChatOps</Text>
+                  <ChatOpsConnectors channelFilterId={channelFilterId} showLineNumber={false} />
+                </VerticalGroup>
              )}

-              <IntegrationBlockItem>
-                <VerticalGroup>
-                  <HorizontalGroup spacing={'xs'}>
+              <VerticalGroup>
+                <HorizontalGroup spacing={'xs'}>
+                  <div data-testid="escalation-chain-select">
                    <InlineLabel
                      width={20}
                      tooltip="The escalation chain determines who and when to notify when an alert group starts."
@ -264,13 +258,13 @@ const ExpandedIntegrationRouteDisplay: React.FC<ExpandedIntegrationRouteDisplayP
                        </HorizontalGroup>
                      </Button>
                    )}
-                  </HorizontalGroup>
+                  </div>
+                </HorizontalGroup>

-                  {!isEscalationCollapsed && (
-                    <ReadOnlyEscalationChain escalationChainId={channelFilter.escalation_chain} />
-                  )}
-                </VerticalGroup>
-              </IntegrationBlockItem>
+                {!isEscalationCollapsed && (
+                  <ReadOnlyEscalationChain escalationChainId={channelFilter.escalation_chain} />
+                )}
+              </VerticalGroup>
            </VerticalGroup>
          }
        />
--- a/grafana-plugin/src/containers/IntegrationContainers/IntegrationTemplatesList.tsx
+++ b/grafana-plugin/src/containers/IntegrationContainers/IntegrationTemplatesList.tsx
@ -66,19 +66,10 @@ const IntegrationTemplateList: React.FC<IntegrationTemplateListProps> = ({
          onDismiss={() => onDismiss()}
        />
      )}
-
-      <IntegrationBlockItem>
-        <Text type="secondary">
-          Set templates to interpret monitoring alerts and minimize noise. Group alerts, enable auto-resolution,
-          customize visualizations and notifications by extracting data from alerts.
-        </Text>
-      </IntegrationBlockItem>
-
      {templatesToRender.map((template, key) => (
        <IntegrationBlockItem key={key}>
          <VerticalBlock>
            {template.name && <Text type={'primary'}>{template.name}</Text>}
-
            {template.contents.map((contents, innerKey) => (
              <IntegrationTemplateBlock
                key={innerKey}
--- a/grafana-plugin/src/containers/TeamsList/TeamsList.tsx
+++ b/grafana-plugin/src/containers/TeamsList/TeamsList.tsx
@ -14,7 +14,6 @@ import { UserActions } from 'utils/authorization';
 const TeamsList = observer(() => {
  const store = useStore();
  const [teamIdToShowModal, setTeamIdToShowModal] = useState<GrafanaTeam['id']>();
-  const { userStore } = store;

  const isTeamDefault = (record: GrafanaTeam) => {
    return (
@ -53,15 +52,9 @@ const TeamsList = observer(() => {
  const renderActionButtons = (record: GrafanaTeam) => {
    const editButton = (
      <HorizontalGroup justify="flex-end">
-        <Tooltip content="Default team will be selected when creating new resources">
-          <Button
-            onClick={async () => {
-              await userStore.updateCurrentUser({ current_team: record.id });
-              store.grafanaTeamStore.updateItems();
-            }}
-            disabled={isTeamDefault(record)}
-            fill="text"
-          >
+        {/* Keep  "Make default" here for backwards compatibility, can be removed in November 2023 */}
+        <Tooltip content="This button is moved to your User Profile">
+          <Button onClick={() => {}} disabled={true} fill="text">
            Make default
          </Button>
        </Tooltip>
--- a/grafana-plugin/src/containers/UserSettings/parts/tabs/UserInfoTab/UserInfoTab.tsx
+++ b/grafana-plugin/src/containers/UserSettings/parts/tabs/UserInfoTab/UserInfoTab.tsx
@ -2,6 +2,7 @@ import React from 'react';

 import { InlineField, Input, Legend } from '@grafana/ui';

+import GrafanaTeamSelect from 'containers/GrafanaTeamSelect/GrafanaTeamSelect';
 import { UserSettingsTab } from 'containers/UserSettings/UserSettings.types';
 import { Connectors } from 'containers/UserSettings/parts/connectors';
 import { User } from 'models/user/user.types';
@ -19,7 +20,7 @@ export const UserInfoTab = (props: UserInfoTabProps) => {
  const { userStore } = store;

  const storeUser = userStore.items[id];
-  let width = 12;
+  let width = 15;

  return (
    <>
@ -39,6 +40,21 @@ export const UserInfoTab = (props: UserInfoTabProps) => {
      <InlineField label="Timezone" labelWidth={width} grow disabled>
        <Input value={storeUser.timezone || ''} />
      </InlineField>
+      <InlineField
+        label="Default Team"
+        labelWidth={width}
+        grow
+        tooltip="Default team will be pre-selected when you create new resources, such as integrations, schedules, etc."
+      >
+        <GrafanaTeamSelect
+          withoutModal
+          defaultValue={storeUser.current_team}
+          onSelect={async (value) => {
+            await userStore.updateCurrentUser({ current_team: value });
+            store.grafanaTeamStore.updateItems();
+          }}
+        />
+      </InlineField>
      <Legend>Notification channels</Legend>
      <Connectors {...props} />
    </>
--- a/grafana-plugin/src/models/user/user.ts
+++ b/grafana-plugin/src/models/user/user.ts
@ -345,7 +345,7 @@ export class UserStore extends BaseStore {
  async deleteNotificationPolicy(userPk: User['pk'], id: NotificationPolicyType['id']) {
    Mixpanel.track('Delete NotificationPolicy', null);

-    await makeRequest(`/notification_policies/${id}`, { method: 'DELETE' });
+    await makeRequest(`/notification_policies/${id}`, { method: 'DELETE' }).catch(this.onApiError);

    this.updateNotificationPolicies(userPk);

--- a/grafana-plugin/src/pages/integration/Integration.tsx
+++ b/grafana-plugin/src/pages/integration/Integration.tsx
@ -160,22 +160,16 @@ class Integration extends React.Component<IntegrationProps, IntegrationState> {
                width="75%"
                scrollableContent
                title="Template Settings"
+                subtitle="Set templates to interpret monitoring alerts and minimize noise. Group alerts, enable auto-resolution, customize visualizations and notifications by extracting data from alerts."
                onClose={() => this.setState({ isTemplateSettingsOpen: false })}
                closeOnMaskClick={false}
              >
-                <IntegrationBlock
-                  className={cx('template-drawer')}
-                  noContent
-                  heading={undefined}
-                  content={
-                    <IntegrationTemplateList
-                      alertReceiveChannelId={alertReceiveChannel.id}
-                      alertReceiveChannelIsBasedOnAlertManager={alertReceiveChannel.is_based_on_alertmanager}
-                      alertReceiveChannelAllowSourceBasedResolving={alertReceiveChannel.allow_source_based_resolving}
-                      openEditTemplateModal={this.openEditTemplateModal}
-                      templates={templates}
-                    />
-                  }
+                <IntegrationTemplateList
+                  alertReceiveChannelId={alertReceiveChannel.id}
+                  alertReceiveChannelIsBasedOnAlertManager={alertReceiveChannel.is_based_on_alertmanager}
+                  alertReceiveChannelAllowSourceBasedResolving={alertReceiveChannel.allow_source_based_resolving}
+                  openEditTemplateModal={this.openEditTemplateModal}
+                  templates={templates}
                />
              </Drawer>
            )}
@ -1104,10 +1098,12 @@ const IntegrationHeader: React.FC<IntegrationHeaderProps> = ({
          <Text type="secondary">Team:</Text>
          <TeamName team={grafanaTeamStore.items[alertReceiveChannel.team]} />
        </div>
-        <div className={cx('headerTop__item')}>
-          <Text type="secondary">Created by:</Text>
-          <UserDisplayWithAvatar id={alertReceiveChannel.author as any}></UserDisplayWithAvatar>
-        </div>
+        {alertReceiveChannel.author && (
+          <div className={cx('headerTop__item')}>
+            <Text type="secondary">Created by:</Text>
+            <UserDisplayWithAvatar id={alertReceiveChannel.author as any}></UserDisplayWithAvatar>
+          </div>
+        )}
      </div>
    </div>
  );
--- a/helm/oncall/values.yaml
+++ b/helm/oncall/values.yaml
@ -80,7 +80,7 @@ engine:
 # Celery workers pods configuration
 celery:
  replicaCount: 1
-  worker_queue: "default,critical,long,slack,telegram,webhook,celery"
+  worker_queue: "default,critical,long,slack,telegram,webhook,celery,grafana"
  worker_concurrency: "1"
  worker_max_tasks_per_child: "100"
  worker_beat_enabled: "True"