Chatops api v3 (#3721)

This PR makes OnCall compatible with chatops-proxy v3. When CHATOPS_V3
is enabled, oncall will use new api client to register tenants and slack
installations. Also I added v3 routes for slack and telegram, so it's
possible to test new chatops proxy.

Currently two versions of chatops-proxy api are deployed, but they are
not compatible. They are doing same thing, using different db model and
tables. Once only v3 version will be left in prod, I'll remove
CHATOPS_V3 env var, all leftovers of previous api client and v3 slack
and telegram routes.

---------

Co-authored-by: Vadim Stepanov <vadimkerr@gmail.com>
This commit is contained in:
Innokentii Konstantinov 2024-01-20 14:56:17 +08:00 committed by GitHub
parent 36f5fdc56e
commit 4a02d83fd1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 435 additions and 22 deletions

View file

@ -19,4 +19,6 @@ urlpatterns = [
path("signup_redirect/<str:subscription>/<str:utm>/", SignupRedirectView.as_view()),
# Trailing / is missing here on purpose. QA the feature if you want to add it. No idea why doesn't it work with it.
path("reset_slack", ResetSlackView.as_view(), name="reset-slack"),
path("v3/event_api_endpoint/", SlackEventApiEndpointView.as_view()),
path("v3/interactive_api_endpoint/", SlackEventApiEndpointView.as_view()),
]

View file

@ -40,7 +40,7 @@ from apps.slack.tasks import clean_slack_integration_leftovers, unpopulate_slack
from apps.slack.types import EventPayload, EventType, MessageEventSubtype, PayloadType, ScenarioRoute
from apps.user_management.models import Organization
from common.insight_log import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log
from common.oncall_gateway import delete_slack_connector
from common.oncall_gateway import unlink_slack_team_wrapper
from .errors import SlackAPITokenError
from .models import SlackMessage, SlackTeamIdentity, SlackUserIdentity
@ -578,7 +578,7 @@ class ResetSlackView(APIView):
if slack_team_identity is not None:
clean_slack_integration_leftovers.apply_async((organization.pk,))
if settings.FEATURE_MULTIREGION_ENABLED:
delete_slack_connector(str(organization.uuid))
unlink_slack_team_wrapper(str(organization.uuid), slack_team_identity.slack_id)
write_chatops_insight_log(
author=request.user,
event_name=ChatOpsEvent.WORKSPACE_DISCONNECTED,

View file

@ -11,7 +11,7 @@ from apps.slack.tasks import populate_slack_channels_for_team, populate_slack_us
from apps.social_auth.exceptions import InstallMultiRegionSlackException
from common.constants.slack_auth import SLACK_AUTH_SLACK_USER_ALREADY_CONNECTED_ERROR, SLACK_AUTH_WRONG_WORKSPACE_ERROR
from common.insight_log import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log
from common.oncall_gateway import check_slack_installation_possible, create_slack_connector
from common.oncall_gateway import can_link_slack_team_wrapper, link_slack_team_wrapper
logger = logging.getLogger(__name__)
@ -95,9 +95,8 @@ def populate_slack_identities(response, backend, user, organization, **kwargs):
return HttpResponse(status=status.HTTP_400_BAD_REQUEST)
slack_team_id = response["team"]["id"]
if settings.FEATURE_MULTIREGION_ENABLED and not check_slack_installation_possible(
str(organization.uuid), slack_team_id, settings.ONCALL_BACKEND_REGION
):
can_link = can_link_slack_team_wrapper(str(organization.uuid), slack_team_id, settings.ONCALL_BACKEND_REGION)
if settings.FEATURE_MULTIREGION_ENABLED and not can_link:
raise InstallMultiRegionSlackException
slack_team_identity, is_slack_team_identity_created = SlackTeamIdentity.objects.get_or_create(
@ -106,7 +105,7 @@ def populate_slack_identities(response, backend, user, organization, **kwargs):
# update slack oauth fields by data from response
slack_team_identity.update_oauth_fields(user, organization, response)
if settings.FEATURE_MULTIREGION_ENABLED:
create_slack_connector(str(organization.uuid), slack_team_id, settings.ONCALL_BACKEND_REGION)
link_slack_team_wrapper(str(organization.uuid), slack_team_id)
populate_slack_channels_for_team.apply_async((slack_team_identity.pk,))
user.slack_user_identity.update_profile_info()
# todo slack: do we need update info for all existing slack users in slack team?

View file

@ -1,6 +1,7 @@
import logging
from typing import Optional, Tuple, Union
from django.conf import settings
from telegram import Bot, InlineKeyboardMarkup, Message, ParseMode
from telegram.error import BadRequest, InvalidToken, Unauthorized
from telegram.utils.request import Request
@ -37,8 +38,15 @@ class TelegramClient:
return False
def register_webhook(self, webhook_url: Optional[str] = None) -> None:
webhook_url = webhook_url or create_engine_url("/telegram/", override_base=live_settings.TELEGRAM_WEBHOOK_HOST)
# Hack to test chatops-proxy v3, remove once v3 is release.
if settings.CHATOPS_V3:
webhook_url = webhook_url or create_engine_url(
"/telegram/v3/", override_base=live_settings.TELEGRAM_WEBHOOK_HOST
)
else:
webhook_url = webhook_url or create_engine_url(
"/telegram/", override_base=live_settings.TELEGRAM_WEBHOOK_HOST
)
# avoid unnecessary set_webhook calls to make sure Telegram rate limits are not exceeded
webhook_info = self.api_client.get_webhook_info()
if webhook_info.url == webhook_url:

View file

@ -6,4 +6,5 @@ app_name = "telegram"
urlpatterns = [
path("", WebHookView.as_view(), name="incoming_webhook"),
path("v3/", WebHookView.as_view(), name="v3_incoming_webhook"),
]

View file

@ -14,7 +14,11 @@ from apps.alerts.models import MaintainableObject
from apps.user_management.constants import AlertGroupTableColumn
from apps.user_management.subscription_strategy import FreePublicBetaSubscriptionStrategy
from common.insight_log import ChatOpsEvent, ChatOpsTypePlug, write_chatops_insight_log
from common.oncall_gateway import create_oncall_connector, delete_oncall_connector, delete_slack_connector
from common.oncall_gateway import (
register_oncall_tenant_wrapper,
unlink_slack_team_wrapper,
unregister_oncall_tenant_wrapper,
)
from common.public_primary_keys import generate_public_primary_key, increase_public_primary_key_length
if typing.TYPE_CHECKING:
@ -61,7 +65,7 @@ class OrganizationQuerySet(models.QuerySet):
def create(self, **kwargs):
instance = super().create(**kwargs)
if settings.FEATURE_MULTIREGION_ENABLED:
create_oncall_connector(str(instance.uuid), settings.ONCALL_BACKEND_REGION)
register_oncall_tenant_wrapper(str(instance.uuid), settings.ONCALL_BACKEND_REGION)
return instance
def delete(self):
@ -104,9 +108,9 @@ class Organization(MaintainableObject):
def delete(self):
if settings.FEATURE_MULTIREGION_ENABLED:
delete_oncall_connector(str(self.uuid))
unregister_oncall_tenant_wrapper(str(self.uuid), settings.ONCALL_BACKEND_REGION)
if self.slack_team_identity:
delete_slack_connector(str(self.uuid))
unlink_slack_team_wrapper(str(self.uuid), self.slack_team_identity.slack_id)
self.deleted_at = timezone.now()
self.save(update_fields=["deleted_at"])

View file

@ -3,9 +3,9 @@ This package is for interaction with OnCall-Gateway, service to provide multireg
"""
from .utils import ( # noqa: F401
check_slack_installation_possible,
create_oncall_connector,
create_slack_connector,
delete_oncall_connector,
delete_slack_connector,
can_link_slack_team_wrapper,
link_slack_team_wrapper,
register_oncall_tenant_wrapper,
unlink_slack_team_wrapper,
unregister_oncall_tenant_wrapper,
)

View file

@ -0,0 +1,154 @@
from dataclasses import dataclass, field
from json import JSONDecodeError
from typing import List
from urllib.parse import urljoin
import requests
SERVICE_TYPE_ONCALL = "oncall"
@dataclass
class SlackLink:
service_type: str
service_tenant_id: str
slack_team_id: str
@dataclass
class MSTeamsLink:
service_type: str
service_tenant_id: str
msteams_id: str
@dataclass
class Tenant:
service_tenant_id: str
service_type: str
cluster_slug: str
slack_links: List[SlackLink] = field(default_factory=list)
msteams_links: List[MSTeamsLink] = field(default_factory=list)
class ChatopsProxyAPIException(Exception):
"""A generic 400 or 500 level exception from the Chatops Proxy API"""
def __init__(self, status, url, msg="", method="GET"):
self.url = url
self.status = status
self.method = method
# Error-message returned by chatops-proxy.
# Since chatops-proxy is internal service messages shouldn't be exposed to the user
self.msg = msg
def __str__(self):
return f"LabelsRepoAPIException: status={self.status} url={self.url} method={self.method} error={self.msg}"
class ChatopsProxyAPIClient:
def __init__(self, url: str, token: str):
self.api_base_url = urljoin(url, "api/v3")
self.api_token = token
# OnCall Tenant
def register_tenant(
self, service_tenant_id: str, cluster_slug: str, service_type: str
) -> tuple[Tenant, requests.models.Response]:
url = f"{self.api_base_url}/tenants/register"
d = {
"tenant": {
"service_tenant_id": service_tenant_id,
"cluster_slug": cluster_slug,
"service_type": service_type,
}
}
response = requests.post(url=url, json=d)
self._check_response(response)
return Tenant(**response.json()["tenant"]), response
def unregister_tenant(
self, service_tenant_id: str, cluster_slug: str, service_type: str
) -> tuple[bool, requests.models.Response]:
url = f"{self.api_base_url}/tenants/unregister"
d = {
"tenant": {
"service_tenant_id": service_tenant_id,
"cluster_slug": cluster_slug,
"service_type": service_type,
}
}
response = requests.post(url=url, json=d)
self._check_response(response)
return response.json()["removed"], response
def can_slack_link(
self, service_tenant_id: str, cluster_slug: str, slack_team_id: str, service_type: str
) -> requests.models.Response:
url = f"{self.api_base_url}/providers/slack/can_link"
d = {
"service_type": service_type,
"service_tenant_id": service_tenant_id,
"cluster_slug": cluster_slug,
"slack_team_id": slack_team_id,
}
response = requests.post(url=url, json=d)
self._check_response(response)
return response
def link_slack_team(
self, service_tenant_id: str, slack_team_id: str, service_type: str
) -> tuple[SlackLink, requests.models.Response]:
url = f"{self.api_base_url}/providers/slack/link"
d = {
"slack_link": {
"service_type": service_type,
"service_tenant_id": service_tenant_id,
"slack_team_id": slack_team_id,
}
}
response = requests.post(url=url, json=d)
self._check_response(response)
return SlackLink(**response.json()["slack_link"]), response
def unlink_slack_team(
self, service_tenant_id: str, slack_team_id: str, service_type: str
) -> tuple[bool, requests.models.Response]:
url = f"{self.api_base_url}/providers/slack/unlink"
d = {
"slack_link": {
"service_type": service_type,
"service_tenant_id": service_tenant_id,
"slack_team_id": slack_team_id,
}
}
response = requests.post(url=url, json=d)
self._check_response(response)
return response.json()["removed"], response
def _check_response(self, response: requests.models.Response):
"""
Wraps an exceptional response to ChatopsProxyAPIException
"""
message = None
if 400 <= response.status_code < 500:
try:
error_data = response.json()
message = error_data.get("error", None)
except JSONDecodeError:
message = response.reason
elif 500 <= response.status_code < 600:
message = response.reason
if message:
raise ChatopsProxyAPIException(
status=response.status_code,
url=response.request.url,
msg=message,
method=response.request.method,
)

View file

@ -31,6 +31,10 @@ DEFAULT_TIMEOUT = 5
class OnCallGatewayAPIClient:
"""
It's a legacy api client, which should be removed after chatops proxy v3 release.
"""
def __init__(self, url: str, token: str):
self.base_url = url
self.api_base_url = urljoin(self.base_url, "api/v1/")

View file

@ -4,7 +4,8 @@ from django.conf import settings
from common.custom_celery_tasks import shared_dedicated_queue_retry_task
from .oncall_gateway_client import OnCallGatewayAPIClient
from .client import ChatopsProxyAPIClient, ChatopsProxyAPIException
from .legacy_client import OnCallGatewayAPIClient
task_logger = get_task_logger(__name__)
@ -40,7 +41,7 @@ def create_oncall_connector_async(oncall_org_id, backend):
def delete_oncall_connector_async(oncall_org_id):
client = OnCallGatewayAPIClient(settings.ONCALL_GATEWAY_URL, settings.ONCALL_GATEWAY_API_TOKEN)
try:
client.delete_slack_connector(oncall_org_id)
client.delete_oncall_connector(oncall_org_id)
except requests.exceptions.HTTPError as http_exc:
if http_exc.response.status_code == 404:
# 404 indicates that connector was deleted already
@ -98,3 +99,103 @@ def delete_slack_connector_async_v2(**kwargs):
except Exception as e:
task_logger.error(f"Failed to delete SlackConnectorV2 oncall_org_id={oncall_org_id} exc={e}")
raise e
# New tasks to use once chatops v3 is landed
@shared_dedicated_queue_retry_task(
autoretry_for=(Exception,),
retry_backoff=True,
max_retries=100,
)
def register_oncall_tenant_async(**kwargs):
service_tenant_id = kwargs.get("service_tenant_id")
cluster_slug = kwargs.get("cluster_slug")
service_type = kwargs.get("service_type")
client = ChatopsProxyAPIClient(settings.ONCALL_GATEWAY_URL, settings.ONCALL_GATEWAY_API_TOKEN)
try:
client.register_tenant(service_tenant_id, cluster_slug, service_type)
except ChatopsProxyAPIException as api_exc:
task_logger.error(
f'msg="Failed to register OnCall tenant: {api_exc.msg}" service_tenant_id={service_tenant_id} cluster_slug={cluster_slug}'
)
if api_exc.status == 409:
# 409 Indicates that it's impossible to register tenant, because tenant already registered.
# Not retrying in this case, becase manual conflict-resolution needed.
return
else:
# Otherwise keep retrying task
raise api_exc
except Exception as e:
# Keep retrying task for any other exceptions too
task_logger.error(
f"Failed to register OnCall tenant: {e} service_tenant_id={service_tenant_id} cluster_slug={cluster_slug}"
)
raise e
@shared_dedicated_queue_retry_task(
autoretry_for=(Exception,),
retry_backoff=True,
max_retries=100,
)
def unregister_oncall_tenant_async(**kwargs):
service_tenant_id = kwargs.get("service_tenant_id")
cluster_slug = kwargs.get("cluster_slug")
service_type = kwargs.get("service_type")
client = ChatopsProxyAPIClient(settings.ONCALL_GATEWAY_URL, settings.ONCALL_GATEWAY_API_TOKEN)
try:
client.unregister_tenant(service_tenant_id, cluster_slug, service_type)
except Exception as e:
task_logger.error(f"Failed to delete OnCallConnector: {e} service_tenant_id={service_tenant_id}")
raise e
@shared_dedicated_queue_retry_task(
autoretry_for=(Exception,),
retry_backoff=True,
max_retries=100,
)
def link_slack_team_async(**kwargs):
service_tenant_id = kwargs.get("service_tenant_id")
service_type = kwargs.get("service_type")
slack_team_id = kwargs.get("slack_team_id")
client = ChatopsProxyAPIClient(settings.ONCALL_GATEWAY_URL, settings.ONCALL_GATEWAY_API_TOKEN)
try:
client.link_slack_team(service_tenant_id, slack_team_id, service_type)
except ChatopsProxyAPIException as api_exc:
task_logger.error(
f'msg="Failed to link slack team: {api_exc.msg}" service_tenant_id={service_tenant_id} slack_team_id={slack_team_id}'
)
if api_exc.status == 409:
# Impossible to register tenant, slack workspace already connected to another cluster.
# Not retrying in this case, because manual conflict-resolution needed.
return
else:
raise api_exc
except Exception as e:
task_logger.error(
f'msg="Failed to link slack team: {e}" service_tenant_id={service_tenant_id} slack_team_id={slack_team_id}'
)
raise e
@shared_dedicated_queue_retry_task(
autoretry_for=(Exception,),
retry_backoff=True,
max_retries=100,
)
def unlink_slack_team_async(**kwargs):
service_tenant_id = kwargs.get("service_tenant_id")
service_type = kwargs.get("service_type")
slack_team_id = kwargs.get("slack_team_id")
client = ChatopsProxyAPIClient(settings.ONCALL_GATEWAY_URL, settings.ONCALL_GATEWAY_API_TOKEN)
try:
client.unlink_slack_team(service_tenant_id, slack_team_id, service_type)
except Exception as e:
task_logger.error(
f'msg="Failed to unlink slack_team: {e}" service_tenant_id={service_tenant_id} slack_team_id={slack_team_id}'
)
raise e

View file

@ -1,19 +1,29 @@
"""
Set of utils to handle oncall and chatops-proxy interaction.
TODO: Once chatops v3 will be released, remove legacy and wrapper functions
"""
import logging
import requests
from django.conf import settings
from .oncall_gateway_client import OnCallGatewayAPIClient
from .client import SERVICE_TYPE_ONCALL, ChatopsProxyAPIClient
from .legacy_client import OnCallGatewayAPIClient
from .tasks import (
create_oncall_connector_async,
create_slack_connector_async_v2,
delete_oncall_connector_async,
delete_slack_connector_async_v2,
link_slack_team_async,
register_oncall_tenant_async,
unlink_slack_team_async,
unregister_oncall_tenant_async,
)
logger = logging.getLogger(__name__)
# Legacy to work with chatops-proxy v1.
def create_oncall_connector(oncall_org_id: str, backend: str):
client = OnCallGatewayAPIClient(settings.ONCALL_GATEWAY_URL, settings.ONCALL_GATEWAY_API_TOKEN)
try:
@ -44,7 +54,7 @@ def check_slack_installation_possible(oncall_org_id: str, slack_id: str, backend
def create_slack_connector(oncall_org_id: str, slack_id: str, backend: str):
client = OnCallGatewayAPIClient(settings.ONCALL_GATEWAY_URL, settings.ONCALL_GATEWAY_API_TOKEN)
client = ChatopsProxyAPIClient(settings.ONCALL_GATEWAY_URL, settings.ONCALL_GATEWAY_API_TOKEN)
try:
client.post_slack_connector(oncall_org_id, slack_id, backend)
except Exception as e:
@ -59,3 +69,128 @@ def create_slack_connector(oncall_org_id: str, slack_id: str, backend: str):
def delete_slack_connector(oncall_org_id: str):
delete_slack_connector_async_v2.delay(oncall_org_id=oncall_org_id)
# utils to work with v3 version
def register_oncall_tenant(service_tenant_id: str, cluster_slug: str):
"""
register_oncall_tenant tries to register oncall tenant synchronously and fall back to task in case of any exceptions
to make sure that tenant is registered.
First attempt is synchronous to register tenant ASAP to not miss any chatops requests.
"""
client = ChatopsProxyAPIClient(settings.ONCALL_GATEWAY_URL, settings.ONCALL_GATEWAY_API_TOKEN)
try:
client.register_tenant(service_tenant_id, cluster_slug, SERVICE_TYPE_ONCALL)
except Exception as e:
logger.error(
f"create_oncall_connector: failed " f"oncall_org_id={service_tenant_id} backend={cluster_slug} exc={e}"
)
register_oncall_tenant_async.apply_async(
kwargs={
"service_tenant_id": service_tenant_id,
"cluster_slug": cluster_slug,
"service_type": SERVICE_TYPE_ONCALL,
},
countdown=2,
)
def unregister_oncall_tenant(service_tenant_id: str, cluster_slug: str):
"""
unregister_oncall_tenant unregisters tenant asynchronously.
"""
unregister_oncall_tenant_async.apply_async(
kwargs={
"service_tenant_id": service_tenant_id,
"cluster_slug": cluster_slug,
"service_type": SERVICE_TYPE_ONCALL,
},
countdown=2,
)
def can_link_slack_team(
service_tenant_id: str,
slack_team_id: str,
cluster_slug: str,
) -> bool:
"""
can_link_slack_team checks if it's possible to link slack workspace to oncall tenant located in cluster.
All oncall tenants linked to same slack team should have same cluster.
"""
client = ChatopsProxyAPIClient(settings.ONCALL_GATEWAY_URL, settings.ONCALL_GATEWAY_API_TOKEN)
try:
response = client.can_slack_link(service_tenant_id, cluster_slug, slack_team_id, SERVICE_TYPE_ONCALL)
return response.status_code == 200
except Exception as e:
logger.error(
f"can_link_slack_team: slack installation impossible: {e} "
f"service_tenant_id={service_tenant_id} slack_team_id={slack_team_id} cluster_slug={cluster_slug}"
)
return False
def link_slack_team(service_tenant_id: str, slack_team_id: str):
client = ChatopsProxyAPIClient(settings.ONCALL_GATEWAY_URL, settings.ONCALL_GATEWAY_API_TOKEN)
try:
client.link_slack_team(service_tenant_id, slack_team_id, SERVICE_TYPE_ONCALL)
except Exception as e:
logger.error(
f'msg="Failed to link slack team: {e}"'
f"service_tenant_id={service_tenant_id} slack_team_id={slack_team_id}"
)
link_slack_team_async.apply_async(
kwargs={
"service_tenant_id": service_tenant_id,
"slack_team_id": slack_team_id,
"service_type": SERVICE_TYPE_ONCALL,
},
countdown=2,
)
def unlink_slack_team(service_tenant_id: str, slack_team_id: str):
unlink_slack_team_async.apply_async(
kwargs={
"service_tenant_id": service_tenant_id,
"slack_team_id": slack_team_id,
"service_type": SERVICE_TYPE_ONCALL,
}
)
# Wrappers to choose whether legacy or v3 function should be call, depending on CHATOPS_V3 env var.
def register_oncall_tenant_wrapper(service_tenant_id: str, cluster_slug: str):
if settings.CHATOPS_V3:
register_oncall_tenant(service_tenant_id, cluster_slug)
else:
create_oncall_connector(service_tenant_id, cluster_slug)
def unregister_oncall_tenant_wrapper(service_tenant_id: str, cluster_slug: str):
if settings.CHATOPS_V3:
unregister_oncall_tenant(service_tenant_id, cluster_slug)
else:
delete_oncall_connector(service_tenant_id)
def can_link_slack_team_wrapper(service_tenant_id: str, slack_team_id, cluster_slug: str):
if settings.CHATOPS_V3:
can_link_slack_team(service_tenant_id, slack_team_id, cluster_slug)
else:
check_slack_installation_possible(service_tenant_id, slack_team_id, cluster_slug)
def link_slack_team_wrapper(service_tenant_id: str, slack_team_id: str):
if settings.CHATOPS_V3:
link_slack_team(service_tenant_id, slack_team_id)
else:
create_slack_connector(service_tenant_id, slack_team_id, settings.ONCALL_BACKEND_REGION)
def unlink_slack_team_wrapper(service_tenant_id: str, slack_team_id: str):
if settings.CHATOPS_V3:
unlink_slack_team(service_tenant_id, slack_team_id)
else:
delete_slack_connector(service_tenant_id)

View file

@ -97,6 +97,7 @@ WEBHOOK_RESPONSE_LIMIT = 50000
ONCALL_GATEWAY_URL = os.environ.get("ONCALL_GATEWAY_URL", "")
ONCALL_GATEWAY_API_TOKEN = os.environ.get("ONCALL_GATEWAY_API_TOKEN", "")
ONCALL_BACKEND_REGION = os.environ.get("ONCALL_BACKEND_REGION")
CHATOPS_V3 = getenv_boolean("CHATOPS_V3", False)
# Prometheus exporter metrics endpoint auth
PROMETHEUS_EXPORTER_SECRET = os.environ.get("PROMETHEUS_EXPORTER_SECRET")

View file

@ -78,6 +78,10 @@ CELERY_TASK_ROUTES = {
"apps.oss_installation.tasks.send_cloud_heartbeat_task": {"queue": "default"},
"apps.oss_installation.tasks.send_usage_stats_report": {"queue": "default"},
"apps.oss_installation.tasks.sync_users_with_cloud": {"queue": "default"},
"common.oncall_gateway.tasks.link_slack_team_async": {"queue": "default"},
"common.oncall_gateway.tasks.unlink_slack_team_async": {"queue": "default"},
"common.oncall_gateway.tasks.register_oncall_tenant_async": {"queue": "default"},
"common.oncall_gateway.tasks.unregister_oncall_tenant_async": {"queue": "default"},
# CRITICAL
"apps.alerts.tasks.acknowledge_reminder.acknowledge_reminder_task": {"queue": "critical"},
"apps.alerts.tasks.acknowledge_reminder.unacknowledge_timeout_task": {"queue": "critical"},