183 lines
8 KiB
Python
183 lines
8 KiB
Python
|
import logging
|
||
|
|
|
||
|
|
from celery.utils.log import get_task_logger
|
||
|
|
from django.conf import settings
|
||
|
|
from rest_framework import status
|
||
|
|
|
||
|
|
from apps.alerts.models import Alert, AlertGroup
|
||
|
|
from apps.mattermost.alert_rendering import AlertGroupMattermostRenderer, MattermostMessageRenderer
|
||
|
|
from apps.mattermost.client import MattermostClient
|
||
|
|
from apps.mattermost.exceptions import MattermostAPIException, MattermostAPITokenInvalid
|
||
|
|
from apps.mattermost.models import MattermostChannel, MattermostMessage
|
||
|
|
from apps.user_management.models import User
|
||
|
|
from common.custom_celery_tasks import shared_dedicated_queue_retry_task
|
||
|
|
from common.utils import OkToRetry
|
||
|
|
|
||
|
|
logger = get_task_logger(__name__)
|
||
|
|
logger.setLevel(logging.DEBUG)
|
||
|
|
|
||
|
|
|
||
|
|
@shared_dedicated_queue_retry_task(
|
||
|
|
bind=True, autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else None
|
||
|
|
)
|
||
|
|
def on_create_alert_async(self, alert_pk):
|
||
|
|
"""
|
||
|
|
It's async in order to prevent Mattermost downtime or formatting issues causing delay with SMS and other destinations.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
alert = Alert.objects.get(pk=alert_pk)
|
||
|
|
except Alert.DoesNotExist as e:
|
||
|
|
if on_create_alert_async.request.retries >= 10:
|
||
|
|
logger.error(f"Alert {alert_pk} was not found. Probably it was deleted. Stop retrying")
|
||
|
|
return
|
||
|
|
else:
|
||
|
|
raise e
|
||
|
|
|
||
|
|
alert_group = alert.group
|
||
|
|
mattermost_channel = MattermostChannel.get_channel_for_alert_group(alert_group=alert_group)
|
||
|
|
if not mattermost_channel:
|
||
|
|
logger.error(f"Mattermost channel not found for alert {alert_pk}. Probably it was deleted. Stop retrying")
|
||
|
|
return
|
||
|
|
|
||
|
|
message = alert_group.mattermost_messages.filter(message_type=MattermostMessage.ALERT_GROUP_MESSAGE).first()
|
||
|
|
if message:
|
||
|
|
logger.error(f"Mattermost message exist with post id {message.post_id} hence skipping")
|
||
|
|
return
|
||
|
|
|
||
|
|
payload = MattermostMessageRenderer(alert_group).render_alert_group_message()
|
||
|
|
|
||
|
|
with OkToRetry(task=self, exc=(MattermostAPIException,), num_retries=3):
|
||
|
|
try:
|
||
|
|
client = MattermostClient()
|
||
|
|
mattermost_post = client.create_post(channel_id=mattermost_channel.channel_id, data=payload)
|
||
|
|
except MattermostAPITokenInvalid:
|
||
|
|
logger.error(f"Mattermost API token is invalid could not create post for alert {alert_pk}")
|
||
|
|
except MattermostAPIException as ex:
|
||
|
|
logger.error(f"Mattermost API error {ex}")
|
||
|
|
if ex.status not in [status.HTTP_401_UNAUTHORIZED]:
|
||
|
|
raise ex
|
||
|
|
else:
|
||
|
|
MattermostMessage.create_message(
|
||
|
|
alert_group=alert_group, post=mattermost_post, message_type=MattermostMessage.ALERT_GROUP_MESSAGE
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@shared_dedicated_queue_retry_task(
|
||
|
|
autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else None
|
||
|
|
)
|
||
|
|
def on_alert_group_action_triggered_async(log_record_id):
|
||
|
|
from apps.alerts.models import AlertGroupLogRecord
|
||
|
|
from apps.mattermost.alert_group_representative import AlertGroupMattermostRepresentative
|
||
|
|
|
||
|
|
try:
|
||
|
|
log_record = AlertGroupLogRecord.objects.get(pk=log_record_id)
|
||
|
|
except AlertGroupLogRecord.DoesNotExist as e:
|
||
|
|
logger.warning(f"Mattermost representative: log record {log_record_id} never created or has been deleted")
|
||
|
|
raise e
|
||
|
|
|
||
|
|
alert_group_id = log_record.alert_group_id
|
||
|
|
|
||
|
|
try:
|
||
|
|
log_record.alert_group.mattermost_messages.get(message_type=MattermostMessage.ALERT_GROUP_MESSAGE)
|
||
|
|
except MattermostMessage.DoesNotExist as e:
|
||
|
|
if on_alert_group_action_triggered_async.request.retries >= 10:
|
||
|
|
logger.error(f"Mattermost message not created for {alert_group_id}. Stop retrying")
|
||
|
|
return
|
||
|
|
else:
|
||
|
|
raise e
|
||
|
|
|
||
|
|
logger.info(
|
||
|
|
f"Start mattermost on_alert_group_action_triggered for alert_group {alert_group_id}, log record {log_record_id}"
|
||
|
|
)
|
||
|
|
representative = AlertGroupMattermostRepresentative(log_record)
|
||
|
|
if representative.is_applicable():
|
||
|
|
handler = representative.get_handler()
|
||
|
|
handler(log_record.alert_group)
|
||
|
|
|
||
|
|
|
||
|
|
@shared_dedicated_queue_retry_task(
|
||
|
|
autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else None
|
||
|
|
)
|
||
|
|
def notify_user_about_alert_async(user_pk, alert_group_pk, notification_policy_pk):
|
||
|
|
from apps.base.models import UserNotificationPolicy, UserNotificationPolicyLogRecord
|
||
|
|
|
||
|
|
def _create_error_log_record(notification_error_code=None):
|
||
|
|
UserNotificationPolicyLogRecord.objects.create(
|
||
|
|
author=user,
|
||
|
|
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
|
||
|
|
notification_policy=notification_policy,
|
||
|
|
alert_group=alert_group,
|
||
|
|
reason="Error during mattermost notification",
|
||
|
|
notification_step=notification_policy.step,
|
||
|
|
notification_channel=notification_policy.notify_by,
|
||
|
|
notification_error_code=notification_error_code,
|
||
|
|
)
|
||
|
|
|
||
|
|
try:
|
||
|
|
user = User.objects.get(pk=user_pk)
|
||
|
|
alert_group = AlertGroup.objects.get(pk=alert_group_pk)
|
||
|
|
notification_policy = UserNotificationPolicy.objects.get(pk=notification_policy_pk)
|
||
|
|
mattermost_messsage = alert_group.mattermost_messages.get(message_type=MattermostMessage.ALERT_GROUP_MESSAGE)
|
||
|
|
except User.DoesNotExist:
|
||
|
|
logger.warning(f"User {user_pk} is not found")
|
||
|
|
return
|
||
|
|
except AlertGroup.DoesNotExist:
|
||
|
|
logger.warning(f"Alert group {alert_group_pk} is not found")
|
||
|
|
return
|
||
|
|
except UserNotificationPolicy.DoesNotExist:
|
||
|
|
logger.warning(f"UserNotificationPolicy {notification_policy_pk} is not found")
|
||
|
|
return
|
||
|
|
except MattermostMessage.DoesNotExist as e:
|
||
|
|
if notify_user_about_alert_async.request.retries >= 10:
|
||
|
|
logger.error(
|
||
|
|
f"Alert group mattermost message is not created {alert_group_pk}. Hence stopped retrying for user notification"
|
||
|
|
)
|
||
|
|
_create_error_log_record(
|
||
|
|
UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_IN_MATTERMOST_ALERT_GROUP_MESSAGE_NOT_FOUND
|
||
|
|
)
|
||
|
|
return
|
||
|
|
else:
|
||
|
|
raise e
|
||
|
|
|
||
|
|
mattermost_channel = MattermostChannel.get_channel_for_alert_group(alert_group=alert_group)
|
||
|
|
if not mattermost_channel:
|
||
|
|
logger.error(f"Mattermost channel not found for user notification {user_pk}")
|
||
|
|
return
|
||
|
|
|
||
|
|
templated_alert = AlertGroupMattermostRenderer(alert_group).alert_renderer.templated_alert
|
||
|
|
|
||
|
|
if not hasattr(user, "mattermost_user_identity"):
|
||
|
|
message = "{}\nTried to invite {} to look at the alert group. Unfortunately {} is not in mattermost.".format(
|
||
|
|
templated_alert.title, user.username, user.username
|
||
|
|
)
|
||
|
|
_create_error_log_record(
|
||
|
|
UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_IN_MATTERMOST_USER_NOT_IN_MATTERMOST
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
message = "{}\nInviting {} to look at the alert group.".format(
|
||
|
|
templated_alert.title, user.mattermost_user_identity.mention_username
|
||
|
|
)
|
||
|
|
|
||
|
|
payload = {"root_id": mattermost_messsage.post_id, "message": message}
|
||
|
|
|
||
|
|
try:
|
||
|
|
client = MattermostClient()
|
||
|
|
client.create_post(channel_id=mattermost_channel.channel_id, data=payload)
|
||
|
|
except MattermostAPITokenInvalid:
|
||
|
|
logger.error(f"Mattermost API token is invalid could not create post for alert {alert_group_pk}")
|
||
|
|
_create_error_log_record(UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_IN_MATTERMOST_API_TOKEN_INVALID)
|
||
|
|
except MattermostAPIException as ex:
|
||
|
|
logger.error(f"Mattermost API error {ex}")
|
||
|
|
if ex.status != status.HTTP_401_UNAUTHORIZED:
|
||
|
|
raise ex
|
||
|
|
_create_error_log_record(UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_IN_MATTERMOST_API_UNAUTHORIZED)
|
||
|
|
else:
|
||
|
|
UserNotificationPolicyLogRecord.objects.create(
|
||
|
|
author=user,
|
||
|
|
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_SUCCESS,
|
||
|
|
notification_policy=notification_policy,
|
||
|
|
alert_group=alert_group,
|
||
|
|
notification_step=notification_policy.step,
|
||
|
|
notification_channel=notification_policy.notify_by,
|
||
|
|
)
|