from datetime import timedelta from functools import partial from django.conf import settings from django.core.cache import cache from django.db import transaction from django.utils import timezone from apps.alerts.signals import post_ack_reminder_message_signal from common.custom_celery_tasks import shared_dedicated_queue_retry_task from .send_alert_group_signal import send_alert_group_signal from .task_logger import task_logger MAX_RETRIES = 1 if settings.DEBUG else 10 def is_allowed_to_send_acknowledge_reminder(alert_group_id, process_id): lock_id = f"acknowledge-reminder-lock-{alert_group_id}" lock_period = 60 * 10 # 10 min # cache.add returns False if the key already exists status = cache.add(lock_id, process_id, lock_period) return status @shared_dedicated_queue_retry_task(autoretry_for=(Exception,), retry_backoff=True, max_retries=MAX_RETRIES) def acknowledge_reminder_task(alert_group_pk: int, unacknowledge_process_id: str) -> None: from apps.alerts.models import AlertGroup, AlertGroupLogRecord from apps.user_management.models import Organization with transaction.atomic(): try: alert_group = AlertGroup.objects.select_for_update().get(pk=alert_group_pk) # Lock alert_group except AlertGroup.DoesNotExist: task_logger.warning(f"AlertGroup {alert_group_pk} does not exist") return if unacknowledge_process_id != alert_group.last_unique_unacknowledge_process_id: return # Don't proceed if acknowledge reminder for this alert group has already been sent recently if not is_allowed_to_send_acknowledge_reminder(alert_group.id, unacknowledge_process_id): task_logger.info(f"Acknowledge reminder for alert_group {alert_group_pk} has already been sent recently.") return organization = alert_group.channel.organization # Get timeout values acknowledge_reminder_timeout = Organization.ACKNOWLEDGE_REMIND_DELAY[organization.acknowledge_remind_timeout] unacknowledge_timeout = Organization.UNACKNOWLEDGE_TIMEOUT_DELAY[organization.unacknowledge_timeout] # Don't proceed if the alert group is not in a state for acknowledgement reminder acknowledge_reminder_required = ( alert_group.is_root_alert_group and alert_group.status == AlertGroup.ACKNOWLEDGED and alert_group.acknowledged_by == AlertGroup.USER and acknowledge_reminder_timeout ) is_organization_deleted = organization.deleted_at is not None log_info = ( f"acknowledge_reminder_timeout option: {acknowledge_reminder_timeout}," f"organization ppk: {organization.public_primary_key}," f"organization is deleted: {is_organization_deleted}" ) if not acknowledge_reminder_required or is_organization_deleted: task_logger.info(f"alert group {alert_group_pk} is not in a state for acknowledgement reminder. {log_info}") return task_logger.info(f"alert group {alert_group_pk} is in a state for acknowledgement reminder. {log_info}") # unacknowledge_timeout_task uses acknowledged_by_confirmed to check if acknowledgement reminder has been confirmed # by the user. Setting to None here to indicate that the user has not confirmed the acknowledgement reminder if alert_group.acknowledged_by_confirmed is not None: alert_group.acknowledged_by_confirmed = None alert_group.save(update_fields=["acknowledged_by_confirmed"]) if unacknowledge_timeout: # "unack in N minutes if no response" is enabled unacknowledge_timeout_task.apply_async( (alert_group.pk, unacknowledge_process_id), countdown=unacknowledge_timeout ) else: if alert_group.started_at < timezone.now() - timedelta(days=settings.ACKNOWLEDGE_REMINDER_TASK_EXPIRY_DAYS): task_logger.info( f"alert group {alert_group_pk} not renewing acknowledgement reminder, started_at is too old. {log_info}" ) return acknowledge_reminder_task.apply_async( (alert_group.pk, unacknowledge_process_id), countdown=acknowledge_reminder_timeout ) with transaction.atomic(): log_record = alert_group.log_records.create( type=AlertGroupLogRecord.TYPE_ACK_REMINDER_TRIGGERED, author=alert_group.acknowledged_by_user ) task_logger.info(f"created log record {log_record.pk}, sending signal...") transaction.on_commit(partial(send_post_ack_reminder_message_signal.delay, log_record.pk)) @shared_dedicated_queue_retry_task(autoretry_for=(Exception,), retry_backoff=True, max_retries=MAX_RETRIES) def unacknowledge_timeout_task(alert_group_pk: int, unacknowledge_process_id: str) -> None: from apps.alerts.models import AlertGroup, AlertGroupLogRecord from apps.user_management.models import Organization with transaction.atomic(): try: alert_group = AlertGroup.objects.select_for_update().get(pk=alert_group_pk) # Lock alert_group except AlertGroup.DoesNotExist: task_logger.warning(f"AlertGroup {alert_group_pk} does not exist") return if unacknowledge_process_id != alert_group.last_unique_unacknowledge_process_id: return organization = alert_group.channel.organization # Get timeout values acknowledge_reminder_timeout = Organization.ACKNOWLEDGE_REMIND_DELAY[organization.acknowledge_remind_timeout] unacknowledge_timeout = Organization.UNACKNOWLEDGE_TIMEOUT_DELAY[organization.unacknowledge_timeout] # Don't proceed if the alert group is not in a state for auto-unacknowledge unacknowledge_required = ( alert_group.is_root_alert_group and alert_group.status == AlertGroup.ACKNOWLEDGED and alert_group.acknowledged_by == AlertGroup.USER and acknowledge_reminder_timeout and unacknowledge_timeout ) is_organization_deleted = organization.deleted_at is not None log_info = ( f"acknowledge_reminder_timeout option: {acknowledge_reminder_timeout}," f"unacknowledge_timeout option: {unacknowledge_timeout}," f"organization ppk: {organization.public_primary_key}," f"organization is deleted: {is_organization_deleted}" ) if not unacknowledge_required or is_organization_deleted: task_logger.info(f"alert group {alert_group_pk} is not in a state for unacknowledge by timeout. {log_info}") return if alert_group.acknowledged_by_confirmed: # acknowledgement reminder was confirmed by the user acknowledge_reminder_task.apply_async( (alert_group_pk, unacknowledge_process_id), countdown=acknowledge_reminder_timeout - unacknowledge_timeout ) task_logger.info( f"Acknowledgement reminder was confirmed by user. Rescheduling acknowledge_reminder_task..." f"alert group: {alert_group_pk}, {log_info}" ) return task_logger.info(f"alert group {alert_group_pk} is in a state for unacknowledge by timeout. {log_info}") # If acknowledgement reminder wasn't confirmed by the user, unacknowledge the alert group and start escalation again log_record = alert_group.log_records.create( type=AlertGroupLogRecord.TYPE_AUTO_UN_ACK, author=alert_group.acknowledged_by_user ) transaction.on_commit(partial(send_alert_group_signal.delay, log_record.pk)) alert_group.unacknowledge() alert_group.start_escalation_if_needed() @shared_dedicated_queue_retry_task(autoretry_for=(Exception,), retry_backoff=True, max_retries=MAX_RETRIES) def send_post_ack_reminder_message_signal(log_record_id): """ Sends signal to post acknowledge reminder message to Slack thread. The signal is connected to AlertGroupSlackRepresentative. """ task_logger.info(f"sending signal for posting ack reminder message, log record {log_record_id}") post_ack_reminder_message_signal.send(sender=send_post_ack_reminder_message_signal, log_record=log_record_id)