oncall-engine/engine/apps/email/tasks.py
Michael Derynck 4c1639c944
Fix retrying email notification task (#4772)
# What this PR does
Email tasks are failing and retrying when they use the fallback default
notification policy as it does not get saved in the DB. This PR uses the
same way as UserNotificationPolicyLogRecord to set that field to null to
avoid `ValueError("save() prohibited to prevent data loss due to unsaved
related object 'notification_policy'.").` when that is the case.

## Which issue(s) this PR closes

Related to [issue link here]

<!--
*Note*: If you want the issue to be auto-closed once the PR is merged,
change "Related to" to "Closes" in the line above.
If you have more than one GitHub issue that this PR closes, be sure to
preface
each issue link with a [closing
keyword](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/using-keywords-in-issues-and-pull-requests#linking-a-pull-request-to-an-issue).
This ensures that the issue(s) are auto-closed once the PR has been
merged.
-->

## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] Added the relevant release notes label (see labels prefixed w/
`release:`). These labels dictate how your PR will
    show up in the autogenerated release notes.
2024-08-01 14:48:39 +00:00

152 lines
6 KiB
Python

from socket import gaierror
from celery.utils.log import get_task_logger
from django.conf import settings
from django.core.mail import BadHeaderError, get_connection, send_mail
from django.utils.html import strip_tags
from apps.alerts.models import AlertGroup
from apps.base.utils import live_settings
from apps.email.alert_rendering import build_subject_and_message
from apps.email.models import EmailMessage
from apps.user_management.models import User
from common.custom_celery_tasks import shared_dedicated_queue_retry_task
MAX_RETRIES = 1 if settings.DEBUG else 10
logger = get_task_logger(__name__)
def get_from_email(user):
if live_settings.EMAIL_FROM_ADDRESS:
return live_settings.EMAIL_FROM_ADDRESS
if settings.LICENSE == settings.CLOUD_LICENSE_NAME:
return "oncall@{}.grafana.net".format(user.organization.stack_slug)
return live_settings.EMAIL_HOST_USER
@shared_dedicated_queue_retry_task(autoretry_for=(Exception,), retry_backoff=True, max_retries=MAX_RETRIES)
def notify_user_async(user_pk, alert_group_pk, notification_policy_pk):
# imported here to avoid circular import error
from apps.base.models import UserNotificationPolicy, UserNotificationPolicyLogRecord
try:
user = User.objects.get(pk=user_pk)
except User.DoesNotExist:
logger.warning(f"User {user_pk} does not exist")
return
try:
alert_group = AlertGroup.objects.get(pk=alert_group_pk)
except AlertGroup.DoesNotExist:
logger.warning(f"Alert group {alert_group_pk} does not exist")
return
using_fallback_default_notification_policy_step = False
if notification_policy_pk is None:
# NOTE: `notification_policy_pk` may be None if the user has no notification policies defined, as
# email is the default backend used. see `UserNotificationPolicy.get_default_fallback_policy` for more details
notification_policy = UserNotificationPolicy.get_default_fallback_policy(user)
using_fallback_default_notification_policy_step = True
else:
try:
notification_policy = UserNotificationPolicy.objects.get(pk=notification_policy_pk)
except UserNotificationPolicy.DoesNotExist:
logger.warning(f"User notification policy {notification_policy_pk} does not exist")
return
def _create_user_notification_policy_log_record(**kwargs):
return UserNotificationPolicyLogRecord.objects.create(
**kwargs, using_fallback_default_notification_policy_step=using_fallback_default_notification_policy_step
)
def _create_email_message(**kwargs):
return EmailMessage.objects.create(
**kwargs, using_fallback_default_notification_policy_step=using_fallback_default_notification_policy_step
)
# create an error log in case EMAIL_HOST is not specified
if not live_settings.EMAIL_HOST:
_create_user_notification_policy_log_record(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
notification_policy=notification_policy,
alert_group=alert_group,
reason="Error while sending email",
notification_step=notification_policy.step,
notification_channel=notification_policy.notify_by,
)
logger.error("Error while sending email: empty EMAIL_HOST env variable")
return
emails_left = user.organization.emails_left(user)
if emails_left <= 0:
_create_user_notification_policy_log_record(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
notification_policy=notification_policy,
alert_group=alert_group,
reason="Error while sending email",
notification_step=notification_policy.step,
notification_channel=notification_policy.notify_by,
notification_error_code=UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_MAIL_LIMIT_EXCEEDED,
)
_create_email_message(
represents_alert_group=alert_group,
notification_policy=notification_policy,
receiver=user,
exceeded_limit=True,
)
return
subject, html_message = build_subject_and_message(alert_group, emails_left)
message = strip_tags(html_message)
from_email = get_from_email(user)
recipient_list = [user.email]
connection = get_connection(
host=live_settings.EMAIL_HOST,
port=live_settings.EMAIL_PORT,
username=live_settings.EMAIL_HOST_USER,
password=live_settings.EMAIL_HOST_PASSWORD,
use_tls=live_settings.EMAIL_USE_TLS,
use_ssl=live_settings.EMAIL_USE_SSL,
fail_silently=False,
timeout=5,
)
try:
send_mail(subject, message, from_email, recipient_list, html_message=html_message, connection=connection)
_create_email_message(
represents_alert_group=alert_group,
notification_policy=notification_policy,
receiver=user,
exceeded_limit=False,
)
except (gaierror, BadHeaderError) as e:
# gaierror is raised when EMAIL_HOST is invalid
# BadHeaderError is raised when there's newlines in the subject
_create_user_notification_policy_log_record(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
notification_policy=notification_policy,
alert_group=alert_group,
reason="Error while sending email",
notification_step=notification_policy.step,
notification_channel=notification_policy.notify_by,
)
logger.error(f"Error while sending email: {e}")
return
# record success log
_create_user_notification_policy_log_record(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_SUCCESS,
notification_policy=notification_policy,
alert_group=alert_group,
notification_step=notification_policy.step,
notification_channel=notification_policy.notify_by,
)