First touch on grafana cloud notifications

This commit is contained in:
Innokentii Konstantinov 2022-06-03 19:47:25 +04:00
parent 6b40f95033
commit 0cdd2d7b8b
14 changed files with 383 additions and 104 deletions

View file

@ -1,5 +1,5 @@
from apps.alerts.incident_appearance.templaters.alert_templater import AlertTemplater
from common.utils import clean_markup
from common.utils import clean_markup, escape_for_twilio_phone_call
class AlertPhoneCallTemplater(AlertTemplater):
@ -24,8 +24,4 @@ class AlertPhoneCallTemplater(AlertTemplater):
return sf.format(data)
def _escape(self, data):
# https://www.twilio.com/docs/api/errors/12100
data = data.replace("&", "&")
data = data.replace(">", ">")
data = data.replace("<", "&lt;")
return data
return escape_for_twilio_phone_call(data)

View file

@ -0,0 +1 @@
CLOUD_URL = "https://a-prod-us-central-0.grafana.net/"

View file

@ -1,2 +1,4 @@
from .cloud_organization_connector import CloudOrganizationConnector # noqa: F401
from .cloud_users import CloudUserIdentity # noqa: F401
from .heartbeat import CloudHeartbeat # noqa: F401
from .oss_installation import OssInstallation # noqa: F401

View file

@ -0,0 +1,138 @@
import logging
from urllib.parse import urljoin
import requests
from django.db import models
from apps.base.utils import live_settings
from apps.oss_installation.constants import CLOUD_URL
from apps.oss_installation.models.cloud_users import CloudUserIdentity
from apps.user_management.models import User
logger = logging.getLogger(__name__)
class CloudOrganizationConnector(models.Model):
"""
CloudOrganizationConnector model represents connection between oss organization and cloud organization.
"""
cloud_url = models.URLField()
organization = models.OneToOneField(
"user_management.organization", related_name="cloud_connector", on_delete=models.CASCADE
)
@classmethod
def sync_with_cloud(cls, organization) -> bool:
"""
sync_with_cloud sync organization with cloud organization defined by provided GRAFANA_CLOUD_ONCALL_TOKEN.
"""
sync_status = False
api_token = live_settings.GRAFANA_CLOUD_ONCALL_TOKEN
if api_token is None:
logger.warning("Unable to sync with cloud. GRAFANA_CLOUD_ONCALL_TOKEN is not set")
else:
info_url = urljoin(CLOUD_URL, "api/v1/info/")
try:
r = requests.get(info_url, headers={"AUTHORIZATION": api_token}, timeout=5)
if r.status_code == 200:
cls.objects.update_or_create(organization=organization, defaults={"cloud_url": r.json()["url"]})
sync_status = True
if r.status_code == 403:
logger.warning("Unable to sync with cloud. GRAFANA_CLOUD_ONCALL_TOKEN is invalid")
except requests.exceptions.RequestException as e:
logger.warning(f"Unable to sync with cloud. Request exception {str(e)}")
return sync_status
def sync_users_with_cloud(self):
api_token = live_settings.GRAFANA_CLOUD_ONCALL_TOKEN
if api_token is None:
logger.warning("Unable to sync with cloud. GRAFANA_CLOUD_ONCALL_TOKEN is not set")
return
existing_emails = list(User.objects.filter(organization=self.organization).values_list("email", flat=True))
# existing_cloud_ids = list(
# CloudUserIdentity.objects.filter(organization=self.organization).values_list("cloud_id", flat=True)
# )
matching_users = []
users_url = urljoin(CLOUD_URL, "api/v1/users")
existing_cloud_identities = list(CloudUserIdentity.objects.filter(organization=self.organization))
existing_cloud_ids = list(map(lambda u: u.cloud_id, existing_cloud_identities))
fetch_next_page = True
page = 1
while fetch_next_page:
try:
url = urljoin(users_url, f"?page={page}&?short=true")
r = requests.get(url, headers={"AUTHORIZATION": api_token}, timeout=5)
if r.status_code != 200:
logger.warning(
f"Unable to fetch page {page} while sync_users_with_cloud. Response status code {r.status_code}"
)
if r.status_code == 429 or r.status_code == 403:
break
data = r.json()
matching_users.extend(list(filter(lambda u: (u["email"] in existing_emails), data["results"])))
page += 1
if data["next"] is None:
fetch_next_page = False
except requests.exceptions.RequestException as e:
logger.warning(f"Unable to sync users with cloud. Request exception {str(e)}")
break
cloud_users_identities_to_update = {}
cloud_users_identities_to_create = []
for user in matching_users:
if user["id"] in existing_cloud_ids:
cloud_users_identities_to_update[user["id"]] = user
else:
cloud_users_identities_to_create.append(
CloudUserIdentity(
cloud_id=user["id"],
email=user["email"],
phone_number_verified=user["is_phone_number_verified"],
organization=self.organization,
)
)
for i in existing_cloud_identities:
i.email = cloud_users_identities_to_update[i.cloud_id]["email"]
i.phone_number_verified = cloud_users_identities_to_update[i.cloud_id]["is_phone_number_verified"]
# TODO: Grafana Twilio: check if data validation needed.
CloudUserIdentity.objects.bulk_create(cloud_users_identities_to_create, batch_size=1000)
CloudUserIdentity.objects.bulk_update(
existing_cloud_identities, ["email", "phone_number_verified"], batch_size=1000
)
def sync_user_with_cloud(self, user):
api_token = live_settings.GRAFANA_CLOUD_ONCALL_TOKEN
if api_token is None:
logger.warning(f"Unable to sync_user_with cloud user_id {user.id}. GRAFANA_CLOUD_ONCALL_TOKEN is not set")
return
url = urljoin(CLOUD_URL, f"api/v1/users/?email={user.email}")
try:
r = requests.get(url, headers={"AUTHORIZATION": api_token}, timeout=5)
if r.status_code != 200:
logger.warning(
f"Unable to sync_user_with_cloud user_id {user.id}. Response status code {r.status_code}"
)
return
data = r.json()
if len(data["results"]) != 0:
cloud_used_data = data["results"][0]
CloudUserIdentity.objects.update_or_create(
email=user.email,
defaults={
"phone_number_verified": cloud_used_data["is_phone_number_verified"],
"cloud_id": cloud_used_data["id"],
},
)
else:
logger.warning(f"Unable to sync_user_with_cloud user_id {user.id}. User with {user.email} not found")
except requests.exceptions.RequestException as e:
logger.warning(f"Unable to sync_user_with cloud user_id {user.id}. Request exception {str(e)}")

View file

@ -0,0 +1,14 @@
from django.db import models
class CloudUserIdentity(models.Model):
phone_number_verified = models.BooleanField(default=False)
cloud_id = models.CharField(max_length=20)
email = models.EmailField()
organization = models.ForeignKey(
"user_management.Organization", on_delete=models.CASCADE, related_name="cloud_users"
)
class Meta:
# TODO: Grafana Twilio: Check if this constraint needed
unique_together = ("cloud_id", "organization")

View file

@ -1,9 +1,16 @@
import logging
import uuid
from django.db import models
logger = logging.getLogger(__name__)
class OssInstallation(models.Model):
"""
OssInstallation is model to track installation of OSS OnCall version.
"""
installation_id = models.UUIDField(default=uuid.uuid4, editable=False)
created_at = models.DateTimeField(auto_now=True)
report_sent_at = models.DateTimeField(null=True, default=None)

View file

@ -30,4 +30,6 @@ router.register(r"teams", views.TeamView, basename="teams")
urlpatterns = [
path("", include(router.urls)),
optional_slash_path("info", views.InfoView.as_view(), name="info"),
optional_slash_path("make_call", views.MakeCallView.as_view(), name="make_call"),
optional_slash_path("send_sms", views.SendSMSView.as_view(), name="send_sms"),
]

View file

@ -8,6 +8,7 @@ from .integrations import IntegrationView # noqa: F401
from .on_call_shifts import CustomOnCallShiftView # noqa: F401
from .organizations import OrganizationView # noqa: F401
from .personal_notifications import PersonalNotificationView # noqa: F401
from .phone_notifications import MakeCallView, SendSMSView # noqa: F401
from .resolution_notes import ResolutionNoteView # noqa: F401
from .routes import ChannelFilterView # noqa: F401
from .schedules import OnCallScheduleChannelView # noqa: F401

View file

@ -0,0 +1,68 @@
# TODO: move to serializers
from rest_framework import serializers, status
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.views import APIView
from twilio.base.exceptions import TwilioRestException
from apps.auth_token.auth import ApiTokenAuthentication
from apps.twilioapp.models import PhoneCall, SMSMessage
class PhoneNotificationDataSerializer(serializers.Serializer):
email = serializers.EmailField()
message = serializers.CharField(max_length=200)
class MakeCallView(APIView):
authentication_classes = (ApiTokenAuthentication,)
permission_classes = (IsAuthenticated,)
# TODO: add ratelimit
def post(self, request):
# TODO: Grafana Twilio:
# 1. Validate user's email
# 2. Validate payload: clean_markup, escape_for_twilio_phone_call
# 3. Create LogRecord (User notification policy or implement new one)
serializer = PhoneNotificationDataSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
organization = self.request.auth.organization
# TODO: filter by verified phone number?
user = organization.users.filter(email=serializer.validated_data["email"]).first()
if not user or not user.verified_phone_number:
return Response(status=status.HTTP_400_BAD_REQUEST)
try:
PhoneCall.make_grafana_cloud_call(user, serializer.validated_data["message"])
except TwilioRestException:
return Response(status=status.HTTP_503_SERVICE_UNAVAILABLE)
except PhoneCall.PhoneCallsLimitExceeded:
return Response(status=status.HTTP_400_BAD_REQUEST)
return Response(status=status.HTTP_200_OK)
class SendSMSView(APIView):
authentication_classes = (ApiTokenAuthentication,)
permission_classes = (IsAuthenticated,)
def post(self, request):
serializer = PhoneNotificationDataSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
organization = self.request.auth.organization
# TODO: filter by verified phone number?
user = organization.users.filter(email=serializer.validated_data["email"]).first()
if not user or not user.verified_phone_number:
return Response(status=status.HTTP_400_BAD_REQUEST)
try:
SMSMessage.send_cloud_sms(user, serializer.validated_data["message"])
except TwilioRestException:
return Response(status=status.HTTP_503_SERVICE_UNAVAILABLE)
except SMSMessage.SMSLimitExceeded:
return Response(status=status.HTTP_400_BAD_REQUEST)
return Response(status=status.HTTP_200_OK)

View file

@ -33,12 +33,16 @@ class UserView(RateLimitHeadersMixin, ShortSerializerMixin, DemoTokenMixin, Read
def get_queryset(self):
username = self.request.query_params.get("username")
email = self.request.query_params.get("email")
is_short_request = self.request.query_params.get("short", "false") == "true"
queryset = self.request.auth.organization.users.filter(role__in=[Role.ADMIN, Role.EDITOR]).distinct()
if username is not None:
queryset = queryset.filter(username=username)
if email is not None:
queryset = queryset.filter(email=email)
if not is_short_request:
queryset = self.serializer_class.setup_eager_loading(queryset)
return queryset.order_by("id")

View file

@ -34,8 +34,10 @@ class PhoneCallManager(models.Manager):
if phone_call_qs.exists() and status:
phone_call_qs.update(status=status)
phone_call = phone_call_qs.first()
if phone_call.grafana_cloud_notification:
# If call was made via grafana twilio it is don't needed to create logs on it's delivery status.
return
log_record = None
if status == TwilioCallStatuses.COMPLETED:
log_record = UserNotificationPolicyLogRecord(
@ -115,6 +117,14 @@ class PhoneCall(models.Model):
created_at = models.DateTimeField(auto_now_add=True)
grafana_cloud_notification = models.BooleanField(default=False)
class PhoneCallsLimitExceeded(Exception):
"""Phone calls limit exceeded"""
class PhoneNumberNotVerifiedError(Exception):
"""Phone number is not verified"""
def process_digit(self, digit):
"""The function process pressed digit at time of call to user
@ -140,55 +150,32 @@ class PhoneCall(models.Model):
@classmethod
def make_call(cls, user, alert_group, notification_policy):
UserNotificationPolicyLogRecord = apps.get_model("base", "UserNotificationPolicyLogRecord")
organization = alert_group.channel.organization
log_record = None
if user.verified_phone_number:
# Create a PhoneCall object in db
phone_call = PhoneCall(
represents_alert_group=alert_group,
receiver=user,
renderer = AlertGroupPhoneCallRenderer(alert_group)
message_body = renderer.render()
try:
cls._make_call(user, message_body, alert_group=alert_group, notification_policy=notification_policy)
except TwilioRestException:
log_record = UserNotificationPolicyLogRecord(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
notification_policy=notification_policy,
alert_group=alert_group,
notification_error_code=UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_NOT_ABLE_TO_CALL,
notification_step=notification_policy.step if notification_policy else None,
notification_channel=notification_policy.notify_by if notification_policy else None,
)
phone_calls_left = organization.phone_calls_left(user)
if phone_calls_left > 0:
phone_call.exceeded_limit = False
renderer = AlertGroupPhoneCallRenderer(alert_group)
message_body = renderer.render()
if phone_calls_left < 3:
message_body += " {} phone calls left. Contact your admin.".format(phone_calls_left)
try:
twilio_call = twilio_client.make_call(message_body, user.verified_phone_number)
except TwilioRestException:
log_record = UserNotificationPolicyLogRecord(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
notification_policy=notification_policy,
alert_group=alert_group,
notification_error_code=UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_NOT_ABLE_TO_CALL,
notification_step=notification_policy.step if notification_policy else None,
notification_channel=notification_policy.notify_by if notification_policy else None,
)
else:
if twilio_call.status and twilio_call.sid:
phone_call.status = TwilioCallStatuses.DETERMINANT.get(twilio_call.status, None)
phone_call.sid = twilio_call.sid
else:
log_record = UserNotificationPolicyLogRecord(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
notification_policy=notification_policy,
alert_group=alert_group,
notification_error_code=UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_PHONE_CALLS_LIMIT_EXCEEDED,
notification_step=notification_policy.step if notification_policy else None,
notification_channel=notification_policy.notify_by if notification_policy else None,
)
phone_call.exceeded_limit = True
phone_call.save()
else:
except PhoneCall.PhoneCallsLimitExceeded:
log_record = UserNotificationPolicyLogRecord(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
notification_policy=notification_policy,
alert_group=alert_group,
notification_error_code=UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_PHONE_CALLS_LIMIT_EXCEEDED,
notification_step=notification_policy.step if notification_policy else None,
notification_channel=notification_policy.notify_by if notification_policy else None,
)
except PhoneCall.PhoneNumberNotVerifiedError:
log_record = UserNotificationPolicyLogRecord(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
@ -203,6 +190,40 @@ class PhoneCall(models.Model):
log_record.save()
user_notification_action_triggered_signal.send(sender=PhoneCall.make_call, log_record=log_record)
@classmethod
def make_grafana_cloud_call(cls, user, message_body):
cls._make_call(user, message_body, grafana_cloud=True)
@classmethod
def _make_call(cls, user, message_body, alert_group=None, notification_policy=None, grafana_cloud=False):
if not user.verified_phone_number:
raise PhoneCall.PhoneNumberNotVerifiedError("User phone number is not verified")
phone_call = PhoneCall(
represents_alert_group=alert_group,
receiver=user,
notification_policy=notification_policy,
grafana_cloud_notification=grafana_cloud,
)
phone_calls_left = user.organization.phone_calls_left(user)
if phone_calls_left <= 0:
phone_call.exceeded_limit = True
phone_call.save()
raise PhoneCall.PhoneCallsLimitExceeded("Organization calls limit exceeded")
phone_call.exceeded_limit = False
if phone_calls_left < 3:
message_body += " {} phone calls left. Contact your admin.".format(phone_calls_left)
twilio_call = twilio_client.make_call(message_body, user.verified_phone_number)
if twilio_call.status and twilio_call.sid:
phone_call.status = TwilioCallStatuses.DETERMINANT.get(twilio_call.status, None)
phone_call.sid = twilio_call.sid
phone_call.save()
return phone_call
@staticmethod
def get_error_code_by_twilio_status(status):
UserNotificationPolicyLogRecord = apps.get_model("base", "UserNotificationPolicyLogRecord")

View file

@ -36,7 +36,9 @@ class SMSMessageManager(models.Manager):
sms_message_qs.update(status=status)
sms_message = sms_message_qs.first()
if sms_message.grafana_cloud_notification:
# If sms was sent via grafana twilio it is don't needed to create logs on it's delivery status.
return
log_record = None
if status == TwilioMessageStatuses.DELIVERED:
@ -90,6 +92,7 @@ class SMSMessage(models.Model):
null=True,
choices=TwilioMessageStatuses.CHOICES,
)
grafana_cloud_notification = models.BooleanField(default=False)
# https://www.twilio.com/docs/sms/api/message-resource#message-properties
sid = models.CharField(
@ -99,6 +102,12 @@ class SMSMessage(models.Model):
created_at = models.DateTimeField(auto_now_add=True)
class SMSLimitExceeded(Exception):
"""SMS limit exceeded"""
class PhoneNumberNotVerifiedError(Exception):
"""Phone number is not verified"""
@property
def created_for_slack(self):
return bool(self.represents_alert_group.slack_message)
@ -107,58 +116,32 @@ class SMSMessage(models.Model):
def send_sms(cls, user, alert_group, notification_policy):
UserNotificationPolicyLogRecord = apps.get_model("base", "UserNotificationPolicyLogRecord")
organization = alert_group.channel.organization
log_record = None
if user.verified_phone_number:
# Create an SMS object in db
sms_message = SMSMessage(
represents_alert_group=alert_group, receiver=user, notification_policy=notification_policy
renderer = AlertGroupSmsRenderer(alert_group)
message_body = renderer.render()
try:
cls._send_sms(user, message_body, alert_group=alert_group, notification_policy=notification_policy)
except TwilioRestException:
log_record = UserNotificationPolicyLogRecord(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
notification_policy=notification_policy,
alert_group=alert_group,
notification_error_code=UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_NOT_ABLE_TO_SEND_SMS,
notification_step=notification_policy.step if notification_policy else None,
notification_channel=notification_policy.notify_by if notification_policy else None,
)
sms_left = organization.sms_left(user)
if sms_left > 0:
# Mark is as successfully sent
sms_message.exceeded_limit = False
# Render alert message for sms
renderer = AlertGroupSmsRenderer(alert_group)
message_body = renderer.render()
# Notify if close to limit
if sms_left < 3:
message_body += " {} sms left. Contact your admin.".format(sms_left)
# Send an sms
try:
twilio_message = twilio_client.send_message(message_body, user.verified_phone_number)
except TwilioRestException:
log_record = UserNotificationPolicyLogRecord(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
notification_policy=notification_policy,
alert_group=alert_group,
notification_error_code=UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_NOT_ABLE_TO_SEND_SMS,
notification_step=notification_policy.step if notification_policy else None,
notification_channel=notification_policy.notify_by if notification_policy else None,
)
else:
if twilio_message.status and twilio_message.sid:
sms_message.status = TwilioMessageStatuses.DETERMINANT.get(twilio_message.status, None)
sms_message.sid = twilio_message.sid
else:
# If no more sms left, mark as exceeded limit
log_record = UserNotificationPolicyLogRecord(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
notification_policy=notification_policy,
alert_group=alert_group,
notification_error_code=UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_SMS_LIMIT_EXCEEDED,
notification_step=notification_policy.step if notification_policy else None,
notification_channel=notification_policy.notify_by if notification_policy else None,
)
sms_message.exceeded_limit = True
# Save object
sms_message.save()
else:
except SMSMessage.SMSLimitExceeded:
log_record = UserNotificationPolicyLogRecord(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
notification_policy=notification_policy,
alert_group=alert_group,
notification_error_code=UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_SMS_LIMIT_EXCEEDED,
notification_step=notification_policy.step if notification_policy else None,
notification_channel=notification_policy.notify_by if notification_policy else None,
)
except SMSMessage.PhoneNumberNotVerifiedError:
log_record = UserNotificationPolicyLogRecord(
author=user,
type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
@ -173,6 +156,40 @@ class SMSMessage(models.Model):
log_record.save()
user_notification_action_triggered_signal.send(sender=SMSMessage.send_sms, log_record=log_record)
@classmethod
def send_grafana_cloud_sms(cls, user, message_body):
cls._send_sms(user, message_body, grafana_cloud=True)
@classmethod
def _send_sms(cls, user, message_body, alert_group=None, notification_policy=None, grafana_cloud=False):
if not user.verified_phone_number:
raise SMSMessage.PhoneNumberNotVerifiedError("User phone number is not verified")
sms_message = SMSMessage(
represents_alert_group=alert_group,
receiver=user,
notification_policy=notification_policy,
grafana_cloud_notification=grafana_cloud,
)
sms_left = user.organization.sms_left(user)
if sms_left <= 0:
sms_message.exceeded_limit = True
sms_message.save()
raise SMSMessage.SMSLimitExceeded("Organization sms limit exceeded")
sms_message.exceeded_limit = False
if sms_left < 3:
message_body += " {} sms left. Contact your admin.".format(sms_left)
twilio_message = twilio_client.send_message(message_body, user.verified_phone_number)
if twilio_message.status and twilio_message.sid:
sms_message.status = TwilioMessageStatuses.DETERMINANT.get(twilio_message.status, None)
sms_message.sid = twilio_message.sid
sms_message.save()
return sms_message
@staticmethod
def get_error_code_by_twilio_status(status):
UserNotificationPolicyLogRecord = apps.get_model("base", "UserNotificationPolicyLogRecord")

View file

@ -177,6 +177,14 @@ def clean_markup(text):
return cleaned
def escape_for_twilio_phone_call(text):
# https://www.twilio.com/docs/api/errors/12100
text = text.replace("&", "&amp;")
text = text.replace(">", "&gt;")
text = text.replace("<", "&lt;")
return text
def escape_html(text):
return html.escape(text)