From fa5d4f2674c880903c2738297b34ae3376cb8b37 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Tue, 11 Oct 2022 12:04:33 -0600 Subject: [PATCH 1/9] Add region_slug column to organization --- engine/apps/api/serializers/organization.py | 12 +++++++++++- engine/apps/grafana_plugin/helpers/gcom.py | 3 +++ .../views/self_hosted_install.py | 1 + .../0004_organization_region_slug.py | 18 ++++++++++++++++++ .../user_management/models/organization.py | 1 + engine/apps/user_management/sync.py | 2 ++ engine/settings/base.py | 1 + 7 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 engine/apps/user_management/migrations/0004_organization_region_slug.py diff --git a/engine/apps/api/serializers/organization.py b/engine/apps/api/serializers/organization.py index 58013052..30241a45 100644 --- a/engine/apps/api/serializers/organization.py +++ b/engine/apps/api/serializers/organization.py @@ -170,4 +170,14 @@ class PluginOrganizationSerializer(serializers.ModelSerializer): class Meta: model = Organization - fields = ["pk", "stack_id", "stack_slug", "grafana_url", "org_id", "org_slug", "org_title", "grafana_token"] + fields = [ + "pk", + "stack_id", + "stack_slug", + "grafana_url", + "org_id", + "org_slug", + "org_title", + "region_slug", + "grafana_token", + ] diff --git a/engine/apps/grafana_plugin/helpers/gcom.py b/engine/apps/grafana_plugin/helpers/gcom.py index 2fbdf44b..58577fc0 100644 --- a/engine/apps/grafana_plugin/helpers/gcom.py +++ b/engine/apps/grafana_plugin/helpers/gcom.py @@ -57,6 +57,7 @@ def check_gcom_permission(token_string: str, context) -> Optional["GcomToken"]: org_id=str(instance_info["orgId"]), org_slug=instance_info["orgSlug"], org_title=instance_info["orgName"], + region_slug=instance_info["regionSlug"], gcom_token=token_string, gcom_token_org_last_time_synced=timezone.now(), ) @@ -64,6 +65,7 @@ def check_gcom_permission(token_string: str, context) -> Optional["GcomToken"]: organization.stack_slug = instance_info["slug"] organization.org_slug = instance_info["orgSlug"] organization.org_title = instance_info["orgName"] + organization.region_slug = instance_info["regionSlug"] organization.grafana_url = instance_info["url"] organization.gcom_token = token_string organization.gcom_token_org_last_time_synced = timezone.now() @@ -72,6 +74,7 @@ def check_gcom_permission(token_string: str, context) -> Optional["GcomToken"]: "stack_slug", "org_slug", "org_title", + "region_slug", "grafana_url", "gcom_token", "gcom_token_org_last_time_synced", diff --git a/engine/apps/grafana_plugin/views/self_hosted_install.py b/engine/apps/grafana_plugin/views/self_hosted_install.py index 16dbd7bc..f4159ea6 100644 --- a/engine/apps/grafana_plugin/views/self_hosted_install.py +++ b/engine/apps/grafana_plugin/views/self_hosted_install.py @@ -46,6 +46,7 @@ class SelfHostedInstallView(GrafanaHeadersMixin, APIView): org_id=org_id, org_slug=settings.SELF_HOSTED_SETTINGS["ORG_SLUG"], org_title=settings.SELF_HOSTED_SETTINGS["ORG_TITLE"], + region_slug=settings.SELF_HOSTED_SETTINGS["REGION_SLUG"], grafana_url=self.instance_context["grafana_url"], api_token=self.instance_context["grafana_token"], ) diff --git a/engine/apps/user_management/migrations/0004_organization_region_slug.py b/engine/apps/user_management/migrations/0004_organization_region_slug.py new file mode 100644 index 00000000..06f02630 --- /dev/null +++ b/engine/apps/user_management/migrations/0004_organization_region_slug.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2022-10-11 17:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('user_management', '0003_user_hide_phone_number'), + ] + + operations = [ + migrations.AddField( + model_name='organization', + name='region_slug', + field=models.CharField(default=None, max_length=300, null=True), + ), + ] diff --git a/engine/apps/user_management/models/organization.py b/engine/apps/user_management/models/organization.py index fd37ba81..a7a8c40b 100644 --- a/engine/apps/user_management/models/organization.py +++ b/engine/apps/user_management/models/organization.py @@ -53,6 +53,7 @@ class Organization(MaintainableObject): stack_slug = models.CharField(max_length=300) org_slug = models.CharField(max_length=300) org_title = models.CharField(max_length=300) + region_slug = models.CharField(max_length=300, null=True, default=None) grafana_url = models.URLField() diff --git a/engine/apps/user_management/sync.py b/engine/apps/user_management/sync.py index 7b0c91d7..80826c5a 100644 --- a/engine/apps/user_management/sync.py +++ b/engine/apps/user_management/sync.py @@ -29,6 +29,7 @@ def sync_organization(organization): "stack_slug", "org_slug", "org_title", + "region_slug", "grafana_url", "last_time_synced", "api_token_status", @@ -47,6 +48,7 @@ def sync_instance_info(organization): organization.stack_slug = instance_info["slug"] organization.org_slug = instance_info["orgSlug"] organization.org_title = instance_info["orgName"] + organization.region_slug = instance_info["regionSlug"] organization.grafana_url = instance_info["url"] organization.gcom_token_org_last_time_synced = timezone.now() diff --git a/engine/settings/base.py b/engine/settings/base.py index d9ec9f36..cdc511f0 100644 --- a/engine/settings/base.py +++ b/engine/settings/base.py @@ -558,6 +558,7 @@ SELF_HOSTED_SETTINGS = { "ORG_ID": 100, "ORG_SLUG": "self_hosted_org", "ORG_TITLE": "Self-Hosted Organization", + "REGION_SLUG": "self_hosted_region", } GRAFANA_INCIDENT_STATIC_API_KEY = os.environ.get("GRAFANA_INCIDENT_STATIC_API_KEY", None) From 0a1a9ab4d80e1127aeb03ce4cbeb892c9379dc79 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Thu, 20 Oct 2022 09:45:48 -0600 Subject: [PATCH 2/9] Add region object --- .../user_management/models/organization.py | 6 ++++ engine/apps/user_management/models/region.py | 33 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 engine/apps/user_management/models/region.py diff --git a/engine/apps/user_management/models/organization.py b/engine/apps/user_management/models/organization.py index 5c283b98..0b0f0662 100644 --- a/engine/apps/user_management/models/organization.py +++ b/engine/apps/user_management/models/organization.py @@ -54,6 +54,12 @@ class Organization(MaintainableObject): org_slug = models.CharField(max_length=300) org_title = models.CharField(max_length=300) region_slug = models.CharField(max_length=300, null=True, default=None) + migration_destination = models.ForeignKey( + to="user_management.Region", + on_delete=models.SET_NULL, + related_name="regions", + default=None, + ) grafana_url = models.URLField() diff --git a/engine/apps/user_management/models/region.py b/engine/apps/user_management/models/region.py new file mode 100644 index 00000000..1b9b2cca --- /dev/null +++ b/engine/apps/user_management/models/region.py @@ -0,0 +1,33 @@ +from django.conf import settings +from django.core.validators import MinLengthValidator +from django.db import models + +from common.public_primary_keys import generate_public_primary_key, increase_public_primary_key_length + + +def generate_public_primary_key_for_region(): + prefix = "R" + new_public_primary_key = generate_public_primary_key(prefix) + + failure_counter = 0 + while Region.objects.filter(public_primary_key=new_public_primary_key).exists(): + new_public_primary_key = increase_public_primary_key_length( + failure_counter=failure_counter, prefix=prefix, model_name="Region" + ) + failure_counter += 1 + + return new_public_primary_key + + +class Region(models.Model): + public_primary_key = models.CharField( + max_length=20, + validators=[MinLengthValidator(settings.PUBLIC_PRIMARY_KEY_MIN_LENGTH + 1)], + unique=True, + default=generate_public_primary_key_for_region, + ) + + name = models.CharField(max_length=300) + slug = models.CharField(max_length=300, unique=True) + oncall_backend_url = models.URLField() + is_default = models.BooleanField(default=False) From febe1b2185eb89f41ec77449f8f153bb45d443d4 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Thu, 20 Oct 2022 15:04:58 -0600 Subject: [PATCH 3/9] Add basic organization moved exception handling and middleware --- engine/apps/auth_token/auth.py | 11 +++++ .../mixins/alert_channel_defining_mixin.py | 5 +++ engine/apps/user_management/middlewares.py | 41 +++++++++++++++++++ .../migrations/0005_auto_20221020_1845.py | 32 +++++++++++++++ .../migrations/0006_alter_region_slug.py | 18 ++++++++ .../apps/user_management/models/__init__.py | 1 + .../user_management/models/organization.py | 1 + engine/apps/user_management/models/region.py | 20 ++++++++- engine/settings/base.py | 1 + 9 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 engine/apps/user_management/middlewares.py create mode 100644 engine/apps/user_management/migrations/0005_auto_20221020_1845.py create mode 100644 engine/apps/user_management/migrations/0006_alter_region_slug.py diff --git a/engine/apps/auth_token/auth.py b/engine/apps/auth_token/auth.py index 551116c6..1c92cebb 100644 --- a/engine/apps/auth_token/auth.py +++ b/engine/apps/auth_token/auth.py @@ -18,6 +18,7 @@ from .exceptions import InvalidToken from .models import ApiAuthToken, PluginAuthToken, ScheduleExportAuthToken, SlackAuthToken, UserScheduleExportAuthToken from .models.mobile_app_auth_token import MobileAppAuthToken from .models.mobile_app_verification_token import MobileAppVerificationToken +from ..user_management.models.region import OrganizationMovedException logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -46,6 +47,10 @@ class ApiTokenAuthentication(BaseAuthentication): auth_token = self.model.validate_token_string(token) except InvalidToken: raise exceptions.AuthenticationFailed("Invalid token.") + + if auth_token.organization.migration_destination is not None: + raise OrganizationMovedException(auth_token.organization) + return auth_token.user, auth_token @@ -167,6 +172,9 @@ class ScheduleExportAuthentication(BaseAuthentication): except InvalidToken: raise exceptions.AuthenticationFailed("Invalid token.") + if auth_token.organization.migration_destination is not None: + raise OrganizationMovedException(auth_token.organization) + if auth_token.schedule.public_primary_key != public_primary_key: raise exceptions.AuthenticationFailed("Invalid schedule export token for schedule") @@ -197,6 +205,9 @@ class UserScheduleExportAuthentication(BaseAuthentication): except InvalidToken: raise exceptions.AuthenticationFailed("Invalid token") + if auth_token.organization.migration_destination is not None: + raise OrganizationMovedException(auth_token.organization) + if auth_token.user.public_primary_key != public_primary_key: raise exceptions.AuthenticationFailed("Invalid schedule export token for user") diff --git a/engine/apps/integrations/mixins/alert_channel_defining_mixin.py b/engine/apps/integrations/mixins/alert_channel_defining_mixin.py index 3e1cc257..8bd79f51 100644 --- a/engine/apps/integrations/mixins/alert_channel_defining_mixin.py +++ b/engine/apps/integrations/mixins/alert_channel_defining_mixin.py @@ -7,6 +7,8 @@ from django.core.cache import cache from django.core.exceptions import PermissionDenied from django.db import OperationalError +from apps.user_management.models.region import OrganizationMovedException + logger = logging.getLogger(__name__) @@ -64,6 +66,9 @@ class AlertChannelDefiningMixin(object): logger.info("Cache is empty!") raise + if alert_receive_channel.organization.migration_destination is not None: + raise OrganizationMovedException(alert_receive_channel.organization) + del kwargs["alert_channel_key"] kwargs["alert_receive_channel"] = alert_receive_channel diff --git a/engine/apps/user_management/middlewares.py b/engine/apps/user_management/middlewares.py new file mode 100644 index 00000000..0fd6d4f3 --- /dev/null +++ b/engine/apps/user_management/middlewares.py @@ -0,0 +1,41 @@ +import logging +import re + +import requests +from django.http import HttpResponse +from django.utils.deprecation import MiddlewareMixin + +from apps.user_management.models.region import OrganizationMovedException +from common.api_helpers.utils import create_engine_url + +logger = logging.getLogger(__name__) + + +class OrganizationMovedMiddleware(MiddlewareMixin): + def process_exception(self, request, exception): + if isinstance(exception, OrganizationMovedException): + region = exception.organization.migration_destination + url = create_engine_url(request.path, override_base=region.oncall_backend_url) + if request.META['QUERY_STRING']: + url = f"{url}?{request.META['QUERY_STRING']}" + + regex = re.compile('^HTTP_') + headers = dict( + (regex.sub('', header), value) for (header, value) in request.META.items() if header.startswith('HTTP_') + ) + + if request.method == "GET": + response = requests.get(url, headers=headers) + elif request.method == "POST": + response = requests.post(url, data=request.body, headers=headers) + elif request.method == "PUT": + response = requests.put(url, data=request.body, headers=headers) + elif request.method == "DELETE": + response = requests.delete(url, headers=headers) + elif request.method == "OPTIONS": + response = requests.options(url, headers=headers) + + response.raise_for_status() + + return HttpResponse(response.content, status=response.status_code) + diff --git a/engine/apps/user_management/migrations/0005_auto_20221020_1845.py b/engine/apps/user_management/migrations/0005_auto_20221020_1845.py new file mode 100644 index 00000000..7cfe249b --- /dev/null +++ b/engine/apps/user_management/migrations/0005_auto_20221020_1845.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2.15 on 2022-10-20 18:45 + +import apps.user_management.models.region +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('user_management', '0004_organization_region_slug'), + ] + + operations = [ + migrations.CreateModel( + name='Region', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('public_primary_key', models.CharField(default=apps.user_management.models.region.generate_public_primary_key_for_region, max_length=20, unique=True, validators=[django.core.validators.MinLengthValidator(13)])), + ('name', models.CharField(max_length=300)), + ('slug', models.CharField(max_length=300, unique=True)), + ('oncall_backend_url', models.URLField()), + ('is_default', models.BooleanField(default=False)), + ], + ), + migrations.AddField( + model_name='organization', + name='migration_destination', + field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='regions', to='user_management.region'), + ), + ] diff --git a/engine/apps/user_management/migrations/0006_alter_region_slug.py b/engine/apps/user_management/migrations/0006_alter_region_slug.py new file mode 100644 index 00000000..780cc9b7 --- /dev/null +++ b/engine/apps/user_management/migrations/0006_alter_region_slug.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2022-10-20 18:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('user_management', '0005_auto_20221020_1845'), + ] + + operations = [ + migrations.AlterField( + model_name='region', + name='slug', + field=models.CharField(max_length=50, unique=True), + ), + ] diff --git a/engine/apps/user_management/models/__init__.py b/engine/apps/user_management/models/__init__.py index 95ed32ab..e2bcd4c7 100644 --- a/engine/apps/user_management/models/__init__.py +++ b/engine/apps/user_management/models/__init__.py @@ -1,3 +1,4 @@ from .user import User # noqa: F401, isort: skip from .organization import Organization # noqa: F401 +from .region import Region # noqa: F401 from .team import Team # noqa: F401 diff --git a/engine/apps/user_management/models/organization.py b/engine/apps/user_management/models/organization.py index 0b0f0662..c1ba8316 100644 --- a/engine/apps/user_management/models/organization.py +++ b/engine/apps/user_management/models/organization.py @@ -59,6 +59,7 @@ class Organization(MaintainableObject): on_delete=models.SET_NULL, related_name="regions", default=None, + null=True, ) grafana_url = models.URLField() diff --git a/engine/apps/user_management/models/region.py b/engine/apps/user_management/models/region.py index 1b9b2cca..b41f4220 100644 --- a/engine/apps/user_management/models/region.py +++ b/engine/apps/user_management/models/region.py @@ -1,10 +1,24 @@ +import logging + from django.conf import settings from django.core.validators import MinLengthValidator from django.db import models +from rest_framework.request import Request +from rest_framework.response import Response +from apps.user_management.models import Organization from common.public_primary_keys import generate_public_primary_key, increase_public_primary_key_length +logger = logging.getLogger(__name__) + + +class OrganizationMovedException(Exception): + + def __init__(self, organization: Organization): + self.organization = organization + + def generate_public_primary_key_for_region(): prefix = "R" new_public_primary_key = generate_public_primary_key(prefix) @@ -19,6 +33,10 @@ def generate_public_primary_key_for_region(): return new_public_primary_key +def redirect_organization_request(organization: Organization, request: Request): + logger.info("**** Redirect! ****") + + class Region(models.Model): public_primary_key = models.CharField( max_length=20, @@ -28,6 +46,6 @@ class Region(models.Model): ) name = models.CharField(max_length=300) - slug = models.CharField(max_length=300, unique=True) + slug = models.CharField(max_length=50, unique=True) oncall_backend_url = models.URLField() is_default = models.BooleanField(default=False) diff --git a/engine/settings/base.py b/engine/settings/base.py index b8d4e9cb..95c090de 100644 --- a/engine/settings/base.py +++ b/engine/settings/base.py @@ -237,6 +237,7 @@ MIDDLEWARE = [ "social_django.middleware.SocialAuthExceptionMiddleware", "apps.social_auth.middlewares.SocialAuthAuthCanceledExceptionMiddleware", "apps.integrations.middlewares.IntegrationExceptionMiddleware", + "apps.user_management.middlewares.OrganizationMovedMiddleware", ] LOG_REQUEST_ID_HEADER = "HTTP_X_CLOUD_TRACE_CONTEXT" From 724278fbc88d81fa80304cb69b3b3f29e8a03c36 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Fri, 21 Oct 2022 14:03:19 -0600 Subject: [PATCH 4/9] Regenerate migrations after complete to consolidate --- .../0004_organization_region_slug.py | 18 ----------- .../migrations/0005_auto_20221020_1845.py | 32 ------------------- .../migrations/0006_alter_region_slug.py | 18 ----------- 3 files changed, 68 deletions(-) delete mode 100644 engine/apps/user_management/migrations/0004_organization_region_slug.py delete mode 100644 engine/apps/user_management/migrations/0005_auto_20221020_1845.py delete mode 100644 engine/apps/user_management/migrations/0006_alter_region_slug.py diff --git a/engine/apps/user_management/migrations/0004_organization_region_slug.py b/engine/apps/user_management/migrations/0004_organization_region_slug.py deleted file mode 100644 index 06f02630..00000000 --- a/engine/apps/user_management/migrations/0004_organization_region_slug.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.15 on 2022-10-11 17:54 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('user_management', '0003_user_hide_phone_number'), - ] - - operations = [ - migrations.AddField( - model_name='organization', - name='region_slug', - field=models.CharField(default=None, max_length=300, null=True), - ), - ] diff --git a/engine/apps/user_management/migrations/0005_auto_20221020_1845.py b/engine/apps/user_management/migrations/0005_auto_20221020_1845.py deleted file mode 100644 index 7cfe249b..00000000 --- a/engine/apps/user_management/migrations/0005_auto_20221020_1845.py +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by Django 3.2.15 on 2022-10-20 18:45 - -import apps.user_management.models.region -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('user_management', '0004_organization_region_slug'), - ] - - operations = [ - migrations.CreateModel( - name='Region', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('public_primary_key', models.CharField(default=apps.user_management.models.region.generate_public_primary_key_for_region, max_length=20, unique=True, validators=[django.core.validators.MinLengthValidator(13)])), - ('name', models.CharField(max_length=300)), - ('slug', models.CharField(max_length=300, unique=True)), - ('oncall_backend_url', models.URLField()), - ('is_default', models.BooleanField(default=False)), - ], - ), - migrations.AddField( - model_name='organization', - name='migration_destination', - field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='regions', to='user_management.region'), - ), - ] diff --git a/engine/apps/user_management/migrations/0006_alter_region_slug.py b/engine/apps/user_management/migrations/0006_alter_region_slug.py deleted file mode 100644 index 780cc9b7..00000000 --- a/engine/apps/user_management/migrations/0006_alter_region_slug.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.15 on 2022-10-20 18:46 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('user_management', '0005_auto_20221020_1845'), - ] - - operations = [ - migrations.AlterField( - model_name='region', - name='slug', - field=models.CharField(max_length=50, unique=True), - ), - ] From 5955b2a81c9312849c6b096860006abc0a21debf Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Mon, 24 Oct 2022 11:15:58 -0600 Subject: [PATCH 5/9] Remove unused function --- engine/apps/user_management/models/region.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/engine/apps/user_management/models/region.py b/engine/apps/user_management/models/region.py index b41f4220..538f6737 100644 --- a/engine/apps/user_management/models/region.py +++ b/engine/apps/user_management/models/region.py @@ -33,10 +33,6 @@ def generate_public_primary_key_for_region(): return new_public_primary_key -def redirect_organization_request(organization: Organization, request: Request): - logger.info("**** Redirect! ****") - - class Region(models.Model): public_primary_key = models.CharField( max_length=20, From 37825059ff16cff0922a48f90988728238e897b2 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Mon, 24 Oct 2022 21:25:32 -0600 Subject: [PATCH 6/9] Add region sync and reverse proxy for migration --- engine/apps/auth_token/auth.py | 2 +- engine/apps/grafana_plugin/helpers/client.py | 3 + engine/apps/grafana_plugin/helpers/gcom.py | 13 ++++ engine/apps/grafana_plugin/tasks/sync.py | 16 ++++- engine/apps/user_management/middlewares.py | 15 +++-- .../migrations/0004_auto_20221025_0316.py | 33 ++++++++++ .../user_management/models/organization.py | 2 + engine/apps/user_management/models/region.py | 65 ++++++++++--------- 8 files changed, 111 insertions(+), 38 deletions(-) create mode 100644 engine/apps/user_management/migrations/0004_auto_20221025_0316.py diff --git a/engine/apps/auth_token/auth.py b/engine/apps/auth_token/auth.py index 1c92cebb..cfa49313 100644 --- a/engine/apps/auth_token/auth.py +++ b/engine/apps/auth_token/auth.py @@ -11,6 +11,7 @@ from rest_framework.request import Request from apps.grafana_plugin.helpers.gcom import check_token from apps.user_management.models import User from apps.user_management.models.organization import Organization +from apps.user_management.models.region import OrganizationMovedException from common.constants.role import Role from .constants import SCHEDULE_EXPORT_TOKEN_NAME, SLACK_AUTH_TOKEN_NAME @@ -18,7 +19,6 @@ from .exceptions import InvalidToken from .models import ApiAuthToken, PluginAuthToken, ScheduleExportAuthToken, SlackAuthToken, UserScheduleExportAuthToken from .models.mobile_app_auth_token import MobileAppAuthToken from .models.mobile_app_verification_token import MobileAppVerificationToken -from ..user_management.models.region import OrganizationMovedException logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/engine/apps/grafana_plugin/helpers/client.py b/engine/apps/grafana_plugin/helpers/client.py index 2dcc4988..f232719e 100644 --- a/engine/apps/grafana_plugin/helpers/client.py +++ b/engine/apps/grafana_plugin/helpers/client.py @@ -143,3 +143,6 @@ class GcomAPIClient(APIClient): def post_active_users(self, body): return self.api_post("app-active-users", body) + + def get_stack_regions(self): + return self.api_get("stack-regions") diff --git a/engine/apps/grafana_plugin/helpers/gcom.py b/engine/apps/grafana_plugin/helpers/gcom.py index 58577fc0..407f70e5 100644 --- a/engine/apps/grafana_plugin/helpers/gcom.py +++ b/engine/apps/grafana_plugin/helpers/gcom.py @@ -112,3 +112,16 @@ def get_active_instance_ids() -> Tuple[Optional[set], bool]: def get_deleted_instance_ids() -> Tuple[Optional[set], bool]: return get_instance_ids(GcomAPIClient.DELETED_INSTANCE_QUERY) + + +def get_stack_regions() -> Tuple[Optional[set], bool]: + if not settings.GRAFANA_COM_API_TOKEN or settings.LICENSE != settings.CLOUD_LICENSE_NAME: + return None, False + + client = GcomAPIClient(settings.GRAFANA_COM_API_TOKEN) + regions, status = client.get_stack_regions() + + if not regions or "items" not in regions: + return None, True + + return regions["items"], True diff --git a/engine/apps/grafana_plugin/tasks/sync.py b/engine/apps/grafana_plugin/tasks/sync.py index a9571335..ed58968d 100644 --- a/engine/apps/grafana_plugin/tasks/sync.py +++ b/engine/apps/grafana_plugin/tasks/sync.py @@ -5,8 +5,9 @@ from django.conf import settings from django.utils import timezone from apps.grafana_plugin.helpers import GcomAPIClient -from apps.grafana_plugin.helpers.gcom import get_active_instance_ids, get_deleted_instance_ids +from apps.grafana_plugin.helpers.gcom import get_active_instance_ids, get_deleted_instance_ids, get_stack_regions from apps.user_management.models import Organization +from apps.user_management.models.region import sync_regions from apps.user_management.sync import cleanup_organization, sync_organization from common.custom_celery_tasks import shared_dedicated_queue_retry_task @@ -103,3 +104,16 @@ def start_cleanup_deleted_organizations(): @shared_dedicated_queue_retry_task(autoretry_for=(Exception,), max_retries=1) def cleanup_organization_async(organization_pk): cleanup_organization(organization_pk) + + +@shared_dedicated_queue_retry_task(autoretry_for=(Exception,), max_retries=1) +def start_sync_regions(): + regions, is_cloud_configured = get_stack_regions() + if not is_cloud_configured: + return + + if not regions: + logger.warning("Did not find any stack-regions!") + return + + sync_regions(regions) diff --git a/engine/apps/user_management/middlewares.py b/engine/apps/user_management/middlewares.py index 0fd6d4f3..8e516999 100644 --- a/engine/apps/user_management/middlewares.py +++ b/engine/apps/user_management/middlewares.py @@ -4,6 +4,7 @@ import re import requests from django.http import HttpResponse from django.utils.deprecation import MiddlewareMixin +from rest_framework.status import HTTP_500_INTERNAL_SERVER_ERROR from apps.user_management.models.region import OrganizationMovedException from common.api_helpers.utils import create_engine_url @@ -15,13 +16,18 @@ class OrganizationMovedMiddleware(MiddlewareMixin): def process_exception(self, request, exception): if isinstance(exception, OrganizationMovedException): region = exception.organization.migration_destination + if not region.oncall_backend_url: + return HttpResponse( + "Organization migration destination undefined URL", status=HTTP_500_INTERNAL_SERVER_ERROR + ) + url = create_engine_url(request.path, override_base=region.oncall_backend_url) - if request.META['QUERY_STRING']: + if request.META["QUERY_STRING"]: url = f"{url}?{request.META['QUERY_STRING']}" - regex = re.compile('^HTTP_') + regex = re.compile("^HTTP_") headers = dict( - (regex.sub('', header), value) for (header, value) in request.META.items() if header.startswith('HTTP_') + (regex.sub("", header), value) for (header, value) in request.META.items() if header.startswith("HTTP_") ) if request.method == "GET": @@ -35,7 +41,4 @@ class OrganizationMovedMiddleware(MiddlewareMixin): elif request.method == "OPTIONS": response = requests.options(url, headers=headers) - response.raise_for_status() - return HttpResponse(response.content, status=response.status_code) - diff --git a/engine/apps/user_management/migrations/0004_auto_20221025_0316.py b/engine/apps/user_management/migrations/0004_auto_20221025_0316.py new file mode 100644 index 00000000..8d1b15c8 --- /dev/null +++ b/engine/apps/user_management/migrations/0004_auto_20221025_0316.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.15 on 2022-10-25 03:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('user_management', '0003_user_hide_phone_number'), + ] + + operations = [ + migrations.CreateModel( + name='Region', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=300)), + ('slug', models.CharField(max_length=50, unique=True)), + ('oncall_backend_url', models.URLField(null=True)), + ], + ), + migrations.AddField( + model_name='organization', + name='region_slug', + field=models.CharField(default=None, max_length=300, null=True), + ), + migrations.AddField( + model_name='organization', + name='migration_destination', + field=models.ForeignKey(db_column='migration_destination_slug', default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='regions', to='user_management.region', to_field='slug'), + ), + ] diff --git a/engine/apps/user_management/models/organization.py b/engine/apps/user_management/models/organization.py index c1ba8316..9cac00a8 100644 --- a/engine/apps/user_management/models/organization.py +++ b/engine/apps/user_management/models/organization.py @@ -56,6 +56,8 @@ class Organization(MaintainableObject): region_slug = models.CharField(max_length=300, null=True, default=None) migration_destination = models.ForeignKey( to="user_management.Region", + to_field="slug", + db_column="migration_destination_slug", on_delete=models.SET_NULL, related_name="regions", default=None, diff --git a/engine/apps/user_management/models/region.py b/engine/apps/user_management/models/region.py index 538f6737..a1d0b312 100644 --- a/engine/apps/user_management/models/region.py +++ b/engine/apps/user_management/models/region.py @@ -1,47 +1,52 @@ import logging -from django.conf import settings -from django.core.validators import MinLengthValidator +from django.apps import apps from django.db import models -from rest_framework.request import Request -from rest_framework.response import Response from apps.user_management.models import Organization -from common.public_primary_keys import generate_public_primary_key, increase_public_primary_key_length - logger = logging.getLogger(__name__) -class OrganizationMovedException(Exception): +def sync_regions(regions: list[dict]): + Region = apps.get_model("user_management", "Region") + gcom_regions = {region["slug"]: region for region in regions} + existing_region_slugs = set(Region.objects.all().values_list("slug", flat=True)) + # create new regions + regions_to_create = tuple( + Region( + name=region["name"], + slug=region["slug"], + oncall_backend_url=region["oncallApiUrl"], + ) + for region in gcom_regions.values() + if region["slug"] not in existing_region_slugs + ) + Region.objects.bulk_create(regions_to_create, batch_size=5000) + + # delete excess regions + regions_to_delete = existing_region_slugs - gcom_regions.keys() + Region.objects.filter(slug__in=regions_to_delete).delete() + + # update existing regions + regions_to_update = [] + for region in Region.objects.filter(slug__in=existing_region_slugs): + gcom_region = gcom_regions[region.slug] + if region.name != gcom_region["name"] or region.oncall_backend_url != gcom_region["oncallApiUrl"]: + region.name = gcom_region["name"] + region.oncall_backend_url = gcom_region["oncallApiUrl"] + regions_to_update.append(region) + + Region.objects.bulk_update(regions_to_update, ["name", "oncall_backend_url"], batch_size=5000) + + +class OrganizationMovedException(Exception): def __init__(self, organization: Organization): self.organization = organization -def generate_public_primary_key_for_region(): - prefix = "R" - new_public_primary_key = generate_public_primary_key(prefix) - - failure_counter = 0 - while Region.objects.filter(public_primary_key=new_public_primary_key).exists(): - new_public_primary_key = increase_public_primary_key_length( - failure_counter=failure_counter, prefix=prefix, model_name="Region" - ) - failure_counter += 1 - - return new_public_primary_key - - class Region(models.Model): - public_primary_key = models.CharField( - max_length=20, - validators=[MinLengthValidator(settings.PUBLIC_PRIMARY_KEY_MIN_LENGTH + 1)], - unique=True, - default=generate_public_primary_key_for_region, - ) - name = models.CharField(max_length=300) slug = models.CharField(max_length=50, unique=True) - oncall_backend_url = models.URLField() - is_default = models.BooleanField(default=False) + oncall_backend_url = models.URLField(null=True) From a912a786de608b3b131856624768d2e2886cc0cf Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Thu, 27 Oct 2022 15:40:46 -0600 Subject: [PATCH 7/9] Add tests --- engine/apps/user_management/middlewares.py | 28 +-- .../apps/user_management/tests/factories.py | 11 +- .../apps/user_management/tests/test_region.py | 219 ++++++++++++++++++ engine/conftest.py | 22 +- 4 files changed, 265 insertions(+), 15 deletions(-) create mode 100644 engine/apps/user_management/tests/test_region.py diff --git a/engine/apps/user_management/middlewares.py b/engine/apps/user_management/middlewares.py index 8e516999..ff6aab56 100644 --- a/engine/apps/user_management/middlewares.py +++ b/engine/apps/user_management/middlewares.py @@ -4,7 +4,7 @@ import re import requests from django.http import HttpResponse from django.utils.deprecation import MiddlewareMixin -from rest_framework.status import HTTP_500_INTERNAL_SERVER_ERROR +from rest_framework import status from apps.user_management.models.region import OrganizationMovedException from common.api_helpers.utils import create_engine_url @@ -18,7 +18,7 @@ class OrganizationMovedMiddleware(MiddlewareMixin): region = exception.organization.migration_destination if not region.oncall_backend_url: return HttpResponse( - "Organization migration destination undefined URL", status=HTTP_500_INTERNAL_SERVER_ERROR + "Organization migration destination undefined URL", status=status.HTTP_500_INTERNAL_SERVER_ERROR ) url = create_engine_url(request.path, override_base=region.oncall_backend_url) @@ -30,15 +30,17 @@ class OrganizationMovedMiddleware(MiddlewareMixin): (regex.sub("", header), value) for (header, value) in request.META.items() if header.startswith("HTTP_") ) - if request.method == "GET": - response = requests.get(url, headers=headers) - elif request.method == "POST": - response = requests.post(url, data=request.body, headers=headers) - elif request.method == "PUT": - response = requests.put(url, data=request.body, headers=headers) - elif request.method == "DELETE": - response = requests.delete(url, headers=headers) - elif request.method == "OPTIONS": - response = requests.options(url, headers=headers) - + response = self.make_request(request.method, url, headers, request.body) return HttpResponse(response.content, status=response.status_code) + + def make_request(self, method, url, headers, body): + if method == "GET": + return requests.get(url, headers=headers) + elif method == "POST": + return requests.post(url, data=body, headers=headers) + elif method == "PUT": + return requests.put(url, data=body, headers=headers) + elif method == "DELETE": + return requests.delete(url, headers=headers) + elif method == "OPTIONS": + return requests.options(url, headers=headers) diff --git a/engine/apps/user_management/tests/factories.py b/engine/apps/user_management/tests/factories.py index 79b20231..c66099c0 100644 --- a/engine/apps/user_management/tests/factories.py +++ b/engine/apps/user_management/tests/factories.py @@ -1,6 +1,6 @@ import factory -from apps.user_management.models import Organization, Team, User +from apps.user_management.models import Organization, Region, Team, User from common.utils import UniqueFaker @@ -31,3 +31,12 @@ class TeamFactory(factory.DjangoModelFactory): class Meta: model = Team + + +class RegionFactory(factory.DjangoModelFactory): + name = factory.Faker("country") + slug = factory.Faker("slug") + oncall_backend_url = factory.Faker("url") + + class Meta: + model = Region diff --git a/engine/apps/user_management/tests/test_region.py b/engine/apps/user_management/tests/test_region.py new file mode 100644 index 00000000..201ca195 --- /dev/null +++ b/engine/apps/user_management/tests/test_region.py @@ -0,0 +1,219 @@ +from unittest.mock import patch + +import pytest +from django.http import HttpResponse +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APIClient + +from apps.alerts.models import AlertReceiveChannel +from apps.auth_token.auth import ApiTokenAuthentication, ScheduleExportAuthentication, UserScheduleExportAuthentication +from apps.auth_token.models import ScheduleExportAuthToken, UserScheduleExportAuthToken +from apps.integrations.views import AlertManagerAPIView +from apps.schedules.models import OnCallScheduleWeb +from apps.user_management.models.region import OrganizationMovedException + + +@pytest.mark.django_db +def test_organization_region_delete( + make_organization_and_region, +): + organization, region = make_organization_and_region() + organization.save() + + organization.refresh_from_db() + assert organization.migration_destination.slug == region.slug + region.delete() + + organization.refresh_from_db() + assert organization.migration_destination is None + + +@pytest.mark.django_db +def test_integration_does_not_raise_exception_organization_moved( + make_organization, + make_alert_receive_channel, +): + organization = make_organization() + alert_receive_channel = make_alert_receive_channel( + organization=organization, + integration=AlertReceiveChannel.INTEGRATION_ALERTMANAGER, + ) + + try: + am = AlertManagerAPIView() + am.dispatch(alert_channel_key=alert_receive_channel.token) + assert False + except OrganizationMovedException: + assert False + except Exception: + assert True + + +@pytest.mark.django_db +def test_integration_raises_exception_organization_moved( + make_organization_and_region, + make_alert_receive_channel, +): + organization, region = make_organization_and_region() + organization.save() + + alert_receive_channel = make_alert_receive_channel( + organization=organization, + integration=AlertReceiveChannel.INTEGRATION_ALERTMANAGER, + ) + + try: + am = AlertManagerAPIView() + am.dispatch(alert_channel_key=alert_receive_channel.token) + assert False + except OrganizationMovedException as e: + assert e.organization == organization + + +@patch("apps.user_management.middlewares.OrganizationMovedMiddleware.make_request") +@pytest.mark.django_db +def test_organization_moved_middleware( + mocked_make_request, + make_organization_and_region, + make_alert_receive_channel, +): + organization, region = make_organization_and_region() + organization.save() + + alert_receive_channel = make_alert_receive_channel( + organization=organization, + integration=AlertReceiveChannel.INTEGRATION_ALERTMANAGER, + ) + + expected_message = bytes(f"Redirected to {region.oncall_backend_url}", 'utf-8') + mocked_make_request.return_value = HttpResponse(expected_message, status=status.HTTP_200_OK) + + client = APIClient() + url = reverse("integrations:alertmanager", kwargs={"alert_channel_key": alert_receive_channel.token}) + + data = {"value": "test"} + response = client.post(url, data, format="json") + assert mocked_make_request.called + assert response.content == expected_message + assert response.status_code == status.HTTP_200_OK + + +@pytest.mark.django_db +def test_api_token_does_not_raise_exception_organization_moved( + make_organization, + make_user_for_organization, + make_public_api_token, +): + organization = make_organization() + + admin = make_user_for_organization(organization) + _, token = make_public_api_token(admin, organization) + + try: + api_auth = ApiTokenAuthentication() + api_auth.authenticate_credentials(token) + assert True + except OrganizationMovedException: + assert False + + +@pytest.mark.django_db +def test_api_token_raises_exception_organization_moved( + make_organization_and_region, + make_user_for_organization, + make_public_api_token, +): + organization, region = make_organization_and_region() + organization.save() + + admin = make_user_for_organization(organization) + _, token = make_public_api_token(admin, organization) + + try: + api_auth = ApiTokenAuthentication() + api_auth.authenticate_credentials(token) + assert False + except OrganizationMovedException as e: + assert e.organization == organization + + +@pytest.mark.django_db +def test_schedule_export_token_does_not_raise_exception_organization_moved( + make_organization, + make_user_for_organization, + make_public_api_token, + make_schedule, +): + organization = make_organization() + schedule = make_schedule(organization, schedule_class=OnCallScheduleWeb) + + admin = make_user_for_organization(organization) + _, token = ScheduleExportAuthToken.create_auth_token(admin, organization, schedule) + + try: + schedule_auth = ScheduleExportAuthentication() + schedule_auth.authenticate_credentials(token, schedule.public_primary_key) + assert True + except OrganizationMovedException: + assert False + + +@pytest.mark.django_db +def test_schedule_export_token_raises_exception_organization_moved( + make_organization_and_region, + make_user_for_organization, + make_public_api_token, + make_schedule, +): + organization, region = make_organization_and_region() + organization.save() + schedule = make_schedule(organization, schedule_class=OnCallScheduleWeb) + + admin = make_user_for_organization(organization) + _, token = ScheduleExportAuthToken.create_auth_token(admin, organization, schedule) + + try: + schedule_auth = ScheduleExportAuthentication() + schedule_auth.authenticate_credentials(token, schedule.public_primary_key) + assert False + except OrganizationMovedException as e: + assert e.organization == organization + + +@pytest.mark.django_db +def test_user_schedule_export_token_does_not_raise_exception_organization_moved( + make_organization, + make_user_for_organization, + make_public_api_token, +): + organization = make_organization() + admin = make_user_for_organization(organization) + _, token = UserScheduleExportAuthToken.create_auth_token(admin, organization) + + try: + user_schedule_auth = UserScheduleExportAuthentication() + user_schedule_auth.authenticate_credentials(token, admin.public_primary_key) + assert True + except OrganizationMovedException: + assert False + + +@pytest.mark.django_db +def test_user_schedule_export_token_raises_exception_organization_moved( + make_organization_and_region, + make_user_for_organization, + make_public_api_token, +): + organization, region = make_organization_and_region() + organization.save() + + admin = make_user_for_organization(organization) + _, token = UserScheduleExportAuthToken.create_auth_token(admin, organization) + + try: + user_schedule_auth = UserScheduleExportAuthentication() + user_schedule_auth.authenticate_credentials(token, admin.public_primary_key) + assert False + except OrganizationMovedException as e: + assert e.organization == organization diff --git a/engine/conftest.py b/engine/conftest.py index 8291d921..4e88b798 100644 --- a/engine/conftest.py +++ b/engine/conftest.py @@ -70,7 +70,7 @@ from apps.telegram.tests.factories import ( ) from apps.twilioapp.tests.factories import PhoneCallFactory, SMSFactory from apps.user_management.models.user import User, listen_for_user_model_save -from apps.user_management.tests.factories import OrganizationFactory, TeamFactory, UserFactory +from apps.user_management.tests.factories import OrganizationFactory, RegionFactory, TeamFactory, UserFactory from common.constants.role import Role register(OrganizationFactory) @@ -666,3 +666,23 @@ def load_slack_urls(settings): reload(sys.modules[urlconf]) else: import_module(urlconf) + + +@pytest.fixture +def make_region(): + def _make_region(**kwargs): + region = RegionFactory(**kwargs) + return region + + return _make_region + + +@pytest.fixture +def make_organization_and_region(make_organization, make_region): + def _make_organization_and_region(): + organization = make_organization() + region = make_region() + organization.migration_destination = region + return organization, region + + return _make_organization_and_region From 942e30bdb3c9ab1278eb9438627089a4bf6f0c2d Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Thu, 27 Oct 2022 15:47:53 -0600 Subject: [PATCH 8/9] Lint --- engine/apps/user_management/tests/test_region.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/apps/user_management/tests/test_region.py b/engine/apps/user_management/tests/test_region.py index 201ca195..02756b9e 100644 --- a/engine/apps/user_management/tests/test_region.py +++ b/engine/apps/user_management/tests/test_region.py @@ -86,7 +86,7 @@ def test_organization_moved_middleware( integration=AlertReceiveChannel.INTEGRATION_ALERTMANAGER, ) - expected_message = bytes(f"Redirected to {region.oncall_backend_url}", 'utf-8') + expected_message = bytes(f"Redirected to {region.oncall_backend_url}", "utf-8") mocked_make_request.return_value = HttpResponse(expected_message, status=status.HTTP_200_OK) client = APIClient() From 6267e31b22e43c12d43b4dadb68c2a6d78a19cbc Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Fri, 28 Oct 2022 15:45:51 -0600 Subject: [PATCH 9/9] Check id instead of object to avoid unnecessary query --- engine/apps/auth_token/auth.py | 6 +++--- .../integrations/mixins/alert_channel_defining_mixin.py | 2 +- engine/apps/user_management/models/organization.py | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/engine/apps/auth_token/auth.py b/engine/apps/auth_token/auth.py index cfa49313..68022469 100644 --- a/engine/apps/auth_token/auth.py +++ b/engine/apps/auth_token/auth.py @@ -48,7 +48,7 @@ class ApiTokenAuthentication(BaseAuthentication): except InvalidToken: raise exceptions.AuthenticationFailed("Invalid token.") - if auth_token.organization.migration_destination is not None: + if auth_token.organization.is_moved: raise OrganizationMovedException(auth_token.organization) return auth_token.user, auth_token @@ -172,7 +172,7 @@ class ScheduleExportAuthentication(BaseAuthentication): except InvalidToken: raise exceptions.AuthenticationFailed("Invalid token.") - if auth_token.organization.migration_destination is not None: + if auth_token.organization.is_moved: raise OrganizationMovedException(auth_token.organization) if auth_token.schedule.public_primary_key != public_primary_key: @@ -205,7 +205,7 @@ class UserScheduleExportAuthentication(BaseAuthentication): except InvalidToken: raise exceptions.AuthenticationFailed("Invalid token") - if auth_token.organization.migration_destination is not None: + if auth_token.organization.is_moved: raise OrganizationMovedException(auth_token.organization) if auth_token.user.public_primary_key != public_primary_key: diff --git a/engine/apps/integrations/mixins/alert_channel_defining_mixin.py b/engine/apps/integrations/mixins/alert_channel_defining_mixin.py index 8bd79f51..0a867595 100644 --- a/engine/apps/integrations/mixins/alert_channel_defining_mixin.py +++ b/engine/apps/integrations/mixins/alert_channel_defining_mixin.py @@ -66,7 +66,7 @@ class AlertChannelDefiningMixin(object): logger.info("Cache is empty!") raise - if alert_receive_channel.organization.migration_destination is not None: + if alert_receive_channel.organization.is_moved: raise OrganizationMovedException(alert_receive_channel.organization) del kwargs["alert_channel_key"] diff --git a/engine/apps/user_management/models/organization.py b/engine/apps/user_management/models/organization.py index a48db58a..51fc5392 100644 --- a/engine/apps/user_management/models/organization.py +++ b/engine/apps/user_management/models/organization.py @@ -283,3 +283,7 @@ class Organization(MaintainableObject): @property def insight_logs_metadata(self): return {} + + @property + def is_moved(self): + return self.migration_destination_id is not None