From 20d3ff2fff9edd9c7830e61072c21fda28e297cb Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Fri, 2 Sep 2022 14:06:42 -0600 Subject: [PATCH] Add task to delete organizations if their stack has been deleted in gcom --- engine/apps/grafana_plugin/tasks/sync.py | 17 ++++++++++- engine/apps/user_management/sync.py | 37 ++++++++++++++---------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/engine/apps/grafana_plugin/tasks/sync.py b/engine/apps/grafana_plugin/tasks/sync.py index 5ee38fe2..29afbce4 100644 --- a/engine/apps/grafana_plugin/tasks/sync.py +++ b/engine/apps/grafana_plugin/tasks/sync.py @@ -7,7 +7,7 @@ from django.utils import timezone from apps.grafana_plugin.helpers import GcomAPIClient from apps.grafana_plugin.helpers.gcom import get_active_instance_ids from apps.user_management.models import Organization -from apps.user_management.sync import sync_organization +from apps.user_management.sync import cleanup_organization, sync_organization from common.custom_celery_tasks import shared_dedicated_queue_retry_task logger = get_task_logger(__name__) @@ -16,6 +16,7 @@ logger.setLevel(logging.DEBUG) # celery beat will schedule start_sync_organizations for every 30 minutes # to make sure that orgs are synced every 30 minutes, SYNC_PERIOD should be a little lower SYNC_PERIOD = timezone.timedelta(minutes=25) +INACTIVE_PERIOD = timezone.timedelta(minutes=60) @shared_dedicated_queue_retry_task(autoretry_for=(Exception,), retry_backoff=True, max_retries=5) @@ -73,3 +74,17 @@ def run_organization_sync(organization_pk, force_sync): sync_organization(organization) logger.info(f"Finish sync Organization {organization_pk}") + + +@shared_dedicated_queue_retry_task(autoretry_for=(Exception,), max_retries=1) +def start_cleanup_deleted_organizations(): + sync_threshold = timezone.now() - INACTIVE_PERIOD + + organization_qs = Organization.objects.filter(last_time_synced__lte=sync_threshold) + organization_pks = organization_qs.values_list("pk", flat=True) + + logger.debug(f"Found {len(organization_pks)} organizations not synced recently") + max_countdown = 60 * 60 # INACTIVE_PERIOD minutes -> Seconds + for idx, organization_pk in enumerate(organization_pks): + countdown = idx % max_countdown # Spread orgs evenly + cleanup_organization.apply_async((organization_pk,), countdown=countdown) diff --git a/engine/apps/user_management/sync.py b/engine/apps/user_management/sync.py index 46f9cda1..b8133186 100644 --- a/engine/apps/user_management/sync.py +++ b/engine/apps/user_management/sync.py @@ -1,8 +1,8 @@ import logging from celery.utils.log import get_task_logger +from django.conf import settings from django.utils import timezone -from rest_framework import status from apps.grafana_plugin.helpers.client import GcomAPIClient, GrafanaAPIClient from apps.user_management.models import Organization, Team, User @@ -15,13 +15,6 @@ def sync_organization(organization): client = GrafanaAPIClient(api_url=organization.grafana_url, api_token=organization.api_token) api_users, call_status = client.get_users() - status_code = call_status["status_code"] - - # if stack is 404ing, delete the organization in case gcom stack is deleted. - if status_code == status.HTTP_404_NOT_FOUND: - is_deleted = delete_organization_if_needed(organization) - if is_deleted: - return sync_instance_info(organization) @@ -82,19 +75,31 @@ def sync_users_and_teams(client, api_users, organization): def delete_organization_if_needed(organization): + # Organization has a manually set API token, it will not be found within GCOM + # and would need to be deleted manually. if organization.gcom_token is None: return False - gcom_client = GcomAPIClient(organization.gcom_token) - is_stack_deleted = gcom_client.is_stack_deleted(organization.stack_id) - + # Use common token as organization.gcom_token could be already revoked + client = GcomAPIClient(settings.GRAFANA_COM_API_TOKEN) + is_stack_deleted = client.is_stack_deleted(organization.stack_id) if not is_stack_deleted: return False - logger.info( - f"Deleting organization due to stack deletion. " - f"pk: {organization.pk}, stack_id: {organization.stack_id}, org_id: {organization.org_id}" - ) organization.delete() - return True + + +def cleanup_organization(organization_pk): + logger.info(f"Start cleanup Organization {organization_pk}") + try: + organization = Organization.objects.get(pk=organization_pk) + if delete_organization_if_needed(organization): + logger.info( + f"Deleting organization due to stack deletion. " + f"pk: {organization.pk}, stack_id: {organization.stack_id}, org_id: {organization.org_id}" + ) + else: + logger.info(f"Organization {organization_pk} not deleted in gcom, no action taken") + except Organization.DoesNotExist: + logger.info(f"Organization {organization_pk} was not found")