Add task to delete organizations if their stack has been deleted in gcom

engine/apps/grafana_plugin/tasks/sync.py

@@ -7,7 +7,7 @@ from django.utils import timezone
 from apps.grafana_plugin.helpers import GcomAPIClient
 from apps.grafana_plugin.helpers.gcom import get_active_instance_ids
 from apps.user_management.models import Organization
-from apps.user_management.sync import sync_organization
+from apps.user_management.sync import cleanup_organization, sync_organization
 from common.custom_celery_tasks import shared_dedicated_queue_retry_task
 
 logger = get_task_logger(__name__)
@@ -16,6 +16,7 @@ logger.setLevel(logging.DEBUG)
 # celery beat will schedule start_sync_organizations for every 30 minutes
 # to make sure that orgs are synced every 30 minutes, SYNC_PERIOD should be a little lower
 SYNC_PERIOD = timezone.timedelta(minutes=25)
+INACTIVE_PERIOD = timezone.timedelta(minutes=60)
 
 
 @shared_dedicated_queue_retry_task(autoretry_for=(Exception,), retry_backoff=True, max_retries=5)
@@ -73,3 +74,17 @@ def run_organization_sync(organization_pk, force_sync):
 
     sync_organization(organization)
     logger.info(f"Finish sync Organization {organization_pk}")
+
+
+@shared_dedicated_queue_retry_task(autoretry_for=(Exception,), max_retries=1)
+def start_cleanup_deleted_organizations():
+    sync_threshold = timezone.now() - INACTIVE_PERIOD
+
+    organization_qs = Organization.objects.filter(last_time_synced__lte=sync_threshold)
+    organization_pks = organization_qs.values_list("pk", flat=True)
+
+    logger.debug(f"Found {len(organization_pks)} organizations not synced recently")
+    max_countdown = 60 * 60  # INACTIVE_PERIOD minutes -> Seconds
+    for idx, organization_pk in enumerate(organization_pks):
+        countdown = idx % max_countdown  # Spread orgs evenly
+        cleanup_organization.apply_async((organization_pk,), countdown=countdown)

engine/apps/user_management/sync.py

@@ -1,8 +1,8 @@
 import logging
 
 from celery.utils.log import get_task_logger
+from django.conf import settings
 from django.utils import timezone
-from rest_framework import status
 
 from apps.grafana_plugin.helpers.client import GcomAPIClient, GrafanaAPIClient
 from apps.user_management.models import Organization, Team, User
@@ -15,13 +15,6 @@ def sync_organization(organization):
     client = GrafanaAPIClient(api_url=organization.grafana_url, api_token=organization.api_token)
 
     api_users, call_status = client.get_users()
-    status_code = call_status["status_code"]
-
-    # if stack is 404ing, delete the organization in case gcom stack is deleted.
-    if status_code == status.HTTP_404_NOT_FOUND:
-        is_deleted = delete_organization_if_needed(organization)
-        if is_deleted:
-            return
 
     sync_instance_info(organization)
 
@@ -82,19 +75,31 @@ def sync_users_and_teams(client, api_users, organization):
 
 
 def delete_organization_if_needed(organization):
+    # Organization has a manually set API token, it will not be found within GCOM
+    # and would need to be deleted manually.
     if organization.gcom_token is None:
         return False
 
-    gcom_client = GcomAPIClient(organization.gcom_token)
-    is_stack_deleted = gcom_client.is_stack_deleted(organization.stack_id)
-
+    # Use common token as organization.gcom_token could be already revoked
+    client = GcomAPIClient(settings.GRAFANA_COM_API_TOKEN)
+    is_stack_deleted = client.is_stack_deleted(organization.stack_id)
     if not is_stack_deleted:
         return False
 
-    logger.info(
-        f"Deleting organization due to stack deletion. "
-        f"pk: {organization.pk}, stack_id: {organization.stack_id}, org_id: {organization.org_id}"
-    )
     organization.delete()
-
     return True
+
+
+def cleanup_organization(organization_pk):
+    logger.info(f"Start cleanup Organization {organization_pk}")
+    try:
+        organization = Organization.objects.get(pk=organization_pk)
+        if delete_organization_if_needed(organization):
+            logger.info(
+                f"Deleting organization due to stack deletion. "
+                f"pk: {organization.pk}, stack_id: {organization.stack_id}, org_id: {organization.org_id}"
+            )
+        else:
+            logger.info(f"Organization {organization_pk} not deleted in gcom, no action taken")
+    except Organization.DoesNotExist:
+        logger.info(f"Organization {organization_pk} was not found")