Sync users and teams on tf requests (#1180)

# What this PR does
This PR add sync with grafana on requests from terraform 

## Which issue(s) this PR fixes
It's needed to fix case when customers want to create team via grafana
terraform provider and use it in the oncall provider without having to
log into Grafana Cloud.

Co-authored-by: Joey Orlando <joey.orlando@grafana.com>

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Add Slack slash command allowing to trigger a direct page via a manually created alert group
+- Add sync with grafana on /users and /teams api calls from terraform plugin
 
 ### Changed
 

engine/apps/grafana_plugin/helpers/client.py

@@ -54,16 +54,16 @@ class APIClient:
         self.api_url = api_url
         self.api_token = api_token
 
-    def api_head(self, endpoint: str, body: dict = None) -> Tuple[Optional[Response], dict]:
+    def api_head(self, endpoint: str, body: dict = None, **kwargs) -> Tuple[Optional[Response], dict]:
-        return self.call_api(endpoint, requests.head, body)
+        return self.call_api(endpoint, requests.head, body, **kwargs)
 
-    def api_get(self, endpoint: str) -> Tuple[Optional[Response], dict]:
+    def api_get(self, endpoint: str, **kwargs) -> Tuple[Optional[Response], dict]:
-        return self.call_api(endpoint, requests.get)
+        return self.call_api(endpoint, requests.get, **kwargs)
 
-    def api_post(self, endpoint: str, body: dict = None) -> Tuple[Optional[Response], dict]:
+    def api_post(self, endpoint: str, body: dict = None, **kwargs) -> Tuple[Optional[Response], dict]:
-        return self.call_api(endpoint, requests.post, body)
+        return self.call_api(endpoint, requests.post, body, **kwargs)
 
-    def call_api(self, endpoint: str, http_method, body: dict = None) -> Tuple[Optional[Response], dict]:
+    def call_api(self, endpoint: str, http_method, body: dict = None, **kwargs) -> Tuple[Optional[Response], dict]:
         request_start = time.perf_counter()
         call_status = {
             "url": urljoin(self.api_url, endpoint),
@@ -72,7 +72,7 @@ class APIClient:
             "message": "",
         }
         try:
-            response = http_method(call_status["url"], json=body, headers=self.request_headers)
+            response = http_method(call_status["url"], json=body, headers=self.request_headers, **kwargs)
             call_status["status_code"] = response.status_code
             response.raise_for_status()
 
@@ -89,6 +89,7 @@ class APIClient:
             requests.exceptions.ConnectionError,
             requests.exceptions.HTTPError,
             requests.exceptions.TooManyRedirects,
+            requests.exceptions.Timeout,
             json.JSONDecodeError,
         ) as e:
             logger.warning("Error connecting to api instance " + str(e))
@@ -153,8 +154,8 @@ class GrafanaAPIClient(APIClient):
         _, resp_status = self.api_head(self.USER_PERMISSION_ENDPOINT)
         return resp_status["status_code"] == status.HTTP_200_OK
 
-    def get_users(self, rbac_is_enabled_for_org: bool) -> List[GrafanaUserWithPermissions]:
+    def get_users(self, rbac_is_enabled_for_org: bool, **kwargs) -> List[GrafanaUserWithPermissions]:
-        users, _ = self.api_get("api/org/users")
+        users, _ = self.api_get("api/org/users", **kwargs)
 
         if not users:
             return []
@@ -166,8 +167,8 @@ class GrafanaAPIClient(APIClient):
             user["permissions"] = user_permissions.get(str(user["userId"]), [])
         return users
 
-    def get_teams(self):
+    def get_teams(self, **kwargs):
-        return self.api_get("api/teams/search?perpage=1000000")
+        return self.api_get("api/teams/search?perpage=1000000", **kwargs)
 
     def get_team_members(self, team_id):
         return self.api_get(f"api/teams/{team_id}/members")

engine/apps/grafana_plugin/tasks/__init__.py

@@ -1 +1,5 @@
-from .sync import start_sync_organizations, sync_organization_async  # noqa: F401
+from .sync import (  # noqa: F401
+    start_sync_organizations,
+    sync_organization_async,
+    sync_team_members_for_organization_async,
+)

engine/apps/grafana_plugin/tasks/sync.py

@@ -5,10 +5,11 @@ from django.conf import settings
 from django.utils import timezone
 
 from apps.grafana_plugin.helpers import GcomAPIClient
+from apps.grafana_plugin.helpers.client import GrafanaAPIClient
 from apps.grafana_plugin.helpers.gcom import get_active_instance_ids, get_deleted_instance_ids, get_stack_regions
 from apps.user_management.models import Organization
 from apps.user_management.models.region import sync_regions
-from apps.user_management.sync import cleanup_organization, sync_organization
+from apps.user_management.sync import cleanup_organization, sync_organization, sync_team_members
 from common.custom_celery_tasks import shared_dedicated_queue_retry_task
 
 logger = get_task_logger(__name__)
@@ -117,3 +118,15 @@ def start_sync_regions():
         return
 
     sync_regions(regions)
+
+
+@shared_dedicated_queue_retry_task(autoretry_for=(Exception,), max_retries=1)
+def sync_team_members_for_organization_async(organization_pk):
+    try:
+        organization = Organization.objects.get(pk=organization_pk)
+    except Organization.DoesNotExist:
+        logger.info(f"Organization {organization_pk} was not found")
+        return
+
+    grafana_api_client = GrafanaAPIClient(api_url=organization.grafana_url, api_token=organization.api_token)
+    sync_team_members(grafana_api_client, organization)

engine/apps/public_api/tf_sync.py

@@ -0,0 +1,35 @@
+import logging
+
+from django.core.cache import cache
+
+from apps.grafana_plugin.helpers.client import GrafanaAPIClient
+from apps.grafana_plugin.tasks import sync_team_members_for_organization_async
+from apps.user_management.sync import sync_teams, sync_users
+
+logger = logging.getLogger(__name__)
+
+SYNC_REQUEST_TIMEOUT = 5
+SYNC_PERIOD = 60
+
+
+def is_request_from_terraform(request) -> bool:
+    return "terraform-provider-grafana" in request.META.get("HTTP_USER_AGENT", "")
+
+
+def sync_users_on_tf_request(organization):
+    cache_key = f"sync_users_on_tf_request_{organization.id}"
+    if not cache.get(cache_key):
+        logger.info(f"Start sync_users_on_tf_request organization_id={organization.id}")
+        client = GrafanaAPIClient(api_url=organization.grafana_url, api_token=organization.api_token)
+        cache.set(cache_key, True, SYNC_PERIOD)
+        sync_users(client, organization, timeout=SYNC_REQUEST_TIMEOUT)
+
+
+def sync_teams_on_tf_request(organization):
+    cache_key = f"sync_teams_on_tf_request_{organization.id}"
+    if not cache.get(cache_key):
+        logger.info(f"Start sync_teams_on_tf_request organization_id={organization.id}")
+        cache.set(cache_key, True, SYNC_PERIOD)
+        client = GrafanaAPIClient(api_url=organization.grafana_url, api_token=organization.api_token)
+        sync_teams(client, organization, timeout=SYNC_REQUEST_TIMEOUT)
+        sync_team_members_for_organization_async.apply_async((organization.id,))

engine/apps/public_api/views/teams.py

@@ -4,6 +4,7 @@ from rest_framework.permissions import IsAuthenticated
 
 from apps.auth_token.auth import ApiTokenAuthentication
 from apps.public_api.serializers.teams import TeamSerializer
+from apps.public_api.tf_sync import is_request_from_terraform, sync_teams_on_tf_request
 from apps.public_api.throttlers.user_throttle import UserThrottle
 from apps.user_management.models import Team
 from common.api_helpers.mixins import PublicPrimaryKeyMixin
@@ -20,6 +21,8 @@ class TeamView(PublicPrimaryKeyMixin, RetrieveModelMixin, ListModelMixin, viewse
     throttle_classes = [UserThrottle]
 
     def get_queryset(self):
+        if is_request_from_terraform(self.request):
+            sync_teams_on_tf_request(self.request.auth.organization)
         name = self.request.query_params.get("name", None)
         queryset = self.request.auth.organization.teams.all()
         if name:

engine/apps/public_api/views/users.py

@@ -9,6 +9,7 @@ from apps.api.permissions import LegacyAccessControlRole
 from apps.auth_token.auth import ApiTokenAuthentication, UserScheduleExportAuthentication
 from apps.public_api.custom_renderers import CalendarRenderer
 from apps.public_api.serializers import FastUserSerializer, UserSerializer
+from apps.public_api.tf_sync import is_request_from_terraform, sync_users_on_tf_request
 from apps.public_api.throttlers.user_throttle import UserThrottle
 from apps.schedules.ical_utils import user_ical_export
 from apps.schedules.models import OnCallSchedule
@@ -48,6 +49,8 @@ class UserView(RateLimitHeadersMixin, ShortSerializerMixin, ReadOnlyModelViewSet
     throttle_classes = [UserThrottle]
 
     def get_queryset(self):
+        if is_request_from_terraform(self.request):
+            sync_users_on_tf_request(self.request.auth.organization)
         is_short_request = self.request.query_params.get("short", "false") == "true"
         queryset = self.request.auth.organization.users.all()
         if not is_short_request:

engine/apps/user_management/sync.py

@@ -30,11 +30,11 @@ def sync_organization(organization):
 
     _sync_instance_info(organization)
 
-    api_users = grafana_api_client.get_users(rbac_is_enabled)
+    _, check_token_call_status = grafana_api_client.check_token()
-
+    if check_token_call_status["status_code"] == 200:
-    if api_users:
         organization.api_token_status = Organization.API_TOKEN_STATUS_OK
-        sync_users_and_teams(grafana_api_client, api_users, organization)
+        sync_users_and_teams(grafana_api_client, organization)
+        organization.last_time_synced = timezone.now()
         organization.is_grafana_incident_enabled = check_grafana_incident_is_enabled(grafana_api_client)
     else:
         organization.api_token_status = Organization.API_TOKEN_STATUS_FAILED
@@ -71,27 +71,42 @@ def _sync_instance_info(organization):
         organization.gcom_token_org_last_time_synced = timezone.now()
 
 
-def sync_users_and_teams(client, api_users, organization):
+def sync_users_and_teams(client: GrafanaAPIClient, organization):
+    sync_users(client, organization)
+    sync_teams(client, organization)
+    sync_team_members(client, organization)
+
+
+def sync_users(client: GrafanaAPIClient, organization, **kwargs):
+    api_users = client.get_users(organization.is_rbac_permissions_enabled, **kwargs)
     # check if api_users are shaped correctly. e.g. for paused instance, the response is not a list.
     if not api_users or not isinstance(api_users, (tuple, list)):
         return
-
     User.objects.sync_for_organization(organization=organization, api_users=api_users)
 
-    api_teams_result, _ = client.get_teams()
+
+def sync_teams(client: GrafanaAPIClient, organization, **kwargs):
+    api_teams_result, _ = client.get_teams(**kwargs)
     if not api_teams_result:
         return
-
     api_teams = api_teams_result["teams"]
     Team.objects.sync_for_organization(organization=organization, api_teams=api_teams)
 
+
+def sync_team_members(client: GrafanaAPIClient, organization):
     for team in organization.teams.all():
         members, _ = client.get_team_members(team.team_id)
         if not members:
             continue
         User.objects.sync_for_team(team=team, api_members=members)
 
-    organization.last_time_synced = timezone.now()
+
+def sync_users_for_teams(client: GrafanaAPIClient, organization, **kwargs):
+    api_teams_result, _ = client.get_teams(**kwargs)
+    if not api_teams_result:
+        return
+    api_teams = api_teams_result["teams"]
+    Team.objects.sync_for_organization(organization=organization, api_teams=api_teams)
 
 
 def check_grafana_incident_is_enabled(client):

engine/apps/user_management/tests/test_sync.py

@@ -134,14 +134,19 @@ def test_sync_organization(make_organization, make_team, make_user_for_organizat
         },
     )
 
+    api_check_token_call_status = {"status_code": 200}
+
     with patch.object(GrafanaAPIClient, "is_rbac_enabled_for_organization", return_value=False):
         with patch.object(GrafanaAPIClient, "get_users", return_value=api_users_response):
             with patch.object(GrafanaAPIClient, "get_teams", return_value=(api_teams_response, None)):
                 with patch.object(GrafanaAPIClient, "get_team_members", return_value=(api_members_response, None)):
                     with patch.object(
-                        GrafanaAPIClient, "get_grafana_plugin_settings", return_value=({"enabled": True}, None)
+                        GrafanaAPIClient, "check_token", return_value=(None, api_check_token_call_status)
                     ):
-                        sync_organization(organization)
+                        with patch.object(
+                            GrafanaAPIClient, "get_grafana_plugin_settings", return_value=({"enabled": True}, None)
+                        ):
+                            sync_organization(organization)
 
     # check that users are populated
     assert organization.users.count() == 1
@@ -167,9 +172,50 @@ def test_sync_organization(make_organization, make_team, make_user_for_organizat
 def test_sync_organization_is_rbac_permissions_enabled_open_source(make_organization, grafana_api_response):
     organization = make_organization()
 
+    api_users_response = (
+        {
+            "userId": 1,
+            "email": "test@test.test",
+            "name": "Test",
+            "login": "test",
+            "role": "admin",
+            "avatarUrl": "test.test/test",
+            "permissions": [],
+        },
+    )
+
+    api_teams_response = {
+        "totalCount": 1,
+        "teams": (
+            {
+                "id": 1,
+                "name": "Test",
+                "email": "test@test.test",
+                "avatarUrl": "test.test/test",
+            },
+        ),
+    }
+
+    api_members_response = (
+        {
+            "orgId": organization.org_id,
+            "teamId": 1,
+            "userId": 1,
+        },
+    )
+    api_check_token_call_status = {"status_code": 200}
+
     with patch.object(GrafanaAPIClient, "is_rbac_enabled_for_organization", return_value=grafana_api_response):
-        with patch.object(GrafanaAPIClient, "get_users", return_value=[]):
+        with patch.object(GrafanaAPIClient, "get_users", return_value=api_users_response):
-            sync_organization(organization)
+            with patch.object(GrafanaAPIClient, "get_teams", return_value=(api_teams_response, None)):
+                with patch.object(GrafanaAPIClient, "get_team_members", return_value=(api_members_response, None)):
+                    with patch.object(
+                        GrafanaAPIClient, "check_token", return_value=(None, api_check_token_call_status)
+                    ):
+                        with patch.object(
+                            GrafanaAPIClient, "get_grafana_plugin_settings", return_value=({"enabled": True}, None)
+                        ):
+                            sync_organization(organization)
 
     organization.refresh_from_db()
     assert organization.is_rbac_permissions_enabled == grafana_api_response
@@ -184,10 +230,50 @@ def test_sync_organization_is_rbac_permissions_enabled_cloud(mocked_gcom_client,
     stack_id = 5
     organization = make_organization(stack_id=stack_id)
 
+    api_check_token_call_status = {"status_code": 200}
+
     mocked_gcom_client.return_value.is_rbac_enabled_for_stack.return_value = gcom_api_response
 
-    with patch.object(GrafanaAPIClient, "get_users", return_value=[]):
+    api_users_response = (
-        sync_organization(organization)
+        {
+            "userId": 1,
+            "email": "test@test.test",
+            "name": "Test",
+            "login": "test",
+            "role": "admin",
+            "avatarUrl": "test.test/test",
+            "permissions": [],
+        },
+    )
+
+    api_teams_response = {
+        "totalCount": 1,
+        "teams": (
+            {
+                "id": 1,
+                "name": "Test",
+                "email": "test@test.test",
+                "avatarUrl": "test.test/test",
+            },
+        ),
+    }
+
+    api_members_response = (
+        {
+            "orgId": organization.org_id,
+            "teamId": 1,
+            "userId": 1,
+        },
+    )
+
+    with patch.object(GrafanaAPIClient, "check_token", return_value=(None, api_check_token_call_status)):
+        with patch.object(GrafanaAPIClient, "get_users", return_value=api_users_response):
+            with patch.object(GrafanaAPIClient, "get_teams", return_value=(api_teams_response, None)):
+                with patch.object(GrafanaAPIClient, "get_team_members", return_value=(api_members_response, None)):
+                    with patch.object(
+                        GrafanaAPIClient, "get_grafana_plugin_settings", return_value=({"enabled": True}, None)
+                    ):
+                        sync_organization(organization)
 
     organization.refresh_from_db()
 

engine/common/oncall_gateway/oncall_gateway_client.py

@@ -129,11 +129,8 @@ class OnCallGatewayAPIClient:
         if response.status_code not in [200, 201, 202, 204]:
             err_msg = cls._get_error_msg_from_response(response)
             if 400 <= response.status_code < 500:
-                print(1)
                 err_msg = "%s Client Error: %s for url: %s" % (response.status_code, err_msg, response.url)
-
             elif 500 <= response.status_code < 600:
-                print(2)
                 err_msg = "%s Server Error: %s for url: %s" % (response.status_code, err_msg, response.url)
             print(err_msg)
             raise requests.exceptions.HTTPError(err_msg, response=response)