fix: return a throttled response if org is being synced for the first time during auth (#5374)

Related to https://github.com/grafana/oncall-private/issues/2826

When Terraform triggers multiple requests and org needs to be synced in
OnCall, the first request will wait for sync to complete but others will
get an immediate response, before a 403, with these changes a 429
indicating to retry (Terraform
[client](https://github.com/grafana/amixr-api-go-client/blob/main/client.go#L310)
will handle the response and perform a retry).

engine/apps/auth_token/auth.py

@@ -381,6 +381,9 @@ class GrafanaServiceAccountAuthentication(BaseAuthentication):
                     # if organization exists, we are good)
                     setup_organization(url, auth)
                     organization = Organization.objects.filter(grafana_url=url).first()
+                    if organization is None:
+                        # sync may still be in progress, client should retry
+                        raise exceptions.Throttled(detail="Organization being synced, please retry.")
                 return organization
 
         if settings.LICENSE == settings.CLOUD_LICENSE_NAME:

engine/apps/auth_token/tests/test_grafana_auth.py

@@ -110,9 +110,9 @@ def test_grafana_authentication_no_org_grafana_url():
     request_sync_url = f"{grafana_url}/api/plugins/{PluginID.ONCALL}/resources/plugin/sync?wait=true&force=true"
     httpretty.register_uri(httpretty.POST, request_sync_url, status=404)
 
-    with pytest.raises(exceptions.AuthenticationFailed) as exc:
+    with pytest.raises(exceptions.Throttled) as exc:
         GrafanaServiceAccountAuthentication().authenticate(request)
-    assert exc.value.detail == "Organization not found."
+    assert exc.value.detail == "Organization being synced, please retry."
 
 
 @pytest.mark.parametrize("grafana_url", ["null;", "foo", ""])