From 962cc34432a77effaf35667d8fce0d1eba7d0be6 Mon Sep 17 00:00:00 2001 From: Michael Derynck Date: Thu, 29 Aug 2024 12:36:35 -0600 Subject: [PATCH] Compress sync data (#4951) # What this PR does - Compresses sync data being sent to engine - Minor fix to log messages when JSON parse errors occur ## Which issue(s) this PR closes Related to [issue link here] ## Checklist - [x] Unit, integration, and e2e (if applicable) tests updated - [x] Documentation added (or `pr:no public docs` PR label added if not required) - [x] Added the relevant release notes label (see labels prefixed w/ `release:`). These labels dictate how your PR will show up in the autogenerated release notes. --- .../grafana_plugin/serializers/sync_data.py | 2 +- .../apps/grafana_plugin/tests/test_sync_v2.py | 60 +++++++++++++++++++ engine/apps/grafana_plugin/views/sync_v2.py | 11 +++- grafana-plugin/pkg/plugin/debug.go | 5 ++ grafana-plugin/pkg/plugin/permissions.go | 4 +- grafana-plugin/pkg/plugin/resources.go | 1 + grafana-plugin/pkg/plugin/settings.go | 2 +- grafana-plugin/pkg/plugin/sync.go | 14 ++++- grafana-plugin/pkg/plugin/teams.go | 6 +- grafana-plugin/pkg/plugin/users.go | 2 +- 10 files changed, 97 insertions(+), 10 deletions(-) diff --git a/engine/apps/grafana_plugin/serializers/sync_data.py b/engine/apps/grafana_plugin/serializers/sync_data.py index bedefea1..71c10d1a 100644 --- a/engine/apps/grafana_plugin/serializers/sync_data.py +++ b/engine/apps/grafana_plugin/serializers/sync_data.py @@ -22,7 +22,7 @@ class SyncUserSerializer(serializers.Serializer): login = serializers.CharField() email = serializers.CharField() role = serializers.CharField() - avatar_url = serializers.CharField() + avatar_url = serializers.CharField(allow_blank=True) permissions = SyncPermissionSerializer(many=True, allow_empty=True, allow_null=True) teams = serializers.ListField(child=serializers.IntegerField(), allow_empty=True, allow_null=True) diff --git a/engine/apps/grafana_plugin/tests/test_sync_v2.py b/engine/apps/grafana_plugin/tests/test_sync_v2.py index 11cd0938..2aed4c3d 100644 --- a/engine/apps/grafana_plugin/tests/test_sync_v2.py +++ b/engine/apps/grafana_plugin/tests/test_sync_v2.py @@ -1,3 +1,6 @@ +import gzip +import json +from dataclasses import asdict from unittest.mock import patch import pytest @@ -6,6 +9,7 @@ from rest_framework import status from rest_framework.test import APIClient from apps.api.permissions import LegacyAccessControlRole +from apps.grafana_plugin.sync_data import SyncData, SyncSettings, SyncUser from apps.grafana_plugin.tasks.sync_v2 import start_sync_organizations_v2 @@ -76,3 +80,59 @@ def test_skip_org_without_api_token(make_organization, api_token, sync_called): ) as mock_sync: start_sync_organizations_v2() assert mock_sync.called == sync_called + + +@pytest.mark.parametrize("format", [("json"), ("gzip")]) +@pytest.mark.django_db +def test_sync_v2_content_encoding( + make_organization_and_user_with_plugin_token, make_user_auth_headers, settings, format +): + organization, user, token = make_organization_and_user_with_plugin_token() + settings.LICENSE = settings.CLOUD_LICENSE_NAME + client = APIClient() + headers = make_user_auth_headers(None, token, organization=organization) + + data = SyncData( + users=[ + SyncUser( + id=user.user_id, + name=user.username, + login=user.username, + email=user.email, + role="Admin", + avatar_url="", + permissions=[], + teams=[], + ) + ], + teams=[], + team_members={}, + settings=SyncSettings( + stack_id=organization.stack_id, + org_id=organization.org_id, + license=settings.CLOUD_LICENSE_NAME, + oncall_api_url="http://localhost", + oncall_token="", + grafana_url="http://localhost", + grafana_token="fake_token", + rbac_enabled=False, + incident_enabled=False, + incident_backend_url="", + labels_enabled=False, + ), + ) + + payload = asdict(data) + headers["HTTP_Content-Type"] = "application/json" + url = reverse("grafana-plugin:sync-v2") + with patch("apps.grafana_plugin.views.sync_v2.apply_sync_data") as mock_sync: + if format == "gzip": + headers["HTTP_Content-Encoding"] = "gzip" + json_data = json.dumps(payload) + payload = gzip.compress(json_data.encode("utf-8")) + response = client.generic("POST", url, data=payload, **headers) + else: + response = client.post(url, format=format, data=payload, **headers) + + assert response.status_code == status.HTTP_200_OK + mock_sync.assert_called() diff --git a/engine/apps/grafana_plugin/views/sync_v2.py b/engine/apps/grafana_plugin/views/sync_v2.py index 1c17cef8..23166207 100644 --- a/engine/apps/grafana_plugin/views/sync_v2.py +++ b/engine/apps/grafana_plugin/views/sync_v2.py @@ -1,3 +1,5 @@ +import gzip +import json import logging from dataclasses import asdict, is_dataclass @@ -25,7 +27,14 @@ class SyncV2View(APIView): authentication_classes = (BasePluginAuthentication,) def do_sync(self, request: Request) -> Organization: - serializer = SyncDataSerializer(data=request.data) + if request.headers.get("Content-Encoding") == "gzip": + gzip_data = gzip.GzipFile(fileobj=request).read() + decoded_data = gzip_data.decode("utf-8") + data = json.loads(decoded_data) + else: + data = request.data + + serializer = SyncDataSerializer(data=data) if not serializer.is_valid(): raise SyncException(serializer.errors) diff --git a/grafana-plugin/pkg/plugin/debug.go b/grafana-plugin/pkg/plugin/debug.go index eb10cd5b..e362c9cc 100644 --- a/grafana-plugin/pkg/plugin/debug.go +++ b/grafana-plugin/pkg/plugin/debug.go @@ -95,3 +95,8 @@ func (a *App) handleDebugStats(w http.ResponseWriter, req *http.Request) { } w.WriteHeader(http.StatusOK) } + +func (a *App) handleDebugUnlock(w http.ResponseWriter, req *http.Request) { + a.OnCallSyncCache.syncMutex.Unlock() + w.WriteHeader(http.StatusOK) +} diff --git a/grafana-plugin/pkg/plugin/permissions.go b/grafana-plugin/pkg/plugin/permissions.go index efeca538..6bb03095 100644 --- a/grafana-plugin/pkg/plugin/permissions.go +++ b/grafana-plugin/pkg/plugin/permissions.go @@ -41,7 +41,7 @@ func (a *App) GetPermissions(settings *OnCallPluginSettings, onCallUser *OnCallU var permissions []OnCallPermission err = json.Unmarshal(body, &permissions) if err != nil { - return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body) + return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body)) } if res.StatusCode == 200 { @@ -88,7 +88,7 @@ func (a *App) GetAllPermissions(settings *OnCallPluginSettings) (map[string]map[ var permissions map[string]map[string]interface{} err = json.Unmarshal(body, &permissions) if err != nil { - return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body) + return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body)) } if res.StatusCode == 200 { diff --git a/grafana-plugin/pkg/plugin/resources.go b/grafana-plugin/pkg/plugin/resources.go index 3fa8a1a0..7dad9564 100644 --- a/grafana-plugin/pkg/plugin/resources.go +++ b/grafana-plugin/pkg/plugin/resources.go @@ -132,6 +132,7 @@ func (a *App) registerRoutes(mux *http.ServeMux) { //mux.HandleFunc("/debug/settings", a.handleDebugSettings) //mux.HandleFunc("/debug/permissions", a.handleDebugPermissions) //mux.HandleFunc("/debug/stats", a.handleDebugStats) + //mux.HandleFunc("/debug/unlock", a.handleDebugUnlock) mux.HandleFunc("/", a.handleInternalApi) } diff --git a/grafana-plugin/pkg/plugin/settings.go b/grafana-plugin/pkg/plugin/settings.go index 70c50e27..700bc423 100644 --- a/grafana-plugin/pkg/plugin/settings.go +++ b/grafana-plugin/pkg/plugin/settings.go @@ -268,7 +268,7 @@ func (a *App) GetOtherPluginSettings(settings *OnCallPluginSettings, pluginID st var result map[string]interface{} err = json.Unmarshal(body, &result) if err != nil { - return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body) + return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body)) } return result, nil diff --git a/grafana-plugin/pkg/plugin/sync.go b/grafana-plugin/pkg/plugin/sync.go index ebe1b7d7..249a10f5 100644 --- a/grafana-plugin/pkg/plugin/sync.go +++ b/grafana-plugin/pkg/plugin/sync.go @@ -2,6 +2,7 @@ package plugin import ( "bytes" + "compress/gzip" "context" "encoding/json" "errors" @@ -136,6 +137,16 @@ func (a *App) makeSyncRequest(ctx context.Context, forceSend bool) error { return fmt.Errorf("error marshalling JSON: %v", err) } + var syncDataBuffer bytes.Buffer + gzipWriter := gzip.NewWriter(&syncDataBuffer) + _, err = gzipWriter.Write(onCallSyncJsonData) + if err != nil { + return fmt.Errorf("error writing sync data to gzip writer: %v", err) + } + if err := gzipWriter.Close(); err != nil { + return fmt.Errorf("error closing gzip writer: %v", err) + } + syncURL, err := url.JoinPath(onCallPluginSettings.OnCallAPIURL, "api/internal/v1/plugin/v2/sync") if err != nil { return fmt.Errorf("error joining path: %v", err) @@ -146,7 +157,7 @@ func (a *App) makeSyncRequest(ctx context.Context, forceSend bool) error { return fmt.Errorf("error parsing path: %v", err) } - syncReq, err := http.NewRequest("POST", parsedSyncURL.String(), bytes.NewBuffer(onCallSyncJsonData)) + syncReq, err := http.NewRequest("POST", parsedSyncURL.String(), &syncDataBuffer) if err != nil { return fmt.Errorf("error creating request: %v", err) } @@ -156,6 +167,7 @@ func (a *App) makeSyncRequest(ctx context.Context, forceSend bool) error { return err } syncReq.Header.Set("Content-Type", "application/json") + syncReq.Header.Set("Content-Encoding", "gzip") res, err := a.httpClient.Do(syncReq) if err != nil { diff --git a/grafana-plugin/pkg/plugin/teams.go b/grafana-plugin/pkg/plugin/teams.go index 4b58781a..c6d6ef67 100644 --- a/grafana-plugin/pkg/plugin/teams.go +++ b/grafana-plugin/pkg/plugin/teams.go @@ -70,7 +70,7 @@ func (a *App) GetTeamsForUser(settings *OnCallPluginSettings, onCallUser *OnCall var result []Team err = json.Unmarshal(body, &result) if err != nil { - return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body) + return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body)) } if res.StatusCode == 200 { @@ -115,7 +115,7 @@ func (a *App) GetAllTeams(settings *OnCallPluginSettings) ([]OnCallTeam, error) var result Teams err = json.Unmarshal(body, &result) if err != nil { - return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body) + return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body)) } if res.StatusCode == 200 { @@ -161,7 +161,7 @@ func (a *App) GetTeamsMembersForTeam(settings *OnCallPluginSettings, onCallTeam var result []OrgUser err = json.Unmarshal(body, &result) if err != nil { - return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body) + return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body)) } if res.StatusCode == 200 { diff --git a/grafana-plugin/pkg/plugin/users.go b/grafana-plugin/pkg/plugin/users.go index 8fdd9166..d5a28299 100644 --- a/grafana-plugin/pkg/plugin/users.go +++ b/grafana-plugin/pkg/plugin/users.go @@ -233,7 +233,7 @@ func (a *App) GetAllUsers(settings *OnCallPluginSettings) ([]OnCallUser, error) var result []OrgUser err = json.Unmarshal(body, &result) if err != nil { - return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body) + return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body)) } if res.StatusCode == 200 {