centralcloud/oncall-engine
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compress sync data (#4951)

Browse source 
# What this PR does
- Compresses sync data being sent to engine
- Minor fix to log messages when JSON parse errors occur

## Which issue(s) this PR closes

Related to [issue link here]

<!--
*Note*: If you want the issue to be auto-closed once the PR is merged,
change "Related to" to "Closes" in the line above.
If you have more than one GitHub issue that this PR closes, be sure to
preface
each issue link with a [closing
keyword](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/using-keywords-in-issues-and-pull-requests#linking-a-pull-request-to-an-issue).
This ensures that the issue(s) are auto-closed once the PR has been
merged.
-->

## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] Added the relevant release notes label (see labels prefixed w/
`release:`). These labels dictate how your PR will
    show up in the autogenerated release notes.
This commit is contained in:
Michael Derynck 2024-08-29 12:36:35 -06:00 • committed by GitHub
parent bbc46e0383
commit 962cc34432
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 97 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
engine/apps/grafana_plugin/serializers/sync_data.py
View file

@ -22,7 +22,7 @@ class SyncUserSerializer(serializers.Serializer):
login = serializers.CharField()
email = serializers.CharField()
role = serializers.CharField()
avatar_url = serializers.CharField()
avatar_url = serializers.CharField(allow_blank=True)
permissions = SyncPermissionSerializer(many=True, allow_empty=True, allow_null=True)
teams = serializers.ListField(child=serializers.IntegerField(), allow_empty=True, allow_null=True)

60
engine/apps/grafana_plugin/tests/test_sync_v2.py
View file

@ -1,3 +1,6 @@
import gzip
import json
from dataclasses import asdict
from unittest.mock import patch
import pytest
@ -6,6 +9,7 @@ from rest_framework import status
from rest_framework.test import APIClient
from apps.api.permissions import LegacyAccessControlRole
from apps.grafana_plugin.sync_data import SyncData, SyncSettings, SyncUser
from apps.grafana_plugin.tasks.sync_v2 import start_sync_organizations_v2
@ -76,3 +80,59 @@ def test_skip_org_without_api_token(make_organization, api_token, sync_called):
) as mock_sync:
start_sync_organizations_v2()
assert mock_sync.called == sync_called
@pytest.mark.parametrize("format", [("json"), ("gzip")])
@pytest.mark.django_db
def test_sync_v2_content_encoding(
make_organization_and_user_with_plugin_token, make_user_auth_headers, settings, format
):
organization, user, token = make_organization_and_user_with_plugin_token()
settings.LICENSE = settings.CLOUD_LICENSE_NAME
client = APIClient()
headers = make_user_auth_headers(None, token, organization=organization)
data = SyncData(
users=[
SyncUser(
id=user.user_id,
name=user.username,
login=user.username,
email=user.email,
role="Admin",
avatar_url="",
permissions=[],
teams=[],
)
],
teams=[],
team_members={},
settings=SyncSettings(
stack_id=organization.stack_id,
org_id=organization.org_id,
license=settings.CLOUD_LICENSE_NAME,
oncall_api_url="http://localhost",
oncall_token="",
grafana_url="http://localhost",
grafana_token="fake_token",
rbac_enabled=False,
incident_enabled=False,
incident_backend_url="",
labels_enabled=False,
),
)
payload = asdict(data)
headers["HTTP_Content-Type"] = "application/json"
url = reverse("grafana-plugin:sync-v2")
with patch("apps.grafana_plugin.views.sync_v2.apply_sync_data") as mock_sync:
if format == "gzip":
headers["HTTP_Content-Encoding"] = "gzip"
json_data = json.dumps(payload)
payload = gzip.compress(json_data.encode("utf-8"))
response = client.generic("POST", url, data=payload, **headers)
else:
response = client.post(url, format=format, data=payload, **headers)
assert response.status_code == status.HTTP_200_OK
mock_sync.assert_called()

11
engine/apps/grafana_plugin/views/sync_v2.py
View file

@ -1,3 +1,5 @@
import gzip
import json
import logging
from dataclasses import asdict, is_dataclass
@ -25,7 +27,14 @@ class SyncV2View(APIView):
authentication_classes = (BasePluginAuthentication,)
def do_sync(self, request: Request) -> Organization:
serializer = SyncDataSerializer(data=request.data)
if request.headers.get("Content-Encoding") == "gzip":
gzip_data = gzip.GzipFile(fileobj=request).read()
decoded_data = gzip_data.decode("utf-8")
data = json.loads(decoded_data)
else:
data = request.data
serializer = SyncDataSerializer(data=data)
if not serializer.is_valid():
raise SyncException(serializer.errors)

5
grafana-plugin/pkg/plugin/debug.go
View file

@ -95,3 +95,8 @@ func (a *App) handleDebugStats(w http.ResponseWriter, req *http.Request) {
}
w.WriteHeader(http.StatusOK)
}
func (a *App) handleDebugUnlock(w http.ResponseWriter, req *http.Request) {
a.OnCallSyncCache.syncMutex.Unlock()
w.WriteHeader(http.StatusOK)
}

4
grafana-plugin/pkg/plugin/permissions.go
View file

@ -41,7 +41,7 @@ func (a *App) GetPermissions(settings *OnCallPluginSettings, onCallUser *OnCallU
var permissions []OnCallPermission
err = json.Unmarshal(body, &permissions)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
}
if res.StatusCode == 200 {
@ -88,7 +88,7 @@ func (a *App) GetAllPermissions(settings *OnCallPluginSettings) (map[string]map[
var permissions map[string]map[string]interface{}
err = json.Unmarshal(body, &permissions)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
}
if res.StatusCode == 200 {

1
grafana-plugin/pkg/plugin/resources.go
View file

@ -132,6 +132,7 @@ func (a *App) registerRoutes(mux *http.ServeMux) {
//mux.HandleFunc("/debug/settings", a.handleDebugSettings)
//mux.HandleFunc("/debug/permissions", a.handleDebugPermissions)
//mux.HandleFunc("/debug/stats", a.handleDebugStats)
//mux.HandleFunc("/debug/unlock", a.handleDebugUnlock)
mux.HandleFunc("/", a.handleInternalApi)
}

2
grafana-plugin/pkg/plugin/settings.go
View file

@ -268,7 +268,7 @@ func (a *App) GetOtherPluginSettings(settings *OnCallPluginSettings, pluginID st
var result map[string]interface{}
err = json.Unmarshal(body, &result)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
}
return result, nil

14
grafana-plugin/pkg/plugin/sync.go
View file

@ -2,6 +2,7 @@ package plugin
import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"errors"
@ -136,6 +137,16 @@ func (a *App) makeSyncRequest(ctx context.Context, forceSend bool) error {
return fmt.Errorf("error marshalling JSON: %v", err)
}
var syncDataBuffer bytes.Buffer
gzipWriter := gzip.NewWriter(&syncDataBuffer)
_, err = gzipWriter.Write(onCallSyncJsonData)
if err != nil {
return fmt.Errorf("error writing sync data to gzip writer: %v", err)
}
if err := gzipWriter.Close(); err != nil {
return fmt.Errorf("error closing gzip writer: %v", err)
}
syncURL, err := url.JoinPath(onCallPluginSettings.OnCallAPIURL, "api/internal/v1/plugin/v2/sync")
if err != nil {
return fmt.Errorf("error joining path: %v", err)
@ -146,7 +157,7 @@ func (a *App) makeSyncRequest(ctx context.Context, forceSend bool) error {
return fmt.Errorf("error parsing path: %v", err)
}
syncReq, err := http.NewRequest("POST", parsedSyncURL.String(), bytes.NewBuffer(onCallSyncJsonData))
syncReq, err := http.NewRequest("POST", parsedSyncURL.String(), &syncDataBuffer)
if err != nil {
return fmt.Errorf("error creating request: %v", err)
}
@ -156,6 +167,7 @@ func (a *App) makeSyncRequest(ctx context.Context, forceSend bool) error {
return err
}
syncReq.Header.Set("Content-Type", "application/json")
syncReq.Header.Set("Content-Encoding", "gzip")
res, err := a.httpClient.Do(syncReq)
if err != nil {

6
grafana-plugin/pkg/plugin/teams.go
View file

@ -70,7 +70,7 @@ func (a *App) GetTeamsForUser(settings *OnCallPluginSettings, onCallUser *OnCall
var result []Team
err = json.Unmarshal(body, &result)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
}
if res.StatusCode == 200 {
@ -115,7 +115,7 @@ func (a *App) GetAllTeams(settings *OnCallPluginSettings) ([]OnCallTeam, error)
var result Teams
err = json.Unmarshal(body, &result)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
}
if res.StatusCode == 200 {
@ -161,7 +161,7 @@ func (a *App) GetTeamsMembersForTeam(settings *OnCallPluginSettings, onCallTeam
var result []OrgUser
err = json.Unmarshal(body, &result)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
}
if res.StatusCode == 200 {

2
grafana-plugin/pkg/plugin/users.go
View file

@ -233,7 +233,7 @@ func (a *App) GetAllUsers(settings *OnCallPluginSettings) ([]OnCallUser, error)
var result []OrgUser
err = json.Unmarshal(body, &result)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
}
if res.StatusCode == 200 {