diff --git a/CHANGELOG.md b/CHANGELOG.md index c970cb98..c11bd126 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update the direct paging feature to page for acknowledged & silenced alert groups, and show a warning for resolved alert groups by @vadimkerr ([#2639](https://github.com/grafana/oncall/pull/2639)) +- Change calls to get instances from GCOM to paginate by @mderynck ([#2669](https://github.com/grafana/oncall/pull/2669)) - Update checking on-call users to use schedule final events ([#2651](https://github.com/grafana/oncall/pull/2651)) ### Fixed diff --git a/engine/apps/grafana_plugin/helpers/client.py b/engine/apps/grafana_plugin/helpers/client.py index 2641821b..064982c2 100644 --- a/engine/apps/grafana_plugin/helpers/client.py +++ b/engine/apps/grafana_plugin/helpers/client.py @@ -253,6 +253,7 @@ class GcomAPIClient(APIClient): DELETED_INSTANCE_QUERY = "instances?status=deleted&includeDeleted=true" STACK_STATUS_DELETED = "deleted" STACK_STATUS_ACTIVE = "active" + PAGE_SIZE = 1000 def __init__(self, api_token: str) -> None: super().__init__(settings.GRAFANA_COM_API_URL, api_token) @@ -315,8 +316,20 @@ class GcomAPIClient(APIClient): return False return self._feature_toggle_is_enabled(instance_info, "accessControlOnCall") - def get_instances(self, query: str): - return self.api_get(query) + def get_instances(self, query: str, page_size=None): + if not page_size: + page, _ = self.api_get(query) + yield page + else: + cursor = 0 + while cursor is not None: + if query: + page_query = query + f"&cursor={cursor}&pageSize={page_size}" + else: + page_query = f"?cursor={cursor}&pageSize={page_size}" + page, _ = self.api_get(page_query) + yield page + cursor = page["nextCursor"] def is_stack_deleted(self, stack_id: str) -> bool: url = f"instances?includeDeleted=true&id={stack_id}" diff --git a/engine/apps/grafana_plugin/helpers/gcom.py b/engine/apps/grafana_plugin/helpers/gcom.py index 67543906..91838b44 100644 --- a/engine/apps/grafana_plugin/helpers/gcom.py +++ b/engine/apps/grafana_plugin/helpers/gcom.py @@ -101,12 +101,13 @@ def get_instance_ids(query: str) -> Tuple[Optional[set], bool]: return None, False client = GcomAPIClient(settings.GRAFANA_COM_API_TOKEN) - instances, status = client.get_instances(query) + instance_pages = client.get_instances(query, GcomAPIClient.PAGE_SIZE) - if not instances: + if not instance_pages: return None, True - ids = set(i["id"] for i in instances["items"]) + ids = set(i["id"] for page in instance_pages for i in page["items"]) + return ids, True diff --git a/engine/apps/grafana_plugin/tests/test_gcom_api_client.py b/engine/apps/grafana_plugin/tests/test_gcom_api_client.py index 3dfc3605..63a2cd21 100644 --- a/engine/apps/grafana_plugin/tests/test_gcom_api_client.py +++ b/engine/apps/grafana_plugin/tests/test_gcom_api_client.py @@ -1,8 +1,11 @@ +import uuid from unittest.mock import patch import pytest from apps.grafana_plugin.helpers.client import GcomAPIClient +from apps.grafana_plugin.helpers.gcom import get_instance_ids +from settings.base import CLOUD_LICENSE_NAME class TestIsRbacEnabledForStack: @@ -82,3 +85,76 @@ class TestIsRbacEnabledForStack: GcomAPIClient("someFakeApiToken")._feature_toggle_is_enabled(instance_info, self.TEST_FEATURE_TOGGLE) == expected ) + + +def build_paged_responses(page_size, pages, total_items): + response = [] + remaining = total_items + for i in range(pages): + if not page_size: + page_item_count = remaining + else: + page_item_count = min(page_size, remaining) + remaining -= page_size + + items = [] + for j in range(page_item_count): + items.append({"id": str(uuid.uuid4())}) + next_cursor = None if i == pages - 1 else i * page_size + response.append(({"items": items, "nextCursor": next_cursor}, {})) + return response + + +@pytest.mark.parametrize( + "page_size, expected_pages, expected_items", + [ + (None, 1, 0), + (None, 1, 5), + (10, 2, 20), + (10, 4, 33), + ], +) +def test_get_instances_pagination(page_size, expected_pages, expected_items): + response = build_paged_responses(page_size, expected_pages, expected_items) + client = GcomAPIClient("someToken") + + pages = [] + items = 0 + with patch( + "apps.grafana_plugin.helpers.client.APIClient.api_get", + side_effect=response, + ): + instance_pages = client.get_instances("", page_size) + for page in instance_pages: + pages.append(page) + items += len(page.get("items", [])) + + assert len(pages) == expected_pages + assert items == expected_items + + +@pytest.mark.parametrize( + "query, expected_pages, expected_items", + [ + (GcomAPIClient.ACTIVE_INSTANCE_QUERY, 1, 0), + ("", 1, 543), + (GcomAPIClient.DELETED_INSTANCE_QUERY, 2, 2000), + ("", 4, 3333), + ], +) +def test_get_instance_ids_pagination(settings, query, expected_pages, expected_items): + settings.GRAFANA_COM_API_TOKEN = "someToken" + settings.LICENSE = CLOUD_LICENSE_NAME + + response = build_paged_responses(GcomAPIClient.PAGE_SIZE, expected_pages, expected_items) + + with patch( + "apps.grafana_plugin.helpers.client.APIClient.api_get", + side_effect=response, + ): + instance_ids, status = get_instance_ids(query) + item_count = len(instance_ids) + assert status is True + assert item_count == expected_items + if item_count > 0: + assert type(next(iter(instance_ids))) is str