Add missing labels permissions, fix tilt ci from ops-devenv, fix expensive e2e tests (#4842)

# What this PR does

- add missing labels-related permissions for external service account
used by new oncall init process
- fix expensive e2e tests in new oncall init process
- unify Grafana versions between standard and expensive e2e tests
- fix running tilt through ops-devenv in new oncall init process
- avoid duplicated standard e2e tests on workflows that run daily and on
merges to main

## Which issue(s) this PR closes

Related to https://github.com/grafana/oncall-private/issues/2656

<!--
*Note*: If you want the issue to be auto-closed once the PR is merged,
change "Related to" to "Closes" in the line above.
If you have more than one GitHub issue that this PR closes, be sure to
preface
each issue link with a [closing
keyword](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/using-keywords-in-issues-and-pull-requests#linking-a-pull-request-to-an-issue).
This ensures that the issue(s) are auto-closed once the PR has been
merged.
-->

## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] Added the relevant release notes label (see labels prefixed w/
`release:`). These labels dictate how your PR will
    show up in the autogenerated release notes.

---------

Co-authored-by: Joey Orlando <joseph.t.orlando@gmail.com>

.github/workflows/e2e-tests.yml

@@ -144,12 +144,14 @@ jobs:
           git clone https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/grafana/ops-devenv.git
           git clone https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/grafana/gops-labels.git
 
-      - name: Tilt CI - standard and expensive E2E tests
+      - name: Tilt CI - Expensive E2E tests
         if: inputs.run-expensive-tests
         shell: bash
         env:
           E2E_TESTS_CMD: "cd ../../grafana-plugin && yarn test:e2e-expensive"
           GRAFANA_VERSION: ${{ inputs.grafana_version }}
+          GF_FEATURE_TOGGLES_ENABLE: "externalServiceAccounts"
+          ONCALL_API_URL: "http://oncall-dev-engine:8080"
           GRAFANA_ADMIN_USERNAME: "irm"
           GRAFANA_ADMIN_PASSWORD: "irm"
           BROWSERS: ${{ inputs.browsers }}

.github/workflows/expensive-e2e-tests.yml

@@ -24,8 +24,8 @@ jobs:
           # - 9.3.16
           # - 9.4.13
           # - 9.5.7
-          - 10.0.11
           - 10.1.7
+          - 10.3.3
           # TODO: fix issues with running e2e tests against Grafana v10.2.x and v10.3.x
           # - 10.2.4
           # - latest
@@ -55,7 +55,7 @@ jobs:
       #
       - uses: slackapi/slack-github-action@v1.24.0
         with:
-          channel-id: gops-oncall-dev
+          channel-id: gops-irm-dev
           # yamllint disable rule:line-length
           payload: |
             {

.github/workflows/linting-and-tests.yml

@@ -239,9 +239,16 @@ jobs:
   end-to-end-tests:
     name: Standard e2e tests
     uses: ./.github/workflows/e2e-tests.yml
+    strategy:
+      matrix:
+        grafana_version:
+          - 10.1.7
+          - 10.3.3
+          # TODO: fix issues with running e2e tests against Grafana v10.2.x and latest
+          # - 10.2.4
+          # - latest
+      fail-fast: false
     with:
-      # TODO: fix issues with running e2e tests against Grafana v10.2.x and v10.3.x
-      grafana_version: 10.1.7
-      # grafana_version: 10.3.3
+      grafana_version: ${{ matrix.grafana_version }}
       run-expensive-tests: false
       browsers: "chromium"

Tiltfile

@@ -1,6 +1,7 @@
 load('ext://uibutton', 'cmd_button', 'location', 'text_input', 'bool_input')
 load("ext://configmap", "configmap_create")
 
+grafana_url = os.getenv("GRAFANA_URL", "http://grafana:3000")
 running_under_parent_tiltfile = os.getenv("TILT_PARENT", "false") == "true"
 twilio_values=[
     "oncall.twilio.accountSid=" + os.getenv("TWILIO_ACCOUNT_SID", ""),
@@ -29,6 +30,14 @@ def plugin_json():
         return plugin_file
     return 'NOT_A_PLUGIN'
 
+def extra_env():
+    return {
+        "GF_APP_URL": grafana_url,
+        "GF_SERVER_ROOT_URL": grafana_url, 
+        "GF_FEATURE_TOGGLES_ENABLE": "externalServiceAccounts",
+        "ONCALL_API_URL": "http://oncall-dev-engine:8080"
+    }
+
 
 allow_k8s_contexts(["kind-kind"])
 
@@ -83,8 +92,6 @@ def load_grafana():
     # The user/pass that you will login to Grafana with
     grafana_admin_user_pass = os.getenv("GRAFANA_ADMIN_USER_PASS", "oncall")
     grafana_version = os.getenv("GRAFANA_VERSION", "latest")
-    grafana_url = os.getenv("GRAFANA_URL", "http://grafana:3000")
-
 
     if 'plugin' in profiles:
         k8s_resource(

grafana-plugin/e2e-tests/alerts/directPaging.test.ts

@@ -1,6 +1,5 @@
-import semver from 'semver';
-
 import { test, expect } from '../fixtures';
+import { isGrafanaVersionLowerThan } from '../utils/constants';
 import { clickButton, fillInInput } from '../utils/forms';
 import { goToOnCallPage } from '../utils/navigation';
 
@@ -22,9 +21,7 @@ test('we can directly page a user', async ({ adminRolePage }) => {
 
   const addRespondersPopup = page.getByTestId('add-responders-popup');
 
-  await addRespondersPopup[semver.lt(process.env.CURRENT_GRAFANA_VERSION, '10.3.0') ? 'getByText' : 'getByLabel'](
-    'Users'
-  ).click();
+  await addRespondersPopup[isGrafanaVersionLowerThan('10.3.0') ? 'getByText' : 'getByLabel']('Users').click();
   await addRespondersPopup.getByText(adminRolePage.userName).first().click();
 
   // If user is not on call, confirm invitation

grafana-plugin/e2e-tests/globalSetup.ts

@@ -6,7 +6,6 @@ import {
   type APIRequestContext,
   Page,
 } from '@playwright/test';
-import semver from 'semver';
 
 import { VIEWER_USER_STORAGE_STATE, EDITOR_USER_STORAGE_STATE, ADMIN_USER_STORAGE_STATE } from '../playwright.config';
 
@@ -21,6 +20,7 @@ import {
   IS_CLOUD,
   IS_OPEN_SOURCE,
   OrgRole,
+  isGrafanaVersionLowerThan,
 } from './utils/constants';
 import { goToOnCallPage } from './utils/navigation';
 
@@ -62,7 +62,7 @@ const idempotentlyInitializePlugin = async (page: Page) => {
   if (await openPluginConfigurationButton.isVisible()) {
     await openPluginConfigurationButton.click();
     // Before 10.3 Admin user needs to create service account manually
-    if (semver.lt(process.env.CURRENT_GRAFANA_VERSION, '10.3.0')) {
+    if (isGrafanaVersionLowerThan('10.3.0')) {
       await page.getByTestId('recreate-service-account').click();
     }
     await page.getByTestId('connect-plugin').click();

grafana-plugin/e2e-tests/insights/insights.test.ts

@@ -1,7 +1,6 @@
-import semver from 'semver';
-
 import { test, expect } from '../fixtures';
 import { resolveFiringAlert } from '../utils/alertGroup';
+import { isGrafanaVersionLowerThan } from '../utils/constants';
 import { createEscalationChain, EscalationStep } from '../utils/escalationChain';
 import { clickButton, generateRandomValue } from '../utils/forms';
 import { createIntegrationAndSendDemoAlert } from '../utils/integrations';
@@ -15,10 +14,7 @@ import { createOnCallSchedule } from '../utils/schedule';
  * and use the currentGrafanaVersion fixture once this bugged is patched in playwright
  * https://github.com/microsoft/playwright/issues/29608
  */
-test.skip(
-  () => semver.lt(process.env.CURRENT_GRAFANA_VERSION, '10.0.0'),
-  'Insights is only available in Grafana 10.0.0 and above'
-);
+test.skip(() => isGrafanaVersionLowerThan('10.0.0'), 'Insights is only available in Grafana 10.0.0 and above');
 
 /**
  * skipping as these tests are currently flaky

grafana-plugin/e2e-tests/labels/createNewLabelKeysAndValues.test.ts

@@ -1,8 +1,14 @@
 import { test, expect } from '../fixtures';
+import { isGrafanaVersionGreaterThan } from '../utils/constants';
 import { clickButton, generateRandomValidLabel, openDropdown } from '../utils/forms';
 import { openCreateIntegrationModal } from '../utils/integrations';
 import { goToOnCallPage } from '../utils/navigation';
 
+test.skip(
+  () => isGrafanaVersionGreaterThan('10.3.0'),
+  'Above 10.3 labels need enterprise version to validate permissions'
+);
+
 test('New label keys and labels can be created @expensive', async ({ adminRolePage }) => {
   const { page } = adminRolePage;
   await goToOnCallPage(page, 'integrations');

grafana-plugin/e2e-tests/pluginInitialization/initialization.test.ts

@@ -1,9 +1,7 @@
-import semver from 'semver';
-
 import { waitInMs } from 'utils/async';
 
 import { test, expect, Page } from '../fixtures';
-import { OrgRole } from '../utils/constants';
+import { OrgRole, isGrafanaVersionLowerThan } from '../utils/constants';
 import { goToGrafanaPage, goToOnCallPage } from '../utils/navigation';
 import { createGrafanaUser, loginAndWaitTillGrafanaIsLoaded } from '../utils/users';
 
@@ -48,10 +46,7 @@ test.describe('Plugin initialization', () => {
   }) => {
     test.slow();
 
-    test.skip(
-      semver.lt(process.env.CURRENT_GRAFANA_VERSION, '10.3.0'),
-      'Extension is only available in Grafana 10.3.0 and above'
-    );
+    test.skip(isGrafanaVersionLowerThan('10.3.0'), 'Extension is only available in Grafana 10.3.0 and above');
 
     // Create new editor user
     const USER_NAME = `editor-${new Date().getTime()}`;

grafana-plugin/e2e-tests/schedules/scheduleView.test.ts

@@ -1,10 +1,9 @@
-import semver from 'semver';
-
 import { scheduleViewToDaysInOneRow } from 'models/schedule/schedule.helpers';
 import { ScheduleView } from 'models/schedule/schedule.types';
 import { HTML_ID } from 'utils/DOM';
 
 import { expect, Page, test } from '../fixtures';
+import { isGrafanaVersionLowerThan } from '../utils/constants';
 import { generateRandomValue } from '../utils/forms';
 import { createOnCallSchedule } from '../utils/schedule';
 
@@ -13,7 +12,7 @@ const getNumberOfWeekdaysInFinalSchedule = async (page: Page) =>
 const getScheduleViewRadioButtonLocator = (page: Page, view: ScheduleView) =>
   page
     .getByTestId('schedule-view-picker')
-    [semver.lt(process.env.CURRENT_GRAFANA_VERSION, '10.2.0') ? 'getByText' : 'getByLabel'](view, { exact: true });
+    [isGrafanaVersionLowerThan('10.2.0') ? 'getByText' : 'getByLabel'](view, { exact: true });
 
 test('schedule view (week/2 weeks/month) toggler works', async ({ adminRolePage }) => {
   const { page, userName } = adminRolePage;

grafana-plugin/e2e-tests/users/usersActions.test.ts

@@ -1,6 +1,5 @@
-import semver from 'semver';
-
 import { test, expect } from '../fixtures';
+import { isGrafanaVersionLowerThan } from '../utils/constants';
 import { goToOnCallPage } from '../utils/navigation';
 import { verifyThatUserCanViewOtherUsers, accessProfileTabs } from '../utils/users';
 
@@ -26,7 +25,7 @@ test.describe('Users screen actions', () => {
     const tabsToCheck = ['tab-phone-verification', 'tab-slack', 'tab-telegram'];
 
     // After 10.3 it's been moved to global user profile
-    if (semver.lt(process.env.CURRENT_GRAFANA_VERSION, '10.3.0')) {
+    if (isGrafanaVersionLowerThan('10.3.0')) {
       tabsToCheck.unshift('tab-mobile-app');
     }
 
@@ -74,10 +73,10 @@ test.describe('Users screen actions', () => {
     await page.waitForTimeout(2000);
 
     await page
-    .locator('div')
-    .filter({ hasText: /^Search or filter results\.\.\.$/ })
-    .nth(1)
-    .click();
+      .locator('div')
+      .filter({ hasText: /^Search or filter results\.\.\.$/ })
+      .nth(1)
+      .click();
     await page.keyboard.insertText(userName);
     await page.keyboard.press('Enter');
     await page.waitForTimeout(2000);

grafana-plugin/e2e-tests/utils/constants.ts

@@ -1,3 +1,5 @@
+import semver from 'semver';
+
 export const BASE_URL = process.env.BASE_URL || 'http://localhost:3000';
 export const MAILSLURP_API_KEY = process.env.MAILSLURP_API_KEY;
 
@@ -19,3 +21,6 @@ export enum OrgRole {
 }
 
 export const MOSCOW_TIMEZONE = 'Europe/Moscow';
+
+export const isGrafanaVersionGreaterThan = (version: string) => semver.gt(process.env.CURRENT_GRAFANA_VERSION, version);
+export const isGrafanaVersionLowerThan = (version: string) => semver.lt(process.env.CURRENT_GRAFANA_VERSION, version);

grafana-plugin/e2e-tests/utils/userSettings.ts

@@ -19,7 +19,7 @@ export const verifyUserPhoneNumber = async (page: Page): Promise<void> => {
   await openUserSettingsModal(page);
 
   // go to the Phone Verification tab
-  await page.locator('a[aria-label="Tab Phone Verification"]').click();
+  await page.getByTestId('tab-phone-verification').click();
 
   // check to see if we've already verified our phone number.. no need to do it more than once
   if (await getForgetPhoneNumberButton(page).isVisible()) {

grafana-plugin/package.json

@@ -18,7 +18,7 @@
     "test:report": "HTML_REPORT_ENABLED=true yarn test",
     "test:silent": "yarn test --silent",
     "test:e2e": "yarn playwright test --grep-invert @expensive",
-    "test:e2e-expensive": "yarn playwright test",
+    "test:e2e-expensive": "yarn playwright test --grep @expensive",
     "test:e2e:watch": "yarn test:e2e --ui",
     "test:e2e-expensive:watch": "yarn test:e2e-expensive --ui",
     "test:e2e:gen": "yarn playwright codegen http://localhost:3000",

grafana-plugin/src/plugin.json

@@ -674,7 +674,11 @@
       {
         "action": "users.roles:read",
         "scope": "users:*"
-      }
+      },
+      { "action": "grafana-labels-app.label:read" },
+      { "action": "grafana-labels-app.label:write" },
+      { "action": "grafana-labels-app.label:create" },
+      { "action": "grafana-labels-app.label:delete" }
     ]
   },
   "dependencies": {