Configure cross plugin e2e tests (#4222)

# What this PR does - Run e2e tests using ops-devenv against environment that includes OnCall & Labels - Add first e2e test for Labels (creating new label key and value) ## Which issue(s) this PR closes Closes https://github.com/grafana/oncall/issues/4083  ## Checklist - [x] Unit, integration, and e2e (if applicable) tests updated - [x] Documentation added (or `pr:no public docs` PR label added if not required) - [x] Added the relevant release notes label (see labels prefixed w/ `release:`). These labels dictate how your PR will show up in the autogenerated release notes.
2024-04-25 08:25:59 +02:00 · 2024-04-25 08:25:59 +02:00 · 9432e9ee44
commit 9432e9ee44
parent 80fcdd7c62
9 changed files with 122 additions and 21 deletions
--- a/.github/helm-ci.yml
+++ b/.github/helm-ci.yml
@ -0,0 +1,10 @@
+env:
+  - name: GRAFANA_CLOUD_NOTIFICATIONS_ENABLED
+    value: "False"
+  - name: FEATURE_PROMETHEUS_EXPORTER_ENABLED
+    value: "True"
+  # enabled to be able to test docker.host.internal in the webhook e2e tests
+  - name: DANGEROUS_WEBHOOKS_ENABLED
+    value: "True"
+  - name: FEATURE_LABELS_ENABLED_FOR_ALL
+    value: "True"
--- a/.github/workflows/daily-e2e-tests.yml
+++ b/.github/workflows/daily-e2e-tests.yml
@ -1,5 +1,5 @@
 name: Daily e2e tests
-"on":
+on:
  # allows manual run on github actions
  workflow_dispatch:
  schedule:
@ -10,7 +10,7 @@ jobs:
    name: End to end tests
    strategy:
      matrix:
-        grafana-image-tag:
+        grafana_version:
          # OnCall doesn't work on the following versions of Grafana
          # - 8.5.22
          # - 9.0.0
@ -29,7 +29,7 @@ jobs:
      fail-fast: false
    uses: ./.github/workflows/e2e-tests.yml
    with:
-      grafana-image-tag: ${{ matrix.grafana-image-tag }}
+      grafana_version: ${{ matrix.grafana_version }}
      run-expensive-tests: true
      # TODO: fix issues with some tests on firefox and webkit
      # example CI build with firefox/webkit failing tests
--- a/.github/workflows/e2e-tests.yml
+++ b/.github/workflows/e2e-tests.yml
@ -1,8 +1,9 @@
 name: e2e tests
-"on":
+
+on:
  workflow_call:
    inputs:
-      grafana-image-tag:
+      grafana_version:
        required: true
        type: string
      browsers:
@ -34,12 +35,13 @@ jobs:
    # the oncall backend, and hence, flaky tests. Let's use CI runners w/ more resources to avoid this (plus
    # this will allow us to run more backend containers and parralelize the tests)
    runs-on: ubuntu-latest-16-cores
-    name: "Grafana: ${{ inputs.grafana-image-tag }}"
+    name: "Grafana: ${{ inputs.grafana_version }}"
    environment:
      name: github-pages
    permissions:
      id-token: write
-      pages: "write"
+      pages: write
+      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@ -120,20 +122,58 @@ jobs:
      - name: Install Playwright deps
        uses: docker://mcr.microsoft.com/playwright:next-jammy

+      # Go and Mage are required to run gops-labels
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.21.5"
+
+      - name: Install Mage
+        run: go install github.com/magefile/mage@v1.15.0
+
+      - name: Get Vault secrets
+        id: get-secrets
+        uses: grafana/shared-workflows/actions/get-vault-secrets@main
+        with:
+          repo_secrets: |
+            GH_APP_ID=github-app:app-id
+            GH_APP_INSTALLATION_ID=github-app:app-installation-id
+            GH_APP_PRIVATE_KEY=github-app:private-key
+
+      - name: Generate Github App token
+        id: generate-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ env.GH_APP_ID }}
+          private-key: ${{ env.GH_APP_PRIVATE_KEY }}
+          owner: grafana
+          repositories: "ops-devenv,gops-labels"
+
+      - name: Clone other repos needed for cross-plugin e2e tests
+        shell: bash
+        run: |
+          cd ..
+          git clone https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/grafana/ops-devenv.git
+          git clone https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/grafana/gops-labels.git
+
      - name: Tilt CI - standard E2E tests
        shell: bash
        if: ${{ inputs.run-expensive-tests == false }}
        env:
-          GRAFANA_IMAGE_TAG: ${{ inputs.grafana-image-tag }}
+          GRAFANA_VERSION: ${{ inputs.grafana_version }}
+          GRAFANA_ADMIN_USERNAME: "irm"
+          GRAFANA_ADMIN_PASSWORD: "irm"
          BROWSERS: ${{ inputs.browsers }}
-        run: tilt ci
+        run: cd ../ops-devenv && tilt ci gops-labels oncall

      - name: Tilt CI - standard and expensive E2E tests
        if: inputs.run-expensive-tests
        shell: bash
        env:
          E2E_TESTS_CMD: "cd grafana-plugin && yarn test:e2e-expensive"
-          GRAFANA_IMAGE_TAG: ${{ inputs.grafana-image-tag }}
+          GRAFANA_VERSION: ${{ inputs.grafana_version }}
+          GRAFANA_ADMIN_USERNAME: "irm"
+          GRAFANA_ADMIN_PASSWORD: "irm"
          BROWSERS: ${{ inputs.browsers }}
          MAILSLURP_API_KEY: ${{ secrets.MAILSLURP_API_KEY }}
          TWILIO_ACCOUNT_SID: ${{ secrets.TWILIO_ACCOUNT_SID }}
@ -141,7 +181,7 @@ jobs:
          # wrapping single quotes are required to prevent stripping leading "+" from the number
          TWILIO_PHONE_NUMBER: '"${{ secrets.TWILIO_PHONE_NUMBER }}"'
          TWILIO_VERIFY_SID: ${{ secrets.TWILIO_VERIFY_SID }}
-        run: tilt ci
+        run: cd ../ops-devenv && tilt ci gops-labels oncall

      - name: Setup Pages
        if: failure()
--- a/.github/workflows/linting-and-tests.yml
+++ b/.github/workflows/linting-and-tests.yml
@ -320,7 +320,7 @@ jobs:
    uses: ./.github/workflows/e2e-tests.yml
    with:
      # TODO: fix issues with running e2e tests against Grafana v10.2.x and v10.3.x
-      grafana-image-tag: 10.1.7
-      # grafana-image-tag: 10.3.3
+      grafana_version: 10.1.7
+      # grafana_version: 10.3.3
      run-expensive-tests: false
      browsers: "chromium"
--- a/9
+++ b/9
@ -2,7 +2,7 @@ load('ext://uibutton', 'cmd_button', 'location', 'text_input', 'bool_input')
 running_under_parent_tiltfile = os.getenv("TILT_PARENT", "false") == "true"
 # The user/pass that you will login to Grafana with
 grafana_admin_user_pass = os.getenv("GRAFANA_ADMIN_USER_PASS", "oncall")
-grafana_image_tag = os.getenv("GRAFANA_IMAGE_TAG", "latest")
+grafana_version = os.getenv("GRAFANA_VERSION", "latest")
 e2e_tests_cmd=os.getenv("E2E_TESTS_CMD", "cd grafana-plugin && yarn test:e2e")
 twilio_values=[
    "oncall.twilio.accountSid=" + os.getenv("TWILIO_ACCOUNT_SID", ""),
@ -113,7 +113,10 @@ cmd_button(
    icon_name="dangerous",
 )

-yaml = helm("helm/oncall", name=HELM_PREFIX, values=["./dev/helm-local.yml", "./dev/helm-local.dev.yml"], set=twilio_values)
+helm_oncall_values = ["./dev/helm-local.yml", "./dev/helm-local.dev.yml"]
+if is_ci:
+    helm_oncall_values = helm_oncall_values + ["./.github/helm-ci.yml"]
+yaml = helm("helm/oncall", name=HELM_PREFIX, values=helm_oncall_values, set=twilio_values)

 k8s_yaml(yaml)

@ -134,7 +137,7 @@ k8s_resource(
 # Use separate grafana helm chart
 if not running_under_parent_tiltfile:
    grafana(
-        grafana_version=grafana_image_tag,
+        grafana_version=grafana_version,
        context="grafana-plugin",
        plugin_files=["grafana-plugin/src/plugin.json"],
        namespace="default",
--- a/grafana-plugin/e2e-tests/labels/createNewLabelKeysAndValues.test.ts
+++ b/grafana-plugin/e2e-tests/labels/createNewLabelKeysAndValues.test.ts
@ -0,0 +1,35 @@
+import { test, expect } from '../fixtures';
+import { clickButton, generateRandomValidLabel, openDropdown } from '../utils/forms';
+import { openCreateIntegrationModal } from '../utils/integrations';
+import { goToOnCallPage } from '../utils/navigation';
+
+test('New label keys and labels can be created', async ({ adminRolePage }) => {
+  const { page } = adminRolePage;
+  await goToOnCallPage(page, 'integrations');
+  await openCreateIntegrationModal(page);
+  const NEW_LABEL_KEY = generateRandomValidLabel();
+  const NEW_LABEL_VALUE = generateRandomValidLabel();
+
+  await page
+    .getByTestId('create-integration-modal')
+    .getByTestId('integration-display-name')
+    .filter({ hasText: 'Webhook' })
+    .first()
+    .click();
+  await clickButton({ page, buttonText: /^Add Labels$/ });
+
+  await openDropdown({ page, text: /^Select key$/ });
+  await page.keyboard.insertText(NEW_LABEL_KEY);
+  await page.getByText('Hit enter to add').waitFor();
+  await page.keyboard.press('Enter');
+
+  await page.waitForTimeout(1000);
+
+  await openDropdown({ page, text: /^Select value$/ });
+  await page.keyboard.insertText(NEW_LABEL_VALUE);
+  await page.getByText('Hit enter to add').waitFor();
+  await page.keyboard.press('Enter');
+
+  await expect(page.getByText(NEW_LABEL_KEY)).toBeVisible();
+  await expect(page.getByText(NEW_LABEL_VALUE)).toBeVisible();
+});
--- a/grafana-plugin/e2e-tests/users/usersActions.test.ts
+++ b/grafana-plugin/e2e-tests/users/usersActions.test.ts
@ -51,14 +51,14 @@ test.describe('Users screen actions', () => {
  });

  test('Search updates the table view', async ({ adminRolePage }) => {
-    const { page } = adminRolePage;
+    const { page, userName } = adminRolePage;
    await goToOnCallPage(page, 'users');

    await page.waitForTimeout(2000);

    const searchInput = page.locator(`[data-testid="search-users"]`);

-    await searchInput.fill('oncall');
+    await searchInput.fill(userName);
    await page.waitForTimeout(2000);

    const result = page.locator(`[data-testid="users-username"]`);
--- a/grafana-plugin/e2e-tests/utils/forms.ts
+++ b/grafana-plugin/e2e-tests/utils/forms.ts
@ -1,6 +1,6 @@
 import type { Locator, Page } from '@playwright/test';

-import { randomUUID } from 'crypto';
+import { randomInt, randomUUID } from 'crypto';

 type SelectorType = 'gSelect' | 'grafanaSelect';
 type SelectDropdownValueArgs = {
@ -97,6 +97,16 @@ export const selectDropdownValue = async (args: SelectDropdownValueArgs): Promis

 export const generateRandomValue = (): string => randomUUID();

+export const generateRandomValidLabel = (length = 10) => {
+  const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
+  let result = '';
+  for (let i = 0; i < length; i++) {
+    const randomIndex = randomInt(0, characters.length);
+    result += characters[randomIndex];
+  }
+  return result;
+};
+
 /**
 * wait for the options to appear
 *
@ -113,3 +123,6 @@ export const selectValuePickerValue = async (
      `div[class*="grafana-select-menu"] >> ${textMatchSelector(optionExactMatch, valuePickerText)}`
    )
  ).click();
+
+export const openDropdown = async ({ page, text }: { page: Page; text: string | RegExp }) =>
+  page.locator('div').filter({ hasText: text }).nth(1).click();
--- a/grafana-plugin/playwright.config.ts
+++ b/grafana-plugin/playwright.config.ts
@ -1,4 +1,4 @@
-import { PlaywrightTestProject, defineConfig, devices, PlaywrightTestConfig } from '@playwright/test';
+import { PlaywrightTestProject, defineConfig, devices } from '@playwright/test';

 import path from 'path';
 /**
@ -49,7 +49,7 @@ export default defineConfig({
   * to flaky tests.. let's allow 1 retry per test
   */
  retries: 1,
-  workers: 2,
+  workers: '25%', // 25% of logical CPU cores, e.g. for 16 CPU cores it will use 4 workers
  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
  // reporter: 'html',
  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */