Brojd/improve e2e tests dx (#3516)

# What this PR does
- introduce e2e tests in Tilt
- support e2e tests commands in Makefile
- stabilize local setup

## Which issue(s) this PR fixes
https://github.com/grafana/oncall/issues/3492

## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not
required)

.gitignore

@@ -10,3 +10,5 @@ venv
 
 yarn.lock
 node_modules
+
+test-results

.prettierrc.js

@@ -0,0 +1,8 @@
+overrides: [
+  {
+    files: ["*.yml", "*.yaml"],
+    options: {
+      singleQuote: false,
+    },
+  },
+];

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+- Support e2e tests in Tilt and Makefile ([#3516](https://github.com/grafana/oncall/pull/3516))
+
 ## v1.3.80 (2023-12-14)
 
 ### Added

Makefile

@@ -197,6 +197,15 @@ engine-manage:  ## run Django's `manage.py` script, inside of a docker container
                 ## https://docs.djangoproject.com/en/4.1/ref/django-admin/#django-admin-makemigrations
 	$(call run_engine_docker_command,python manage.py $(CMD))
 
+test-e2e:  ## run the e2e tests in headless mode
+	yarn --cwd grafana-plugin test:e2e
+
+test-e2e-watch:  ## start e2e tests in watch mode
+	yarn --cwd grafana-plugin test:e2e:watch
+
+test-e2e-show-report:  ## open last e2e test report
+	yarn --cwd grafana-plugin playwright show-report
+
 ui-test:  ## run the UI tests
 	$(call run_ui_docker_command,yarn test)
 

Tiltfile

@@ -1,3 +1,4 @@
+load('ext://uibutton', 'cmd_button', 'location', 'text_input', 'bool_input')
 running_under_parent_tiltfile = os.getenv("TILT_PARENT", "false") == "true"
 # The user/pass that you will login to Grafana with
 grafana_admin_user_pass = os.getenv("GRAFANA_ADMIN_USER_PASS", "oncall")
@@ -36,7 +37,7 @@ docker_build_sub(
     "localhost:63628/oncall/engine:dev",
     context="./engine",
     cache_from=["grafana/oncall:latest", "grafana/oncall:dev"],
-    ignore=["./grafana-plugin/test-results/", "./grafana-plugin/dist/", "./grafana-plugin/e2e-tests/"],
+    ignore=["./test-results/", "./grafana-plugin/dist/", "./grafana-plugin/e2e-tests/"],
     child_context=".",
     target="dev",
     extra_cmds=["ADD ./grafana-plugin/src/plugin.json /etc/grafana-plugin/src/plugin.json"],
@@ -54,10 +55,56 @@ local_resource(
     "build-ui",
     labels=["OnCallUI"],
     cmd="cd grafana-plugin && yarn install && yarn build:dev",
-    serve_cmd="cd grafana-plugin && ONCALL_API_URL=http://oncall-dev-engine:8080 yarn watch",
+    serve_cmd="cd grafana-plugin && yarn watch",
     allow_parallel=True,
 )
 
+local_resource(
+    "e2e-tests",
+    labels=["E2eTests"],
+    cmd="cd grafana-plugin && yarn test:e2e",
+    trigger_mode=TRIGGER_MODE_MANUAL,
+    auto_init=False,
+    resource_deps=["build-ui", "grafana", "grafana-oncall-app-provisioning-configmap", "engine"]
+)
+
+cmd_button(
+    name="E2E Tests - headless run",
+    argv=["sh", "-c", "yarn --cwd ./grafana-plugin test:e2e $STOP_ON_FIRST_FAILURE"],
+    text="Restart headless run",
+    resource="e2e-tests",
+    icon_name="replay",
+    inputs=[
+        text_input("BROWSERS", "Browsers (e.g. \"chromium,firefox,webkit\")", "chromium", "chromium,firefox,webkit"), 
+        bool_input("REPORTER", "Use HTML reporter", True, 'html', 'line'),
+        bool_input("STOP_ON_FIRST_FAILURE", "Stop on first failure", True, "-x", ""),
+    ]
+)
+
+cmd_button(
+    name="E2E Tests - open watch mode",
+    argv=["sh", "-c", "yarn --cwd grafana-plugin test:e2e:watch"],
+    text="Open watch mode",
+    resource="e2e-tests",
+    icon_name="visibility",
+)
+
+cmd_button(
+    name="E2E Tests - show report",
+    argv=["sh", "-c", "yarn --cwd grafana-plugin playwright show-report"],
+    text="Show last HTML report",
+    resource="e2e-tests",
+    icon_name="assignment",
+)
+
+cmd_button(
+    name="E2E Tests - stop current run",
+    argv=["sh", "-c", "kill -9 $(pgrep -f test:e2e)"],
+    text="Stop",
+    resource="e2e-tests",
+    icon_name="dangerous",
+)
+
 yaml = helm("helm/oncall", name=HELM_PREFIX, values=["./dev/helm-local.yml", "./dev/helm-local.dev.yml"])
 
 k8s_yaml(yaml)

dev/README.md

@@ -243,13 +243,18 @@ are run on pull request CI builds. New features should ideally include a new/mod
 
 To run these tests locally simply do the following:
 
-```bash
+1. Install Playwright dependencies with `npx playwright install`
-npx playwright install  # install playwright dependencies
+2. [Launch the environment](#launch-the-environment)
-cp ./grafana-plugin/e2e-tests/.env.example ./grafana-plugin/e2e-tests/.env
+3. Then you interact with tests in 2 different ways:
-# you may need to tweak the values in ./grafana-plugin/.env according to your local setup
+   1. Using `Tilt` - open _E2eTests_ section where you will find 4 buttons:
-cd grafana-plugin
+      1. Restart headless run (you can configure browsers, reporter and failure allowance there)
-yarn test:e2e
+      2. Open watch mode
-```
+      3. Show last HTML report
+      4. Stop (stops any pending e2e test process)
+   2. Using `make`:
+      1. `make test:e2e` to start headless run
+      2. `make test:e2e:watch` to open watch mode
+      3. `make test:e2e:show:report` to open last HTML report
 
 ## Helm unit tests
 

dev/helm-local.yml

@@ -1,4 +1,4 @@
-base_url: localhost:30001
+base_url: localhost:8080
 base_url_protocol: http
 env:
   - name: GRAFANA_CLOUD_NOTIFICATIONS_ENABLED

grafana-plugin/.dockerignore

@@ -1,5 +1,4 @@
 node_modules
 frontend_enterprise
 .DS_Store
-test-results
 playwright-report

grafana-plugin/.gitignore

@@ -16,7 +16,6 @@ grafana-plugin.yml
 frontend_enterprise
 
 # playwright
-/test-results/
 /playwright-report/
 /playwright/.cache/
 /e2e-tests/storageState.json

grafana-plugin/e2e-tests/.env.example

@@ -1,5 +1,3 @@
-BASE_URL=http://localhost:30002/grafana
-ONCALL_API_URL=http://oncall-dev-engine-external:8080/
 GRAFANA_VIEWER_USERNAME=viewer
 GRAFANA_VIEWER_PASSWORD=viewer
 GRAFANA_EDITOR_USERNAME=editor

grafana-plugin/e2e-tests/alerts/onCallSchedule.test.ts

@@ -6,9 +6,6 @@ import { createIntegrationAndSendDemoAlert } from '../utils/integrations';
 import { createOnCallSchedule } from '../utils/schedule';
 
 test('we can create an oncall schedule + receive an alert', async ({ adminRolePage }) => {
-  // this test does a lot of stuff, lets give it adequate time to do its thing
-  test.slow();
-
   const { page, userName } = adminRolePage;
   const escalationChainName = generateRandomValue();
   const integrationName = generateRandomValue();

grafana-plugin/e2e-tests/escalationChains/escalationPolicy.test.ts

@@ -1,6 +1,6 @@
-import {expect, test} from "../fixtures";
+import { expect, test } from '../fixtures';
-import {createEscalationChain, EscalationStep, selectEscalationStepValue} from "../utils/escalationChain";
+import { createEscalationChain, EscalationStep, selectEscalationStepValue } from '../utils/escalationChain';
-import {generateRandomValue} from "../utils/forms";
+import { generateRandomValue } from '../utils/forms';
 
 test('escalation policy does not go back to "Default" after adding users to notify', async ({ adminRolePage }) => {
   const { page, userName } = adminRolePage;
@@ -13,7 +13,5 @@ test('escalation policy does not go back to "Default" after adding users to noti
 
   // reload and check if important is still selected
   await page.reload();
-  await page.waitForLoadState('networkidle');
+  await expect(page.getByText('Important')).toBeVisible();
-
-  expect(await page.locator('text=Important').isVisible()).toBe(true);
 });

grafana-plugin/e2e-tests/globalSetup.ts

@@ -1,6 +1,8 @@
 import { OrgRole } from '@grafana/data';
 import { test as setup, chromium, expect, Page, BrowserContext, FullConfig, APIRequestContext } from '@playwright/test';
 
+import { getOnCallApiUrl } from 'utils/consts';
+
 import { VIEWER_USER_STORAGE_STATE, EDITOR_USER_STORAGE_STATE, ADMIN_USER_STORAGE_STATE } from '../playwright.config';
 
 import GrafanaAPIClient from './utils/clients/grafana';
@@ -13,7 +15,6 @@ import {
   GRAFANA_VIEWER_USERNAME,
   IS_CLOUD,
   IS_OPEN_SOURCE,
-  ONCALL_API_URL,
 } from './utils/constants';
 import { clickButton, getInputByName } from './utils/forms';
 import { goToGrafanaPage } from './utils/navigation';
@@ -59,17 +60,26 @@ const configureOnCallPlugin = async (page: Page): Promise<void> => {
    * go to the oncall plugin configuration page and wait for the page to be loaded
    */
   await goToGrafanaPage(page, '/plugins/grafana-oncall-app');
-  await page.waitForSelector('text=Configure Grafana OnCall');
+  await page.waitForTimeout(2000);
 
-  /**
+  // if plugin is configured, go to OnCall
-   * we may need to fill in the OnCall API URL if it is not set in the process.env
+  const isConfigured = (await page.getByText('Connected to OnCall').count()) >= 1;
-   * of the frontend build
+  if (isConfigured) {
-   */
+    await page.getByRole('link', { name: 'Open Grafana OnCall' }).click();
-  const onCallApiUrlInput = getInputByName(page, 'onCallApiUrl');
+    return;
-  const pluginIsAutoConfigured = (await onCallApiUrlInput.count()) === 0;
+  }
 
-  if (!pluginIsAutoConfigured) {
+  // otherwise we may need to reconfigure the plugin
-    await onCallApiUrlInput.fill(ONCALL_API_URL);
+  const needToReconfigure = (await page.getByText('try removing your plugin configuration').count()) >= 1;
+  if (needToReconfigure) {
+    await clickButton({ page, buttonText: 'Remove current configuration' });
+    await clickButton({ page, buttonText: /^Remove$/ });
+  }
+  await page.waitForTimeout(2000);
+
+  const needToEnterOnCallApiUrl = await page.getByText(/Connected to OnCall/).isHidden();
+  if (needToEnterOnCallApiUrl) {
+    await getInputByName(page, 'onCallApiUrl').fill(getOnCallApiUrl() || 'http://oncall-dev-engine:8080');
     await clickButton({ page, buttonText: 'Connect' });
   }
 
@@ -88,13 +98,6 @@ const configureOnCallPlugin = async (page: Page): Promise<void> => {
  * https://github.com/grafana/incident/blob/main/plugin/e2e/global-setup.ts
  */
 setup('Configure Grafana OnCall plugin', async ({ request }, { config }) => {
-  /**
-   * Unconditionally marks the setup as "slow", giving it triple the default timeout.
-   * This is mostly useful for the rare case for Cloud Grafana instances where the instance may be down/unavailable
-   * and we need to poll it until it is available
-   */
-  setup.slow();
-
   if (IS_CLOUD) {
     await grafanaApiClient.pollInstanceUntilItIsHealthy(request);
   }

grafana-plugin/e2e-tests/integrations/heartbeat.test.ts

@@ -1,6 +1,7 @@
 import { test, Page, expect } from '../fixtures';
 import { generateRandomValue, selectDropdownValue } from '../utils/forms';
-import { createIntegration } from '../utils/integrations';
+import { createIntegration, searchIntegrationAndAssertItsPresence } from '../utils/integrations';
+import { goToOnCallPage } from '../utils/navigation';
 
 const HEARTBEAT_SETTINGS_FORM_TEST_ID = 'heartbeat-settings-form';
 
@@ -12,7 +13,8 @@ test.describe("updating an integration's heartbeat interval works", async () =>
   };
 
   test('change heartbeat interval', async ({ adminRolePage: { page } }) => {
-    await createIntegration({ page, integrationName: generateRandomValue() });
+    const integrationName = generateRandomValue();
+    await createIntegration({ page, integrationName });
 
     await _openHeartbeatSettingsForm(page);
 
@@ -42,7 +44,8 @@ test.describe("updating an integration's heartbeat interval works", async () =>
   });
 
   test('send heartbeat', async ({ adminRolePage: { page } }) => {
-    await createIntegration({ page, integrationName: generateRandomValue() });
+    const integrationName = generateRandomValue();
+    await createIntegration({ page, integrationName });
 
     await _openHeartbeatSettingsForm(page);
 
@@ -59,6 +62,9 @@ test.describe("updating an integration's heartbeat interval works", async () =>
      */
     await page.request.get(endpoint);
     await page.reload({ waitUntil: 'networkidle' });
+
+    await goToOnCallPage(page, 'integrations');
+    await searchIntegrationAndAssertItsPresence({ page, integrationName });
     await page.getByTestId('heartbeat-badge').waitFor();
   });
 });

grafana-plugin/e2e-tests/integrations/integrationsTable.test.ts

@@ -17,7 +17,6 @@ test('Integrations table shows data in Monitoring Systems and Direct Paging tabs
   await createIntegration({
     page,
     integrationSearchText: 'Alertmanager',
-    shouldGoToIntegrationsPage: false,
     integrationName: ALERTMANAGER_INTEGRATION_NAME,
   });
   await page.waitForTimeout(1000);
@@ -32,7 +31,6 @@ test('Integrations table shows data in Monitoring Systems and Direct Paging tabs
     await createIntegration({
       page,
       integrationSearchText: 'Direct paging',
-      shouldGoToIntegrationsPage: false,
       integrationName: DIRECT_PAGING_INTEGRATION_NAME,
     });
     await page.waitForTimeout(1000);
@@ -40,15 +38,13 @@ test('Integrations table shows data in Monitoring Systems and Direct Paging tabs
   await page.getByRole('tab', { name: 'Tab Integrations' }).click();
 
   // By default Monitoring Systems tab is opened and newly created integrations are visible except Direct Paging one
-  await searchIntegrationAndAssertItsPresence({ page, integrationsTable, integrationName: WEBHOOK_INTEGRATION_NAME });
+  await searchIntegrationAndAssertItsPresence({ page, integrationName: WEBHOOK_INTEGRATION_NAME });
   await searchIntegrationAndAssertItsPresence({
     page,
-    integrationsTable,
     integrationName: ALERTMANAGER_INTEGRATION_NAME,
   });
   await searchIntegrationAndAssertItsPresence({
     page,
-    integrationsTable,
     integrationName: DIRECT_PAGING_INTEGRATION_NAME,
     visibleExpected: false,
   });
@@ -57,19 +53,16 @@ test('Integrations table shows data in Monitoring Systems and Direct Paging tabs
   await page.getByRole('tab', { name: 'Tab Manual Direct Paging' }).click();
   await searchIntegrationAndAssertItsPresence({
     page,
-    integrationsTable,
     integrationName: WEBHOOK_INTEGRATION_NAME,
     visibleExpected: false,
   });
   await searchIntegrationAndAssertItsPresence({
     page,
-    integrationsTable,
     integrationName: ALERTMANAGER_INTEGRATION_NAME,
     visibleExpected: false,
   });
   await searchIntegrationAndAssertItsPresence({
     page,
-    integrationsTable,
     integrationName: 'Direct paging',
   });
 });

grafana-plugin/e2e-tests/integrations/maintenanceMode.test.ts

@@ -103,6 +103,7 @@ test.describe('maintenance mode works', () => {
 
     await createEscalationChain(page, escalationChainName, EscalationStep.NotifyUsers, userName);
     await createIntegration({ page, integrationName });
+    await page.waitForTimeout(1000);
     await assignEscalationChainToIntegration(page, escalationChainName);
     await enableMaintenanceMode(page, maintenanceModeType);
 
@@ -110,8 +111,6 @@ test.describe('maintenance mode works', () => {
   };
 
   test('debug mode', async ({ adminRolePage: { page, userName } }) => {
-    test.slow();
-
     const { escalationChainName, integrationName } = await createIntegrationAndEscalationChainAndEnableMaintenanceMode(
       page,
       userName,
@@ -128,7 +127,6 @@ test.describe('maintenance mode works', () => {
   });
 
   test('"maintenance" mode', async ({ adminRolePage: { page, userName } }) => {
-    test.slow();
     const { integrationName } = await createIntegrationAndEscalationChainAndEnableMaintenanceMode(
       page,
       userName,

grafana-plugin/e2e-tests/utils/constants.ts

@@ -1,5 +1,4 @@
 export const BASE_URL = process.env.BASE_URL || 'http://localhost:3000';
-export const ONCALL_API_URL = process.env.ONCALL_API_URL || 'http://host.docker.internal:8080';
 export const MAILSLURP_API_KEY = process.env.MAILSLURP_API_KEY;
 
 export const GRAFANA_VIEWER_USERNAME = process.env.GRAFANA_VIEWER_USERNAME || 'viewer';

grafana-plugin/e2e-tests/utils/forms.ts

@@ -22,7 +22,7 @@ type SelectDropdownValueArgs = {
 
 type ClickButtonArgs = {
   page: Page;
-  buttonText: string;
+  buttonText: string | RegExp;
   // if provided, use this Locator as the root of our search for the button
   startingLocator?: Locator;
 };

grafana-plugin/e2e-tests/utils/integrations.ts

@@ -1,4 +1,4 @@
-import { Locator, Page, expect } from '@playwright/test';
+import { Page, expect } from '@playwright/test';
 
 import { clickButton, generateRandomValue, selectDropdownValue } from './forms';
 import { goToOnCallPage } from './navigation';
@@ -38,17 +38,24 @@ export const createIntegration = async ({
     .click();
 
   // fill in the required inputs
-  (await page.waitForSelector('input[name="verbal_name"]', { state: 'attached' })).fill(integrationName);
+  await page.getByPlaceholder('Integration Name').fill(integrationName);
-  (await page.waitForSelector('textarea[name="description_short"]', { state: 'attached' })).fill(
+  await page.getByPlaceholder('Integration Description').fill('Here goes your integration description');
-    'Here goes your integration description'
+  await page.getByTestId('update-integration-button').focus();
-  );
+  await page.getByTestId('update-integration-button').click();
 
-  const grafanaUpdateBtn = page.getByTestId('update-integration-button');
+  await goToOnCallPage(page, 'integrations');
-  await grafanaUpdateBtn.click();
+  await searchIntegrationAndAssertItsPresence({ page, integrationName });
+
+  await page.getByRole('link', { name: integrationName }).click();
 };
 
 export const assignEscalationChainToIntegration = async (page: Page, escalationChainName: string): Promise<void> => {
-  await page.getByTestId('integration-escalation-chain-not-selected').click();
+  const notSelected = page.getByTestId('integration-escalation-chain-not-selected');
+  if (await notSelected.isHidden()) {
+    await clickButton({ page, buttonText: 'Add route' });
+    await page.waitForTimeout(500);
+  }
+  await notSelected.last().click();
 
   // assign the escalation chain to the integration
   await selectDropdownValue({
@@ -56,7 +63,7 @@ export const assignEscalationChainToIntegration = async (page: Page, escalationC
     selectType: 'grafanaSelect',
     placeholderText: 'Select Escalation Chain',
     value: escalationChainName,
-    startingLocator: page.getByTestId('escalation-chain-select'),
+    startingLocator: page.getByTestId('escalation-chain-select').last(),
   });
 };
 
@@ -92,11 +99,9 @@ export const filterIntegrationsTableAndGoToDetailPage = async (page: Page, integ
 export const searchIntegrationAndAssertItsPresence = async ({
   page,
   integrationName,
-  integrationsTable,
   visibleExpected = true,
 }: {
   page: Page;
-  integrationsTable: Locator;
   integrationName: string;
   visibleExpected?: boolean;
 }) => {
@@ -105,6 +110,7 @@ export const searchIntegrationAndAssertItsPresence = async ({
     .filter({ hasText: /^Search or filter results\.\.\.$/ })
     .nth(1)
     .click();
+  const integrationsTable = page.getByTestId('integrations-table');
   await page.keyboard.insertText(integrationName);
   await page.keyboard.press('Enter');
   await page.waitForTimeout(2000);

grafana-plugin/e2e-tests/utils/navigation.ts

@@ -1,13 +1,15 @@
-import type { Page, Response } from '@playwright/test';
+import type { Page } from '@playwright/test';
 
 import { BASE_URL } from './constants';
 
 type GrafanaPage = '/plugins/grafana-oncall-app';
 type OnCallPage = 'alert-groups' | 'integrations' | 'escalations' | 'schedules' | 'users';
 
-const _goToPage = (page: Page, url = ''): Promise<Response> => page.goto(`${BASE_URL}${url}`);
+const _goToPage = async (page: Page, url = '') => page.goto(`${BASE_URL}${url}`);
 
-export const goToGrafanaPage = (page: Page, url: GrafanaPage): Promise<Response> => _goToPage(page, url);
+export const goToGrafanaPage = async (page: Page, url: GrafanaPage) => _goToPage(page, url);
 
-export const goToOnCallPage = (page: Page, onCallPage: OnCallPage): Promise<Response> =>
+export const goToOnCallPage = async (page: Page, onCallPage: OnCallPage) => {
-  _goToPage(page, `/a/grafana-oncall-app/${onCallPage}`);
+  await _goToPage(page, `/a/grafana-oncall-app/${onCallPage}`);
+  await page.waitForTimeout(1000);
+};

grafana-plugin/playwright.config.ts

@@ -1,4 +1,4 @@
-import { PlaywrightTestProject, defineConfig, devices } from '@playwright/test';
+import { PlaywrightTestProject, defineConfig, devices, PlaywrightTestConfig } from '@playwright/test';
 
 import path from 'path';
 /**
@@ -12,7 +12,11 @@ export const EDITOR_USER_STORAGE_STATE = path.join(__dirname, 'e2e-tests/.auth/e
 export const ADMIN_USER_STORAGE_STATE = path.join(__dirname, 'e2e-tests/.auth/admin.json');
 
 const IS_CI = !!process.env.CI;
-const BROWSERS = process.env.BROWSERS || 'chromium firefox webkit';
+const BROWSERS = process.env.BROWSERS || 'chromium';
+const REPORTER_WITH_DEFAULT = process.env.REPORTER || 'html';
+const REPORTER = (
+  process.env.REPORTER === 'html' ? [['html', { open: 'never' }]] : REPORTER_WITH_DEFAULT
+) as PlaywrightTestConfig['reporter'];
 
 const SETUP_PROJECT_NAME = 'setup';
 const getEnabledBrowsers = (browsers: PlaywrightTestProject[]) =>
@@ -25,16 +29,18 @@ export default defineConfig({
   testDir: './e2e-tests',
 
   /* Maximum time all the tests can run for. */
-  globalTimeout: 20 * 60 * 1000, // 20 minutes
+  globalTimeout: 20 * 60 * 1_000, // 20 minutes
+
+  reporter: REPORTER,
 
   /* Maximum time one test can run for. */
-  timeout: 60 * 1000,
+  timeout: 60_000,
   expect: {
     /**
      * Maximum time expect() should wait for the condition to be met.
      * For example in `await expect(locator).toHaveText();`
      */
-    timeout: 10000,
+    timeout: 6_000,
   },
   /* Run tests in files in parallel */
   fullyParallel: false,
@@ -46,10 +52,10 @@ export default defineConfig({
    * NOTE: until we fix this issue (https://github.com/grafana/oncall/issues/1692) which occasionally leads
    * to flaky tests.. let's allow 1 retry per test
    */
-  retries: IS_CI ? 1 : 0,
+  retries: 1,
   workers: 2,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'html',
+  // reporter: 'html',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
@@ -59,7 +65,7 @@ export default defineConfig({
 
     trace: 'on',
     video: 'on',
-    headless: IS_CI,
+    headless: true,
   },
 
   /* Configure projects for major browsers. The final list is filtered based on BROWSERS env var */
@@ -109,8 +115,9 @@ export default defineConfig({
     // },
   ]),
 
-  /* Folder for test artifacts such as screenshots, videos, traces, etc. */
+  /* Folder for test artifacts such as screenshots, videos, traces, etc. 
-  // outputDir: 'test-results/',
+  Set outside of grafana-plugin to prevent refreshing Grafana UI during e2e test runs */
+  outputDir: '../test-results/',
 
   /* Run your local dev server before starting the tests */
   // webServer: {

grafana-plugin/src/components/TooltipBadge/TooltipBadge.tsx

@@ -18,6 +18,7 @@ interface TooltipBadgeProps {
   customIcon?: React.ReactNode;
   addPadding?: boolean;
   placement?;
+  testId?: string;
 
   onHover?: () => void;
 }
@@ -36,11 +37,9 @@ const TooltipBadge: FC<TooltipBadgeProps> = (props) => {
     icon,
     customIcon,
     className,
-    ...rest
+    testId,
   } = props;
 
-  const testId = rest['data-testid'];
-
   return (
     <Tooltip
       placement={placement || 'bottom-start'}

grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx

@@ -5,7 +5,7 @@ import { useLocation } from 'react-router-dom';
 import { OnCallPluginConfigPageProps } from 'types';
 
 import PluginState, { PluginStatusResponseBase } from 'state/plugin';
-import { FALLBACK_LICENSE, GRAFANA_LICENSE_OSS } from 'utils/consts';
+import { FALLBACK_LICENSE, getOnCallApiUrl, GRAFANA_LICENSE_OSS, hasPluginBeenConfigured } from 'utils/consts';
 
 import ConfigurationForm from './parts/ConfigurationForm';
 import RemoveCurrentConfigurationButton from './parts/RemoveCurrentConfigurationButton';
@@ -46,7 +46,8 @@ export const removePluginConfiguredQueryParams = (pluginIsEnabled: boolean): voi
 
 const PluginConfigPage: FC<OnCallPluginConfigPageProps> = ({
   plugin: {
-    meta: { jsonData, enabled: pluginIsEnabled },
+    meta,
+    meta: { enabled: pluginIsEnabled },
   },
 }) => {
   const { search } = useLocation();
@@ -75,11 +76,8 @@ const PluginConfigPage: FC<OnCallPluginConfigPageProps> = ({
 
   const [resettingPlugin, setResettingPlugin] = useState<boolean>(false);
   const [pluginResetError, setPluginResetError] = useState<string>(null);
-
-  const pluginMetaOnCallApiUrl = jsonData?.onCallApiUrl;
-  const processEnvOnCallApiUrl = process.env.ONCALL_API_URL; // don't destructure this, will break how webpack supplies this
-  const onCallApiUrl = pluginMetaOnCallApiUrl || processEnvOnCallApiUrl;
   const licenseType = pluginIsConnected?.license || FALLBACK_LICENSE;
+  const onCallApiUrl = getOnCallApiUrl(meta);
 
   const resetQueryParams = useCallback(() => removePluginConfiguredQueryParams(pluginIsEnabled), [pluginIsEnabled]);
 
@@ -110,12 +108,12 @@ const PluginConfigPage: FC<OnCallPluginConfigPageProps> = ({
        * Supplying the env var basically allows to skip the configuration form
        * (check webpack.config.js to see how this is set)
        */
-      if (!pluginMetaOnCallApiUrl && processEnvOnCallApiUrl) {
+      if (!hasPluginBeenConfigured(meta) && onCallApiUrl) {
         /**
          * onCallApiUrl is not yet saved in the grafana plugin settings, but has been supplied as an env var
          * lets auto-trigger a self-hosted plugin install w/ the onCallApiUrl passed in as an env var
          */
-        const errorMsg = await PluginState.selfHostedInstallPlugin(processEnvOnCallApiUrl, true);
+        const errorMsg = await PluginState.selfHostedInstallPlugin(onCallApiUrl, true);
         if (errorMsg) {
           setPluginConnectionCheckError(errorMsg);
           setCheckingIfPluginIsConnected(false);
@@ -146,7 +144,7 @@ const PluginConfigPage: FC<OnCallPluginConfigPageProps> = ({
     if (!pluginConfiguredRedirect) {
       configurePluginAndUpdatePluginStatus();
     }
-  }, [pluginMetaOnCallApiUrl, processEnvOnCallApiUrl, onCallApiUrl, pluginConfiguredRedirect]);
+  }, [onCallApiUrl, pluginConfiguredRedirect]);
 
   const resetMessages = useCallback(() => {
     setPluginResetError(null);
@@ -210,9 +208,7 @@ const PluginConfigPage: FC<OnCallPluginConfigPageProps> = ({
       </>
     );
   } else if (!pluginIsConnected) {
-    content = (
+    content = <ConfigurationForm onSuccessfulSetup={triggerUpdatePluginStatus} defaultOnCallApiUrl={onCallApiUrl} />;
-      <ConfigurationForm onSuccessfulSetup={triggerUpdatePluginStatus} defaultOnCallApiUrl={processEnvOnCallApiUrl} />
-    );
   } else {
     // plugin is fully connected and synced
     const pluginLink = (

grafana-plugin/src/pages/integration/Integration.tsx

@@ -420,7 +420,7 @@ class Integration extends React.Component<IntegrationProps, IntegrationState> {
                         Autoresolve:
                       </Text>
                       <Text type="primary">
-                        {IntegrationHelper.truncateLine(templates['resolve_condition_template'] || 'disabled')}
+                        {IntegrationHelper.truncateLine(templates?.['resolve_condition_template'] || 'disabled')}
                       </Text>
                     </div>
 
@@ -1131,7 +1131,7 @@ const IntegrationHeader: React.FC<IntegrationHeaderProps> = ({
 
       {alertReceiveChannel.maintenance_till && (
         <TooltipBadge
-          data-testid="maintenance-mode-remaining-time-tooltip"
+          testId="maintenance-mode-remaining-time-tooltip"
           borderType="primary"
           icon="pause"
           text={IntegrationHelper.getMaintenanceText(alertReceiveChannel.maintenance_till)}
@@ -1193,7 +1193,6 @@ const IntegrationHeader: React.FC<IntegrationHeaderProps> = ({
 
     return (
       <TooltipBadge
-        data-testid="heartbeat-badge"
         text={undefined}
         className={cx('heartbeat-badge')}
         borderType={heartbeatStatus ? 'success' : 'danger'}

grafana-plugin/src/pages/integrations/Integrations.tsx

@@ -440,6 +440,7 @@ class Integrations extends React.Component<IntegrationsProps, IntegrationsState>
       <div>
         {alertReceiveChannel.is_available_for_integration_heartbeat && heartbeat?.last_heartbeat_time_verbal && (
           <TooltipBadge
+            testId="heartbeat-badge"
             text={undefined}
             className={cx('heartbeat-badge')}
             placement="top"

grafana-plugin/src/state/rootBaseStore/index.ts

@@ -37,6 +37,7 @@ import { retryFailingPromises } from 'utils/async';
 import {
   APP_VERSION,
   CLOUD_VERSION_REGEX,
+  getOnCallApiUrl,
   GRAFANA_LICENSE_CLOUD,
   GRAFANA_LICENSE_OSS,
   PLUGIN_ROOT,
@@ -167,7 +168,7 @@ export class RootBaseStore {
    */
   async setupPlugin(meta: OnCallAppPluginMeta) {
     this.initializationError = null;
-    this.onCallApiUrl = meta.jsonData?.onCallApiUrl;
+    this.onCallApiUrl = getOnCallApiUrl(meta);
 
     if (!FaroHelper.faro) {
       FaroHelper.initializeFaro(this.onCallApiUrl);
@@ -180,7 +181,7 @@ export class RootBaseStore {
 
     if (this.isOpenSource() && !meta.secureJsonFields?.onCallApiToken) {
       // Reinstall plugin if onCallApiToken is missing
-      const errorMsg = await PluginState.selfHostedInstallPlugin(process.env.ONCALL_API_URL, true);
+      const errorMsg = await PluginState.selfHostedInstallPlugin(this.onCallApiUrl, true);
       if (errorMsg) {
         return this.setupPluginError(errorMsg);
       }
@@ -310,8 +311,8 @@ export class RootBaseStore {
     await this.slackStore.installSlackIntegration();
   }
 
+  @action.bound
   async getApiUrlForSettings() {
-    const settings = await PluginState.getGrafanaPluginSettings();
+    return this.onCallApiUrl;
-    return settings.jsonData?.onCallApiUrl;
   }
 }

grafana-plugin/src/state/rootBaseStore/rootBaseStore.test.ts

@@ -17,6 +17,8 @@ jest.mock('grafana/app/core/core', () => ({
   },
 }));
 
+const onCallApiUrl = 'http://oncall-dev-engine:8080';
+
 const isUserActionAllowed = isUserActionAllowedOriginal as jest.Mock<ReturnType<typeof isUserActionAllowedOriginal>>;
 
 const generatePluginData = (
@@ -32,7 +34,6 @@ describe('rootBaseStore', () => {
   });
 
   test("onCallApiUrl is not set in the plugin's meta jsonData", async () => {
-    // mocks/setup
     const rootBaseStore = new RootBaseStore();
 
     // test
@@ -43,9 +44,7 @@ describe('rootBaseStore', () => {
   });
 
   test('when there is an issue checking the plugin connection, the error is properly handled', async () => {
-    // mocks/setup
     const errorMsg = 'ohhh noooo error';
-    const onCallApiUrl = 'http://asdfasdf.com';
     const rootBaseStore = new RootBaseStore();
 
     PluginState.updatePluginStatus = jest.fn().mockResolvedValueOnce(errorMsg);
@@ -61,8 +60,6 @@ describe('rootBaseStore', () => {
   });
 
   test('currently undergoing maintenance', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
     const rootBaseStore = new RootBaseStore();
     const maintenanceMessage = 'mncvnmvcmnvkjdjkd';
 
@@ -82,8 +79,6 @@ describe('rootBaseStore', () => {
   });
 
   test('anonymous user', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
     const rootBaseStore = new RootBaseStore();
 
     PluginState.updatePluginStatus = jest.fn().mockResolvedValueOnce({
@@ -108,8 +103,6 @@ describe('rootBaseStore', () => {
   });
 
   test('the plugin is not installed, and allow_signup is false', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
     const rootBaseStore = new RootBaseStore();
 
     PluginState.updatePluginStatus = jest.fn().mockResolvedValueOnce({
@@ -137,8 +130,6 @@ describe('rootBaseStore', () => {
   });
 
   test('plugin is not installed, user is not an Admin', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
     const rootBaseStore = new RootBaseStore();
 
     contextSrv.user.orgRole = OrgRole.Viewer;
@@ -174,8 +165,6 @@ describe('rootBaseStore', () => {
     { is_installed: false, token_ok: true },
     { is_installed: true, token_ok: false },
   ])('signup is allowed, user is an admin, plugin installation is triggered', async (scenario) => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
     const rootBaseStore = new RootBaseStore();
     const mockedLoadCurrentUser = jest.fn();
 
@@ -219,8 +208,6 @@ describe('rootBaseStore', () => {
       expected_result: false,
     },
   ])('signup is allowed, licensedAccessControlEnabled, various roles and permissions', async (scenario) => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
     const rootBaseStore = new RootBaseStore();
     const mockedLoadCurrentUser = jest.fn();
 
@@ -261,8 +248,6 @@ describe('rootBaseStore', () => {
   });
 
   test('plugin is not installed, signup is allowed, the user is an admin, and plugin installation throws an error', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
     const rootBaseStore = new RootBaseStore();
     const installPluginError = new Error('asdasdfasdfasf');
     const humanReadableErrorMsg = 'asdfasldkfjaksdjflk';
@@ -304,8 +289,6 @@ describe('rootBaseStore', () => {
   });
 
   test('when the plugin is installed, a data sync is triggered', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
     const rootBaseStore = new RootBaseStore();
     const mockedLoadCurrentUser = jest.fn();
 
@@ -333,8 +316,6 @@ describe('rootBaseStore', () => {
   });
 
   test('when the plugin is installed, and the data sync returns an error, it is properly handled', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
     const rootBaseStore = new RootBaseStore();
     const mockedLoadCurrentUser = jest.fn();
     const updatePluginStatusError = 'asdasdfasdfasf';

grafana-plugin/src/utils/consts.ts

@@ -1,3 +1,5 @@
+import { OnCallAppPluginMeta } from 'types';
+
 import plugin from '../../package.json'; // eslint-disable-line
 
 // Navbar
@@ -30,6 +32,13 @@ export const ONCALL_PROD = 'https://oncall-prod-us-central-0.grafana.net/oncall'
 export const ONCALL_OPS = 'https://oncall-ops-us-east-0.grafana.net/oncall';
 export const ONCALL_DEV = 'https://oncall-dev-us-central-0.grafana.net/oncall';
 
+// Single source of truth on the frontend for OnCall API URL
+export const getOnCallApiUrl = (meta?: OnCallAppPluginMeta) =>
+  meta?.jsonData?.onCallApiUrl || process.env.ONCALL_API_URL;
+
+// If the plugin has never been configured, onCallApiUrl will be undefined in the plugin's jsonData
+export const hasPluginBeenConfigured = (meta?: OnCallAppPluginMeta) => Boolean(meta?.jsonData?.onCallApiUrl);
+
 // Faro
 export const FARO_ENDPOINT_DEV =
   'https://faro-collector-prod-us-central-0.grafana.net/collect/fb03e474a96cf867f4a34590c002984c';