Brojd/improve e2e tests dx (#3516)

# What this PR does - introduce e2e tests in Tilt - support e2e tests commands in Makefile - stabilize local setup ## Which issue(s) this PR fixes https://github.com/grafana/oncall/issues/3492 ## Checklist - [x] Unit, integration, and e2e (if applicable) tests updated - [x] Documentation added (or `pr:no public docs` PR label added if not required) - [x] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not required)
2023-12-15 09:58:25 +01:00 · 2023-12-15 09:58:25 +01:00 · 92fa509d22
commit 92fa509d22
parent 96d3e18eb6
28 changed files with 193 additions and 125 deletions
--- a/.gitignore
+++ b/.gitignore
@ -10,3 +10,5 @@ venv

 yarn.lock
 node_modules
+
+test-results
--- a/.prettierrc.js
+++ b/.prettierrc.js
@ -0,0 +1,8 @@
+overrides: [
+  {
+    files: ["*.yml", "*.yaml"],
+    options: {
+      singleQuote: false,
+    },
+  },
+];
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

+## Unreleased
+
+### Added
+
+- Support e2e tests in Tilt and Makefile ([#3516](https://github.com/grafana/oncall/pull/3516))
+
 ## v1.3.80 (2023-12-14)

 ### Added
--- a/9
+++ b/9
@ -197,6 +197,15 @@ engine-manage:  ## run Django's `manage.py` script, inside of a docker container
                ## https://docs.djangoproject.com/en/4.1/ref/django-admin/#django-admin-makemigrations
 	$(call run_engine_docker_command,python manage.py $(CMD))

+test-e2e:  ## run the e2e tests in headless mode
+	yarn --cwd grafana-plugin test:e2e
+
+test-e2e-watch:  ## start e2e tests in watch mode
+	yarn --cwd grafana-plugin test:e2e:watch
+
+test-e2e-show-report:  ## open last e2e test report
+	yarn --cwd grafana-plugin playwright show-report
+
 ui-test:  ## run the UI tests
 	$(call run_ui_docker_command,yarn test)

--- a/51
+++ b/51
@ -1,3 +1,4 @@
+load('ext://uibutton', 'cmd_button', 'location', 'text_input', 'bool_input')
 running_under_parent_tiltfile = os.getenv("TILT_PARENT", "false") == "true"
 # The user/pass that you will login to Grafana with
 grafana_admin_user_pass = os.getenv("GRAFANA_ADMIN_USER_PASS", "oncall")
@ -36,7 +37,7 @@ docker_build_sub(
    "localhost:63628/oncall/engine:dev",
    context="./engine",
    cache_from=["grafana/oncall:latest", "grafana/oncall:dev"],
-    ignore=["./grafana-plugin/test-results/", "./grafana-plugin/dist/", "./grafana-plugin/e2e-tests/"],
+    ignore=["./test-results/", "./grafana-plugin/dist/", "./grafana-plugin/e2e-tests/"],
    child_context=".",
    target="dev",
    extra_cmds=["ADD ./grafana-plugin/src/plugin.json /etc/grafana-plugin/src/plugin.json"],
@ -54,10 +55,56 @@ local_resource(
    "build-ui",
    labels=["OnCallUI"],
    cmd="cd grafana-plugin && yarn install && yarn build:dev",
-    serve_cmd="cd grafana-plugin && ONCALL_API_URL=http://oncall-dev-engine:8080 yarn watch",
+    serve_cmd="cd grafana-plugin && yarn watch",
    allow_parallel=True,
 )

+local_resource(
+    "e2e-tests",
+    labels=["E2eTests"],
+    cmd="cd grafana-plugin && yarn test:e2e",
+    trigger_mode=TRIGGER_MODE_MANUAL,
+    auto_init=False,
+    resource_deps=["build-ui", "grafana", "grafana-oncall-app-provisioning-configmap", "engine"]
+)
+
+cmd_button(
+    name="E2E Tests - headless run",
+    argv=["sh", "-c", "yarn --cwd ./grafana-plugin test:e2e $STOP_ON_FIRST_FAILURE"],
+    text="Restart headless run",
+    resource="e2e-tests",
+    icon_name="replay",
+    inputs=[
+        text_input("BROWSERS", "Browsers (e.g. \"chromium,firefox,webkit\")", "chromium", "chromium,firefox,webkit"), 
+        bool_input("REPORTER", "Use HTML reporter", True, 'html', 'line'),
+        bool_input("STOP_ON_FIRST_FAILURE", "Stop on first failure", True, "-x", ""),
+    ]
+)
+
+cmd_button(
+    name="E2E Tests - open watch mode",
+    argv=["sh", "-c", "yarn --cwd grafana-plugin test:e2e:watch"],
+    text="Open watch mode",
+    resource="e2e-tests",
+    icon_name="visibility",
+)
+
+cmd_button(
+    name="E2E Tests - show report",
+    argv=["sh", "-c", "yarn --cwd grafana-plugin playwright show-report"],
+    text="Show last HTML report",
+    resource="e2e-tests",
+    icon_name="assignment",
+)
+
+cmd_button(
+    name="E2E Tests - stop current run",
+    argv=["sh", "-c", "kill -9 $(pgrep -f test:e2e)"],
+    text="Stop",
+    resource="e2e-tests",
+    icon_name="dangerous",
+)
+
 yaml = helm("helm/oncall", name=HELM_PREFIX, values=["./dev/helm-local.yml", "./dev/helm-local.dev.yml"])

 k8s_yaml(yaml)
--- a/dev/README.md
+++ b/dev/README.md
@ -243,13 +243,18 @@ are run on pull request CI builds. New features should ideally include a new/mod

 To run these tests locally simply do the following:

-```bash
-npx playwright install  # install playwright dependencies
-cp ./grafana-plugin/e2e-tests/.env.example ./grafana-plugin/e2e-tests/.env
-# you may need to tweak the values in ./grafana-plugin/.env according to your local setup
-cd grafana-plugin
-yarn test:e2e
-```
+1. Install Playwright dependencies with `npx playwright install`
+2. [Launch the environment](#launch-the-environment)
+3. Then you interact with tests in 2 different ways:
+   1. Using `Tilt` - open _E2eTests_ section where you will find 4 buttons:
+      1. Restart headless run (you can configure browsers, reporter and failure allowance there)
+      2. Open watch mode
+      3. Show last HTML report
+      4. Stop (stops any pending e2e test process)
+   2. Using `make`:
+      1. `make test:e2e` to start headless run
+      2. `make test:e2e:watch` to open watch mode
+      3. `make test:e2e:show:report` to open last HTML report

 ## Helm unit tests

--- a/dev/helm-local.yml
+++ b/dev/helm-local.yml
@ -1,4 +1,4 @@
-base_url: localhost:30001
+base_url: localhost:8080
 base_url_protocol: http
 env:
  - name: GRAFANA_CLOUD_NOTIFICATIONS_ENABLED
--- a/grafana-plugin/.dockerignore
+++ b/grafana-plugin/.dockerignore
@ -1,5 +1,4 @@
 node_modules
 frontend_enterprise
 .DS_Store
-test-results
 playwright-report
--- a/grafana-plugin/.gitignore
+++ b/grafana-plugin/.gitignore
@ -16,7 +16,6 @@ grafana-plugin.yml
 frontend_enterprise

 # playwright
-/test-results/
 /playwright-report/
 /playwright/.cache/
 /e2e-tests/storageState.json
--- a/grafana-plugin/e2e-tests/.env.example
+++ b/grafana-plugin/e2e-tests/.env.example
@ -1,5 +1,3 @@
-BASE_URL=http://localhost:30002/grafana
-ONCALL_API_URL=http://oncall-dev-engine-external:8080/
 GRAFANA_VIEWER_USERNAME=viewer
 GRAFANA_VIEWER_PASSWORD=viewer
 GRAFANA_EDITOR_USERNAME=editor
--- a/grafana-plugin/e2e-tests/alerts/onCallSchedule.test.ts
+++ b/grafana-plugin/e2e-tests/alerts/onCallSchedule.test.ts
@ -6,9 +6,6 @@ import { createIntegrationAndSendDemoAlert } from '../utils/integrations';
 import { createOnCallSchedule } from '../utils/schedule';

 test('we can create an oncall schedule + receive an alert', async ({ adminRolePage }) => {
-  // this test does a lot of stuff, lets give it adequate time to do its thing
-  test.slow();
-
  const { page, userName } = adminRolePage;
  const escalationChainName = generateRandomValue();
  const integrationName = generateRandomValue();
--- a/grafana-plugin/e2e-tests/escalationChains/escalationPolicy.test.ts
+++ b/grafana-plugin/e2e-tests/escalationChains/escalationPolicy.test.ts
@ -1,6 +1,6 @@
-import {expect, test} from "../fixtures";
-import {createEscalationChain, EscalationStep, selectEscalationStepValue} from "../utils/escalationChain";
-import {generateRandomValue} from "../utils/forms";
+import { expect, test } from '../fixtures';
+import { createEscalationChain, EscalationStep, selectEscalationStepValue } from '../utils/escalationChain';
+import { generateRandomValue } from '../utils/forms';

 test('escalation policy does not go back to "Default" after adding users to notify', async ({ adminRolePage }) => {
  const { page, userName } = adminRolePage;
@ -13,7 +13,5 @@ test('escalation policy does not go back to "Default" after adding users to noti

  // reload and check if important is still selected
  await page.reload();
-  await page.waitForLoadState('networkidle');
-
-  expect(await page.locator('text=Important').isVisible()).toBe(true);
+  await expect(page.getByText('Important')).toBeVisible();
 });
--- a/grafana-plugin/e2e-tests/globalSetup.ts
+++ b/grafana-plugin/e2e-tests/globalSetup.ts
@ -1,6 +1,8 @@
 import { OrgRole } from '@grafana/data';
 import { test as setup, chromium, expect, Page, BrowserContext, FullConfig, APIRequestContext } from '@playwright/test';

+import { getOnCallApiUrl } from 'utils/consts';
+
 import { VIEWER_USER_STORAGE_STATE, EDITOR_USER_STORAGE_STATE, ADMIN_USER_STORAGE_STATE } from '../playwright.config';

 import GrafanaAPIClient from './utils/clients/grafana';
@ -13,7 +15,6 @@ import {
  GRAFANA_VIEWER_USERNAME,
  IS_CLOUD,
  IS_OPEN_SOURCE,
-  ONCALL_API_URL,
 } from './utils/constants';
 import { clickButton, getInputByName } from './utils/forms';
 import { goToGrafanaPage } from './utils/navigation';
@ -59,17 +60,26 @@ const configureOnCallPlugin = async (page: Page): Promise<void> => {
   * go to the oncall plugin configuration page and wait for the page to be loaded
   */
  await goToGrafanaPage(page, '/plugins/grafana-oncall-app');
-  await page.waitForSelector('text=Configure Grafana OnCall');
+  await page.waitForTimeout(2000);

-  /**
-   * we may need to fill in the OnCall API URL if it is not set in the process.env
-   * of the frontend build
-   */
-  const onCallApiUrlInput = getInputByName(page, 'onCallApiUrl');
-  const pluginIsAutoConfigured = (await onCallApiUrlInput.count()) === 0;
+  // if plugin is configured, go to OnCall
+  const isConfigured = (await page.getByText('Connected to OnCall').count()) >= 1;
+  if (isConfigured) {
+    await page.getByRole('link', { name: 'Open Grafana OnCall' }).click();
+    return;
+  }

-  if (!pluginIsAutoConfigured) {
-    await onCallApiUrlInput.fill(ONCALL_API_URL);
+  // otherwise we may need to reconfigure the plugin
+  const needToReconfigure = (await page.getByText('try removing your plugin configuration').count()) >= 1;
+  if (needToReconfigure) {
+    await clickButton({ page, buttonText: 'Remove current configuration' });
+    await clickButton({ page, buttonText: /^Remove$/ });
+  }
+  await page.waitForTimeout(2000);
+
+  const needToEnterOnCallApiUrl = await page.getByText(/Connected to OnCall/).isHidden();
+  if (needToEnterOnCallApiUrl) {
+    await getInputByName(page, 'onCallApiUrl').fill(getOnCallApiUrl() || 'http://oncall-dev-engine:8080');
    await clickButton({ page, buttonText: 'Connect' });
  }

@ -88,13 +98,6 @@ const configureOnCallPlugin = async (page: Page): Promise<void> => {
 * https://github.com/grafana/incident/blob/main/plugin/e2e/global-setup.ts
 */
 setup('Configure Grafana OnCall plugin', async ({ request }, { config }) => {
-  /**
-   * Unconditionally marks the setup as "slow", giving it triple the default timeout.
-   * This is mostly useful for the rare case for Cloud Grafana instances where the instance may be down/unavailable
-   * and we need to poll it until it is available
-   */
-  setup.slow();
-
  if (IS_CLOUD) {
    await grafanaApiClient.pollInstanceUntilItIsHealthy(request);
  }
--- a/grafana-plugin/e2e-tests/integrations/heartbeat.test.ts
+++ b/grafana-plugin/e2e-tests/integrations/heartbeat.test.ts
@ -1,6 +1,7 @@
 import { test, Page, expect } from '../fixtures';
 import { generateRandomValue, selectDropdownValue } from '../utils/forms';
-import { createIntegration } from '../utils/integrations';
+import { createIntegration, searchIntegrationAndAssertItsPresence } from '../utils/integrations';
+import { goToOnCallPage } from '../utils/navigation';

 const HEARTBEAT_SETTINGS_FORM_TEST_ID = 'heartbeat-settings-form';

@ -12,7 +13,8 @@ test.describe("updating an integration's heartbeat interval works", async () =>
  };

  test('change heartbeat interval', async ({ adminRolePage: { page } }) => {
-    await createIntegration({ page, integrationName: generateRandomValue() });
+    const integrationName = generateRandomValue();
+    await createIntegration({ page, integrationName });

    await _openHeartbeatSettingsForm(page);

@ -42,7 +44,8 @@ test.describe("updating an integration's heartbeat interval works", async () =>
  });

  test('send heartbeat', async ({ adminRolePage: { page } }) => {
-    await createIntegration({ page, integrationName: generateRandomValue() });
+    const integrationName = generateRandomValue();
+    await createIntegration({ page, integrationName });

    await _openHeartbeatSettingsForm(page);

@ -59,6 +62,9 @@ test.describe("updating an integration's heartbeat interval works", async () =>
     */
    await page.request.get(endpoint);
    await page.reload({ waitUntil: 'networkidle' });
+
+    await goToOnCallPage(page, 'integrations');
+    await searchIntegrationAndAssertItsPresence({ page, integrationName });
    await page.getByTestId('heartbeat-badge').waitFor();
  });
 });
--- a/grafana-plugin/e2e-tests/integrations/integrationsTable.test.ts
+++ b/grafana-plugin/e2e-tests/integrations/integrationsTable.test.ts
@ -17,7 +17,6 @@ test('Integrations table shows data in Monitoring Systems and Direct Paging tabs
  await createIntegration({
    page,
    integrationSearchText: 'Alertmanager',
-    shouldGoToIntegrationsPage: false,
    integrationName: ALERTMANAGER_INTEGRATION_NAME,
  });
  await page.waitForTimeout(1000);
@ -32,7 +31,6 @@ test('Integrations table shows data in Monitoring Systems and Direct Paging tabs
    await createIntegration({
      page,
      integrationSearchText: 'Direct paging',
-      shouldGoToIntegrationsPage: false,
      integrationName: DIRECT_PAGING_INTEGRATION_NAME,
    });
    await page.waitForTimeout(1000);
@ -40,15 +38,13 @@ test('Integrations table shows data in Monitoring Systems and Direct Paging tabs
  await page.getByRole('tab', { name: 'Tab Integrations' }).click();

  // By default Monitoring Systems tab is opened and newly created integrations are visible except Direct Paging one
-  await searchIntegrationAndAssertItsPresence({ page, integrationsTable, integrationName: WEBHOOK_INTEGRATION_NAME });
+  await searchIntegrationAndAssertItsPresence({ page, integrationName: WEBHOOK_INTEGRATION_NAME });
  await searchIntegrationAndAssertItsPresence({
    page,
-    integrationsTable,
    integrationName: ALERTMANAGER_INTEGRATION_NAME,
  });
  await searchIntegrationAndAssertItsPresence({
    page,
-    integrationsTable,
    integrationName: DIRECT_PAGING_INTEGRATION_NAME,
    visibleExpected: false,
  });
@ -57,19 +53,16 @@ test('Integrations table shows data in Monitoring Systems and Direct Paging tabs
  await page.getByRole('tab', { name: 'Tab Manual Direct Paging' }).click();
  await searchIntegrationAndAssertItsPresence({
    page,
-    integrationsTable,
    integrationName: WEBHOOK_INTEGRATION_NAME,
    visibleExpected: false,
  });
  await searchIntegrationAndAssertItsPresence({
    page,
-    integrationsTable,
    integrationName: ALERTMANAGER_INTEGRATION_NAME,
    visibleExpected: false,
  });
  await searchIntegrationAndAssertItsPresence({
    page,
-    integrationsTable,
    integrationName: 'Direct paging',
  });
 });
--- a/grafana-plugin/e2e-tests/integrations/maintenanceMode.test.ts
+++ b/grafana-plugin/e2e-tests/integrations/maintenanceMode.test.ts
@ -103,6 +103,7 @@ test.describe('maintenance mode works', () => {

    await createEscalationChain(page, escalationChainName, EscalationStep.NotifyUsers, userName);
    await createIntegration({ page, integrationName });
+    await page.waitForTimeout(1000);
    await assignEscalationChainToIntegration(page, escalationChainName);
    await enableMaintenanceMode(page, maintenanceModeType);

@ -110,8 +111,6 @@ test.describe('maintenance mode works', () => {
  };

  test('debug mode', async ({ adminRolePage: { page, userName } }) => {
-    test.slow();
-
    const { escalationChainName, integrationName } = await createIntegrationAndEscalationChainAndEnableMaintenanceMode(
      page,
      userName,
@ -128,7 +127,6 @@ test.describe('maintenance mode works', () => {
  });

  test('"maintenance" mode', async ({ adminRolePage: { page, userName } }) => {
-    test.slow();
    const { integrationName } = await createIntegrationAndEscalationChainAndEnableMaintenanceMode(
      page,
      userName,
--- a/grafana-plugin/e2e-tests/utils/constants.ts
+++ b/grafana-plugin/e2e-tests/utils/constants.ts
@ -1,5 +1,4 @@
 export const BASE_URL = process.env.BASE_URL || 'http://localhost:3000';
-export const ONCALL_API_URL = process.env.ONCALL_API_URL || 'http://host.docker.internal:8080';
 export const MAILSLURP_API_KEY = process.env.MAILSLURP_API_KEY;

 export const GRAFANA_VIEWER_USERNAME = process.env.GRAFANA_VIEWER_USERNAME || 'viewer';
--- a/grafana-plugin/e2e-tests/utils/forms.ts
+++ b/grafana-plugin/e2e-tests/utils/forms.ts
@ -22,7 +22,7 @@ type SelectDropdownValueArgs = {

 type ClickButtonArgs = {
  page: Page;
-  buttonText: string;
+  buttonText: string | RegExp;
  // if provided, use this Locator as the root of our search for the button
  startingLocator?: Locator;
 };
--- a/grafana-plugin/e2e-tests/utils/integrations.ts
+++ b/grafana-plugin/e2e-tests/utils/integrations.ts
@ -1,4 +1,4 @@
-import { Locator, Page, expect } from '@playwright/test';
+import { Page, expect } from '@playwright/test';

 import { clickButton, generateRandomValue, selectDropdownValue } from './forms';
 import { goToOnCallPage } from './navigation';
@ -38,17 +38,24 @@ export const createIntegration = async ({
    .click();

  // fill in the required inputs
-  (await page.waitForSelector('input[name="verbal_name"]', { state: 'attached' })).fill(integrationName);
-  (await page.waitForSelector('textarea[name="description_short"]', { state: 'attached' })).fill(
-    'Here goes your integration description'
-  );
+  await page.getByPlaceholder('Integration Name').fill(integrationName);
+  await page.getByPlaceholder('Integration Description').fill('Here goes your integration description');
+  await page.getByTestId('update-integration-button').focus();
+  await page.getByTestId('update-integration-button').click();

-  const grafanaUpdateBtn = page.getByTestId('update-integration-button');
-  await grafanaUpdateBtn.click();
+  await goToOnCallPage(page, 'integrations');
+  await searchIntegrationAndAssertItsPresence({ page, integrationName });
+
+  await page.getByRole('link', { name: integrationName }).click();
 };

 export const assignEscalationChainToIntegration = async (page: Page, escalationChainName: string): Promise<void> => {
-  await page.getByTestId('integration-escalation-chain-not-selected').click();
+  const notSelected = page.getByTestId('integration-escalation-chain-not-selected');
+  if (await notSelected.isHidden()) {
+    await clickButton({ page, buttonText: 'Add route' });
+    await page.waitForTimeout(500);
+  }
+  await notSelected.last().click();

  // assign the escalation chain to the integration
  await selectDropdownValue({
@ -56,7 +63,7 @@ export const assignEscalationChainToIntegration = async (page: Page, escalationC
    selectType: 'grafanaSelect',
    placeholderText: 'Select Escalation Chain',
    value: escalationChainName,
-    startingLocator: page.getByTestId('escalation-chain-select'),
+    startingLocator: page.getByTestId('escalation-chain-select').last(),
  });
 };

@ -92,11 +99,9 @@ export const filterIntegrationsTableAndGoToDetailPage = async (page: Page, integ
 export const searchIntegrationAndAssertItsPresence = async ({
  page,
  integrationName,
-  integrationsTable,
  visibleExpected = true,
 }: {
  page: Page;
-  integrationsTable: Locator;
  integrationName: string;
  visibleExpected?: boolean;
 }) => {
@ -105,6 +110,7 @@ export const searchIntegrationAndAssertItsPresence = async ({
    .filter({ hasText: /^Search or filter results\.\.\.$/ })
    .nth(1)
    .click();
+  const integrationsTable = page.getByTestId('integrations-table');
  await page.keyboard.insertText(integrationName);
  await page.keyboard.press('Enter');
  await page.waitForTimeout(2000);
--- a/grafana-plugin/e2e-tests/utils/navigation.ts
+++ b/grafana-plugin/e2e-tests/utils/navigation.ts
@ -1,13 +1,15 @@
-import type { Page, Response } from '@playwright/test';
+import type { Page } from '@playwright/test';

 import { BASE_URL } from './constants';

 type GrafanaPage = '/plugins/grafana-oncall-app';
 type OnCallPage = 'alert-groups' | 'integrations' | 'escalations' | 'schedules' | 'users';

-const _goToPage = (page: Page, url = ''): Promise<Response> => page.goto(`${BASE_URL}${url}`);
+const _goToPage = async (page: Page, url = '') => page.goto(`${BASE_URL}${url}`);

-export const goToGrafanaPage = (page: Page, url: GrafanaPage): Promise<Response> => _goToPage(page, url);
+export const goToGrafanaPage = async (page: Page, url: GrafanaPage) => _goToPage(page, url);

-export const goToOnCallPage = (page: Page, onCallPage: OnCallPage): Promise<Response> =>
-  _goToPage(page, `/a/grafana-oncall-app/${onCallPage}`);
+export const goToOnCallPage = async (page: Page, onCallPage: OnCallPage) => {
+  await _goToPage(page, `/a/grafana-oncall-app/${onCallPage}`);
+  await page.waitForTimeout(1000);
+};
--- a/grafana-plugin/playwright.config.ts
+++ b/grafana-plugin/playwright.config.ts
@ -1,4 +1,4 @@
-import { PlaywrightTestProject, defineConfig, devices } from '@playwright/test';
+import { PlaywrightTestProject, defineConfig, devices, PlaywrightTestConfig } from '@playwright/test';

 import path from 'path';
 /**
@ -12,7 +12,11 @@ export const EDITOR_USER_STORAGE_STATE = path.join(__dirname, 'e2e-tests/.auth/e
 export const ADMIN_USER_STORAGE_STATE = path.join(__dirname, 'e2e-tests/.auth/admin.json');

 const IS_CI = !!process.env.CI;
-const BROWSERS = process.env.BROWSERS || 'chromium firefox webkit';
+const BROWSERS = process.env.BROWSERS || 'chromium';
+const REPORTER_WITH_DEFAULT = process.env.REPORTER || 'html';
+const REPORTER = (
+  process.env.REPORTER === 'html' ? [['html', { open: 'never' }]] : REPORTER_WITH_DEFAULT
+) as PlaywrightTestConfig['reporter'];

 const SETUP_PROJECT_NAME = 'setup';
 const getEnabledBrowsers = (browsers: PlaywrightTestProject[]) =>
@ -25,16 +29,18 @@ export default defineConfig({
  testDir: './e2e-tests',

  /* Maximum time all the tests can run for. */
-  globalTimeout: 20 * 60 * 1000, // 20 minutes
+  globalTimeout: 20 * 60 * 1_000, // 20 minutes
+
+  reporter: REPORTER,

  /* Maximum time one test can run for. */
-  timeout: 60 * 1000,
+  timeout: 60_000,
  expect: {
    /**
     * Maximum time expect() should wait for the condition to be met.
     * For example in `await expect(locator).toHaveText();`
     */
-    timeout: 10000,
+    timeout: 6_000,
  },
  /* Run tests in files in parallel */
  fullyParallel: false,
@ -46,10 +52,10 @@ export default defineConfig({
   * NOTE: until we fix this issue (https://github.com/grafana/oncall/issues/1692) which occasionally leads
   * to flaky tests.. let's allow 1 retry per test
   */
-  retries: IS_CI ? 1 : 0,
+  retries: 1,
  workers: 2,
  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'html',
+  // reporter: 'html',
  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
  use: {
    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
@ -59,7 +65,7 @@ export default defineConfig({

    trace: 'on',
    video: 'on',
-    headless: IS_CI,
+    headless: true,
  },

  /* Configure projects for major browsers. The final list is filtered based on BROWSERS env var */
@ -109,8 +115,9 @@ export default defineConfig({
    // },
  ]),

-  /* Folder for test artifacts such as screenshots, videos, traces, etc. */
-  // outputDir: 'test-results/',
+  /* Folder for test artifacts such as screenshots, videos, traces, etc. 
+  Set outside of grafana-plugin to prevent refreshing Grafana UI during e2e test runs */
+  outputDir: '../test-results/',

  /* Run your local dev server before starting the tests */
  // webServer: {
--- a/grafana-plugin/src/components/TooltipBadge/TooltipBadge.tsx
+++ b/grafana-plugin/src/components/TooltipBadge/TooltipBadge.tsx
@ -18,6 +18,7 @@ interface TooltipBadgeProps {
  customIcon?: React.ReactNode;
  addPadding?: boolean;
  placement?;
+  testId?: string;

  onHover?: () => void;
 }
@ -36,11 +37,9 @@ const TooltipBadge: FC<TooltipBadgeProps> = (props) => {
    icon,
    customIcon,
    className,
-    ...rest
+    testId,
  } = props;

-  const testId = rest['data-testid'];
-
  return (
    <Tooltip
      placement={placement || 'bottom-start'}
--- a/grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx
+++ b/grafana-plugin/src/containers/PluginConfigPage/PluginConfigPage.tsx
@ -5,7 +5,7 @@ import { useLocation } from 'react-router-dom';
 import { OnCallPluginConfigPageProps } from 'types';

 import PluginState, { PluginStatusResponseBase } from 'state/plugin';
-import { FALLBACK_LICENSE, GRAFANA_LICENSE_OSS } from 'utils/consts';
+import { FALLBACK_LICENSE, getOnCallApiUrl, GRAFANA_LICENSE_OSS, hasPluginBeenConfigured } from 'utils/consts';

 import ConfigurationForm from './parts/ConfigurationForm';
 import RemoveCurrentConfigurationButton from './parts/RemoveCurrentConfigurationButton';
@ -46,7 +46,8 @@ export const removePluginConfiguredQueryParams = (pluginIsEnabled: boolean): voi

 const PluginConfigPage: FC<OnCallPluginConfigPageProps> = ({
  plugin: {
-    meta: { jsonData, enabled: pluginIsEnabled },
+    meta,
+    meta: { enabled: pluginIsEnabled },
  },
 }) => {
  const { search } = useLocation();
@ -75,11 +76,8 @@ const PluginConfigPage: FC<OnCallPluginConfigPageProps> = ({

  const [resettingPlugin, setResettingPlugin] = useState<boolean>(false);
  const [pluginResetError, setPluginResetError] = useState<string>(null);
-
-  const pluginMetaOnCallApiUrl = jsonData?.onCallApiUrl;
-  const processEnvOnCallApiUrl = process.env.ONCALL_API_URL; // don't destructure this, will break how webpack supplies this
-  const onCallApiUrl = pluginMetaOnCallApiUrl || processEnvOnCallApiUrl;
  const licenseType = pluginIsConnected?.license || FALLBACK_LICENSE;
+  const onCallApiUrl = getOnCallApiUrl(meta);

  const resetQueryParams = useCallback(() => removePluginConfiguredQueryParams(pluginIsEnabled), [pluginIsEnabled]);

@ -110,12 +108,12 @@ const PluginConfigPage: FC<OnCallPluginConfigPageProps> = ({
       * Supplying the env var basically allows to skip the configuration form
       * (check webpack.config.js to see how this is set)
       */
-      if (!pluginMetaOnCallApiUrl && processEnvOnCallApiUrl) {
+      if (!hasPluginBeenConfigured(meta) && onCallApiUrl) {
        /**
         * onCallApiUrl is not yet saved in the grafana plugin settings, but has been supplied as an env var
         * lets auto-trigger a self-hosted plugin install w/ the onCallApiUrl passed in as an env var
         */
-        const errorMsg = await PluginState.selfHostedInstallPlugin(processEnvOnCallApiUrl, true);
+        const errorMsg = await PluginState.selfHostedInstallPlugin(onCallApiUrl, true);
        if (errorMsg) {
          setPluginConnectionCheckError(errorMsg);
          setCheckingIfPluginIsConnected(false);
@ -146,7 +144,7 @@ const PluginConfigPage: FC<OnCallPluginConfigPageProps> = ({
    if (!pluginConfiguredRedirect) {
      configurePluginAndUpdatePluginStatus();
    }
-  }, [pluginMetaOnCallApiUrl, processEnvOnCallApiUrl, onCallApiUrl, pluginConfiguredRedirect]);
+  }, [onCallApiUrl, pluginConfiguredRedirect]);

  const resetMessages = useCallback(() => {
    setPluginResetError(null);
@ -210,9 +208,7 @@ const PluginConfigPage: FC<OnCallPluginConfigPageProps> = ({
      </>
    );
  } else if (!pluginIsConnected) {
-    content = (
-      <ConfigurationForm onSuccessfulSetup={triggerUpdatePluginStatus} defaultOnCallApiUrl={processEnvOnCallApiUrl} />
-    );
+    content = <ConfigurationForm onSuccessfulSetup={triggerUpdatePluginStatus} defaultOnCallApiUrl={onCallApiUrl} />;
  } else {
    // plugin is fully connected and synced
    const pluginLink = (
--- a/grafana-plugin/src/pages/integration/Integration.tsx
+++ b/grafana-plugin/src/pages/integration/Integration.tsx
@ -420,7 +420,7 @@ class Integration extends React.Component<IntegrationProps, IntegrationState> {
                        Autoresolve:
                      </Text>
                      <Text type="primary">
-                        {IntegrationHelper.truncateLine(templates['resolve_condition_template'] || 'disabled')}
+                        {IntegrationHelper.truncateLine(templates?.['resolve_condition_template'] || 'disabled')}
                      </Text>
                    </div>

@ -1131,7 +1131,7 @@ const IntegrationHeader: React.FC<IntegrationHeaderProps> = ({

      {alertReceiveChannel.maintenance_till && (
        <TooltipBadge
-          data-testid="maintenance-mode-remaining-time-tooltip"
+          testId="maintenance-mode-remaining-time-tooltip"
          borderType="primary"
          icon="pause"
          text={IntegrationHelper.getMaintenanceText(alertReceiveChannel.maintenance_till)}
@ -1193,7 +1193,6 @@ const IntegrationHeader: React.FC<IntegrationHeaderProps> = ({

    return (
      <TooltipBadge
-        data-testid="heartbeat-badge"
        text={undefined}
        className={cx('heartbeat-badge')}
        borderType={heartbeatStatus ? 'success' : 'danger'}
--- a/grafana-plugin/src/pages/integrations/Integrations.tsx
+++ b/grafana-plugin/src/pages/integrations/Integrations.tsx
@ -440,6 +440,7 @@ class Integrations extends React.Component<IntegrationsProps, IntegrationsState>
      <div>
        {alertReceiveChannel.is_available_for_integration_heartbeat && heartbeat?.last_heartbeat_time_verbal && (
          <TooltipBadge
+            testId="heartbeat-badge"
            text={undefined}
            className={cx('heartbeat-badge')}
            placement="top"
--- a/grafana-plugin/src/state/rootBaseStore/index.ts
+++ b/grafana-plugin/src/state/rootBaseStore/index.ts
@ -37,6 +37,7 @@ import { retryFailingPromises } from 'utils/async';
 import {
  APP_VERSION,
  CLOUD_VERSION_REGEX,
+  getOnCallApiUrl,
  GRAFANA_LICENSE_CLOUD,
  GRAFANA_LICENSE_OSS,
  PLUGIN_ROOT,
@ -167,7 +168,7 @@ export class RootBaseStore {
   */
  async setupPlugin(meta: OnCallAppPluginMeta) {
    this.initializationError = null;
-    this.onCallApiUrl = meta.jsonData?.onCallApiUrl;
+    this.onCallApiUrl = getOnCallApiUrl(meta);

    if (!FaroHelper.faro) {
      FaroHelper.initializeFaro(this.onCallApiUrl);
@ -180,7 +181,7 @@ export class RootBaseStore {

    if (this.isOpenSource() && !meta.secureJsonFields?.onCallApiToken) {
      // Reinstall plugin if onCallApiToken is missing
-      const errorMsg = await PluginState.selfHostedInstallPlugin(process.env.ONCALL_API_URL, true);
+      const errorMsg = await PluginState.selfHostedInstallPlugin(this.onCallApiUrl, true);
      if (errorMsg) {
        return this.setupPluginError(errorMsg);
      }
@ -310,8 +311,8 @@ export class RootBaseStore {
    await this.slackStore.installSlackIntegration();
  }

+  @action.bound
  async getApiUrlForSettings() {
-    const settings = await PluginState.getGrafanaPluginSettings();
-    return settings.jsonData?.onCallApiUrl;
+    return this.onCallApiUrl;
  }
 }
--- a/grafana-plugin/src/state/rootBaseStore/rootBaseStore.test.ts
+++ b/grafana-plugin/src/state/rootBaseStore/rootBaseStore.test.ts
@ -17,6 +17,8 @@ jest.mock('grafana/app/core/core', () => ({
  },
 }));

+const onCallApiUrl = 'http://oncall-dev-engine:8080';
+
 const isUserActionAllowed = isUserActionAllowedOriginal as jest.Mock<ReturnType<typeof isUserActionAllowedOriginal>>;

 const generatePluginData = (
@ -32,7 +34,6 @@ describe('rootBaseStore', () => {
  });

  test("onCallApiUrl is not set in the plugin's meta jsonData", async () => {
-    // mocks/setup
    const rootBaseStore = new RootBaseStore();

    // test
@ -43,9 +44,7 @@ describe('rootBaseStore', () => {
  });

  test('when there is an issue checking the plugin connection, the error is properly handled', async () => {
-    // mocks/setup
    const errorMsg = 'ohhh noooo error';
-    const onCallApiUrl = 'http://asdfasdf.com';
    const rootBaseStore = new RootBaseStore();

    PluginState.updatePluginStatus = jest.fn().mockResolvedValueOnce(errorMsg);
@ -61,8 +60,6 @@ describe('rootBaseStore', () => {
  });

  test('currently undergoing maintenance', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
    const rootBaseStore = new RootBaseStore();
    const maintenanceMessage = 'mncvnmvcmnvkjdjkd';

@ -82,8 +79,6 @@ describe('rootBaseStore', () => {
  });

  test('anonymous user', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
    const rootBaseStore = new RootBaseStore();

    PluginState.updatePluginStatus = jest.fn().mockResolvedValueOnce({
@ -108,8 +103,6 @@ describe('rootBaseStore', () => {
  });

  test('the plugin is not installed, and allow_signup is false', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
    const rootBaseStore = new RootBaseStore();

    PluginState.updatePluginStatus = jest.fn().mockResolvedValueOnce({
@ -137,8 +130,6 @@ describe('rootBaseStore', () => {
  });

  test('plugin is not installed, user is not an Admin', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
    const rootBaseStore = new RootBaseStore();

    contextSrv.user.orgRole = OrgRole.Viewer;
@ -174,8 +165,6 @@ describe('rootBaseStore', () => {
    { is_installed: false, token_ok: true },
    { is_installed: true, token_ok: false },
  ])('signup is allowed, user is an admin, plugin installation is triggered', async (scenario) => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
    const rootBaseStore = new RootBaseStore();
    const mockedLoadCurrentUser = jest.fn();

@ -219,8 +208,6 @@ describe('rootBaseStore', () => {
      expected_result: false,
    },
  ])('signup is allowed, licensedAccessControlEnabled, various roles and permissions', async (scenario) => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
    const rootBaseStore = new RootBaseStore();
    const mockedLoadCurrentUser = jest.fn();

@ -261,8 +248,6 @@ describe('rootBaseStore', () => {
  });

  test('plugin is not installed, signup is allowed, the user is an admin, and plugin installation throws an error', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
    const rootBaseStore = new RootBaseStore();
    const installPluginError = new Error('asdasdfasdfasf');
    const humanReadableErrorMsg = 'asdfasldkfjaksdjflk';
@ -304,8 +289,6 @@ describe('rootBaseStore', () => {
  });

  test('when the plugin is installed, a data sync is triggered', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
    const rootBaseStore = new RootBaseStore();
    const mockedLoadCurrentUser = jest.fn();

@ -333,8 +316,6 @@ describe('rootBaseStore', () => {
  });

  test('when the plugin is installed, and the data sync returns an error, it is properly handled', async () => {
-    // mocks/setup
-    const onCallApiUrl = 'http://asdfasdf.com';
    const rootBaseStore = new RootBaseStore();
    const mockedLoadCurrentUser = jest.fn();
    const updatePluginStatusError = 'asdasdfasdfasf';
--- a/grafana-plugin/src/utils/consts.ts
+++ b/grafana-plugin/src/utils/consts.ts
@ -1,3 +1,5 @@
+import { OnCallAppPluginMeta } from 'types';
+
 import plugin from '../../package.json'; // eslint-disable-line

 // Navbar
@ -30,6 +32,13 @@ export const ONCALL_PROD = 'https://oncall-prod-us-central-0.grafana.net/oncall'
 export const ONCALL_OPS = 'https://oncall-ops-us-east-0.grafana.net/oncall';
 export const ONCALL_DEV = 'https://oncall-dev-us-central-0.grafana.net/oncall';

+// Single source of truth on the frontend for OnCall API URL
+export const getOnCallApiUrl = (meta?: OnCallAppPluginMeta) =>
+  meta?.jsonData?.onCallApiUrl || process.env.ONCALL_API_URL;
+
+// If the plugin has never been configured, onCallApiUrl will be undefined in the plugin's jsonData
+export const hasPluginBeenConfigured = (meta?: OnCallAppPluginMeta) => Boolean(meta?.jsonData?.onCallApiUrl);
+
 // Faro
 export const FARO_ENDPOINT_DEV =
  'https://faro-collector-prod-us-central-0.grafana.net/collect/fb03e474a96cf867f4a34590c002984c';