improve e2e tests authentication flow (#1470)

# What this PR does

This PR makes the Grafana login portion of the e2e tests much
faster/more reliable.

Currently we use CSS selectors to go to the login form, input the
username/password, and proceed as such. This PR refactors to instead
make a call to `POST ${GRAFANA_API_URL}/login` and then stores that
authentication state that is then reused by subsequent browsers. This
was inspired by [how the Incident team does their playwright
authentication](https://github.com/grafana/incident/blob/main/plugin/e2e/global-setup.ts)
+ the recommendation from the [Playwright
docs](https://playwright.dev/docs/auth#basic-shared-account-in-all-tests)

## Which issue(s) this PR fixes

Slow/flaky Grafana login flow

## Checklist

- [x] Tests updated
- [ ] Documentation added (N/A)
- [ ] `CHANGELOG.md` updated (N/A)

grafana-plugin/.gitignore

@@ -19,3 +19,4 @@ frontend_enterprise
 /test-results/
 /playwright-report/
 /playwright/.cache/
+storageState.json

grafana-plugin/integration-tests/alerts/onCallSchedule.test.ts

@@ -1,5 +1,5 @@
 import { test } from '@playwright/test';
-import { openOnCallPlugin } from '../utils';
+import { configureOnCallPlugin } from '../utils/configurePlugin';
 import { verifyThatAlertGroupIsTriggered } from '../utils/alertGroup';
 import { createEscalationChain, EscalationStep } from '../utils/escalationChain';
 import { generateRandomValue } from '../utils/forms';
@@ -7,7 +7,7 @@ import { createIntegrationAndSendDemoAlert } from '../utils/integrations';
 import { createOnCallSchedule } from '../utils/schedule';
 
 test.beforeEach(async ({ page }) => {
-  await openOnCallPlugin(page);
+  await configureOnCallPlugin(page);
 });
 
 test('we can create an oncall schedule + receive an alert', async ({ page }) => {

grafana-plugin/integration-tests/alerts/sms.test.ts

@@ -1,5 +1,5 @@
 import { test, expect } from '@playwright/test';
-import { openOnCallPlugin } from '../utils';
+import { configureOnCallPlugin } from '../utils/configurePlugin';
 import { GRAFANA_USERNAME } from '../utils/constants';
 import { createEscalationChain, EscalationStep } from '../utils/escalationChain';
 import { generateRandomValue } from '../utils/forms';
@@ -8,7 +8,7 @@ import { waitForSms } from '../utils/phone';
 import { configureUserNotificationSettings, verifyUserPhoneNumber } from '../utils/userSettings';
 
 test.beforeEach(async ({ page }) => {
-  await openOnCallPlugin(page);
+  await configureOnCallPlugin(page);
 });
 
 // TODO: enable once we've signed up for a MailSlurp account to receieve SMSes

grafana-plugin/integration-tests/globalSetup.ts

@@ -0,0 +1,26 @@
+import { chromium, FullConfig, expect } from '@playwright/test';
+
+import { BASE_URL, GRAFANA_PASSWORD, GRAFANA_USERNAME } from './utils/constants';
+
+/**
+ * Borrowed from our friends on the Incident team
+ * https://github.com/grafana/incident/blob/main/plugin/e2e/global-setup.ts
+ */
+const globalSetup = async (config: FullConfig): Promise<void> => {
+  const { headless } = config.projects[0]!.use;
+  const browser = await chromium.launch({ headless, slowMo: headless ? 0 : 100 });
+  const browserContext = await browser.newContext();
+
+  const res = await browserContext.request.post(`${BASE_URL}/login`, {
+    data: {
+      user: GRAFANA_USERNAME,
+      password: GRAFANA_PASSWORD,
+    },
+  });
+
+  expect(res.ok()).toBeTruthy();
+  await browserContext.storageState({ path: './storageState.json' });
+  await browserContext.close();
+};
+
+export default globalSetup;

grafana-plugin/integration-tests/schedules/addOverride.test.ts

@@ -1,11 +1,11 @@
 import { test, expect } from '@playwright/test';
-import { openOnCallPlugin } from '../utils';
+import { configureOnCallPlugin } from '../utils/configurePlugin';
 import { clickButton, generateRandomValue } from '../utils/forms';
 import { createOnCallSchedule, getOverrideFormDateInputs } from '../utils/schedule';
-import dayjs from "dayjs";
+import dayjs from 'dayjs';
 
 test.beforeEach(async ({ page }) => {
-  await openOnCallPlugin(page);
+  await configureOnCallPlugin(page);
 });
 
 test('default dates in override creation modal are correct', async ({ page }) => {
@@ -16,8 +16,8 @@ test('default dates in override creation modal are correct', async ({ page }) =>
 
   const overrideFormDateInputs = await getOverrideFormDateInputs(page);
 
-  const expectedStart = dayjs().startOf('day');  // start of today
-  const expectedEnd = expectedStart.add(1, 'day');  // end of today
+  const expectedStart = dayjs().startOf('day'); // start of today
+  const expectedEnd = expectedStart.add(1, 'day'); // end of today
 
   expect(overrideFormDateInputs.start.isSame(expectedStart)).toBe(true);
   expect(overrideFormDateInputs.end.isSame(expectedEnd)).toBe(true);

grafana-plugin/integration-tests/utils/alertGroup.ts

@@ -1,6 +1,6 @@
 import { Page, expect } from '@playwright/test';
 import { selectDropdownValue, selectValuePickerValue } from './forms';
-import { goToOnCallPageByClickingOnTab } from './navigation';
+import { goToOnCallPage } from './navigation';
 
 const MAX_RETRIES = 5;
 
@@ -27,7 +27,7 @@ export const verifyThatAlertGroupIsTriggered = async (
   integrationName: string,
   triggeredStepText: string
 ): Promise<void> => {
-  await goToOnCallPageByClickingOnTab(page, 'Alert Groups');
+  await goToOnCallPage(page, 'incidents');
 
   // filter by integration
   await selectDropdownValue({

grafana-plugin/integration-tests/utils/configurePlugin.ts

@@ -1,5 +1,5 @@
 import type { Page } from '@playwright/test';
-import { ONCALL_API_URL, ONCALL_LEFT_HAND_NAV_ICON_SELECTOR } from './constants';
+import { ONCALL_API_URL } from './constants';
 import { clickButton, getInputByName } from './forms';
 import { goToGrafanaPage } from './navigation';
 
@@ -7,7 +7,11 @@ import { goToGrafanaPage } from './navigation';
  * go to config page and wait for plugin icon to be available on left-hand navigation
  */
 export const configureOnCallPlugin = async (page: Page): Promise<void> => {
+  /**
+   * go to the oncall plugin configuration page and wait for the page to be loaded
+   */
   await goToGrafanaPage(page, '/plugins/grafana-oncall-app');
+  await page.waitForSelector('text=Configure Grafana OnCall');
 
   /**
    * we may need to fill in the OnCall API URL if it is not set in the process.env
@@ -25,5 +29,5 @@ export const configureOnCallPlugin = async (page: Page): Promise<void> => {
    * wait for the page to be refreshed and the icon to show up, this means the plugin
    * has been successfully configured
    */
-  await page.waitForSelector(ONCALL_LEFT_HAND_NAV_ICON_SELECTOR);
+  await page.waitForSelector('div.scrollbar-view img[src*="grafana-oncall-app/img/logo.svg"]');
 };

grafana-plugin/integration-tests/utils/constants.ts

@@ -2,7 +2,4 @@ export const BASE_URL = process.env.BASE_URL || 'http://localhost:3000';
 export const ONCALL_API_URL = process.env.ONCALL_API_URL || 'http://host.docker.internal:8080';
 export const GRAFANA_USERNAME = process.env.GRAFANA_USERNAME || 'oncall';
 export const GRAFANA_PASSWORD = process.env.GRAFANA_PASSWORD || 'oncall';
-
 export const MAILSLURP_API_KEY = process.env.MAILSLURP_API_KEY;
-
-export const ONCALL_LEFT_HAND_NAV_ICON_SELECTOR = 'div.scrollbar-view img[src*="grafana-oncall-app/img/logo.svg"]';

grafana-plugin/integration-tests/utils/escalationChain.ts

@@ -1,7 +1,7 @@
 import { Page } from '@playwright/test';
 
 import { clickButton, fillInInput, selectDropdownValue } from './forms';
-import { goToOnCallPageByClickingOnTab } from './navigation';
+import { goToOnCallPage } from './navigation';
 
 export enum EscalationStep {
   NotifyUsers = 'Notify users',
@@ -20,7 +20,7 @@ export const createEscalationChain = async (
   escalationStepValue: string
 ): Promise<void> => {
   // go to the escalation chains page
-  await goToOnCallPageByClickingOnTab(page, 'Escalation Chains');
+  await goToOnCallPage(page, 'escalations');
 
   // open the create escalation chain modal
   (await page.waitForSelector('text=New Escalation Chain')).click();

grafana-plugin/integration-tests/utils/index.ts

@@ -1,10 +0,0 @@
-import type { Page } from '@playwright/test';
-import { configureOnCallPlugin } from './configurePlugin';
-import { login } from './login';
-import { goToOnCallPage } from './navigation';
-
-export const openOnCallPlugin = async (page: Page): Promise<void> => {
-  await login(page);
-  await configureOnCallPlugin(page);
-  await goToOnCallPage(page);
-};

grafana-plugin/integration-tests/utils/integrations.ts

@@ -1,6 +1,6 @@
 import { Page } from '@playwright/test';
 import { clickButton, fillInInput, selectDropdownValue } from './forms';
-import { goToOnCallPageByClickingOnTab } from './navigation';
+import { goToOnCallPage } from './navigation';
 
 export const createIntegrationAndSendDemoAlert = async (
   page: Page,
@@ -8,7 +8,7 @@ export const createIntegrationAndSendDemoAlert = async (
   escalationChainName: string
 ): Promise<void> => {
   // go to the integrations page
-  await goToOnCallPageByClickingOnTab(page, 'Integrations');
+  await goToOnCallPage(page, 'integrations');
 
   // open the create integration modal
   (await page.waitForSelector('text=New integration to receive alerts')).click();

grafana-plugin/integration-tests/utils/login.ts

@@ -1,13 +0,0 @@
-import type { Page } from '@playwright/test';
-import { GRAFANA_PASSWORD, GRAFANA_USERNAME } from './constants';
-import { clickButton, fillInInputByPlaceholderValue } from './forms';
-import { goToGrafanaPage, waitForNoNetworkActivity } from './navigation';
-
-export const login = async (page: Page): Promise<void> => {
-  await goToGrafanaPage(page, '/login', 'load');
-
-  await fillInInputByPlaceholderValue(page, 'email or username', GRAFANA_USERNAME);
-  await fillInInputByPlaceholderValue(page, 'password', GRAFANA_PASSWORD);
-  await clickButton({ page, buttonText: 'Log in' });
-  await waitForNoNetworkActivity(page);
-};

grafana-plugin/integration-tests/utils/navigation.ts

@@ -1,21 +1,13 @@
 import type { Page, Response } from '@playwright/test';
 import { BASE_URL } from './constants';
 
-type WaitUntil = 'networkidle' | 'load';
-type GrafanaPage = '/login' | '/plugins/grafana-oncall-app';
-type OnCallPage = 'incidents' | 'integrations' | 'escalations';
-type OnCallPluginTab = 'Integrations' | 'Escalation Chains' | 'Users' | 'Schedules' | 'Alert Groups';
+type GrafanaPage = '/plugins/grafana-oncall-app';
+type OnCallPage = 'incidents' | 'integrations' | 'escalations' | 'schedules' | 'users';
 
-const _goToPage = (page: Page, url = '', waitUntil: WaitUntil = 'networkidle'): Promise<Response> =>
-  page.goto(`${BASE_URL}${url}`, { waitUntil });
+const _goToPage = (page: Page, url = ''): Promise<Response> =>
+  page.goto(`${BASE_URL}${url}`, { waitUntil: 'networkidle' });
 
-export const goToGrafanaPage = (page: Page, url?: GrafanaPage, waitUntil?: WaitUntil): Promise<Response> =>
-  _goToPage(page, url, waitUntil);
+export const goToGrafanaPage = (page: Page, url: GrafanaPage): Promise<Response> => _goToPage(page, url);
 
-export const goToOnCallPage = (page: Page, onCallPage: OnCallPage = 'incidents'): Promise<Response> =>
+export const goToOnCallPage = (page: Page, onCallPage: OnCallPage): Promise<Response> =>
   _goToPage(page, `/a/grafana-oncall-app/${onCallPage}`);
-
-export const goToOnCallPageByClickingOnTab = async (page: Page, onCallTab: OnCallPluginTab): Promise<void> =>
-  (await page.waitForSelector(`div[class*="LegacyNavTabsBar"] >> text=${onCallTab}`)).click();
-
-export const waitForNoNetworkActivity = (page: Page): Promise<void> => page.waitForLoadState('networkidle');

grafana-plugin/integration-tests/utils/schedule.ts

@@ -1,12 +1,12 @@
 import { Page } from '@playwright/test';
 import { GRAFANA_USERNAME } from './constants';
 import { clickButton, fillInInput, selectDropdownValue, selectValuePickerValue } from './forms';
-import { goToOnCallPageByClickingOnTab } from './navigation';
-import dayjs from "dayjs";
+import { goToOnCallPage } from './navigation';
+import dayjs from 'dayjs';
 
 export const createOnCallSchedule = async (page: Page, scheduleName: string): Promise<void> => {
-  // go to the escalation chains page
-  await goToOnCallPageByClickingOnTab(page, 'Schedules');
+  // go to the schedules page
+  await goToOnCallPage(page, 'schedules');
 
   // create an oncall-rotation schedule
   await clickButton({ page, buttonText: 'New Schedule' });
@@ -38,9 +38,9 @@ export interface OverrideFormDateInputs {
 
 export const getOverrideFormDateInputs = async (page: Page): Promise<OverrideFormDateInputs> => {
   const getInputValue = async (inputNumber: number): Promise<string> => {
-    const element = await page.waitForSelector(`div[data-testid=\"override-inputs\"] >> input >> nth=${inputNumber}`)
+    const element = await page.waitForSelector(`div[data-testid=\"override-inputs\"] >> input >> nth=${inputNumber}`);
     return await element.inputValue();
-  }
+  };
 
   const startDate = await getInputValue(0);
   const startTime = await getInputValue(1);

grafana-plugin/integration-tests/utils/userSettings.ts

@@ -2,13 +2,13 @@ import { Locator, Page } from '@playwright/test';
 
 import { clickButton, fillInInputByPlaceholderValue, selectDropdownValue } from './forms';
 import { closeModal } from './modals';
-import { goToOnCallPageByClickingOnTab } from './navigation';
+import { goToOnCallPage } from './navigation';
 import { getPhoneNumber, getVerificationCodeFromSms, waitForSms } from './phone';
 
 type NotifyBy = 'SMS' | 'Phone call';
 
 const openUserSettingsModal = async (page: Page): Promise<void> => {
-  await goToOnCallPageByClickingOnTab(page, 'Users');
+  await goToOnCallPage(page, 'users');
   await clickButton({ page, buttonText: 'View my profile' });
   await page.locator('text=To edit user details such as Username, email, and role').waitFor({ state: 'visible' });
 };

grafana-plugin/playwright.config.ts

@@ -12,6 +12,7 @@ require('dotenv').config();
  */
 const config: PlaywrightTestConfig = {
   testDir: './integration-tests',
+  globalSetup: './integration-tests/globalSetup.ts',
   /* Maximum time one test can run for. */
   timeout: 60 * 1000,
   expect: {
@@ -33,6 +34,8 @@ const config: PlaywrightTestConfig = {
   reporter: 'html',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
+    storageState: './storageState.json',
+
     /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
     actionTimeout: 0,
     /* Base URL to use in actions like `await page.goto('/')`. */