From b6615c087fbb2cd37c32ef09bc91a32491fe1de8 Mon Sep 17 00:00:00 2001 From: Joey Orlando Date: Fri, 10 Mar 2023 06:45:15 +0100 Subject: [PATCH] improve e2e tests authentication flow (#1470) # What this PR does This PR makes the Grafana login portion of the e2e tests much faster/more reliable. Currently we use CSS selectors to go to the login form, input the username/password, and proceed as such. This PR refactors to instead make a call to `POST ${GRAFANA_API_URL}/login` and then stores that authentication state that is then reused by subsequent browsers. This was inspired by [how the Incident team does their playwright authentication](https://github.com/grafana/incident/blob/main/plugin/e2e/global-setup.ts) + the recommendation from the [Playwright docs](https://playwright.dev/docs/auth#basic-shared-account-in-all-tests) ## Which issue(s) this PR fixes Slow/flaky Grafana login flow ## Checklist - [x] Tests updated - [ ] Documentation added (N/A) - [ ] `CHANGELOG.md` updated (N/A) --- grafana-plugin/.gitignore | 1 + .../alerts/onCallSchedule.test.ts | 4 +-- .../integration-tests/alerts/sms.test.ts | 4 +-- .../integration-tests/globalSetup.ts | 26 +++++++++++++++++++ .../schedules/addOverride.test.ts | 10 +++---- .../integration-tests/utils/alertGroup.ts | 4 +-- .../utils/configurePlugin.ts | 8 ++++-- .../integration-tests/utils/constants.ts | 3 --- .../utils/escalationChain.ts | 4 +-- .../integration-tests/utils/index.ts | 10 ------- .../integration-tests/utils/integrations.ts | 4 +-- .../integration-tests/utils/login.ts | 13 ---------- .../integration-tests/utils/navigation.ts | 20 +++++--------- .../integration-tests/utils/schedule.ts | 12 ++++----- .../integration-tests/utils/userSettings.ts | 4 +-- grafana-plugin/playwright.config.ts | 3 +++ 16 files changed, 65 insertions(+), 65 deletions(-) create mode 100644 grafana-plugin/integration-tests/globalSetup.ts delete mode 100644 grafana-plugin/integration-tests/utils/index.ts delete mode 100644 grafana-plugin/integration-tests/utils/login.ts diff --git a/grafana-plugin/.gitignore b/grafana-plugin/.gitignore index 8d0c1577..16f10cb1 100644 --- a/grafana-plugin/.gitignore +++ b/grafana-plugin/.gitignore @@ -19,3 +19,4 @@ frontend_enterprise /test-results/ /playwright-report/ /playwright/.cache/ +storageState.json diff --git a/grafana-plugin/integration-tests/alerts/onCallSchedule.test.ts b/grafana-plugin/integration-tests/alerts/onCallSchedule.test.ts index 91026f4b..a8b13efd 100644 --- a/grafana-plugin/integration-tests/alerts/onCallSchedule.test.ts +++ b/grafana-plugin/integration-tests/alerts/onCallSchedule.test.ts @@ -1,5 +1,5 @@ import { test } from '@playwright/test'; -import { openOnCallPlugin } from '../utils'; +import { configureOnCallPlugin } from '../utils/configurePlugin'; import { verifyThatAlertGroupIsTriggered } from '../utils/alertGroup'; import { createEscalationChain, EscalationStep } from '../utils/escalationChain'; import { generateRandomValue } from '../utils/forms'; @@ -7,7 +7,7 @@ import { createIntegrationAndSendDemoAlert } from '../utils/integrations'; import { createOnCallSchedule } from '../utils/schedule'; test.beforeEach(async ({ page }) => { - await openOnCallPlugin(page); + await configureOnCallPlugin(page); }); test('we can create an oncall schedule + receive an alert', async ({ page }) => { diff --git a/grafana-plugin/integration-tests/alerts/sms.test.ts b/grafana-plugin/integration-tests/alerts/sms.test.ts index 4b22eeda..dac42bb8 100644 --- a/grafana-plugin/integration-tests/alerts/sms.test.ts +++ b/grafana-plugin/integration-tests/alerts/sms.test.ts @@ -1,5 +1,5 @@ import { test, expect } from '@playwright/test'; -import { openOnCallPlugin } from '../utils'; +import { configureOnCallPlugin } from '../utils/configurePlugin'; import { GRAFANA_USERNAME } from '../utils/constants'; import { createEscalationChain, EscalationStep } from '../utils/escalationChain'; import { generateRandomValue } from '../utils/forms'; @@ -8,7 +8,7 @@ import { waitForSms } from '../utils/phone'; import { configureUserNotificationSettings, verifyUserPhoneNumber } from '../utils/userSettings'; test.beforeEach(async ({ page }) => { - await openOnCallPlugin(page); + await configureOnCallPlugin(page); }); // TODO: enable once we've signed up for a MailSlurp account to receieve SMSes diff --git a/grafana-plugin/integration-tests/globalSetup.ts b/grafana-plugin/integration-tests/globalSetup.ts new file mode 100644 index 00000000..3fc05399 --- /dev/null +++ b/grafana-plugin/integration-tests/globalSetup.ts @@ -0,0 +1,26 @@ +import { chromium, FullConfig, expect } from '@playwright/test'; + +import { BASE_URL, GRAFANA_PASSWORD, GRAFANA_USERNAME } from './utils/constants'; + +/** + * Borrowed from our friends on the Incident team + * https://github.com/grafana/incident/blob/main/plugin/e2e/global-setup.ts + */ +const globalSetup = async (config: FullConfig): Promise => { + const { headless } = config.projects[0]!.use; + const browser = await chromium.launch({ headless, slowMo: headless ? 0 : 100 }); + const browserContext = await browser.newContext(); + + const res = await browserContext.request.post(`${BASE_URL}/login`, { + data: { + user: GRAFANA_USERNAME, + password: GRAFANA_PASSWORD, + }, + }); + + expect(res.ok()).toBeTruthy(); + await browserContext.storageState({ path: './storageState.json' }); + await browserContext.close(); +}; + +export default globalSetup; diff --git a/grafana-plugin/integration-tests/schedules/addOverride.test.ts b/grafana-plugin/integration-tests/schedules/addOverride.test.ts index f682c581..3104cc68 100644 --- a/grafana-plugin/integration-tests/schedules/addOverride.test.ts +++ b/grafana-plugin/integration-tests/schedules/addOverride.test.ts @@ -1,11 +1,11 @@ import { test, expect } from '@playwright/test'; -import { openOnCallPlugin } from '../utils'; +import { configureOnCallPlugin } from '../utils/configurePlugin'; import { clickButton, generateRandomValue } from '../utils/forms'; import { createOnCallSchedule, getOverrideFormDateInputs } from '../utils/schedule'; -import dayjs from "dayjs"; +import dayjs from 'dayjs'; test.beforeEach(async ({ page }) => { - await openOnCallPlugin(page); + await configureOnCallPlugin(page); }); test('default dates in override creation modal are correct', async ({ page }) => { @@ -16,8 +16,8 @@ test('default dates in override creation modal are correct', async ({ page }) => const overrideFormDateInputs = await getOverrideFormDateInputs(page); - const expectedStart = dayjs().startOf('day'); // start of today - const expectedEnd = expectedStart.add(1, 'day'); // end of today + const expectedStart = dayjs().startOf('day'); // start of today + const expectedEnd = expectedStart.add(1, 'day'); // end of today expect(overrideFormDateInputs.start.isSame(expectedStart)).toBe(true); expect(overrideFormDateInputs.end.isSame(expectedEnd)).toBe(true); diff --git a/grafana-plugin/integration-tests/utils/alertGroup.ts b/grafana-plugin/integration-tests/utils/alertGroup.ts index f07e4d36..f729f1a7 100644 --- a/grafana-plugin/integration-tests/utils/alertGroup.ts +++ b/grafana-plugin/integration-tests/utils/alertGroup.ts @@ -1,6 +1,6 @@ import { Page, expect } from '@playwright/test'; import { selectDropdownValue, selectValuePickerValue } from './forms'; -import { goToOnCallPageByClickingOnTab } from './navigation'; +import { goToOnCallPage } from './navigation'; const MAX_RETRIES = 5; @@ -27,7 +27,7 @@ export const verifyThatAlertGroupIsTriggered = async ( integrationName: string, triggeredStepText: string ): Promise => { - await goToOnCallPageByClickingOnTab(page, 'Alert Groups'); + await goToOnCallPage(page, 'incidents'); // filter by integration await selectDropdownValue({ diff --git a/grafana-plugin/integration-tests/utils/configurePlugin.ts b/grafana-plugin/integration-tests/utils/configurePlugin.ts index 17d4a91d..5f2869c5 100644 --- a/grafana-plugin/integration-tests/utils/configurePlugin.ts +++ b/grafana-plugin/integration-tests/utils/configurePlugin.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { ONCALL_API_URL, ONCALL_LEFT_HAND_NAV_ICON_SELECTOR } from './constants'; +import { ONCALL_API_URL } from './constants'; import { clickButton, getInputByName } from './forms'; import { goToGrafanaPage } from './navigation'; @@ -7,7 +7,11 @@ import { goToGrafanaPage } from './navigation'; * go to config page and wait for plugin icon to be available on left-hand navigation */ export const configureOnCallPlugin = async (page: Page): Promise => { + /** + * go to the oncall plugin configuration page and wait for the page to be loaded + */ await goToGrafanaPage(page, '/plugins/grafana-oncall-app'); + await page.waitForSelector('text=Configure Grafana OnCall'); /** * we may need to fill in the OnCall API URL if it is not set in the process.env @@ -25,5 +29,5 @@ export const configureOnCallPlugin = async (page: Page): Promise => { * wait for the page to be refreshed and the icon to show up, this means the plugin * has been successfully configured */ - await page.waitForSelector(ONCALL_LEFT_HAND_NAV_ICON_SELECTOR); + await page.waitForSelector('div.scrollbar-view img[src*="grafana-oncall-app/img/logo.svg"]'); }; diff --git a/grafana-plugin/integration-tests/utils/constants.ts b/grafana-plugin/integration-tests/utils/constants.ts index f6a9f9d2..aa62ee5a 100644 --- a/grafana-plugin/integration-tests/utils/constants.ts +++ b/grafana-plugin/integration-tests/utils/constants.ts @@ -2,7 +2,4 @@ export const BASE_URL = process.env.BASE_URL || 'http://localhost:3000'; export const ONCALL_API_URL = process.env.ONCALL_API_URL || 'http://host.docker.internal:8080'; export const GRAFANA_USERNAME = process.env.GRAFANA_USERNAME || 'oncall'; export const GRAFANA_PASSWORD = process.env.GRAFANA_PASSWORD || 'oncall'; - export const MAILSLURP_API_KEY = process.env.MAILSLURP_API_KEY; - -export const ONCALL_LEFT_HAND_NAV_ICON_SELECTOR = 'div.scrollbar-view img[src*="grafana-oncall-app/img/logo.svg"]'; diff --git a/grafana-plugin/integration-tests/utils/escalationChain.ts b/grafana-plugin/integration-tests/utils/escalationChain.ts index 8f7f5317..e8553f86 100644 --- a/grafana-plugin/integration-tests/utils/escalationChain.ts +++ b/grafana-plugin/integration-tests/utils/escalationChain.ts @@ -1,7 +1,7 @@ import { Page } from '@playwright/test'; import { clickButton, fillInInput, selectDropdownValue } from './forms'; -import { goToOnCallPageByClickingOnTab } from './navigation'; +import { goToOnCallPage } from './navigation'; export enum EscalationStep { NotifyUsers = 'Notify users', @@ -20,7 +20,7 @@ export const createEscalationChain = async ( escalationStepValue: string ): Promise => { // go to the escalation chains page - await goToOnCallPageByClickingOnTab(page, 'Escalation Chains'); + await goToOnCallPage(page, 'escalations'); // open the create escalation chain modal (await page.waitForSelector('text=New Escalation Chain')).click(); diff --git a/grafana-plugin/integration-tests/utils/index.ts b/grafana-plugin/integration-tests/utils/index.ts deleted file mode 100644 index 65513913..00000000 --- a/grafana-plugin/integration-tests/utils/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { Page } from '@playwright/test'; -import { configureOnCallPlugin } from './configurePlugin'; -import { login } from './login'; -import { goToOnCallPage } from './navigation'; - -export const openOnCallPlugin = async (page: Page): Promise => { - await login(page); - await configureOnCallPlugin(page); - await goToOnCallPage(page); -}; diff --git a/grafana-plugin/integration-tests/utils/integrations.ts b/grafana-plugin/integration-tests/utils/integrations.ts index eab5a475..eb93ceff 100644 --- a/grafana-plugin/integration-tests/utils/integrations.ts +++ b/grafana-plugin/integration-tests/utils/integrations.ts @@ -1,6 +1,6 @@ import { Page } from '@playwright/test'; import { clickButton, fillInInput, selectDropdownValue } from './forms'; -import { goToOnCallPageByClickingOnTab } from './navigation'; +import { goToOnCallPage } from './navigation'; export const createIntegrationAndSendDemoAlert = async ( page: Page, @@ -8,7 +8,7 @@ export const createIntegrationAndSendDemoAlert = async ( escalationChainName: string ): Promise => { // go to the integrations page - await goToOnCallPageByClickingOnTab(page, 'Integrations'); + await goToOnCallPage(page, 'integrations'); // open the create integration modal (await page.waitForSelector('text=New integration to receive alerts')).click(); diff --git a/grafana-plugin/integration-tests/utils/login.ts b/grafana-plugin/integration-tests/utils/login.ts deleted file mode 100644 index 172d7732..00000000 --- a/grafana-plugin/integration-tests/utils/login.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { Page } from '@playwright/test'; -import { GRAFANA_PASSWORD, GRAFANA_USERNAME } from './constants'; -import { clickButton, fillInInputByPlaceholderValue } from './forms'; -import { goToGrafanaPage, waitForNoNetworkActivity } from './navigation'; - -export const login = async (page: Page): Promise => { - await goToGrafanaPage(page, '/login', 'load'); - - await fillInInputByPlaceholderValue(page, 'email or username', GRAFANA_USERNAME); - await fillInInputByPlaceholderValue(page, 'password', GRAFANA_PASSWORD); - await clickButton({ page, buttonText: 'Log in' }); - await waitForNoNetworkActivity(page); -}; diff --git a/grafana-plugin/integration-tests/utils/navigation.ts b/grafana-plugin/integration-tests/utils/navigation.ts index a43ad706..120b0ee3 100644 --- a/grafana-plugin/integration-tests/utils/navigation.ts +++ b/grafana-plugin/integration-tests/utils/navigation.ts @@ -1,21 +1,13 @@ import type { Page, Response } from '@playwright/test'; import { BASE_URL } from './constants'; -type WaitUntil = 'networkidle' | 'load'; -type GrafanaPage = '/login' | '/plugins/grafana-oncall-app'; -type OnCallPage = 'incidents' | 'integrations' | 'escalations'; -type OnCallPluginTab = 'Integrations' | 'Escalation Chains' | 'Users' | 'Schedules' | 'Alert Groups'; +type GrafanaPage = '/plugins/grafana-oncall-app'; +type OnCallPage = 'incidents' | 'integrations' | 'escalations' | 'schedules' | 'users'; -const _goToPage = (page: Page, url = '', waitUntil: WaitUntil = 'networkidle'): Promise => - page.goto(`${BASE_URL}${url}`, { waitUntil }); +const _goToPage = (page: Page, url = ''): Promise => + page.goto(`${BASE_URL}${url}`, { waitUntil: 'networkidle' }); -export const goToGrafanaPage = (page: Page, url?: GrafanaPage, waitUntil?: WaitUntil): Promise => - _goToPage(page, url, waitUntil); +export const goToGrafanaPage = (page: Page, url: GrafanaPage): Promise => _goToPage(page, url); -export const goToOnCallPage = (page: Page, onCallPage: OnCallPage = 'incidents'): Promise => +export const goToOnCallPage = (page: Page, onCallPage: OnCallPage): Promise => _goToPage(page, `/a/grafana-oncall-app/${onCallPage}`); - -export const goToOnCallPageByClickingOnTab = async (page: Page, onCallTab: OnCallPluginTab): Promise => - (await page.waitForSelector(`div[class*="LegacyNavTabsBar"] >> text=${onCallTab}`)).click(); - -export const waitForNoNetworkActivity = (page: Page): Promise => page.waitForLoadState('networkidle'); diff --git a/grafana-plugin/integration-tests/utils/schedule.ts b/grafana-plugin/integration-tests/utils/schedule.ts index d41afbe8..7754699e 100644 --- a/grafana-plugin/integration-tests/utils/schedule.ts +++ b/grafana-plugin/integration-tests/utils/schedule.ts @@ -1,12 +1,12 @@ import { Page } from '@playwright/test'; import { GRAFANA_USERNAME } from './constants'; import { clickButton, fillInInput, selectDropdownValue, selectValuePickerValue } from './forms'; -import { goToOnCallPageByClickingOnTab } from './navigation'; -import dayjs from "dayjs"; +import { goToOnCallPage } from './navigation'; +import dayjs from 'dayjs'; export const createOnCallSchedule = async (page: Page, scheduleName: string): Promise => { - // go to the escalation chains page - await goToOnCallPageByClickingOnTab(page, 'Schedules'); + // go to the schedules page + await goToOnCallPage(page, 'schedules'); // create an oncall-rotation schedule await clickButton({ page, buttonText: 'New Schedule' }); @@ -38,9 +38,9 @@ export interface OverrideFormDateInputs { export const getOverrideFormDateInputs = async (page: Page): Promise => { const getInputValue = async (inputNumber: number): Promise => { - const element = await page.waitForSelector(`div[data-testid=\"override-inputs\"] >> input >> nth=${inputNumber}`) + const element = await page.waitForSelector(`div[data-testid=\"override-inputs\"] >> input >> nth=${inputNumber}`); return await element.inputValue(); - } + }; const startDate = await getInputValue(0); const startTime = await getInputValue(1); diff --git a/grafana-plugin/integration-tests/utils/userSettings.ts b/grafana-plugin/integration-tests/utils/userSettings.ts index e671f027..a3141549 100644 --- a/grafana-plugin/integration-tests/utils/userSettings.ts +++ b/grafana-plugin/integration-tests/utils/userSettings.ts @@ -2,13 +2,13 @@ import { Locator, Page } from '@playwright/test'; import { clickButton, fillInInputByPlaceholderValue, selectDropdownValue } from './forms'; import { closeModal } from './modals'; -import { goToOnCallPageByClickingOnTab } from './navigation'; +import { goToOnCallPage } from './navigation'; import { getPhoneNumber, getVerificationCodeFromSms, waitForSms } from './phone'; type NotifyBy = 'SMS' | 'Phone call'; const openUserSettingsModal = async (page: Page): Promise => { - await goToOnCallPageByClickingOnTab(page, 'Users'); + await goToOnCallPage(page, 'users'); await clickButton({ page, buttonText: 'View my profile' }); await page.locator('text=To edit user details such as Username, email, and role').waitFor({ state: 'visible' }); }; diff --git a/grafana-plugin/playwright.config.ts b/grafana-plugin/playwright.config.ts index fb366a59..4977535d 100644 --- a/grafana-plugin/playwright.config.ts +++ b/grafana-plugin/playwright.config.ts @@ -12,6 +12,7 @@ require('dotenv').config(); */ const config: PlaywrightTestConfig = { testDir: './integration-tests', + globalSetup: './integration-tests/globalSetup.ts', /* Maximum time one test can run for. */ timeout: 60 * 1000, expect: { @@ -33,6 +34,8 @@ const config: PlaywrightTestConfig = { reporter: 'html', /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { + storageState: './storageState.json', + /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ actionTimeout: 0, /* Base URL to use in actions like `await page.goto('/')`. */