eefe7be56a
Occasionally, the Playwright global setup step (which authenticates w/ the Grafana API + configures the plugin) would fail, leading to the CI job to instantly fail (playwright doesn't retry global setup if it fails). My current hypothesis as to why this is happening is because the `oncall-engine` and `oncall-celery` pods aren't _actually_ ready in these cases based on the way the `jupyterhub/action-k8s-await-workloads` action await k8s workloads: <img width="1076" alt="Screenshot 2023-05-23 at 18 24 36" src="https://github.com/grafana/oncall/assets/9406895/68d8d2d9-4274-4749-8788-e0a9a3dbad83"> By using the `kubectl rollout status deployment/<deployment-name> --timeout=300s` instead, we can be sure that these pods are _actually_ ready to receive traffic before we start the tests. ```bash ❯ kubectl rollout status --help Show the status of the rollout. By default 'rollout status' will watch the status of the latest rollout until it's done. If you don't want to wait for the rollout to finish then you can use --watch=false. Note that if a new rollout starts in-between, then 'rollout status' will continue watching the latest revision. If you want to pin to a specific revision and abort if it is rolled over by another revision, use --revision=N where N is the revision you need to watch for. ``` Lastly, even despite this, sometimes the `POST /api/internal/v1/plugin/sync` endpoint will return HTTP 500 ([example logs](https://github.com/grafana/oncall/actions/runs/5062712137/jobs/9088529416#step:19:2536) from failed CI job). In this case, let's setup the Playwright global setup to retry 3 times.
114 lines
2.9 KiB
TypeScript
114 lines
2.9 KiB
TypeScript
import type { PlaywrightTestConfig } from '@playwright/test';
|
|
import { devices } from '@playwright/test';
|
|
|
|
/**
|
|
* Read environment variables from file.
|
|
* https://github.com/motdotla/dotenv
|
|
*/
|
|
require('dotenv').config();
|
|
|
|
/**
|
|
* See https://playwright.dev/docs/test-configuration.
|
|
*/
|
|
const config: PlaywrightTestConfig = {
|
|
testDir: './integration-tests',
|
|
globalSetup: './integration-tests/globalSetup.ts',
|
|
/* Maximum time one test can run for. */
|
|
timeout: 60 * 1000,
|
|
expect: {
|
|
/**
|
|
* Maximum time expect() should wait for the condition to be met.
|
|
* For example in `await expect(locator).toHaveText();`
|
|
*/
|
|
timeout: 5000,
|
|
},
|
|
/* Run tests in files in parallel */
|
|
fullyParallel: true,
|
|
/* Fail the build on CI if you accidentally left test.only in the source code. */
|
|
forbidOnly: !!process.env.CI,
|
|
/**
|
|
* Retry on CI only
|
|
*
|
|
* NOTE: until we fix this issue (https://github.com/grafana/oncall/issues/1692) which occasionally leads
|
|
* to flaky tests.. let's just retry failed tests. If the same test fails 3 times, you know something must be up
|
|
*/
|
|
retries: !!process.env.CI ? 3 : 0,
|
|
workers: !!process.env.CI ? 2 : 1,
|
|
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
|
|
reporter: 'html',
|
|
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
|
|
use: {
|
|
storageState: './storageState.json',
|
|
|
|
/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
|
|
actionTimeout: 0,
|
|
/* Base URL to use in actions like `await page.goto('/')`. */
|
|
// baseURL: 'http://localhost:3000',
|
|
|
|
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
|
|
trace: 'on',
|
|
video: 'on',
|
|
headless: !!process.env.CI,
|
|
},
|
|
|
|
/* Configure projects for major browsers */
|
|
projects: [
|
|
{
|
|
name: 'chromium',
|
|
use: {
|
|
...devices['Desktop Chrome'],
|
|
},
|
|
},
|
|
{
|
|
name: 'firefox',
|
|
use: {
|
|
...devices['Desktop Firefox'],
|
|
},
|
|
},
|
|
{
|
|
name: 'webkit',
|
|
use: {
|
|
...devices['Desktop Safari'],
|
|
},
|
|
},
|
|
|
|
/* Test against mobile viewports. */
|
|
// {
|
|
// name: 'Mobile Chrome',
|
|
// use: {
|
|
// ...devices['Pixel 5'],
|
|
// },
|
|
// },
|
|
// {
|
|
// name: 'Mobile Safari',
|
|
// use: {
|
|
// ...devices['iPhone 12'],
|
|
// },
|
|
// },
|
|
|
|
/* Test against branded browsers. */
|
|
// {
|
|
// name: 'Microsoft Edge',
|
|
// use: {
|
|
// channel: 'msedge',
|
|
// },
|
|
// },
|
|
// {
|
|
// name: 'Google Chrome',
|
|
// use: {
|
|
// channel: 'chrome',
|
|
// },
|
|
// },
|
|
],
|
|
|
|
/* Folder for test artifacts such as screenshots, videos, traces, etc. */
|
|
// outputDir: 'test-results/',
|
|
|
|
/* Run your local dev server before starting the tests */
|
|
// webServer: {
|
|
// command: 'npm run start',
|
|
// port: 3000,
|
|
// },
|
|
};
|
|
|
|
export default config;
|