From 2a8240ab55ee2eddd70acadcaee6337330403628 Mon Sep 17 00:00:00 2001 From: Dominik Broj Date: Fri, 26 Apr 2024 23:41:08 +0200 Subject: [PATCH] Use ops-devenv only in expensive tests (#4285) # What this PR does - Use cross-plugin e2e tests setup (cloning ops-devenv, gops-labels) only on a daily runs and on dev/main branch pipelines (exclude it from PRs so that community PRs don't rely on secrets) - Rename "Daily e2e tests" to "Expensive e2e tests" and run them both daily and when PRs are merged to dev/main - Post Slack message only if e2e tests fail ## Checklist - [x] Unit, integration, and e2e (if applicable) tests updated - [x] Documentation added (or `pr:no public docs` PR label added if not required) - [x] Added the relevant release notes label (see labels prefixed w/ `release:`). These labels dictate how your PR will show up in the autogenerated release notes. --------- Co-authored-by: Joey Orlando --- .github/workflows/e2e-tests.yml | 26 +++++++++++-------- ...-e2e-tests.yml => expensive-e2e-tests.yml} | 9 ++++--- .github/workflows/linting-and-tests.yml | 4 +-- .../createNewLabelKeysAndValues.test.ts | 2 +- 4 files changed, 24 insertions(+), 17 deletions(-) rename .github/workflows/{daily-e2e-tests.yml => expensive-e2e-tests.yml} (95%) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index ea2405f4..adccedcf 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -122,16 +122,19 @@ jobs: - name: Install Playwright deps uses: docker://mcr.microsoft.com/playwright:next-jammy - # Go and Mage are required to run gops-labels + # ---------- Expensive e2e tests steps start ----------- - name: Install Go + if: inputs.run-expensive-tests uses: actions/setup-go@v4 with: go-version: "1.21.5" - name: Install Mage + if: inputs.run-expensive-tests run: go install github.com/magefile/mage@v1.15.0 - name: Get Vault secrets + if: inputs.run-expensive-tests id: get-secrets uses: grafana/shared-workflows/actions/get-vault-secrets@main with: @@ -141,6 +144,7 @@ jobs: GH_APP_PRIVATE_KEY=github-app:private-key - name: Generate Github App token + if: inputs.run-expensive-tests id: generate-token uses: actions/create-github-app-token@v1 with: @@ -150,22 +154,13 @@ jobs: repositories: "ops-devenv,gops-labels" - name: Clone other repos needed for cross-plugin e2e tests + if: inputs.run-expensive-tests shell: bash run: | cd .. git clone https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/grafana/ops-devenv.git git clone https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/grafana/gops-labels.git - - name: Tilt CI - standard E2E tests - shell: bash - if: ${{ inputs.run-expensive-tests == false }} - env: - GRAFANA_VERSION: ${{ inputs.grafana_version }} - GRAFANA_ADMIN_USERNAME: "irm" - GRAFANA_ADMIN_PASSWORD: "irm" - BROWSERS: ${{ inputs.browsers }} - run: cd ../ops-devenv && tilt ci gops-labels oncall - - name: Tilt CI - standard and expensive E2E tests if: inputs.run-expensive-tests shell: bash @@ -182,6 +177,15 @@ jobs: TWILIO_PHONE_NUMBER: '"${{ secrets.TWILIO_PHONE_NUMBER }}"' TWILIO_VERIFY_SID: ${{ secrets.TWILIO_VERIFY_SID }} run: cd ../ops-devenv && tilt ci gops-labels oncall + # ---------- Expensive e2e tests steps end ----------- + + - name: Tilt CI - standard E2E tests + shell: bash + if: ${{ inputs.run-expensive-tests == false }} + env: + GRAFANA_VERSION: ${{ inputs.grafana_version }} + BROWSERS: ${{ inputs.browsers }} + run: tilt ci - name: Setup Pages if: failure() diff --git a/.github/workflows/daily-e2e-tests.yml b/.github/workflows/expensive-e2e-tests.yml similarity index 95% rename from .github/workflows/daily-e2e-tests.yml rename to .github/workflows/expensive-e2e-tests.yml index 8f03cf2c..5bda45e3 100644 --- a/.github/workflows/daily-e2e-tests.yml +++ b/.github/workflows/expensive-e2e-tests.yml @@ -1,5 +1,8 @@ -name: Daily e2e tests +name: Expensive e2e tests on: + push: + branches: + - main # allows manual run on github actions workflow_dispatch: schedule: @@ -7,7 +10,7 @@ on: jobs: end-to-end-tests: - name: End to end tests + name: Expensive e2e tests strategy: matrix: grafana_version: @@ -41,7 +44,7 @@ jobs: post-status-to-slack: runs-on: ubuntu-latest needs: end-to-end-tests - if: always() + if: failure steps: # Useful references # https://stackoverflow.com/questions/59073850/github-actions-get-url-of-test-build diff --git a/.github/workflows/linting-and-tests.yml b/.github/workflows/linting-and-tests.yml index 4a5dc38e..795ac982 100644 --- a/.github/workflows/linting-and-tests.yml +++ b/.github/workflows/linting-and-tests.yml @@ -1,4 +1,4 @@ -name: Linting and Unit/e2e Tests +name: Linting and Tests "on": push: @@ -316,7 +316,7 @@ jobs: mypy . end-to-end-tests: - name: End to end tests + name: Standard e2e tests uses: ./.github/workflows/e2e-tests.yml with: # TODO: fix issues with running e2e tests against Grafana v10.2.x and v10.3.x diff --git a/grafana-plugin/e2e-tests/labels/createNewLabelKeysAndValues.test.ts b/grafana-plugin/e2e-tests/labels/createNewLabelKeysAndValues.test.ts index 6daeac61..96226620 100644 --- a/grafana-plugin/e2e-tests/labels/createNewLabelKeysAndValues.test.ts +++ b/grafana-plugin/e2e-tests/labels/createNewLabelKeysAndValues.test.ts @@ -3,7 +3,7 @@ import { clickButton, generateRandomValidLabel, openDropdown } from '../utils/fo import { openCreateIntegrationModal } from '../utils/integrations'; import { goToOnCallPage } from '../utils/navigation'; -test('New label keys and labels can be created', async ({ adminRolePage }) => { +test('New label keys and labels can be created @expensive', async ({ adminRolePage }) => { const { page } = adminRolePage; await goToOnCallPage(page, 'integrations'); await openCreateIntegrationModal(page);