Use ops-devenv only in expensive tests (#4285)

# What this PR does

- Use cross-plugin e2e tests setup (cloning ops-devenv, gops-labels)
only on a daily runs and on dev/main branch pipelines (exclude it from
PRs so that community PRs don't rely on secrets)
- Rename "Daily e2e tests" to "Expensive e2e tests" and run them both
daily and when PRs are merged to dev/main
- Post Slack message only if e2e tests fail

## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] Added the relevant release notes label (see labels prefixed w/
`release:`). These labels dictate how your PR will
    show up in the autogenerated release notes.

---------

Co-authored-by: Joey Orlando <joey.orlando@grafana.com>

.github/workflows/e2e-tests.yml

@@ -122,16 +122,19 @@ jobs:
       - name: Install Playwright deps
         uses: docker://mcr.microsoft.com/playwright:next-jammy
 
-      # Go and Mage are required to run gops-labels
+      # ---------- Expensive e2e tests steps start -----------
       - name: Install Go
+        if: inputs.run-expensive-tests
         uses: actions/setup-go@v4
         with:
           go-version: "1.21.5"
 
       - name: Install Mage
+        if: inputs.run-expensive-tests
         run: go install github.com/magefile/mage@v1.15.0
 
       - name: Get Vault secrets
+        if: inputs.run-expensive-tests
         id: get-secrets
         uses: grafana/shared-workflows/actions/get-vault-secrets@main
         with:
@@ -141,6 +144,7 @@ jobs:
             GH_APP_PRIVATE_KEY=github-app:private-key
 
       - name: Generate Github App token
+        if: inputs.run-expensive-tests
         id: generate-token
         uses: actions/create-github-app-token@v1
         with:
@@ -150,22 +154,13 @@ jobs:
           repositories: "ops-devenv,gops-labels"
 
       - name: Clone other repos needed for cross-plugin e2e tests
+        if: inputs.run-expensive-tests
         shell: bash
         run: |
           cd ..
           git clone https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/grafana/ops-devenv.git
           git clone https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/grafana/gops-labels.git
 
-      - name: Tilt CI - standard E2E tests
-        shell: bash
-        if: ${{ inputs.run-expensive-tests == false }}
-        env:
-          GRAFANA_VERSION: ${{ inputs.grafana_version }}
-          GRAFANA_ADMIN_USERNAME: "irm"
-          GRAFANA_ADMIN_PASSWORD: "irm"
-          BROWSERS: ${{ inputs.browsers }}
-        run: cd ../ops-devenv && tilt ci gops-labels oncall
-
       - name: Tilt CI - standard and expensive E2E tests
         if: inputs.run-expensive-tests
         shell: bash
@@ -182,6 +177,15 @@ jobs:
           TWILIO_PHONE_NUMBER: '"${{ secrets.TWILIO_PHONE_NUMBER }}"'
           TWILIO_VERIFY_SID: ${{ secrets.TWILIO_VERIFY_SID }}
         run: cd ../ops-devenv && tilt ci gops-labels oncall
+      # ---------- Expensive e2e tests steps end -----------
+
+      - name: Tilt CI - standard E2E tests
+        shell: bash
+        if: ${{ inputs.run-expensive-tests == false }}
+        env:
+          GRAFANA_VERSION: ${{ inputs.grafana_version }}
+          BROWSERS: ${{ inputs.browsers }}
+        run: tilt ci
 
       - name: Setup Pages
         if: failure()

.github/workflows/expensive-e2e-tests.yml

@@ -1,5 +1,8 @@
-name: Daily e2e tests
+name: Expensive e2e tests
 on:
+  push:
+    branches:
+      - main
   # allows manual run on github actions
   workflow_dispatch:
   schedule:
@@ -7,7 +10,7 @@ on:
 
 jobs:
   end-to-end-tests:
-    name: End to end tests
+    name: Expensive e2e tests
     strategy:
       matrix:
         grafana_version:
@@ -41,7 +44,7 @@ jobs:
   post-status-to-slack:
     runs-on: ubuntu-latest
     needs: end-to-end-tests
-    if: always()
+    if: failure
     steps:
       # Useful references
       # https://stackoverflow.com/questions/59073850/github-actions-get-url-of-test-build

.github/workflows/linting-and-tests.yml

@@ -1,4 +1,4 @@
-name: Linting and Unit/e2e Tests
+name: Linting and Tests
 
 "on":
   push:
@@ -316,7 +316,7 @@ jobs:
           mypy .
 
   end-to-end-tests:
-    name: End to end tests
+    name: Standard e2e tests
     uses: ./.github/workflows/e2e-tests.yml
     with:
       # TODO: fix issues with running e2e tests against Grafana v10.2.x and v10.3.x

grafana-plugin/e2e-tests/labels/createNewLabelKeysAndValues.test.ts

@@ -3,7 +3,7 @@ import { clickButton, generateRandomValidLabel, openDropdown } from '../utils/fo
 import { openCreateIntegrationModal } from '../utils/integrations';
 import { goToOnCallPage } from '../utils/navigation';
 
-test('New label keys and labels can be created', async ({ adminRolePage }) => {
+test('New label keys and labels can be created @expensive', async ({ adminRolePage }) => {
   const { page } = adminRolePage;
   await goToOnCallPage(page, 'integrations');
   await openCreateIntegrationModal(page);