diff --git a/.github/helm-ci.yml b/.github/helm-ci.yml
new file mode 100644
index 00000000..7fecb906
--- /dev/null
+++ b/.github/helm-ci.yml
@@ -0,0 +1,10 @@
+env:
+  - name: GRAFANA_CLOUD_NOTIFICATIONS_ENABLED
+    value: "False"
+  - name: FEATURE_PROMETHEUS_EXPORTER_ENABLED
+    value: "True"
+  # enabled to be able to test docker.host.internal in the webhook e2e tests
+  - name: DANGEROUS_WEBHOOKS_ENABLED
+    value: "True"
+  - name: FEATURE_LABELS_ENABLED_FOR_ALL
+    value: "True"
diff --git a/.github/workflows/daily-e2e-tests.yml b/.github/workflows/daily-e2e-tests.yml
index ead16a15..8f03cf2c 100644
--- a/.github/workflows/daily-e2e-tests.yml
+++ b/.github/workflows/daily-e2e-tests.yml
@@ -1,5 +1,5 @@
 name: Daily e2e tests
-"on":
+on:
   # allows manual run on github actions
   workflow_dispatch:
   schedule:
@@ -10,7 +10,7 @@ jobs:
     name: End to end tests
     strategy:
       matrix:
-        grafana-image-tag:
+        grafana_version:
           # OnCall doesn't work on the following versions of Grafana
           # - 8.5.22
           # - 9.0.0
@@ -29,7 +29,7 @@ jobs:
       fail-fast: false
     uses: ./.github/workflows/e2e-tests.yml
     with:
-      grafana-image-tag: ${{ matrix.grafana-image-tag }}
+      grafana_version: ${{ matrix.grafana_version }}
       run-expensive-tests: true
       # TODO: fix issues with some tests on firefox and webkit
       # example CI build with firefox/webkit failing tests
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
index 6124e5f4..ea2405f4 100644
--- a/.github/workflows/e2e-tests.yml
+++ b/.github/workflows/e2e-tests.yml
@@ -1,8 +1,9 @@
 name: e2e tests
-"on":
+
+on:
   workflow_call:
     inputs:
-      grafana-image-tag:
+      grafana_version:
         required: true
         type: string
       browsers:
@@ -34,12 +35,13 @@ jobs:
     # the oncall backend, and hence, flaky tests. Let's use CI runners w/ more resources to avoid this (plus
     # this will allow us to run more backend containers and parralelize the tests)
     runs-on: ubuntu-latest-16-cores
-    name: "Grafana: ${{ inputs.grafana-image-tag }}"
+    name: "Grafana: ${{ inputs.grafana_version }}"
     environment:
       name: github-pages
     permissions:
       id-token: write
-      pages: "write"
+      pages: write
+      contents: read
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -120,20 +122,58 @@ jobs:
       - name: Install Playwright deps
         uses: docker://mcr.microsoft.com/playwright:next-jammy
 
+      # Go and Mage are required to run gops-labels
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.21.5"
+
+      - name: Install Mage
+        run: go install github.com/magefile/mage@v1.15.0
+
+      - name: Get Vault secrets
+        id: get-secrets
+        uses: grafana/shared-workflows/actions/get-vault-secrets@main
+        with:
+          repo_secrets: |
+            GH_APP_ID=github-app:app-id
+            GH_APP_INSTALLATION_ID=github-app:app-installation-id
+            GH_APP_PRIVATE_KEY=github-app:private-key
+
+      - name: Generate Github App token
+        id: generate-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ env.GH_APP_ID }}
+          private-key: ${{ env.GH_APP_PRIVATE_KEY }}
+          owner: grafana
+          repositories: "ops-devenv,gops-labels"
+
+      - name: Clone other repos needed for cross-plugin e2e tests
+        shell: bash
+        run: |
+          cd ..
+          git clone https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/grafana/ops-devenv.git
+          git clone https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/grafana/gops-labels.git
+
       - name: Tilt CI - standard E2E tests
         shell: bash
         if: ${{ inputs.run-expensive-tests == false }}
         env:
-          GRAFANA_IMAGE_TAG: ${{ inputs.grafana-image-tag }}
+          GRAFANA_VERSION: ${{ inputs.grafana_version }}
+          GRAFANA_ADMIN_USERNAME: "irm"
+          GRAFANA_ADMIN_PASSWORD: "irm"
           BROWSERS: ${{ inputs.browsers }}
-        run: tilt ci
+        run: cd ../ops-devenv && tilt ci gops-labels oncall
 
       - name: Tilt CI - standard and expensive E2E tests
         if: inputs.run-expensive-tests
         shell: bash
         env:
           E2E_TESTS_CMD: "cd grafana-plugin && yarn test:e2e-expensive"
-          GRAFANA_IMAGE_TAG: ${{ inputs.grafana-image-tag }}
+          GRAFANA_VERSION: ${{ inputs.grafana_version }}
+          GRAFANA_ADMIN_USERNAME: "irm"
+          GRAFANA_ADMIN_PASSWORD: "irm"
           BROWSERS: ${{ inputs.browsers }}
           MAILSLURP_API_KEY: ${{ secrets.MAILSLURP_API_KEY }}
           TWILIO_ACCOUNT_SID: ${{ secrets.TWILIO_ACCOUNT_SID }}
@@ -141,7 +181,7 @@ jobs:
           # wrapping single quotes are required to prevent stripping leading "+" from the number
           TWILIO_PHONE_NUMBER: '"${{ secrets.TWILIO_PHONE_NUMBER }}"'
           TWILIO_VERIFY_SID: ${{ secrets.TWILIO_VERIFY_SID }}
-        run: tilt ci
+        run: cd ../ops-devenv && tilt ci gops-labels oncall
 
       - name: Setup Pages
         if: failure()
diff --git a/.github/workflows/linting-and-tests.yml b/.github/workflows/linting-and-tests.yml
index 6e51b360..2e2fd81e 100644
--- a/.github/workflows/linting-and-tests.yml
+++ b/.github/workflows/linting-and-tests.yml
@@ -320,7 +320,7 @@ jobs:
     uses: ./.github/workflows/e2e-tests.yml
     with:
       # TODO: fix issues with running e2e tests against Grafana v10.2.x and v10.3.x
-      grafana-image-tag: 10.1.7
-      # grafana-image-tag: 10.3.3
+      grafana_version: 10.1.7
+      # grafana_version: 10.3.3
       run-expensive-tests: false
       browsers: "chromium"
diff --git a/Tiltfile b/Tiltfile
index a366862f..2c7c1205 100644
--- a/Tiltfile
+++ b/Tiltfile
@@ -2,7 +2,7 @@ load('ext://uibutton', 'cmd_button', 'location', 'text_input', 'bool_input')
 running_under_parent_tiltfile = os.getenv("TILT_PARENT", "false") == "true"
 # The user/pass that you will login to Grafana with
 grafana_admin_user_pass = os.getenv("GRAFANA_ADMIN_USER_PASS", "oncall")
-grafana_image_tag = os.getenv("GRAFANA_IMAGE_TAG", "latest")
+grafana_version = os.getenv("GRAFANA_VERSION", "latest")
 e2e_tests_cmd=os.getenv("E2E_TESTS_CMD", "cd grafana-plugin && yarn test:e2e")
 twilio_values=[
     "oncall.twilio.accountSid=" + os.getenv("TWILIO_ACCOUNT_SID", ""),
@@ -113,7 +113,10 @@ cmd_button(
     icon_name="dangerous",
 )
 
-yaml = helm("helm/oncall", name=HELM_PREFIX, values=["./dev/helm-local.yml", "./dev/helm-local.dev.yml"], set=twilio_values)
+helm_oncall_values = ["./dev/helm-local.yml", "./dev/helm-local.dev.yml"]
+if is_ci:
+    helm_oncall_values = helm_oncall_values + ["./.github/helm-ci.yml"]
+yaml = helm("helm/oncall", name=HELM_PREFIX, values=helm_oncall_values, set=twilio_values)
 
 k8s_yaml(yaml)
 
@@ -134,7 +137,7 @@ k8s_resource(
 # Use separate grafana helm chart
 if not running_under_parent_tiltfile:
     grafana(
-        grafana_version=grafana_image_tag,
+        grafana_version=grafana_version,
         context="grafana-plugin",
         plugin_files=["grafana-plugin/src/plugin.json"],
         namespace="default",
diff --git a/grafana-plugin/e2e-tests/labels/createNewLabelKeysAndValues.test.ts b/grafana-plugin/e2e-tests/labels/createNewLabelKeysAndValues.test.ts
new file mode 100644
index 00000000..6daeac61
--- /dev/null
+++ b/grafana-plugin/e2e-tests/labels/createNewLabelKeysAndValues.test.ts
@@ -0,0 +1,35 @@
+import { test, expect } from '../fixtures';
+import { clickButton, generateRandomValidLabel, openDropdown } from '../utils/forms';
+import { openCreateIntegrationModal } from '../utils/integrations';
+import { goToOnCallPage } from '../utils/navigation';
+
+test('New label keys and labels can be created', async ({ adminRolePage }) => {
+  const { page } = adminRolePage;
+  await goToOnCallPage(page, 'integrations');
+  await openCreateIntegrationModal(page);
+  const NEW_LABEL_KEY = generateRandomValidLabel();
+  const NEW_LABEL_VALUE = generateRandomValidLabel();
+
+  await page
+    .getByTestId('create-integration-modal')
+    .getByTestId('integration-display-name')
+    .filter({ hasText: 'Webhook' })
+    .first()
+    .click();
+  await clickButton({ page, buttonText: /^Add Labels$/ });
+
+  await openDropdown({ page, text: /^Select key$/ });
+  await page.keyboard.insertText(NEW_LABEL_KEY);
+  await page.getByText('Hit enter to add').waitFor();
+  await page.keyboard.press('Enter');
+
+  await page.waitForTimeout(1000);
+
+  await openDropdown({ page, text: /^Select value$/ });
+  await page.keyboard.insertText(NEW_LABEL_VALUE);
+  await page.getByText('Hit enter to add').waitFor();
+  await page.keyboard.press('Enter');
+
+  await expect(page.getByText(NEW_LABEL_KEY)).toBeVisible();
+  await expect(page.getByText(NEW_LABEL_VALUE)).toBeVisible();
+});
diff --git a/grafana-plugin/e2e-tests/users/usersActions.test.ts b/grafana-plugin/e2e-tests/users/usersActions.test.ts
index 129f517b..3134f262 100644
--- a/grafana-plugin/e2e-tests/users/usersActions.test.ts
+++ b/grafana-plugin/e2e-tests/users/usersActions.test.ts
@@ -51,14 +51,14 @@ test.describe('Users screen actions', () => {
   });
 
   test('Search updates the table view', async ({ adminRolePage }) => {
-    const { page } = adminRolePage;
+    const { page, userName } = adminRolePage;
     await goToOnCallPage(page, 'users');
 
     await page.waitForTimeout(2000);
 
     const searchInput = page.locator(`[data-testid="search-users"]`);
 
-    await searchInput.fill('oncall');
+    await searchInput.fill(userName);
     await page.waitForTimeout(2000);
 
     const result = page.locator(`[data-testid="users-username"]`);
diff --git a/grafana-plugin/e2e-tests/utils/forms.ts b/grafana-plugin/e2e-tests/utils/forms.ts
index 82aa4d8e..cc53d62b 100644
--- a/grafana-plugin/e2e-tests/utils/forms.ts
+++ b/grafana-plugin/e2e-tests/utils/forms.ts
@@ -1,6 +1,6 @@
 import type { Locator, Page } from '@playwright/test';
 
-import { randomUUID } from 'crypto';
+import { randomInt, randomUUID } from 'crypto';
 
 type SelectorType = 'gSelect' | 'grafanaSelect';
 type SelectDropdownValueArgs = {
@@ -97,6 +97,16 @@ export const selectDropdownValue = async (args: SelectDropdownValueArgs): Promis
 
 export const generateRandomValue = (): string => randomUUID();
 
+export const generateRandomValidLabel = (length = 10) => {
+  const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
+  let result = '';
+  for (let i = 0; i < length; i++) {
+    const randomIndex = randomInt(0, characters.length);
+    result += characters[randomIndex];
+  }
+  return result;
+};
+
 /**
  * wait for the options to appear
  *
@@ -113,3 +123,6 @@ export const selectValuePickerValue = async (
       `div[class*="grafana-select-menu"] >> ${textMatchSelector(optionExactMatch, valuePickerText)}`
     )
   ).click();
+
+export const openDropdown = async ({ page, text }: { page: Page; text: string | RegExp }) =>
+  page.locator('div').filter({ hasText: text }).nth(1).click();
diff --git a/grafana-plugin/playwright.config.ts b/grafana-plugin/playwright.config.ts
index fe52569c..42a37f25 100644
--- a/grafana-plugin/playwright.config.ts
+++ b/grafana-plugin/playwright.config.ts
@@ -1,4 +1,4 @@
-import { PlaywrightTestProject, defineConfig, devices, PlaywrightTestConfig } from '@playwright/test';
+import { PlaywrightTestProject, defineConfig, devices } from '@playwright/test';
 
 import path from 'path';
 /**
@@ -49,7 +49,7 @@ export default defineConfig({
    * to flaky tests.. let's allow 1 retry per test
    */
   retries: 1,
-  workers: 2,
+  workers: '25%', // 25% of logical CPU cores, e.g. for 16 CPU cores it will use 4 workers
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   // reporter: 'html',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */