diff --git a/.github/workflows/helm_release.yml b/.github/workflows/helm_release.yml
deleted file mode 100644
index 63aa4104..00000000
--- a/.github/workflows/helm_release.yml
+++ /dev/null
@@ -1,16 +0,0 @@
-name: Helm Release
-
-"on":
-  push:
-    branches:
-      - main
-
-jobs:
-  call-update-helm-repo:
-    uses: grafana/helm-charts/.github/workflows/update-helm-repo.yaml@main
-    with:
-      charts_dir: helm
-      cr_configfile: helm/cr.yaml
-      ct_configfile: helm/ct.yaml
-    secrets:
-      helm_repo_token: ${{ secrets.GH_HELM_RELEASE }}
diff --git a/.github/workflows/helm_release_pr.yml b/.github/workflows/helm_release_pr.yml
deleted file mode 100644
index 9087a60c..00000000
--- a/.github/workflows/helm_release_pr.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-name: Create PR to release updated oncall Helm chart
-
-"on":
-  push:
-    tags:
-      - "v[0-9]+.[0-9]+.[0-9]+"
-jobs:
-  update-helm-chart-versions:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Define app_version and helm version
-        id: tags
-        run: |
-          # Strip git ref prefix from version
-          APP_VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-          # Strip "v" prefix from tag name
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && HELM_VERSION=$(echo $APP_VERSION | sed -e 's/^v//')
-          echo "::set-output name=app_version::$APP_VERSION"
-          echo "::set-output name=helm_version::$HELM_VERSION"
-
-      - name: Update oncall Helm chart Chart.yaml
-        uses: fjogeleit/yaml-update-action@v0.12.3
-        with:
-          valueFile: "helm/oncall/Chart.yaml"
-          branch: helm-release/${{ steps.tags.outputs.helm_version }}
-          targetBranch: main
-          masterBranchName: main
-          createPR: "true"
-          description: "Merge this PR to `main` branch to start another
-            [github actions job](https://github.com/grafana/oncall/blob/dev/.github/workflows/helm_release.yml)
-            that will release the updated version of the chart
-            (version: ${{ steps.tags.outputs.helm_version }}, appVersion: ${{ steps.tags.outputs.app_version }})
-            into `grafana/helm-charts` helm repository. \n\n
-            This PR was created automatically by this
-            [github action](https://github.com/grafana/oncall/blob/dev/.github/workflows/helm_release_pr.yml)."
-          message: "Release oncall Helm chart ${{ steps.tags.outputs.helm_version }}"
-          changes: |
-            {
-              "version": "${{ steps.tags.outputs.helm_version }}",
-              "appVersion": "${{ steps.tags.outputs.app_version }}"
-            }
diff --git a/.github/workflows/on-release-published.yml b/.github/workflows/on-release-published.yml
index b58151cf..b47fc496 100644
--- a/.github/workflows/on-release-published.yml
+++ b/.github/workflows/on-release-published.yml
@@ -99,17 +99,47 @@ jobs:
         with:
           repo_secrets: |
             GITHUB_API_KEY=github_actions:github-api-key
-      - name: Slice v from the tag
-        id: prepare-version-tag
+      - name: Prepare version tags
+        id: prepare-version-tags
         run: |
-          echo processed-tag="${GITHUB_REF_NAME:1}" >> $GITHUB_OUTPUT
-      - name: Find Helm release pull request
-        uses: juliangruber/find-pull-request-action@v1
-        id: find-pull-request
+          echo app-version="${GITHUB_REF_NAME}" >> $GITHUB_OUTPUT
+          echo version="${GITHUB_REF_NAME:1}" >> $GITHUB_OUTPUT
+      - name: Update oncall Helm chart Chart.yaml
+        id: update-helm-chart-pr
+        uses: fjogeleit/yaml-update-action@v0.12.3
         with:
-          branch: helm-release/${{ steps.prepare-version-tag.outputs.processed-tag }}
+          valueFile: helm/oncall/Chart.yaml
+          branch: helm-release/${{ steps.prepare-version-tags.outputs.version }}
+          targetBranch: main
+          masterBranchName: main
+          createPR: true
+          description: |
+            This PR was created automatically by this
+            [github action](https://github.com/grafana/oncall/blob/dev/.github/workflows/on-release-published.yml).
+
+            It will be auto-merged very soon, which will then release the updated version of the chart
+            (version: ${{ steps.prepare-version-tags.outputs.version }}, appVersion: ${{ steps.prepare-version-tags.outputs.app-version }})
+            into the `grafana/helm-charts` helm repository.
+          message: "Release oncall Helm chart ${{ steps.prepare-version-tags.outputs.version }}"
+          changes: |
+            {
+              "version": "${{ steps.prepare-version-tags.outputs.version }}",
+              "appVersion": "${{ steps.prepare-version-tags.outputs.app-version }}"
+            }
       - name: Merge pull Request
         uses: juliangruber/merge-pull-request-action@v1
         with:
           github-token: ${{ env.GITHUB_API_KEY }}
-          number: ${{ steps.find-pull-request.outputs.number }}
+          number: ${{ fromJSON(steps.update-helm-chart-pr.outputs.pull_request).number }}
+
+  update-helm-repo:
+    name: Update Helm Repo
+    uses: grafana/helm-charts/.github/workflows/update-helm-repo.yaml@main
+    needs:
+      - merge-helm-release-pr
+    with:
+      charts_dir: helm
+      cr_configfile: helm/cr.yaml
+      ct_configfile: helm/ct.yaml
+    secrets:
+      helm_repo_token: ${{ secrets.GH_HELM_RELEASE }}