diff --git a/helm/oncall/Chart.lock b/helm/oncall/Chart.lock new file mode 100644 index 00000000..a91cbcb0 --- /dev/null +++ b/helm/oncall/Chart.lock @@ -0,0 +1,24 @@ +dependencies: +- name: cert-manager + repository: https://charts.jetstack.io + version: v1.8.0 +- name: mariadb + repository: https://charts.bitnami.com/bitnami + version: 11.0.10 +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 11.9.10 +- name: rabbitmq + repository: https://charts.bitnami.com/bitnami + version: 10.1.1 +- name: redis + repository: https://charts.bitnami.com/bitnami + version: 16.10.1 +- name: grafana + repository: https://grafana.github.io/helm-charts + version: 6.29.6 +- name: ingress-nginx + repository: https://kubernetes.github.io/ingress-nginx + version: 4.1.4 +digest: sha256:8e17f2f6a087b6db52670458fc0e1cb39b0a3f7962ff7ebbc7be4c982a4e1720 +generated: "2022-10-18T11:22:39.061819+02:00" diff --git a/helm/oncall/Chart.yaml b/helm/oncall/Chart.yaml index f7708ca6..55564328 100644 --- a/helm/oncall/Chart.yaml +++ b/helm/oncall/Chart.yaml @@ -25,6 +25,10 @@ dependencies: version: 11.0.10 repository: https://charts.bitnami.com/bitnami condition: mariadb.enabled + - name: postgresql + version: 11.9.10 + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled - name: rabbitmq version: 10.1.1 repository: https://charts.bitnami.com/bitnami diff --git a/helm/oncall/README.md b/helm/oncall/README.md index 87a41f0d..d64e6c89 100644 --- a/helm/oncall/README.md +++ b/helm/oncall/README.md @@ -6,23 +6,27 @@ It will also deploy cert manager and nginx ingress controller, as Grafana OnCall to receive alerts from other monitoring systems. Grafana OnCall engine acts as a backend and can be connected to the Grafana frontend plugin named Grafana OnCall. Architecture diagram can be found [here](https://raw.githubusercontent.com/grafana/oncall/dev/docs/img/architecture_diagram.png) +## Production usage -### Production usage **Default helm chart configuration is not intended for production.** The helm chart includes all the services into a single release, which is not recommended for production usage. It is recommended to run stateful services such as MySQL and RabbitMQ separately from this release or use managed PaaS solutions. It will significantly reduce the overhead of managing them. Here are the instructions on how to set up your own [ingress](#set-up-external-access), [MySQL](#connect-external-mysql), [RabbitMQ](#connect-external-rabbitmq), [Redis](#connect-external-redis) - ### Cluster requirements + * ensure you can run x86-64/amd64 workloads. arm64 architecture is currently not supported * kubernetes version 1.25+ is not supported, if cert-manager is enabled ## Install + ### Prepare the repo -``` + +```bash # Add the repository helm repo add grafana https://grafana.github.io/helm-charts helm repo update ``` + ### Installing the helm chart + ```bash # Install the chart helm install \ @@ -34,7 +38,8 @@ helm install \ ``` Follow the `helm install` output to finish setting up Grafana OnCall backend and Grafana OnCall frontend plugin e.g. -``` + +```bash 👋 Your Grafana OnCall instance has been successfully deployed ❗ Set up a DNS record for your domain (use A Record and "@" to point a root domain to the IP address) @@ -73,6 +78,7 @@ Follow the `helm install` output to finish setting up Grafana OnCall backend and ## Configuration You can edit values.yml to make changes to the helm chart configuration and re-deploy the release with the following command: + ```bash helm upgrade \ --install \ @@ -87,7 +93,7 @@ helm upgrade \ You can set up Slack connection via following variables: -``` +```yaml oncall: slack: enabled: true @@ -103,7 +109,7 @@ oncall: To set up Telegram tokem and webhook url use: -``` +```yaml oncall: telegram: enabled: true @@ -112,13 +118,14 @@ oncall: ``` ### Set up external access + Grafana OnCall can be connected to the external monitoring systems or grafana deployed to the other cluster. Nginx Ingress Controller and Cert Manager charts are included in the helm chart with the default configuration. -If you set the DNS A Record pointing to the external IP address of the installation with the Hostname matching base_url parameter, https will be automatically set up. If grafana is enabled in the chart values, it will also be available on https:///grafana/. See the details in `helm install` output. +If you set the DNS A Record pointing to the external IP address of the installation with the Hostname matching base_url parameter, https will be automatically set up. If grafana is enabled in the chart values, it will also be available on `https:///grafana/`. See the details in `helm install` output. To use a different ingress controller or tls certificate management system, set the following values to false and edit ingress settings -``` +```yaml ingress-nginx: enabled: false @@ -132,18 +139,36 @@ ingress: cert-manager.io/issuer: "letsencrypt-prod" ``` +### Use PostgreSQL instead of MySQL + +It is possible to use PostgreSQL instead of MySQL. To do so, set mariadb.enabled to `false`, +postgresql.enabled to `true` and database.type to `postgresql`. + +```yaml +mariadb: + enabled: false + +postgresql: + enabled: true + +database: + type: postgresql +``` + ### Connect external MySQL It is recommended to use the managed MySQL 5.7 database provided by your cloud provider Make sure to create the database with the following parameters before installing this chart -``` + +```sql CREATE DATABASE oncall CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; ``` -To use an external MySQL instance set mysql.enabled to `false` and configure the `externalMysql` parameters. -``` +To use an external MySQL instance set mariadb.enabled to `false` and configure the `externalMysql` parameters. + +```yaml mariadb: - enabled: true + enabled: false # Make sure to create the database with the following parameters: # CREATE DATABASE oncall CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; @@ -155,13 +180,42 @@ externalMysql: password: ``` +### Connect external PostgreSQL + +To use an external PostgreSQL instance set mariadb.enabled to `false`, +postgresql.enabled to `false`, database.type to `postgresql` and configure +the `externalPostgresql` parameters. + +```yaml +mariadb: + enabled: false + +postgresql: + enabled: false + +database: + type: postgresql + +# Make sure to create the database with the following parameters: +# CREATE DATABASE oncall CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +externalPostgresql: + host: + port: + db_name: + user: + password: + existingSecret: "" + passwordKey: password + ``` + ### Connect external RabbitMQ Option 1. Install RabbitMQ separately into the cluster using the [official documentation](https://www.rabbitmq.com/kubernetes/operator/operator-overview.html) Option 2. Use managed solution such as [CloudAMPQ](https://www.cloudamqp.com/) To use an external RabbitMQ instance set rabbitmq.enabled to `false` and configure the `externalRabbitmq` parameters. -``` + +```yaml rabbitmq: enabled: false # Disable the RabbitMQ dependency from the release @@ -175,7 +229,8 @@ externalRabbitmq: ### Connect external Redis To use an external Redis instance set redis.enabled to `false` and configure the `externalRedis` parameters. -``` + +```yaml redis: enabled: false # Disable the Redis dependency from the release @@ -185,7 +240,8 @@ externalRedis: ``` ## Update -```shell + +```bash # Add & upgrade the repository helm repo add grafana https://grafana.github.io/helm-charts helm repo update @@ -203,19 +259,23 @@ helm upgrade \ After re-deploying, please also update the Grafana OnCall plugin on the plugin version page. See [Grafana docs](https://grafana.com/docs/grafana/latest/administration/plugin-management/#update-a-plugin) for more info on updating Grafana plugins. ## Uninstall + ### Uninstalling the helm chart + ```bash helm delete release-oncall ``` ### Clean up PVC's + ```bash kubectl delete pvc data-release-oncall-mariadb-0 data-release-oncall-rabbitmq-0 \ redis-data-release-oncall-redis-master-0 redis-data-release-oncall-redis-replicas-0 \ redis-data-release-oncall-redis-replicas-1 redis-data-release-oncall-redis-replicas-2 ``` - + ### Clean up secrets + ```bash kubectl delete secrets certificate-tls release-oncall-cert-manager-webhook-ca release-oncall-ingress-nginx-admission ``` diff --git a/helm/oncall/charts/postgresql-11.9.10.tgz b/helm/oncall/charts/postgresql-11.9.10.tgz new file mode 100644 index 00000000..2e267653 Binary files /dev/null and b/helm/oncall/charts/postgresql-11.9.10.tgz differ diff --git a/helm/oncall/templates/NOTES.txt b/helm/oncall/templates/NOTES.txt index 675e20a5..d2a3290c 100644 --- a/helm/oncall/templates/NOTES.txt +++ b/helm/oncall/templates/NOTES.txt @@ -37,7 +37,7 @@ Issue the one-time token to connect Grafana OnCall backend and Grafana OnCall plugin by running these commands: export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "oncall.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=engine" -o jsonpath="{.items[0].metadata.name}") - kubectl exec -it $POD_NAME -- bash -c "python manage.py issue_invite_for_the_frontend --override" + kubectl exec -it $POD_NAME --namespace {{ .Release.Namespace }} -- bash -c "python manage.py issue_invite_for_the_frontend --override" Fill the Grafana OnCall Backend URL: diff --git a/helm/oncall/templates/_env.tpl b/helm/oncall/templates/_env.tpl index 25072fc1..93fbc7d8 100644 --- a/helm/oncall/templates/_env.tpl +++ b/helm/oncall/templates/_env.tpl @@ -136,6 +136,74 @@ {{- end -}} {{- end -}} +{{- define "snippet.postgresql.env" -}} +- name: DATABASE_TYPE + value: {{ .Values.database.type }} +- name: DATABASE_HOST + value: {{ include "snippet.postgresql.host" . }} +- name: DATABASE_PORT + value: {{ include "snippet.postgresql.port" . }} +- name: DATABASE_NAME + value: {{ include "snippet.postgresql.db" . }} +- name: DATABASE_USER + value: {{ include "snippet.postgresql.user" . }} +- name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "snippet.postgresql.password.secret.name" . }} + key: {{ include "snippet.postgresql.password.secret.key" . }} +{{- end }} + +{{- define "snippet.postgresql.password.secret.name" -}} +{{- if and (not .Values.postgresql.enabled) .Values.externalPostgresql.password -}} +{{ include "oncall.fullname" . }}-postgresql-external +{{- else if and (not .Values.postgresql.enabled) .Values.externalPostgresql.existingSecret -}} +{{ .Values.externalPostgresql.existingSecret }} +{{- else -}} +{{ include "oncall.postgresql.fullname" . }} +{{- end -}} +{{- end -}} + +{{- define "snippet.postgresql.password.secret.key" -}} +{{- if and (not .Values.postgresql.enabled) .Values.externalPostgresql.passwordKey -}} +{{ .Values.externalPostgresql.passwordKey }} +{{- else -}} +"postgres-password" +{{- end -}} +{{- end -}} + +{{- define "snippet.postgresql.host" -}} +{{- if and (not .Values.postgresql.enabled) .Values.externalPostgresql.host -}} +{{- required "externalPostgresql.host is required if not postgresql.enabled" .Values.externalPostgresql.host | quote }} +{{- else -}} +{{ include "oncall.postgresql.fullname" . }} +{{- end -}} +{{- end -}} + +{{- define "snippet.postgresql.port" -}} +{{- if and (not .Values.mariadb.enabled) .Values.externalPostgresql.port -}} +{{- required "externalPostgresql.port is required if not postgresql.enabled" .Values.externalPostgresql.port | quote }} +{{- else -}} +"5432" +{{- end -}} +{{- end -}} + +{{- define "snippet.postgresql.db" -}} +{{- if and (not .Values.postgresql.enabled) .Values.externalPostgresql.db -}} +{{- required "externalPostgresql.db is required if not postgresql.enabled" .Values.externalPostgresql.db | quote}} +{{- else -}} +"oncall" +{{- end -}} +{{- end -}} + +{{- define "snippet.postgresql.user" -}} +{{- if and (not .Values.postgresql.enabled) .Values.externalPostgresql.user -}} +{{- .Values.externalPostgresql.user | quote}} +{{- else -}} +"postgres" +{{- end -}} +{{- end -}} + {{- define "snippet.rabbitmq.env" -}} - name: RABBITMQ_USERNAME value: {{ include "snippet.rabbitmq.user" . }} diff --git a/helm/oncall/templates/_helpers.tpl b/helm/oncall/templates/_helpers.tpl index 18bcee42..b5f4dbcc 100644 --- a/helm/oncall/templates/_helpers.tpl +++ b/helm/oncall/templates/_helpers.tpl @@ -66,6 +66,11 @@ Create the name of the service account to use {{- printf "%s-%s" .Release.Name "mariadb" | trunc 63 | trimSuffix "-" }} {{- end }} +{{/* Generate the fullname of postgresql subchart */}} +{{- define "oncall.postgresql.fullname" -}} +{{- printf "%s-%s" .Release.Name "postgresql" | trunc 63 | trimSuffix "-" }} +{{- end }} + {{- define "oncall.grafana.fullname" -}} {{- printf "%s-%s" .Release.Name "grafana" | trunc 63 | trimSuffix "-" }} {{- end }} @@ -96,3 +101,20 @@ Create the name of the service account to use {{- toYaml .Values.env | nindent 12 }} {{- end }} {{- end }} + +{{- define "oncall.postgresql.wait-for-db" }} +- name: wait-for-db + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ['sh', '-c', "until (python manage.py migrate --check); do echo Waiting for database migrations; sleep 2; done"] + securityContext: + {{ toYaml .Values.init.securityContext| nindent 4}} + env: + {{- include "snippet.oncall.env" . | nindent 12 }} + {{- include "snippet.postgresql.env" . | nindent 12 }} + {{- include "snippet.rabbitmq.env" . | nindent 12 }} + {{- include "snippet.redis.env" . | nindent 12 }} + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} +{{- end }} diff --git a/helm/oncall/templates/celery/_deployment.tpl b/helm/oncall/templates/celery/_deployment.tpl index 0892c0cb..e25d4adc 100644 --- a/helm/oncall/templates/celery/_deployment.tpl +++ b/helm/oncall/templates/celery/_deployment.tpl @@ -28,7 +28,12 @@ spec: securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} initContainers: + {{- if eq .Values.database.type "mysql" }} {{- include "oncall.mariadb.wait-for-db" . | indent 8 }} + {{- end }} + {{- if eq .Values.database.type "postgresql" }} + {{- include "oncall.postgresql.wait-for-db" . | indent 8 }} + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: @@ -42,7 +47,12 @@ spec: {{- include "snippet.oncall.slack.env" . | nindent 12 }} {{- include "snippet.oncall.telegram.env" . | nindent 12 }} {{- include "snippet.oncall.smtp.env" . | nindent 12 }} + {{- if eq .Values.database.type "mysql" }} {{- include "snippet.mysql.env" . | nindent 12 }} + {{- end }} + {{- if eq .Values.database.type "postgresql" }} + {{- include "snippet.postgresql.env" . | nindent 12 }} + {{- end }} {{- include "snippet.rabbitmq.env" . | nindent 12 }} {{- include "snippet.redis.env" . | nindent 12 }} {{- if .Values.env }} diff --git a/helm/oncall/templates/engine/deployment.yaml b/helm/oncall/templates/engine/deployment.yaml index 5b1f7776..b06462e4 100644 --- a/helm/oncall/templates/engine/deployment.yaml +++ b/helm/oncall/templates/engine/deployment.yaml @@ -32,7 +32,12 @@ spec: securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} initContainers: + {{- if eq .Values.database.type "mysql" }} {{- include "oncall.mariadb.wait-for-db" . | indent 8 }} + {{- end }} + {{- if eq .Values.database.type "postgresql" }} + {{- include "oncall.postgresql.wait-for-db" . | indent 8 }} + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: @@ -48,7 +53,12 @@ spec: {{- include "snippet.oncall.slack.env" . | nindent 12 }} {{- include "snippet.oncall.telegram.env" . | nindent 12 }} {{- include "snippet.oncall.smtp.env" . | nindent 12 }} + {{- if eq .Values.database.type "mysql" }} {{- include "snippet.mysql.env" . | nindent 12 }} + {{- end }} + {{- if eq .Values.database.type "postgresql" }} + {{- include "snippet.postgresql.env" . | nindent 12 }} + {{- end }} {{- include "snippet.rabbitmq.env" . | nindent 12 }} {{- include "snippet.redis.env" . | nindent 12 }} {{- if .Values.env }} diff --git a/helm/oncall/templates/engine/job-migrate.yaml b/helm/oncall/templates/engine/job-migrate.yaml index 8778f420..86fcbaa9 100644 --- a/helm/oncall/templates/engine/job-migrate.yaml +++ b/helm/oncall/templates/engine/job-migrate.yaml @@ -35,16 +35,30 @@ spec: command: - /bin/sh - -c + {{- if eq .Values.database.type "mysql" }} - | until (nc -vz $MYSQL_HOST $MYSQL_PORT); do echo "waiting for MySQL"; sleep 1; done python manage.py migrate + {{- else if eq .Values.database.type "postgresql" }} + - | + until (nc -vz $DATABASE_HOST $DATABASE_PORT); + do + echo "waiting for PostgreSQL"; sleep 1; + done + python manage.py migrate + {{- end }} env: {{- include "snippet.oncall.env" . | nindent 12 }} {{- include "snippet.oncall.smtp.env" . | nindent 12 }} + {{- if eq .Values.database.type "mysql" }} {{- include "snippet.mysql.env" . | nindent 12 }} + {{- end }} + {{- if eq .Values.database.type "postgresql" }} + {{- include "snippet.postgresql.env" . | nindent 12 }} + {{- end }} {{- include "snippet.rabbitmq.env" . | nindent 12 }} {{- include "snippet.redis.env" . | nindent 12 }} {{- if .Values.env }} diff --git a/helm/oncall/templates/secrets.yaml b/helm/oncall/templates/secrets.yaml index 64d1e22c..39e48a00 100644 --- a/helm/oncall/templates/secrets.yaml +++ b/helm/oncall/templates/secrets.yaml @@ -11,7 +11,7 @@ data: MIRAGE_CIPHER_IV: {{ randAlphaNum 40 | b64enc | quote }} --- -{{ if not .Values.mariadb.enabled -}} +{{ if and (not .Values.mariadb.enabled) (eq .Values.database.type "mysql") -}} apiVersion: v1 kind: Secret metadata: @@ -50,3 +50,13 @@ type: Opaque data: smtp-password: {{ required "oncall.smtp.password is required if oncall.smtp.enabled" .Values.oncall.smtp.password | b64enc | quote }} {{- end }} +--- +{{ if and (not .Values.postgresql.enabled) (eq .Values.database.type "postgresql") (not .Values.externalPostgresql.existingSecret) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "oncall.fullname" . }}-postgresql-external +type: Opaque +data: + postgres-password: {{ required "externalPostgresql.password is required if not postgresql.enabled and not externalPostgresql.existingSecret" .Values.externalPostgresql.password | b64enc | quote }} +{{- end }} diff --git a/helm/oncall/values.yaml b/helm/oncall/values.yaml index db665bd6..5939a9e1 100644 --- a/helm/oncall/values.yaml +++ b/helm/oncall/values.yaml @@ -111,7 +111,7 @@ ingress: annotations: kubernetes.io/ingress.class: "nginx" cert-manager.io/issuer: "letsencrypt-prod" - tls: + tls: - hosts: - "{{ .Values.base_url }}" secretName: certificate-tls @@ -153,6 +153,10 @@ cert-manager: - 8.8.8.8 - 1.1.1.1 +database: + # can be either mysql or postgresql + type: mysql + # MySQL is included into this release for the convenience. # It is recommended to host it separately from this release # Set mariadb.enabled = false and configure externalMysql @@ -182,6 +186,27 @@ externalMysql: user: password: +# PostgreSQL is included into this release for the convenience. +# It is recommended to host it separately from this release +# Set postgresql.enabled = false and configure externalPostgresql +postgresql: + enabled: false + auth: + database: oncall + +# Make sure to create the database with the following parameters: +# CREATE DATABASE oncall CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +externalPostgresql: + host: + port: + db_name: + user: + password: + # use an existing secret for the database password + existingSecret: "" + # the key in the secret containing the database password + passwordKey: password + # RabbitMQ is included into this release for the convenience. # It is recommended to host it separately from this release # Set rabbitmq.enabled = false and configure externalRabbitmq