diff --git a/helm/oncall/Chart.yaml b/helm/oncall/Chart.yaml index 75cb31bf..3f2cf5dd 100644 --- a/helm/oncall/Chart.yaml +++ b/helm/oncall/Chart.yaml @@ -23,6 +23,11 @@ version: 0.1.0 # It is recommended to use it with quotes. appVersion: "1.16.0" dependencies: + - name: cert-manager + version: v1.8.0 + repository: https://charts.jetstack.io + condition: cert-manager.enabled + - name: mariadb version: 11.0.10 repository: https://charts.bitnami.com/bitnami @@ -39,3 +44,7 @@ dependencies: version: 6.29.6 repository: https://grafana.github.io/helm-charts condition: grafana.enabled + - name: ingress-nginx + version: 4.1.4 + repository: https://kubernetes.github.io/ingress-nginx + condition: ingress-nginx.enabled \ No newline at end of file diff --git a/helm/oncall/QUICKSTART.md b/helm/oncall/QUICKSTART.md new file mode 100644 index 00000000..6576bd3f --- /dev/null +++ b/helm/oncall/QUICKSTART.md @@ -0,0 +1,50 @@ +Quick Start + +Cluster requirements: +* ensure you can run x86-64/amd64 workloads. arm64 architecture is currently not supported + + +NOTE: + +Default chart places the stateful services into the current installation into the cluster. +This services are provided for the convenience and are not intended for production. +They need to be properly managed, maintained and backed up. +We recommend to run stateful applications, such as MySql and RabbitMQ separately or use managed solutions +as grafana does in Grafana Cloud. +https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/doc/installation/index.md + +Prerequisites: +Tools: +* kubectl v1.22 +* helm v3 + +Infrastructure: +* kubernetes cluster. Recomended resources: X vcpu and Y Gb of RAM +Stateful services are recommended outside of the cluster using managed solutions or compute nodes + 1. MySQL 5.7 database +We recommend using + 2. Rabbitmq + + + +1. Prepare the chart values + +2. Install the chart + +3. Finish the configuration + +3.1. Get the external ip address + +3.2. Set up the DNS +The external IP that is allocated to the ingress-controller is the IP to which all incoming traffic should be routed. To enable this, add it to a DNS zone you control, for example as www.example.com. +This quick-start assumes you know how to assign a DNS entry to an IP address and will do so. + +3.3. Open Grafana and connect Grafana OnCall plugin to Grafana OnCall Backend + + +Troubleshooting: +Error: failed post-install: warning: Hook post-install oncall/templates/cert-issuer.yaml failed: Internal error occurred: failed calling webhook "webhook.cert-manager.io": failed to call webhook: Post "https://oncall-ildar-cert-manager-webhook.default.svc:443/mutate?timeout=30s": no endpoints available for service "oncall-ildar-cert-manager-webhook" +Upgrade the release + +Error: failed post-install: warning: Hook post-install oncall/templates/cert-issuer.yaml failed: Internal error occurred: failed calling webhook "webhook.cert-manager.io": failed to call webhook: Post "https://oncall-ildar-cert-manager-webhook.default.svc:443/mutate?timeout=30s": no endpoints available for service "oncall-ildar-cert-manager-webhook" + diff --git a/helm/oncall/README.md b/helm/oncall/README.md new file mode 100644 index 00000000..bbec683f --- /dev/null +++ b/helm/oncall/README.md @@ -0,0 +1,73 @@ +# oncall + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) + +A Helm chart for Kubernetes + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | mariadb | 11.0.10 | +| https://charts.bitnami.com/bitnami | rabbitmq | 10.1.1 | +| https://charts.bitnami.com/bitnami | redis | 16.10.1 | +| https://charts.jetstack.io | cert-manager | v1.8.0 | +| https://grafana.github.io/helm-charts | grafana | 6.29.6 | +| https://helm.nginx.com/stable | nginx-ingress | 0.13.2 | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| base_url | string | `"ildari.me"` | | +| celery.replicaCount | int | `1` | | +| celery.resources | object | `{}` | | +| cert-manager.enabled | bool | `true` | | +| cert-manager.installCRDs | bool | `true` | | +| cert-manager.webhook.securePort | int | `10260` | | +| cert-manager.webhook.timeoutSeconds | int | `30` | | +| engine.replicaCount | int | `1` | | +| engine.resources | object | `{}` | | +| env | list | `[]` | | +| externalMysql.db_name | string | `nil` | | +| externalMysql.host | string | `nil` | | +| externalMysql.password | string | `nil` | | +| externalMysql.port | string | `nil` | | +| externalMysql.user | string | `nil` | | +| externalRabbitmq.host | string | `nil` | | +| externalRabbitmq.password | string | `nil` | | +| externalRabbitmq.port | string | `nil` | | +| externalRabbitmq.user | string | `nil` | | +| external_redis.host | string | `nil` | | +| external_redis.password | string | `nil` | | +| fullnameOverride | string | `""` | | +| grafana."grafana.ini".server.domain | string | `"example.com"` | | +| grafana."grafana.ini".server.root_url | string | `"%(protocol)s://%(domain)s/grafana"` | | +| grafana."grafana.ini".server.serve_from_sub_path | bool | `true` | | +| grafana.enabled | bool | `true` | | +| grafana.persistence.enabled | bool | `true` | | +| grafana.plugins[0] | string | `"grafana-oncall-app"` | | +| ildar.enabled | bool | `true` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"registry.digitalocean.com/ildar-testing/hobby-oncall-2"` | | +| image.tag | string | `"latest"` | | +| imagePullSecrets[0].name | string | `"registry-ildar-testing"` | | +| ingress.enabled | bool | `true` | | +| mariadb.auth.database | string | `"oncall"` | | +| mariadb.enabled | bool | `true` | | +| nameOverride | string | `""` | | +| nginx-ingress.enabled | bool | `true` | | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| rabbitmq.enabled | bool | `true` | | +| redis.enabled | bool | `true` | | +| securityContext | object | `{}` | | +| service.enabled | bool | `false` | | +| service.port | int | `8080` | | +| service.type | string | `"LoadBalancer"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.10.0](https://github.com/norwoodj/helm-docs/releases/v1.10.0) diff --git a/helm/oncall/charts/cert-manager-v1.8.0.tgz b/helm/oncall/charts/cert-manager-v1.8.0.tgz new file mode 100644 index 00000000..c04b5b79 Binary files /dev/null and b/helm/oncall/charts/cert-manager-v1.8.0.tgz differ diff --git a/helm/oncall/charts/ingress-nginx-4.1.4.tgz b/helm/oncall/charts/ingress-nginx-4.1.4.tgz new file mode 100644 index 00000000..2c999e7a Binary files /dev/null and b/helm/oncall/charts/ingress-nginx-4.1.4.tgz differ diff --git a/helm/oncall/templates/NOTES.txt b/helm/oncall/templates/NOTES.txt index 1b63be87..6d6ce177 100644 --- a/helm/oncall/templates/NOTES.txt +++ b/helm/oncall/templates/NOTES.txt @@ -1,71 +1,36 @@ -👋 Your Grafana OnCall instance has been successfully deployed and a few steps left to finish the configuration +👋 Your Grafana OnCall instance has been successfully deployed -1. Get the Grafana OnCall backend URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "oncall.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - export GRAFANA_ONCALL_BE_URL=http://$NODE_IP:$NODE_PORT - echo $GRAFANA_ONCALL_BE_URL -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available - You can watch the status of by running the following command - kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "oncall.engine.fullname" . }} - Wait until LoadBalancer is ready (EXTERNAL-IP exists) and run the following commands - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "oncall.engine.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - export GRAFANA_ONCALL_BE_URL=http://$SERVICE_IP:{{ .Values.service.port }} - echo $GRAFANA_ONCALL_BE_URL -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "oncall.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=engine" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - export GRAFANA_ONCALL_BE_URL=http://{{ include "oncall.grafana.fullname" . }}:$CONTAINER_PORT - - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} - -2. Migrate the database by running these commands: - NOTE: Until then grafana oncall containers will remain with Status Init:0/1. +A few steps left to finish the configuration, Copy-paste this these command to get the instructions: export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "oncall.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=engine" -o jsonpath="{.items[0].metadata.name}") kubectl exec -it $POD_NAME -c wait-for-db -- bash -c "python manage.py migrate;" + echo Database was successfully migrated 👍 -3. Issue the token to connect Grafana OnCall backend and Grafana OnCall plugin by running this command + echo Issuing the token to connect Grafana OnCall backend and Grafana OnCall plugin kubectl exec -it $POD_NAME -- bash -c "python manage.py issue_invite_for_the_frontend --override" -4. Open Grafana in your browser and Enable Grafana OnCall plugin there + echo Open Grafana in your browser and Enable Grafana OnCall plugin there {{- if .Values.grafana.enabled }} - Grafana was installed as a part of this helm release. We will need to connect Grafana OnCall plugin and Grafana OnCall backend. - Get the access to grafana by running the following commands + echo Grafana was installed as a part of this helm release. + echo We will need to connect Grafana OnCall plugin and Grafana OnCall backend. - 3.1. Open a new terminal session and run the following command - kubectl --namespace {{ .Release.Namespace }} port-forward svc/{{ include "oncall.grafana.fullname" . }} 8083:80 - - 3.2 Open http://127.0.0.1:8083 to access Grafana from your browser - - 3.3 Use the username '{{ .Values.grafana.adminUser }}' and get the password by running the following command - kubectl get secret --namespace {{ .Release.Namespace }} {{ template "oncall.grafana.fullname" . }} -o jsonpath="{.data.admin-password}" | base64 --decode ; echo - - 3.4. Within the cluster Grafana is available on: http://{{ include "oncall.grafana.fullname" . }}:3000. You can use this address to connect Grafana OnCall backend + echo Open https://{{ .Values.base_url }}/grafana/plugins/grafana-oncall-app + echo Username: {{ .Values.grafana.adminUser }} + echo Password $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "oncall.grafana.fullname" . }} -o jsonpath="{.data.admin-password}" | base64 --decode ; echo) {{- else }} - Grafana was not installed as a part of this helm release. - Make sure you have external Grafana that is available by the network for the containers installed by this release. + echo Grafana was not installed as a part of this helm release. Open your own Grafana in the browser. + echo Make sure your external Grafana is available by the network for the containers installed by this release. {{- end }} -5. Configure Grafana OnCall plugin to work with Grafana OnCall backend - 4.1. Open Grafana, go to "Configuration" -> "Plugins" and find Grafana OnCall plugin - 4.2. Fill the "Invite token" issued on step 3 - NOTE: you can re-issue the token by running this command: kubectl exec -it $POD_NAME -- bash -c "python manage.py issue_invite_for_the_frontend --override" - 4.3. Fill the Grafana OnCall Backend URL from Step 1 - NOTE: Run this command to get it one more time: - echo $GRAFANA_ONCALL_BE_URL - 4.4. Fill the Grafana URL from step 4. - NOTE: this URL should be accessible by Grafana OnCall Backend container{{ if .Values.grafana.enabled }}: http://{{ include "oncall.grafana.fullname" . }}:3000{{- end }} +4. Configure Grafana OnCall plugin to work with Grafana OnCall backend + echo Open Grafana, go to "Configuration" -> "Plugins" and find Grafana OnCall plugin + echo Fill the "Invite token" issued on step 2 + echo NOTE: you can re-issue the token by running this command: kubectl exec -it $POD_NAME -- bash -c "python manage.py issue_invite_for_the_frontend --override" + echo Fill the Grafana OnCall Backend URL: http://{{ include "oncall.engine.fullname" . }}:8080 + echo Fill the Grafana URL from step 3{{ if .Values.grafana.enabled }}: http://{{ include "oncall.grafana.fullname" . }}{{- end }} + echo NOTE: this URL should be accessible by Grafana OnCall Backend container -🎉🎉🎉 Done! 🎉🎉🎉 \ No newline at end of file + + +echo 🎉🎉🎉 Done! 🎉🎉🎉 diff --git a/helm/oncall/templates/_env.tpl b/helm/oncall/templates/_env.tpl index 6c964813..aafe05e1 100644 --- a/helm/oncall/templates/_env.tpl +++ b/helm/oncall/templates/_env.tpl @@ -1,4 +1,6 @@ {{- define "snippet.oncall.env" -}} +- name: BASE_URL + value: {{ .Values.base_url | quote }} - name: SECRET_KEY valueFrom: secretKeyRef: diff --git a/helm/oncall/templates/cert-issuer.yaml b/helm/oncall/templates/cert-issuer.yaml new file mode 100644 index 00000000..8cad693b --- /dev/null +++ b/helm/oncall/templates/cert-issuer.yaml @@ -0,0 +1,22 @@ +{{- if .Values.ildar.enabled }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: letsencrypt-prod + annotations: + "helm.sh/hook": post-install,post-upgrade +spec: + acme: + # The ACME server URL + server: https://acme-v02.api.letsencrypt.org/directory + # Email address used for ACME registration + email: no-reply@{{ .Values.base_url }} + # Name of a secret used to store the ACME account private key + privateKeySecretRef: + name: letsencrypt-prod + # Enable the HTTP-01 challenge provider + solvers: + - http01: + ingress: + class: nginx +{{- end }} diff --git a/helm/oncall/templates/engine/ingress.yaml b/helm/oncall/templates/engine/ingress.yaml deleted file mode 100644 index 4d81f19e..00000000 --- a/helm/oncall/templates/engine/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "oncall.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "oncall.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/oncall/templates/engine/service-external.yaml b/helm/oncall/templates/engine/service-external.yaml new file mode 100644 index 00000000..9c204a24 --- /dev/null +++ b/helm/oncall/templates/engine/service-external.yaml @@ -0,0 +1,17 @@ +{{- if .Values.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "oncall.engine.fullname" . }}-external + labels: + {{- include "oncall.engine.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "oncall.engine.selectorLabels" . | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/helm/oncall/templates/engine/service.yaml b/helm/oncall/templates/engine/service-internal.yaml similarity index 80% rename from helm/oncall/templates/engine/service.yaml rename to helm/oncall/templates/engine/service-internal.yaml index 49787cd5..07785035 100644 --- a/helm/oncall/templates/engine/service.yaml +++ b/helm/oncall/templates/engine/service-internal.yaml @@ -5,9 +5,9 @@ metadata: labels: {{- include "oncall.engine.labels" . | nindent 4 }} spec: - type: {{ .Values.service.type }} + type: ClusterIP ports: - - port: {{ .Values.service.port }} + - port: 8080 targetPort: http protocol: TCP name: http diff --git a/helm/oncall/templates/ingress-regular.yaml b/helm/oncall/templates/ingress-regular.yaml new file mode 100644 index 00000000..d98519ac --- /dev/null +++ b/helm/oncall/templates/ingress-regular.yaml @@ -0,0 +1,34 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "oncall.engine.fullname" . }} + labels: + {{- include "oncall.labels" . | nindent 4 }} + annotations: + kubernetes.io/ingress.class: "nginx" + cert-manager.io/issuer: "letsencrypt-prod" +spec: + tls: + - hosts: + - {{ .Values.base_url | quote }} + secretName: quickstart-example-tls + rules: + - host: {{ .Values.base_url | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "oncall.engine.fullname" . }} + port: + number: 8080 + - path: /grafana + pathType: Prefix + backend: + service: + name: {{ include "oncall.grafana.fullname" . }} + port: + number: 80 +{{- end }} diff --git a/helm/oncall/values.yaml b/helm/oncall/values.yaml index 8affaf0d..93d750d2 100644 --- a/helm/oncall/values.yaml +++ b/helm/oncall/values.yaml @@ -1,4 +1,8 @@ -# Grafana OnCall image used by engine and celery +# Default values for Grafana OnCall + +# i.e. example.com +base_url: ildari.me + image: # TODO: use our public repo repository: registry.digitalocean.com/ildar-testing/hobby-oncall-2 @@ -8,10 +12,7 @@ image: # TODO: remove this after we use public image imagePullSecrets: - - name: "ildar-testing" - -nameOverride: "" -fullnameOverride: "" + - name: "registry-ildar-testing" engine: replicaCount: 1 @@ -33,52 +34,40 @@ celery: # cpu: 100m # memory: 128Mi -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -env: - - name: BASE_URL - value: "localhost:8000" +env: [] service: + enabled: false type: LoadBalancer -# type: ClusterIP port: 8080 ingress: - enabled: false - className: "" - annotations: {} -# kubernetes.io/ingress.class: nginx -# kubernetes.io/tls-acme: "true" - hosts: [] -# - host: oncall.example.com -# paths: -# - path: / -# pathType: ImplementationSpecific - tls: # [] -# - secretName: oncall-example-com-tls -# hosts: -# - oncall.example.com + enabled: true +# className: "" +# annotations: {} +## kubernetes.io/ingress.class: nginx +## kubernetes.io/tls-acme: "true" +# hosts: [] +## - host: oncall.example.com +## paths: +## - path: / +## pathType: ImplementationSpecific +# tls: # [] +## - secretName: oncall-example-com-tls +## hosts: +## - oncall.example.com + +nginx-ingress: + enabled: true + +cert-manager: + enabled: true + installCRDs: true + webhook: + timeoutSeconds: 30 + # cert-manager tries to use the already used port, changing to another one + # https://github.com/cert-manager/cert-manager/issues/3237 + securePort: 10260 # Additional services configuration # We recommend using external services to reduce the overhead of managing statefule services @@ -112,7 +101,41 @@ external_redis: grafana: enabled: true + grafana.ini: + server: + domain: example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true persistence: enabled: true plugins: - grafana-oncall-app + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + +ildar: + enabled: true \ No newline at end of file