diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index fcc9e030..675d11ec 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -150,7 +150,7 @@ jobs: if: inputs.run-expensive-tests shell: bash env: - E2E_TESTS_CMD: "cd grafana-plugin && yarn test:e2e-expensive" + E2E_TESTS_CMD: "cd ../../grafana-plugin && yarn test:e2e-expensive" GRAFANA_VERSION: ${{ inputs.grafana_version }} GRAFANA_ADMIN_USERNAME: "irm" GRAFANA_ADMIN_PASSWORD: "irm" diff --git a/.tilt/backend/Tiltfile b/.tilt/backend/Tiltfile new file mode 100644 index 00000000..52493436 --- /dev/null +++ b/.tilt/backend/Tiltfile @@ -0,0 +1,16 @@ +label = "OnCall.Backend" + +k8s_resource( + workload="celery", + resource_deps=["mariadb", "redis-master"], + labels=[label], +) + +k8s_resource( + workload="engine", + port_forwards=8080, + resource_deps=["mariadb", "redis-master"], + labels=[label], +) + +k8s_resource(workload="engine-migrate", labels=[label]) \ No newline at end of file diff --git a/.tilt/deps/Tiltfile b/.tilt/deps/Tiltfile new file mode 100644 index 00000000..acd4e44e --- /dev/null +++ b/.tilt/deps/Tiltfile @@ -0,0 +1,11 @@ +label = "OnCall.Deps" + +k8s_resource(workload="redis-master", labels=[label]) + +k8s_resource(workload="prometheus-server", labels=[label]) + +k8s_resource( + workload="mariadb", + port_forwards='3307:3306', # : + labels=[label], +) \ No newline at end of file diff --git a/.tilt/plugin/Tiltfile b/.tilt/plugin/Tiltfile new file mode 100644 index 00000000..858ecffd --- /dev/null +++ b/.tilt/plugin/Tiltfile @@ -0,0 +1,26 @@ +label = "OnCall.Plugin" + +is_ci=config.tilt_subcommand == "ci" +grafana_plugin_dir="../../grafana-plugin" + +# On CI dependencies are installed separately so we just build prod bundle to be consumed by Grafana dev server +if is_ci: + local_resource( + "build-ui", + labels=[label], + dir=grafana_plugin_dir, + cmd="yarn build", + allow_parallel=True, + ) + +# Locally we install dependencies and we run watch mode +if not is_ci: + local_resource( + "build-ui", + labels=[label], + dir=grafana_plugin_dir, + cmd="yarn install", + serve_dir=grafana_plugin_dir, + serve_cmd="yarn watch", + allow_parallel=True, + ) \ No newline at end of file diff --git a/.tilt/tests/Tiltfile b/.tilt/tests/Tiltfile new file mode 100644 index 00000000..7fec693b --- /dev/null +++ b/.tilt/tests/Tiltfile @@ -0,0 +1,80 @@ +label = "OnCall.AllTests" + +load('ext://uibutton', 'cmd_button', 'location', 'text_input', 'bool_input') + +e2e_tests_cmd=os.getenv("E2E_TESTS_CMD", "cd ../../grafana-plugin && yarn test:e2e") +is_ci=config.tilt_subcommand == "ci" + +local_resource( + "e2e-tests", + labels=[label], + cmd=e2e_tests_cmd, + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=is_ci, + resource_deps=["build-ui", "grafana", "grafana-oncall-app-provisioning-configmap", "engine", "celery"] +) + +cmd_button( + name="E2E Tests - headless run", + argv=["sh", "-c", "yarn --cwd ./grafana-plugin test:e2e $STOP_ON_FIRST_FAILURE $TESTS_FILTER"], + text="Restart headless run", + resource="e2e-tests", + icon_name="replay", + inputs=[ + text_input("BROWSERS", "Browsers (e.g. \"chromium,firefox,webkit\")", "chromium", "chromium,firefox,webkit"), + text_input("TESTS_FILTER", "Test filter (e.g. \"timezones.test quality.test\")", "", "Test file names to run"), + bool_input("STOP_ON_FIRST_FAILURE", "Stop on first failure", True, "-x", ""), + ] +) + +cmd_button( + name="E2E Tests - open watch mode", + argv=["sh", "-c", "yarn --cwd grafana-plugin test:e2e:watch"], + text="Open watch mode", + resource="e2e-tests", + icon_name="visibility", +) + +cmd_button( + name="E2E Tests - show report", + argv=["sh", "-c", "yarn --cwd grafana-plugin playwright show-report"], + text="Show last HTML report", + resource="e2e-tests", + icon_name="assignment", +) + +cmd_button( + name="E2E Tests - stop current run", + argv=["sh", "-c", "kill -9 $(pgrep -f test:e2e)"], + text="Stop", + resource="e2e-tests", + icon_name="dangerous", +) + +# Inspired by https://github.com/grafana/slo/blob/main/Tiltfile#L72 +pod_engine_pytest_script = ''' +set -eu +# get engine k8s pod name from tilt resource name +POD_NAME="$(tilt get kubernetesdiscovery "engine" -ojsonpath='{.status.pods[0].name}')" +kubectl exec "$POD_NAME" -- pytest . $STOP_ON_FIRST_FAILURE $TESTS_FILTER +''' +local_resource( + "pytest-tests", + labels=[label], + cmd=['sh', '-c', pod_engine_pytest_script], + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + resource_deps=["engine"] +) + +cmd_button( + name="pytest Tests - headless run", + argv=['sh', '-c', pod_engine_pytest_script], + text="Run pytest", + resource="pytest-tests", + icon_name="replay", + inputs=[ + text_input("TESTS_FILTER", "pytest optional arguments (e.g. \"apps/webhooks/tests/test_webhook.py::test_build_url_private_raises\")", "", "Test file names to run"), + bool_input("STOP_ON_FIRST_FAILURE", "Stop on first failure", True, "-x", ""), + ] +) \ No newline at end of file diff --git a/Tiltfile b/Tiltfile index df0e53fa..ab4a1f64 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,9 +1,7 @@ load('ext://uibutton', 'cmd_button', 'location', 'text_input', 'bool_input') +load("ext://configmap", "configmap_create") + running_under_parent_tiltfile = os.getenv("TILT_PARENT", "false") == "true" -# The user/pass that you will login to Grafana with -grafana_admin_user_pass = os.getenv("GRAFANA_ADMIN_USER_PASS", "oncall") -grafana_version = os.getenv("GRAFANA_VERSION", "latest") -e2e_tests_cmd=os.getenv("E2E_TESTS_CMD", "cd grafana-plugin && yarn test:e2e") twilio_values=[ "oncall.twilio.accountSid=" + os.getenv("TWILIO_ACCOUNT_SID", ""), "oncall.twilio.authToken=" + os.getenv("TWILIO_AUTH_TOKEN", ""), @@ -16,25 +14,11 @@ HELM_PREFIX = "oncall-dev" # Use docker registery generated by ctlptl (dev/kind-config.yaml) DOCKER_REGISTRY = "localhost:63628/" -if not running_under_parent_tiltfile: - # Load the custom Grafana extensions - v1alpha1.extension_repo( - name="grafana-tilt-extensions", - ref="v1.2.0", - url="https://github.com/grafana/tilt-extensions", - ) -v1alpha1.extension( - name="grafana", repo_name="grafana-tilt-extensions", repo_path="grafana" -) - -load("ext://grafana", "grafana") -load("ext://configmap", "configmap_create") load("ext://docker_build_sub", "docker_build_sub") # Tell ops-devenv/Tiltifle where our plugin.json file lives plugin_file = os.path.abspath("grafana-plugin/src/plugin.json") - def plugin_json(): return plugin_file @@ -59,108 +43,15 @@ docker_build_sub( ], ) -# On CI dependencies are installed separately so we just build prod bundle to be consumed by Grafana dev server -if is_ci: - local_resource( - "build-ui", - labels=["OnCallUI"], - dir="grafana-plugin", - cmd="yarn build", - allow_parallel=True, - ) -# Locally we install dependencies and we run watch mode -if not is_ci: - local_resource( - "build-ui", - labels=["OnCallUI"], - dir="grafana-plugin", - cmd="yarn install", - serve_dir="grafana-plugin", - serve_cmd="yarn watch", - allow_parallel=True, - ) +def load_oncall_helm(): + helm_oncall_values = ["./dev/helm-local.yml", "./dev/helm-local.dev.yml"] + if is_ci: + helm_oncall_values = helm_oncall_values + ["./.github/helm-ci.yml"] + yaml = helm("helm/oncall", name=HELM_PREFIX, values=helm_oncall_values, set=twilio_values, namespace="default") + k8s_yaml(yaml) -local_resource( - "e2e-tests", - labels=["allTests"], - cmd=e2e_tests_cmd, - trigger_mode=TRIGGER_MODE_MANUAL, - auto_init=is_ci, - resource_deps=["build-ui", "grafana", "grafana-oncall-app-provisioning-configmap", "engine", "celery"] -) - -cmd_button( - name="E2E Tests - headless run", - argv=["sh", "-c", "yarn --cwd ./grafana-plugin test:e2e $STOP_ON_FIRST_FAILURE $TESTS_FILTER"], - text="Restart headless run", - resource="e2e-tests", - icon_name="replay", - inputs=[ - text_input("BROWSERS", "Browsers (e.g. \"chromium,firefox,webkit\")", "chromium", "chromium,firefox,webkit"), - text_input("TESTS_FILTER", "Test filter (e.g. \"timezones.test quality.test\")", "", "Test file names to run"), - bool_input("STOP_ON_FIRST_FAILURE", "Stop on first failure", True, "-x", ""), - ] -) - -cmd_button( - name="E2E Tests - open watch mode", - argv=["sh", "-c", "yarn --cwd grafana-plugin test:e2e:watch"], - text="Open watch mode", - resource="e2e-tests", - icon_name="visibility", -) - -cmd_button( - name="E2E Tests - show report", - argv=["sh", "-c", "yarn --cwd grafana-plugin playwright show-report"], - text="Show last HTML report", - resource="e2e-tests", - icon_name="assignment", -) - -cmd_button( - name="E2E Tests - stop current run", - argv=["sh", "-c", "kill -9 $(pgrep -f test:e2e)"], - text="Stop", - resource="e2e-tests", - icon_name="dangerous", -) - -# Inspired by https://github.com/grafana/slo/blob/main/Tiltfile#L72 -pod_engine_pytest_script = ''' -set -eu -# get engine k8s pod name from tilt resource name -POD_NAME="$(tilt get kubernetesdiscovery "engine" -ojsonpath='{.status.pods[0].name}')" -kubectl exec "$POD_NAME" -- pytest . $STOP_ON_FIRST_FAILURE $TESTS_FILTER -''' -local_resource( - "pytest-tests", - labels=["allTests"], - cmd=['sh', '-c', pod_engine_pytest_script], - trigger_mode=TRIGGER_MODE_MANUAL, - auto_init=False, - resource_deps=["engine"] -) - -cmd_button( - name="pytest Tests - headless run", - argv=['sh', '-c', pod_engine_pytest_script], - text="Run pytest", - resource="pytest-tests", - icon_name="replay", - inputs=[ - text_input("TESTS_FILTER", "pytest optional arguments (e.g. \"apps/webhooks/tests/test_webhook.py::test_build_url_private_raises\")", "", "Test file names to run"), - bool_input("STOP_ON_FIRST_FAILURE", "Stop on first failure", True, "-x", ""), - ] -) - -helm_oncall_values = ["./dev/helm-local.yml", "./dev/helm-local.dev.yml"] -if is_ci: - helm_oncall_values = helm_oncall_values + ["./.github/helm-ci.yml"] -yaml = helm("helm/oncall", name=HELM_PREFIX, values=helm_oncall_values, set=twilio_values, namespace="default") - -k8s_yaml(yaml) +# --- GRAFANA START ---- # Generate and load the grafana deploy yaml configmap_create( @@ -169,49 +60,63 @@ configmap_create( from_file="dev/grafana/provisioning/plugins/grafana-oncall-app-provisioning.yaml", ) -k8s_resource( - objects=["grafana-oncall-app-provisioning:configmap"], - new_name="grafana-oncall-app-provisioning-configmap", - resource_deps=["build-ui"], - labels=["Grafana"], -) - -# Use separate grafana helm chart if not running_under_parent_tiltfile: - grafana( - grafana_version=grafana_version, - context="grafana-plugin", - plugin_files=["grafana-plugin/src/plugin.json"], - namespace="default", - deps=["grafana-oncall-app-provisioning-configmap", "build-ui"], - extra_env={ - "GF_SECURITY_ADMIN_PASSWORD": "oncall", - "GF_SECURITY_ADMIN_USER": "oncall", - "GF_AUTH_ANONYMOUS_ENABLED": "false", - }, + # Load the custom Grafana extensions + v1alpha1.extension_repo( + name="grafana-tilt-extensions", + ref="v1.2.0", + url="https://github.com/grafana/tilt-extensions", + ) +v1alpha1.extension( + name="grafana", repo_name="grafana-tilt-extensions", repo_path="grafana" +) +load("ext://grafana", "grafana") + +def load_grafana(): + # The user/pass that you will login to Grafana with + grafana_admin_user_pass = os.getenv("GRAFANA_ADMIN_USER_PASS", "oncall") + grafana_version = os.getenv("GRAFANA_VERSION", "latest") + + + k8s_resource( + objects=["grafana-oncall-app-provisioning:configmap"], + new_name="grafana-oncall-app-provisioning-configmap", + resource_deps=["build-ui"], + labels=["Grafana"], ) -k8s_resource( - workload="celery", - resource_deps=["mariadb", "redis-master"], - labels=["OnCallBackend"], -) -k8s_resource( - workload="engine", - port_forwards=8080, - resource_deps=["mariadb", "redis-master"], - labels=["OnCallBackend"], -) -k8s_resource(workload="engine-migrate", labels=["OnCallBackend"]) + # Use separate grafana helm chart + if not running_under_parent_tiltfile: + grafana( + grafana_version=grafana_version, + context="grafana-plugin", + plugin_files=["grafana-plugin/src/plugin.json"], + namespace="default", + deps=["grafana-oncall-app-provisioning-configmap", "build-ui"], + extra_env={ + "GF_SECURITY_ADMIN_PASSWORD": "oncall", + "GF_SECURITY_ADMIN_USER": "oncall", + "GF_AUTH_ANONYMOUS_ENABLED": "false", + }, + ) +# --- GRAFANA END ---- -k8s_resource(workload="redis-master", labels=["OnCallDeps"]) -k8s_resource(workload="prometheus-server", labels=["OnCallDeps"]) -k8s_resource( - workload="mariadb", - port_forwards='3307:3306', # : - labels=["OnCallDeps"], -) +def get_profiles(): + profiles = os.getenv('ONCALL_PROFILES', 'grafana,plugin,backend,tests') + return profiles.split(',') +profiles = get_profiles() + +if 'grafana' in profiles: + load_grafana() +if 'plugin' in profiles: + include(".tilt/plugin/Tiltfile") +if 'backend' in profiles: + load_oncall_helm() + include(".tilt/backend/Tiltfile") + include(".tilt/deps/Tiltfile") +if 'tests' in profiles: + include(".tilt/tests/Tiltfile") # name all tilt resources after the k8s object namespace + name def resource_name(id): diff --git a/dev/README.md b/dev/README.md index c6f82c5a..7b0b308f 100644 --- a/dev/README.md +++ b/dev/README.md @@ -36,6 +36,14 @@ Related: [How to develop integrations](/engine/config_integrations/README.md) value: "True" ``` + You can also choose set of resources that will be installed in your local cluster, e.g.: + + ```bash + ONCALL_PROFILES=grafana,plugin,backend tilt up + ``` + + Available profiles are: `grafana,plugin,backend,tests`, by default all the profiles are enabled. + 3. Wait until all resources are green and open (user: oncall, password: oncall) 4. Modify source code, backend and frontend will be hot reloaded