Configure pyroscope (#1638)

# What this PR does

## Which issue(s) this PR fixes

## Checklist

- [ ] Unit, integration, and e2e (if applicable) tests updated
- [ ] Documentation added (or `pr:no public docs` PR label added if not
required)
- [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not
required)

engine/engine/celery.py

@@ -79,3 +79,18 @@ if settings.DEBUG_CELERY_TASKS_PROFILING:
         logger.info("ended: {} of {} with cpu={} at {}".format(task_id, task.name, time.perf_counter(), time.time()))
         sample_mem()
         memdump()
+
+
+if settings.PYROSCOPE_PROFILER_ENABLED:
+
+    @celery.signals.worker_process_init.connect(weak=False)
+    def init_pyroscope(*args, **kwargs):
+        import pyroscope
+
+        pyroscope.configure(
+            application_name=settings.PYROSCOPE_APPLICATION_NAME,
+            server_address=settings.PYROSCOPE_SERVER_ADDRESS,
+            auth_token=settings.PYROSCOPE_AUTH_TOKEN,
+            detect_subprocesses=True,  # detect subprocesses started by the main process; default is False
+            tags={"type": "celery", "celery_worker": settings.CELERY_WORKER_QUEUE},
+        )

engine/engine/wsgi.py

@@ -52,8 +52,7 @@ if settings.PYROSCOPE_PROFILER_ENABLED:
                 application_name=settings.PYROSCOPE_APPLICATION_NAME,
                 server_address=settings.PYROSCOPE_SERVER_ADDRESS,
                 auth_token=settings.PYROSCOPE_AUTH_TOKEN,
-                detect_subprocesses=True,
-                tags={"celery_worker": settings.PYROSCOPE_CELERY_WORKER_QUEUE},
+                tags={"type": "uwsgi"},
             )
 
     except ModuleNotFoundError:

engine/settings/base.py

@@ -681,4 +681,3 @@ PYROSCOPE_PROFILER_ENABLED = getenv_boolean("PYROSCOPE_PROFILER_ENABLED", defaul
 PYROSCOPE_APPLICATION_NAME = os.getenv("PYROSCOPE_APPLICATION_NAME", "oncall")
 PYROSCOPE_SERVER_ADDRESS = os.getenv("PYROSCOPE_SERVER_ADDRESS", "http://pyroscope:4040")
 PYROSCOPE_AUTH_TOKEN = os.getenv("PYROSCOPE_AUTH_TOKEN", "")
-PYROSCOPE_CELERY_WORKER_QUEUE = os.getenv("CELERY_WORKER_QUEUE", None)

engine/uwsgi.ini

@@ -1,4 +1,5 @@
 [uwsgi]
+strict=true
 chdir=/etc/app
 module=engine.wsgi:application
 master=True
@@ -14,6 +15,7 @@ buffer-size=65535
 http-auto-chunked=True
 http-timeout=620
 post-buffering=1
+enable-threads=true
 
 logger=stdio
 log-format=source=engine:uwsgi status=%(status) method=%(method) path=%(uri) latency=%(secs) google_trace_id=%(var.HTTP_X_CLOUD_TRACE_CONTEXT) protocol=%(proto) resp_size=%(size) req_body_size=%(cl)