Instrument requests lib (#4008)

# What this PR does

Instument requests lib to provide consistent trace_id

engine/engine/wsgi.py

@@ -14,6 +14,7 @@ from django.core.wsgi import get_wsgi_application
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
 from opentelemetry.instrumentation.wsgi import OpenTelemetryMiddleware
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
@@ -27,7 +28,8 @@ application = WhiteNoise(application)
 # check both OTEL_TRACING_ENABLED and OTEL_EXPORTER_OTLP_ENDPOINT
 # since OTLPSpanExporter expects endpoint to send data to
 if settings.OTEL_TRACING_ENABLED and settings.OTEL_EXPORTER_OTLP_ENDPOINT:
-    # Set up tracing and logging instrumentation under uwsgi web server environment
+    # Set up tracing and logging instrumentation under uwsgi web server environment.
+    # Since it's wsgi setup, it will be used in prod.
     try:
         from uwsgidecorators import postfork
 
@@ -38,7 +40,8 @@ if settings.OTEL_TRACING_ENABLED and settings.OTEL_EXPORTER_OTLP_ENDPOINT:
             trace.set_tracer_provider(TracerProvider())
             span_processor = BatchSpanProcessor(OTLPSpanExporter())
             trace.get_tracer_provider().add_span_processor(span_processor)
-            LoggingInstrumentor().instrument()
+            LoggingInstrumentor().instrument()  # Instrument logs to add trace_id to log lines
+            RequestsInstrumentor().instrument()  # Instrument requests to instrument downstream calls
 
     except ModuleNotFoundError:
         pass

engine/manage.py

@@ -4,7 +4,9 @@ import sys
 
 from django.conf import settings
 from opentelemetry import trace
+from opentelemetry.instrumentation.django import DjangoInstrumentor
 from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor
 
@@ -12,10 +14,15 @@ if __name__ == "__main__":
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.dev")
     if settings.OTEL_TRACING_ENABLED:
         # Set up tracing and logging instrumentation under manage.ru runserver command.
+        # It's used to provide simple way to test tracing locally.
         trace.set_tracer_provider(TracerProvider())
-        span_processor = SimpleSpanProcessor(ConsoleSpanExporter())
+        span_processor = SimpleSpanProcessor(ConsoleSpanExporter())  # Log spans to console to simplify local setup
         trace.get_tracer_provider().add_span_processor(span_processor)
-        LoggingInstrumentor().instrument()
+        # DjangoInstrumentor instruments incoming requests and starts root span.
+        # It's used instead of wsgi middleware in local setup. Not sure if it should be used in prod
+        DjangoInstrumentor().instrument()
+        LoggingInstrumentor().instrument()  # Instrument logs to add trace_id to log lines
+        RequestsInstrumentor().instrument()  # Instrument requests to instrument downstream calls
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

engine/requirements.in

@@ -39,6 +39,8 @@ opentelemetry-api==1.23.0
 opentelemetry-exporter-otlp-proto-grpc==1.15.0
 opentelemetry-instrumentation-logging==0.44b0
 opentelemetry-instrumentation-wsgi==0.44b0
+opentelemetry-instrumentation-requests==0.44b0
+opentelemetry-instrumentation-django==0.44b0
 phonenumbers==8.10.0
 prometheus_client==0.16.0
 psutil==5.9.4

engine/requirements.txt

@@ -264,19 +264,29 @@ opentelemetry-api==1.23.0
     #   -r ./engine/requirements.in
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-django
     #   opentelemetry-instrumentation-logging
+    #   opentelemetry-instrumentation-requests
     #   opentelemetry-instrumentation-wsgi
     #   opentelemetry-sdk
 opentelemetry-exporter-otlp-proto-grpc==1.15.0
     # via -r ./engine/requirements.in
 opentelemetry-instrumentation==0.44b0
     # via
+    #   opentelemetry-instrumentation-django
     #   opentelemetry-instrumentation-logging
+    #   opentelemetry-instrumentation-requests
     #   opentelemetry-instrumentation-wsgi
+opentelemetry-instrumentation-django==0.44b0
+    # via -r ./engine/requirements.in
 opentelemetry-instrumentation-logging==0.44b0
     # via -r ./engine/requirements.in
-opentelemetry-instrumentation-wsgi==0.44b0
+opentelemetry-instrumentation-requests==0.44b0
     # via -r ./engine/requirements.in
+opentelemetry-instrumentation-wsgi==0.44b0
+    # via
+    #   -r ./engine/requirements.in
+    #   opentelemetry-instrumentation-django
 opentelemetry-proto==1.15.0
     # via opentelemetry-exporter-otlp-proto-grpc
 opentelemetry-sdk==1.23.0
@@ -285,10 +295,15 @@ opentelemetry-sdk==1.23.0
     #   opentelemetry-exporter-otlp-proto-grpc
 opentelemetry-semantic-conventions==0.44b0
     # via
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-requests
     #   opentelemetry-instrumentation-wsgi
     #   opentelemetry-sdk
 opentelemetry-util-http==0.44b0
-    # via opentelemetry-instrumentation-wsgi
+    # via
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-requests
+    #   opentelemetry-instrumentation-wsgi
 pem==23.1.0
     # via django-sns-view
 phonenumbers==8.10.0