multiple opentelemetry fixups

.github/workflows/famedly-tests.yml

@@ -359,22 +359,23 @@ jobs:
 
       - run: cargo test
 
-   otlp:
+  otlp:
     if: ${{ !failure() && !cancelled() }}
     runs-on: ubuntu-latest
 
     steps:
       - name: Run actions/checkout@v4 for synapse
         uses: actions/checkout@v4
-        with:
-          path: synapse
       - run: |
           set -e
-          DOCKER_BUILDKIT=1 sudo docker build -t famedly/synapse -f docker/Dockerfile .
+          DOCKER_BUILDKIT=1 docker build -t famedly/synapse -f docker/Dockerfile .
           cd otlp-test
           success() { [ -s out/traces.json ] && [ -s out/logs.json ]; }
-          (until success; do sleep 1; done && sudo docker compose down >/dev/null 2>&1) &
-          sudo timeout 30 docker compose up && success
+          (until success; do sleep 1; done && docker compose down >/dev/null 2>&1) &
+          timeout 30 docker compose up
+          ! sudo grep -q '"ERROR"' out/logs.json  # ensure no errors occurred
+          success  # ensure both traces and logs are present
+          sudo grep -q 'Main loop terminated.' out/logs.json  # ensure log is complete (to the end)
 
   # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
   tests-done:

otlp-test/docker-compose.yaml

@@ -10,6 +10,7 @@ services:
       - SYNAPSE_SERVER_NAME=example.com
       - SYNAPSE_REPORT_STATS=no
       - SYNAPSE_NO_TLS=1
+      - SYNAPSE_LOG_LEVEL=DEBUG
       - OTLP_BACKEND=1
 
   synapse:
@@ -19,6 +20,13 @@ services:
       - ./synapse-data:/data
     environment:
        - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+    healthcheck:
+      test: curl --fail http://localhost:8008/health
+      interval: 50s
+      timeout: 5s
+      retries: 3
+      start_period: 50s
+      start_interval: 1s
     depends_on:
       - init
       - otel-collector

pyproject.toml

@@ -251,17 +251,17 @@ Pympler = { version = "*", optional = true }
 parameterized = { version = ">=0.7.4", optional = true }
 idna = { version = ">=2.5", optional = true }
 pyicu = { version = ">=2.10.2", optional = true }
-opentelemetry-api = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "34b3ac68cccd63ba44c05a568be94218cd6ed198", subdirectory = "opentelemetry-api", optional = true }
-opentelemetry-sdk = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "34b3ac68cccd63ba44c05a568be94218cd6ed198", subdirectory = "opentelemetry-sdk", optional = true }
+opentelemetry-api = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "742171e50c70f4a4540a01d2c6c3cbcf4882d810", subdirectory = "opentelemetry-api", optional = true }
+opentelemetry-sdk = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "742171e50c70f4a4540a01d2c6c3cbcf4882d810", subdirectory = "opentelemetry-sdk", optional = true }
 opentelemetry-opentracing-shim = { version = "^0.52b0", optional = true }
-opentelemetry-exporter-otlp = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "34b3ac68cccd63ba44c05a568be94218cd6ed198", subdirectory = "exporter/opentelemetry-exporter-otlp", optional = true }
+opentelemetry-exporter-otlp = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "742171e50c70f4a4540a01d2c6c3cbcf4882d810", subdirectory = "exporter/opentelemetry-exporter-otlp", optional = true }
 
 # The lines below and also the corresponding extra dependencies can be removed as soon as we switch to a released version of opentelemetry-python (1.32.0):
-opentelemetry-exporter-otlp-proto-grpc = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "34b3ac68cccd63ba44c05a568be94218cd6ed198", subdirectory = "exporter/opentelemetry-exporter-otlp-proto-grpc", optional = true }
-opentelemetry-exporter-otlp-proto-http = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "34b3ac68cccd63ba44c05a568be94218cd6ed198", subdirectory = "exporter/opentelemetry-exporter-otlp-proto-http", optional = true }
-opentelemetry-exporter-otlp-proto-common = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "34b3ac68cccd63ba44c05a568be94218cd6ed198", subdirectory = "exporter/opentelemetry-exporter-otlp-proto-common", optional = true }
-opentelemetry-proto = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "34b3ac68cccd63ba44c05a568be94218cd6ed198", subdirectory = "opentelemetry-proto", optional = true }
-opentelemetry-semantic-conventions = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "34b3ac68cccd63ba44c05a568be94218cd6ed198", subdirectory = "opentelemetry-semantic-conventions", optional = true }
+opentelemetry-exporter-otlp-proto-grpc = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "742171e50c70f4a4540a01d2c6c3cbcf4882d810", subdirectory = "exporter/opentelemetry-exporter-otlp-proto-grpc", optional = true }
+opentelemetry-exporter-otlp-proto-http = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "742171e50c70f4a4540a01d2c6c3cbcf4882d810", subdirectory = "exporter/opentelemetry-exporter-otlp-proto-http", optional = true }
+opentelemetry-exporter-otlp-proto-common = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "742171e50c70f4a4540a01d2c6c3cbcf4882d810", subdirectory = "exporter/opentelemetry-exporter-otlp-proto-common", optional = true }
+opentelemetry-proto = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "742171e50c70f4a4540a01d2c6c3cbcf4882d810", subdirectory = "opentelemetry-proto", optional = true }
+opentelemetry-semantic-conventions = { git = "https://github.com/open-telemetry/opentelemetry-python", rev = "742171e50c70f4a4540a01d2c6c3cbcf4882d810", subdirectory = "opentelemetry-semantic-conventions", optional = true }
 
 [tool.poetry.extras]
 # NB: Packages that should be part of `pip install matrix-synapse[all]` need to be specified

synapse/config/logger.py

@@ -112,6 +112,12 @@
         class: logging.StreamHandler
         formatter: precise
 
+    # A commented-out OtlpHandler based on opentelemetry-sdk that reads
+    # its configuration from the common `OTLP_` environment variables:
+    # Requires the extra "opentelemetry-log-handler".
+    #otlp:
+    #    class: synapse.logging.handlers.OtlpHandler
+
 loggers:
     synapse.storage.SQL:
         # beware: increasing this to DEBUG will make synapse log sensitive
@@ -126,6 +132,8 @@
     #
     # Replace "buffer" with "console" to log to stderr instead.
     #
+    # Add or set the handler to "otlp" after enabling it in the "handlers" section.
+    #
     handlers: [buffer]
 
 disable_existing_loggers: false

synapse/logging/handlers.py

@@ -92,20 +92,31 @@ def close(self) -> None:
 
 
 try:
-    from opentelemetry._logs import set_logger_provider
     from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
     from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
     from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
     from opentelemetry.sdk.resources import Resource
+
+    USE_REAL_OTLP = True
+
+    class RealOtlpHandler(LoggingHandler):
+        def __init__(self, level: int = logging.NOTSET) -> None:
+            self.logger_provider = LoggerProvider(
+                resource=Resource(attributes={"service.name": "synapse"})
+            )
+            self.logger_provider.add_log_record_processor(
+                BatchLogRecordProcessor(OTLPLogExporter())
+            )
+            super().__init__(level, self.logger_provider)
 except ImportError as e:
     OTLP_IMPORT_EXC = e
-    class OtlpHandler:
+
+    USE_REAL_OTLP = False
+
+    class StubOtlpHandler:
         def __init__(self) -> None:
             check_requirements("opentelemetry-log-handler")
             raise OTLP_IMPORT_EXC
-else:
-    class OtlpHandler(LoggingHandler):
-        def __init__(self, level=logging.NOTSET, logger_provider=None) -> None:
-            self.logger_provider = LoggerProvider(resource=Resource(attributes={"service.name": "synapse"}))
-            self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
-            super().__init__(level, self.logger_provider)
+
+
+OtlpHandler = RealOtlpHandler if USE_REAL_OTLP else StubOtlpHandler

synapse/logging/opentracing.py

@@ -515,13 +515,15 @@ def init_tracer_otlp(hs: "HomeServer") -> None:
     assert opentracing is not None
 
     from opentelemetry import trace
-    from opentelemetry.sdk.trace import TracerProvider
-    from opentelemetry.sdk.trace.export import BatchSpanProcessor
     from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
     from opentelemetry.sdk.resources import Resource
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
     from opentelemetry.shim.opentracing_shim import create_tracer
 
-    tracer_provider = TracerProvider(resource=Resource(attributes={"service.name": "synapse"}))
+    tracer_provider = TracerProvider(
+        resource=Resource(attributes={"service.name": "synapse"})
+    )
     tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
     trace.set_tracer_provider(tracer_provider)
     opentracing.tracer = create_tracer(trace.get_tracer_provider())