Be able to correlate timeouts in reverse-proxy layer in front of Synapse (tag specific request headers)

Fix https://github.com/matrix-org/synapse/issues/13685

docs/usage/configuration/config_documentation.md

@@ -3612,6 +3612,11 @@ Sub-options include:
 * `jaeger_config`: Jaeger can be configured to sample traces at different rates.
    All configuration options provided by Jaeger can be set here. Jaeger's configuration is
    mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/).
+* `request_headers_to_tag`: A list of headers to extract from the request and
+  add to to the top-level servlet tracing span as tags. Useful when you're using
+  a reverse proxy service like Cloudflare to protect your Synapse instance in
+  order to correlate and match up requests that timed out at the Cloudflare
+  layer to the Synapse traces.
 
 Example configuration:
 ```yaml
@@ -3629,6 +3634,9 @@ opentracing:
         param: 1
       logging:
         false
+
+    request_headers_to_tag:
+      - "cf-ray"
 ```
 ---
 ## Workers ##

synapse/api/auth.py

@@ -173,6 +173,20 @@ class Auth:
                     parent_span.set_tag("device_id", requester.device_id)
                 if requester.app_service is not None:
                     parent_span.set_tag("appservice_id", requester.app_service.id)
+
+                # Tag any headers that we need to extract from the request. This
+                # is useful to specify any headers that a reverse-proxy in front
+                # of Synapse may be sending to correlate and match up something
+                # in that layer to a Synapse trace. ex. when Cloudflare times
+                # out it gives a `cf-ray` header which we can also tag here to
+                # find the trace.
+                for header_key in self.hs.config.tracing.request_headers_to_tag:
+                    headers = request.requestHeaders.getRawHeaders(header_key)
+                    if len(headers):
+                        parent_span.set_tag(
+                            SynapseTags.REQUEST_HEADER_PREFIX + header_key, headers[0]
+                        )
+
             return requester
 
     @cancellable

synapse/config/tracer.py

@@ -37,6 +37,16 @@ class TracerConfig(Config):
 
         self.force_tracing_for_users: Set[str] = set()
 
+        # A list of headers to extract from the request and add to to the
+        # top-level servlet tracing span as tags. Useful when you're using a
+        # reverse proxy service like Cloudflare to protect your Synapse instance
+        # in order to correlate and match up requests that timed out at the
+        # Cloudflare layer to the Synapse traces.
+        self.request_headers_to_tag = opentracing_config.get(
+            "request_headers_to_tag",
+            [],
+        )
+
         if not self.opentracer_enabled:
             return