From 2a3f3c7c284af44adde654cfee234fed1d366249 Mon Sep 17 00:00:00 2001
From: Eric Eastwood <erice@element.io>
Date: Mon, 12 Sep 2022 17:54:01 -0500
Subject: [PATCH] Be able to correlate timeouts in reverse-proxy layer in front
 of Synapse (tag specific request headers)

Fix https://github.com/matrix-org/synapse/issues/13685
---
 docs/usage/configuration/config_documentation.md |  8 ++++++++
 synapse/api/auth.py                              | 14 ++++++++++++++
 synapse/config/tracer.py                         | 10 ++++++++++
 3 files changed, 32 insertions(+)

diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md
index cd546041b2..8823727024 100644
--- a/docs/usage/configuration/config_documentation.md
+++ b/docs/usage/configuration/config_documentation.md
@@ -3612,6 +3612,11 @@ Sub-options include:
 * `jaeger_config`: Jaeger can be configured to sample traces at different rates.
    All configuration options provided by Jaeger can be set here. Jaeger's configuration is
    mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/).
+* `request_headers_to_tag`: A list of headers to extract from the request and
+  add to to the top-level servlet tracing span as tags. Useful when you're using
+  a reverse proxy service like Cloudflare to protect your Synapse instance in
+  order to correlate and match up requests that timed out at the Cloudflare
+  layer to the Synapse traces.
 
 Example configuration:
 ```yaml
@@ -3629,6 +3634,9 @@ opentracing:
         param: 1
       logging:
         false
+
+    request_headers_to_tag:
+      - "cf-ray"
 ```
 ---
 ## Workers ##
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 4a75eb6b21..6ef29b201f 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -173,6 +173,20 @@ class Auth:
                     parent_span.set_tag("device_id", requester.device_id)
                 if requester.app_service is not None:
                     parent_span.set_tag("appservice_id", requester.app_service.id)
+
+                # Tag any headers that we need to extract from the request. This
+                # is useful to specify any headers that a reverse-proxy in front
+                # of Synapse may be sending to correlate and match up something
+                # in that layer to a Synapse trace. ex. when Cloudflare times
+                # out it gives a `cf-ray` header which we can also tag here to
+                # find the trace.
+                for header_key in self.hs.config.tracing.request_headers_to_tag:
+                    headers = request.requestHeaders.getRawHeaders(header_key)
+                    if len(headers):
+                        parent_span.set_tag(
+                            SynapseTags.REQUEST_HEADER_PREFIX + header_key, headers[0]
+                        )
+
             return requester
 
     @cancellable
diff --git a/synapse/config/tracer.py b/synapse/config/tracer.py
index c19270c6c5..6d750e397f 100644
--- a/synapse/config/tracer.py
+++ b/synapse/config/tracer.py
@@ -37,6 +37,16 @@ class TracerConfig(Config):
 
         self.force_tracing_for_users: Set[str] = set()
 
+        # A list of headers to extract from the request and add to to the
+        # top-level servlet tracing span as tags. Useful when you're using a
+        # reverse proxy service like Cloudflare to protect your Synapse instance
+        # in order to correlate and match up requests that timed out at the
+        # Cloudflare layer to the Synapse traces.
+        self.request_headers_to_tag = opentracing_config.get(
+            "request_headers_to_tag",
+            [],
+        )
+
         if not self.opentracer_enabled:
             return