Stop sending or processing the origin field in PDUs (#18418)

Co-authored-by: Quentin Gliech <quenting@element.io>
Co-authored-by: Eric Eastwood <erice@element.io>

changelog.d/18418.removal

@@ -0,0 +1 @@
+Stop adding the "origin" field to newly-created events (PDUs).

contrib/graph/graph.py

@@ -45,6 +45,10 @@ def make_graph(pdus: List[dict], filename_prefix: str) -> None:
     colors = {"red", "green", "blue", "yellow", "purple"}
 
     for pdu in pdus:
+        # TODO: The "origin" field has since been removed from events generated
+        # by Synapse. We should consider removing it here as well but since this
+        # is part of `contrib/`, it is left for the community to revise and ensure things
+        # still work correctly.
         origins.add(pdu.get("origin"))
 
     color_map = {color: color for color in colors if color in origins}

docs/admin_api/event_reports.md

@@ -117,7 +117,6 @@ It returns a JSON body like the following:
         "hashes": {
             "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw"
         },
-        "origin": "matrix.org",
         "origin_server_ts": 1592291711430,
         "prev_events": [
             "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M"

synapse/events/__init__.py

@@ -208,7 +208,6 @@ class EventBase(metaclass=abc.ABCMeta):
     depth: DictProperty[int] = DictProperty("depth")
     content: DictProperty[JsonDict] = DictProperty("content")
     hashes: DictProperty[Dict[str, str]] = DictProperty("hashes")
-    origin: DictProperty[str] = DictProperty("origin")
     origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts")
     room_id: DictProperty[str] = DictProperty("room_id")
     sender: DictProperty[str] = DictProperty("sender")

synapse/events/builder.py

@@ -302,8 +302,8 @@ def create_local_event_from_event_dict(
     event_dict: JsonDict,
     internal_metadata_dict: Optional[JsonDict] = None,
 ) -> EventBase:
-    """Takes a fully formed event dict, ensuring that fields like `origin`
-    and `origin_server_ts` have correct values for a locally produced event,
+    """Takes a fully formed event dict, ensuring that fields like
+    `origin_server_ts` have correct values for a locally produced event,
     then signs and hashes it.
     """
 
@@ -319,7 +319,6 @@ def create_local_event_from_event_dict(
     if format_version == EventFormatVersions.ROOM_V1_V2:
         event_dict["event_id"] = _create_event_id(clock, hostname)
 
-    event_dict["origin"] = hostname
     event_dict.setdefault("origin_server_ts", time_now)
 
     event_dict.setdefault("unsigned", {})

synapse/events/validator.py

@@ -67,7 +67,6 @@ class EventValidator:
             "auth_events",
             "content",
             "hashes",
-            "origin",
             "prev_events",
             "sender",
             "type",
@@ -77,13 +76,6 @@ class EventValidator:
             if k not in event:
                 raise SynapseError(400, "Event does not have key %s" % (k,))
 
-        # Check that the following keys have string values
-        event_strings = ["origin"]
-
-        for s in event_strings:
-            if not isinstance(getattr(event, s), str):
-                raise SynapseError(400, "'%s' not a string type" % (s,))
-
         # Depending on the room version, ensure the data is spec compliant JSON.
         if event.room_version.strict_canonicaljson:
             validate_canonicaljson(event.get_pdu_json())

synapse/federation/federation_base.py

@@ -322,8 +322,7 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB
         SynapseError: if the pdu is missing required fields or is otherwise
             not a valid matrix event
     """
-    # we could probably enforce a bunch of other fields here (room_id, sender,
-    # origin, etc etc)
+    # we could probably enforce a bunch of other fields here (room_id, sender, etc.)
     assert_params_in_dict(pdu_json, ("type", "depth"))
 
     # Strip any unauthorized values from "unsigned" if they exist

synapse/util/caches/__init__.py

@@ -225,7 +225,7 @@ KNOWN_KEYS = {
         "depth",
         "event_id",
         "hashes",
-        "origin",
+        "origin",  # old events were created with an origin field.
         "origin_server_ts",
         "prev_events",
         "room_id",

tests/crypto/test_event_signing.py

@@ -48,7 +48,6 @@ class EventSigningTestCase(unittest.TestCase):
     def test_sign_minimal(self) -> None:
         event_dict = {
             "event_id": "$0:domain",
-            "origin": "domain",
             "origin_server_ts": 1000000,
             "signatures": {},
             "type": "X",
@@ -64,7 +63,7 @@ class EventSigningTestCase(unittest.TestCase):
         self.assertTrue(hasattr(event, "hashes"))
         self.assertIn("sha256", event.hashes)
         self.assertEqual(
-            event.hashes["sha256"], "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI"
+            event.hashes["sha256"], "A6Nco6sqoy18PPfPDVdYvoowfc0PVBk9g9OiyT3ncRM"
         )
 
         self.assertTrue(hasattr(event, "signatures"))
@@ -72,15 +71,14 @@ class EventSigningTestCase(unittest.TestCase):
         self.assertIn(KEY_NAME, event.signatures["domain"])
         self.assertEqual(
             event.signatures[HOSTNAME][KEY_NAME],
-            "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+"
-            "aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA",
+            "PBc48yDVszWB9TRaB/+CZC1B+pDAC10F8zll006j+NN"
+            "fe4PEMWcVuLaG63LFTK9e4rwJE8iLZMPtCKhDTXhpAQ",
         )
 
     def test_sign_message(self) -> None:
         event_dict = {
             "content": {"body": "Here is the message content"},
             "event_id": "$0:domain",
-            "origin": "domain",
             "origin_server_ts": 1000000,
             "type": "m.room.message",
             "room_id": "!r:domain",
@@ -98,7 +96,7 @@ class EventSigningTestCase(unittest.TestCase):
         self.assertTrue(hasattr(event, "hashes"))
         self.assertIn("sha256", event.hashes)
         self.assertEqual(
-            event.hashes["sha256"], "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g"
+            event.hashes["sha256"], "rDCeYBepPlI891h/RkI2/Lkf9bt7u0TxFku4tMs7WKk"
         )
 
         self.assertTrue(hasattr(event, "signatures"))
@@ -106,6 +104,6 @@ class EventSigningTestCase(unittest.TestCase):
         self.assertIn(KEY_NAME, event.signatures["domain"])
         self.assertEqual(
             event.signatures[HOSTNAME][KEY_NAME],
-            "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUw"
-            "u6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA",
+            "Ay4aj2b5oJ1k8INYZ9n3KnszCflM0emwcmQQ7vxpbdc"
+            "Sv9bkJxIZdWX1IJllcZLq89+D3sSabE+vqPtZs9akDw",
         )

tests/events/test_utils.py

@@ -122,7 +122,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
                 "prev_events": "prev_events",
                 "prev_state": "prev_state",
                 "auth_events": "auth_events",
-                "origin": "domain",
+                "origin": "domain",  # historical top-level field that still exists on old events
                 "origin_server_ts": 1234,
                 "membership": "join",
                 # Also include a key that should be removed.
@@ -139,7 +139,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
                 "prev_events": "prev_events",
                 "prev_state": "prev_state",
                 "auth_events": "auth_events",
-                "origin": "domain",
+                "origin": "domain",  # historical top-level field that still exists on old events
                 "origin_server_ts": 1234,
                 "membership": "join",
                 "content": {},
@@ -148,13 +148,12 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
             },
         )
 
-        # As of room versions we now redact the membership, prev_states, and origin keys.
+        # As of room versions we now redact the membership and prev_states keys.
         self.run_test(
             {
                 "type": "A",
                 "prev_state": "prev_state",
                 "membership": "join",
-                "origin": "example.com",
             },
             {"type": "A", "content": {}, "signatures": {}, "unsigned": {}},
             room_version=RoomVersions.V11,
@@ -238,7 +237,6 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
             {
                 "type": "m.room.create",
                 "content": {"not_a_real_key": True},
-                "origin": "some_homeserver",
                 "nonsense_field": "some_random_garbage",
             },
             {

tests/federation/test_federation_server.py

@@ -535,7 +535,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase):
             "depth": 1000,
             "origin_server_ts": 1,
             "type": "m.room.member",
-            "origin": "test.servx",
             "content": {"membership": "join"},
             "auth_events": [],
             "unsigned": {"malicious garbage": "hackz", "more warez": "more hackz"},
@@ -552,7 +551,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase):
             "depth": 1000,
             "origin_server_ts": 1,
             "type": "m.room.member",
-            "origin": "test.servx",
             "auth_events": [],
             "content": {"membership": "join"},
             "unsigned": {
@@ -579,7 +577,6 @@ class StripUnsignedFromEventsTestCase(unittest.TestCase):
             "depth": 1000,
             "origin_server_ts": 1,
             "type": "m.room.power_levels",
-            "origin": "test.servx",
             "content": {},
             "auth_events": [],
             "unsigned": {