diff --git a/synapse/storage/databases/main/schema/delta/58/11recovery_after_outage.sql b/synapse/storage/databases/main/schema/delta/58/11recovery_after_outage.sql
index 5e331cc7c6..e3644367a2 100644
--- a/synapse/storage/databases/main/schema/delta/58/11recovery_after_outage.sql
+++ b/synapse/storage/databases/main/schema/delta/58/11recovery_after_outage.sql
@@ -18,7 +18,10 @@
 -- This stores, for each (destination, room) pair, the event_id and stream_ordering
 -- of the latest event for that destination.
 CREATE TABLE IF NOT EXISTS destination_rooms (
-  -- the destination in question
+  -- the destination in question.
+  --    Can not be a foreign key because rows in the `destinations` table will
+  --    only be created when we back off or when we successfully send a
+  --    transaction.
   destination TEXT NOT NULL,
   -- the ID of the room in question
   room_id TEXT NOT NULL,
@@ -26,7 +29,8 @@ CREATE TABLE IF NOT EXISTS destination_rooms (
   stream_ordering INTEGER,
   -- the event_id of the event
   event_id TEXT NOT NULL,
-  PRIMARY KEY (destination, room_id)
+  PRIMARY KEY (destination, room_id),
+  FOREIGN KEY (room_id) REFERENCES rooms (room_id)
 );
 
 -- this column tracks the stream_ordering of the event that was most recently