kaslo/synapse
1
0
Fork 0
Code Wiki Activity

Fix backfill being able to cleanly branch into history and back to "live"

Browse Source 
Fix https://github.com/matrix-org/synapse/pull/11114#discussion_r731363046

 - When we see a connected insertion event indicating a branch of history,
   we now don't accidentally leak the base event we're branching from
   in the backfill chunk until all of the events in the historical brach
   are exhausted.
 - Backfill will now check whether the federating homeserver asking
   has a given insertion event when we see one connected to the DAG.
   This allows backfill to initially decide to give all the historical
   messages in the branch, then next time when the base event we branched
   from comes up in backfill again, it sees that federated homeserver
   already has the insertion event, and we can continue down the normal
   DAG prev_event path instead.

Example Backfill response (in order yay):

Overview:
```
eventIDsAfter
batch0
batch1
eventIDsBefore
```

```
new implementation backfill events(100)=[
  "event_id=$q3w2dF5mWBkgUF1ADdVgS_bX7YQuqODJ2HC33bln4Ho,depth=11,body=Message 1 (eventIDsAfter),prevs=['$Com3-ZIR35FvYcBQvHdpxGsS1iW-DO_Z5x1zZ5R5Az8']",
  "event_id=$Com3-ZIR35FvYcBQvHdpxGsS1iW-DO_Z5x1zZ5R5Az8,depth=10,body=Message 0 (eventIDsAfter),prevs=['$T5-OUXAW3CCYAHskqk5s0V2ZzJoQy5D8lQJ7LZFWvTI']",
  "event_id=$78Yy5bqa9XuQ5fhcw9AF36Xf4W5EDcjsraSInif7nYU,depth=10,body=org.matrix.msc2716.insertion,prevs=['$T5-OUXAW3CCYAHskqk5s0V2ZzJoQy5D8lQJ7LZFWvTI']",
  "event_id=$GEzF4euV8StAtdCDCtWmpdhxRE3KQWgreyr1_ePqlpM,depth=10,body=org.matrix.msc2716.batch,prevs=['$ot97LEPqM55EV1ka8YM445c6q-ny4jBBRxjgQnGxihQ']",
  "event_id=$ot97LEPqM55EV1ka8YM445c6q-ny4jBBRxjgQnGxihQ,depth=10,body=Historical 99 (batch=0),prevs=['$i53GQczFiX-tqXgcxudp5AWR9C-A0yJiET_ZXe9D49I']",
  "event_id=$i53GQczFiX-tqXgcxudp5AWR9C-A0yJiET_ZXe9D49I,depth=10,body=Historical 98 (batch=0),prevs=['$lqjhSBLzbi2WfWNAIm1mngStqsPfQJUwPO0NAFtB8lo']",
  "event_id=$lqjhSBLzbi2WfWNAIm1mngStqsPfQJUwPO0NAFtB8lo,depth=10,body=Historical 97 (batch=0),prevs=['$k8OC791p_vB-Hz36Byn-o3vK9x_rc0qkVufBpxYRcaA']",
  "event_id=$k8OC791p_vB-Hz36Byn-o3vK9x_rc0qkVufBpxYRcaA,depth=10,body=Historical 96 (batch=0),prevs=['$9obrnkfP2HpDfKUkWyHrtHML5_31MZ3T9sKQOb6eUV8']",
  "event_id=$9obrnkfP2HpDfKUkWyHrtHML5_31MZ3T9sKQOb6eUV8,depth=10,body=Historical 95 (batch=0),prevs=['$PSNLMD1RhBvR9ffpbEOKtUHE9PUUiacJ2g8_DA9nSjM']",
  "event_id=$PSNLMD1RhBvR9ffpbEOKtUHE9PUUiacJ2g8_DA9nSjM,depth=10,body=Historical 94 (batch=0),prevs=['$vGkUiP049MLeapDGHegqdiGZRkuc6WxEJ-iOAN7mqr4']",
  "event_id=$vGkUiP049MLeapDGHegqdiGZRkuc6WxEJ-iOAN7mqr4,depth=10,body=Historical 93 (batch=0),prevs=['$reYXzhTXHlqqk6XMLK_mEIFQGIesn4NJ70JYa7EziQc']",
  "event_id=$reYXzhTXHlqqk6XMLK_mEIFQGIesn4NJ70JYa7EziQc,depth=10,body=Historical 92 (batch=0),prevs=['$dg8ZPPPqh1SbIhcg-GMvEGz49h53TUMoZ8yO4zbfA4U']",
  "event_id=$dg8ZPPPqh1SbIhcg-GMvEGz49h53TUMoZ8yO4zbfA4U,depth=10,body=Historical 91 (batch=0),prevs=['$QW7wied5SyHtJElJTo4SUkc-zNLlAh6tbCiAa5EJXg4']",
  "event_id=$QW7wied5SyHtJElJTo4SUkc-zNLlAh6tbCiAa5EJXg4,depth=10,body=Historical 90 (batch=0),prevs=['$COQuVmW8z0geXFZ2fPdo_kW0q1NK1LLZSI7fZNIwvgM']",
  "event_id=$COQuVmW8z0geXFZ2fPdo_kW0q1NK1LLZSI7fZNIwvgM,depth=10,body=Historical 89 (batch=0),prevs=['$04k2wOdm-L41JDQ73KPpDe1njG3L9FYjOsNVamFsDQ0']",
  "event_id=$04k2wOdm-L41JDQ73KPpDe1njG3L9FYjOsNVamFsDQ0,depth=10,body=Historical 88 (batch=0),prevs=['$70BsBGyeg4Pf3VhGhXjtk4N5XKNNSit4vjnrirwTmic']",
  "event_id=$70BsBGyeg4Pf3VhGhXjtk4N5XKNNSit4vjnrirwTmic,depth=10,body=Historical 87 (batch=0),prevs=['$Y_6AET6hMZY42rPki4r0GxMfs8E_TCJyepNL1Vfwnh4']",
  "event_id=$Y_6AET6hMZY42rPki4r0GxMfs8E_TCJyepNL1Vfwnh4,depth=10,body=Historical 86 (batch=0),prevs=['$jSk0bO6PFUcqxHsjTvFPyJZoXL4N9nEGex7I-yyvBj8']",
  "event_id=$jSk0bO6PFUcqxHsjTvFPyJZoXL4N9nEGex7I-yyvBj8,depth=10,body=Historical 85 (batch=0),prevs=['$wPV_BB23RIolkHz0yGTvwt1-TJbqpGPqZc33vR5DUMw']",
  "event_id=$wPV_BB23RIolkHz0yGTvwt1-TJbqpGPqZc33vR5DUMw,depth=10,body=Historical 84 (batch=0),prevs=['$X2FJ_Jmoy5sx9u0UWTN1bkds6diO0uDxxRGco4r9Msw']",
  "event_id=$X2FJ_Jmoy5sx9u0UWTN1bkds6diO0uDxxRGco4r9Msw,depth=10,body=Historical 83 (batch=0),prevs=['$jrmzKa3aDmpZg5HgDHQqQ_lgl9L4rDWiWk7xd1igfAE']",
  "event_id=$jrmzKa3aDmpZg5HgDHQqQ_lgl9L4rDWiWk7xd1igfAE,depth=10,body=Historical 82 (batch=0),prevs=['$8Ov9djlzX3ixwCWaULf6-SJNBuV8eeslfBVvt5b1Uds']",
  "event_id=$8Ov9djlzX3ixwCWaULf6-SJNBuV8eeslfBVvt5b1Uds,depth=10,body=Historical 81 (batch=0),prevs=['$Xflk7Gara-ISOAblxE3Vh5qbRDxJal8NLWSD6BC3wfY']",
  "event_id=$Xflk7Gara-ISOAblxE3Vh5qbRDxJal8NLWSD6BC3wfY,depth=10,body=Historical 80 (batch=0),prevs=['$cd4vq5xB0kqHa7HziMmI1U5njSwDr6NOz0qkvCrUeiA']",
  "event_id=$cd4vq5xB0kqHa7HziMmI1U5njSwDr6NOz0qkvCrUeiA,depth=10,body=Historical 79 (batch=0),prevs=['$xblwBKeHnnXlcK9nusq0mWw712br9ySr3elpt5vqYG8']",
  "event_id=$xblwBKeHnnXlcK9nusq0mWw712br9ySr3elpt5vqYG8,depth=10,body=Historical 78 (batch=0),prevs=['$cAnav2OUp8QsmQRbPc-PcX6A4rRW9UUsGFEGb1lgSyI']",
  "event_id=$cAnav2OUp8QsmQRbPc-PcX6A4rRW9UUsGFEGb1lgSyI,depth=10,body=Historical 77 (batch=0),prevs=['$3janYqE-QmrenClP9K2jxr-TM2AvG5H-kcgkM6F9UgA']",
  "event_id=$3janYqE-QmrenClP9K2jxr-TM2AvG5H-kcgkM6F9UgA,depth=10,body=Historical 76 (batch=0),prevs=['$UqMZVF90ZR0kzwt5q2noHEGCS8UFp_IAN4EbiMYVl4o']",
  "event_id=$UqMZVF90ZR0kzwt5q2noHEGCS8UFp_IAN4EbiMYVl4o,depth=10,body=Historical 75 (batch=0),prevs=['$D6IL83wLN4nkNwwikin8olaPEZCpzUh7-1PAOdD7HSY']",
  "event_id=$D6IL83wLN4nkNwwikin8olaPEZCpzUh7-1PAOdD7HSY,depth=10,body=Historical 74 (batch=0),prevs=['$MWVNPcLeQ_INUKlCdoOaySGlGeP5WwDt0myistqnTVk']",
  "event_id=$MWVNPcLeQ_INUKlCdoOaySGlGeP5WwDt0myistqnTVk,depth=10,body=Historical 73 (batch=0),prevs=['$Co1eNgILBaxBR_Gt7Vz_tIs0LutTc9CPvtYRyElNMj8']",
  "event_id=$Co1eNgILBaxBR_Gt7Vz_tIs0LutTc9CPvtYRyElNMj8,depth=10,body=Historical 72 (batch=0),prevs=['$ahJyDyEY1Oo8OAkrMZAeYDeTWNzx0B36GsQxcZWchFE']",
  "event_id=$ahJyDyEY1Oo8OAkrMZAeYDeTWNzx0B36GsQxcZWchFE,depth=10,body=Historical 71 (batch=0),prevs=['$JfGy5KmG-3KFMJFfcHsci6YvhvLq9pWok6GgcZlaXX8']",
  "event_id=$JfGy5KmG-3KFMJFfcHsci6YvhvLq9pWok6GgcZlaXX8,depth=10,body=Historical 70 (batch=0),prevs=['$VEfiWOEPPyCpN9z305k4IJySpEj5tl4InJb5mhFs7EA']",
  "event_id=$VEfiWOEPPyCpN9z305k4IJySpEj5tl4InJb5mhFs7EA,depth=10,body=Historical 69 (batch=0),prevs=['$FZswJ09DFvJSzGDFiA3JIdLuzFnG_TfObeEOES1fVTA']",
  "event_id=$FZswJ09DFvJSzGDFiA3JIdLuzFnG_TfObeEOES1fVTA,depth=10,body=Historical 68 (batch=0),prevs=['$fnzsLJ8duVIJ_S74QIozUaEwOp4G7j7fN5yRV-bzDHQ']",
  "event_id=$fnzsLJ8duVIJ_S74QIozUaEwOp4G7j7fN5yRV-bzDHQ,depth=10,body=Historical 67 (batch=0),prevs=['$vBWTNsLkJHP9bK0L1Wx3HdND6Rh8HhPmxBtXP7klwvE']",
  "event_id=$vBWTNsLkJHP9bK0L1Wx3HdND6Rh8HhPmxBtXP7klwvE,depth=10,body=Historical 66 (batch=0),prevs=['$y5PGi9vY9UcTEwTzeE18JB6-bjhf1GVeSEOph4ko-SE']",
  "event_id=$y5PGi9vY9UcTEwTzeE18JB6-bjhf1GVeSEOph4ko-SE,depth=10,body=Historical 65 (batch=0),prevs=['$7pM8GWYvhzxZ3zGkc76RLtsw7w02IT-sAiV8JkKeQCU']",
  "event_id=$7pM8GWYvhzxZ3zGkc76RLtsw7w02IT-sAiV8JkKeQCU,depth=10,body=Historical 64 (batch=0),prevs=['$wKaeK8hHhwLb1zHZaRJFE1C-n2kOtWnf076sNeWqaMc']",
  "event_id=$wKaeK8hHhwLb1zHZaRJFE1C-n2kOtWnf076sNeWqaMc,depth=10,body=Historical 63 (batch=0),prevs=['$8OHC-bni0whbdGfis5cfWL3--JACd0fx-pUBs5LuBXM']",
  "event_id=$8OHC-bni0whbdGfis5cfWL3--JACd0fx-pUBs5LuBXM,depth=10,body=Historical 62 (batch=0),prevs=['$nZ1WO04o85g1lkLQYlb72bZfx195YwuA-mQ3SsLEsuI']",
  "event_id=$nZ1WO04o85g1lkLQYlb72bZfx195YwuA-mQ3SsLEsuI,depth=10,body=Historical 61 (batch=0),prevs=['$rdA6ksM5YpvBrm9vHREi-d7x5AJCPZzrAAqxNN5I2ms']",
  "event_id=$rdA6ksM5YpvBrm9vHREi-d7x5AJCPZzrAAqxNN5I2ms,depth=10,body=Historical 60 (batch=0),prevs=['$v-3prgvdtmX8ySVzXby4Lw6zrncPxKdiLrDnM94y3K4']",
  "event_id=$v-3prgvdtmX8ySVzXby4Lw6zrncPxKdiLrDnM94y3K4,depth=10,body=Historical 59 (batch=0),prevs=['$ahhYJXd-_gSSZ_mROiEvBI37nYgYCi4WsrvG8sfEQ6g']",
  "event_id=$ahhYJXd-_gSSZ_mROiEvBI37nYgYCi4WsrvG8sfEQ6g,depth=10,body=Historical 58 (batch=0),prevs=['$JHFT5ugpxzbU6MmFJZ3wXN0hK33EX3-9As0MemxjsX0']",
  "event_id=$JHFT5ugpxzbU6MmFJZ3wXN0hK33EX3-9As0MemxjsX0,depth=10,body=Historical 57 (batch=0),prevs=['$JYLcmedG2JdmtBQnnXbIEI34QyeaJrMGLAJt_Mi_vuY']",
  "event_id=$JYLcmedG2JdmtBQnnXbIEI34QyeaJrMGLAJt_Mi_vuY,depth=10,body=Historical 56 (batch=0),prevs=['$nFeaEcPilAYFoXcid_5mz5jDEqIyvTZLYX4erRf2jlM']",
  "event_id=$nFeaEcPilAYFoXcid_5mz5jDEqIyvTZLYX4erRf2jlM,depth=10,body=Historical 55 (batch=0),prevs=['$ig5cEWTkkNG-ox84UUjG35G6WEHu-qpqAwYKQSOwHxQ']",
  "event_id=$ig5cEWTkkNG-ox84UUjG35G6WEHu-qpqAwYKQSOwHxQ,depth=10,body=Historical 54 (batch=0),prevs=['$2MDm0b5ztPCyLlAThSBT38Ie3F0ZG3sAE0r3to3yhtc']",
  "event_id=$2MDm0b5ztPCyLlAThSBT38Ie3F0ZG3sAE0r3to3yhtc,depth=10,body=Historical 53 (batch=0),prevs=['$gZ_e1ws9bh_uKzY4UskhVCO4IFikMi-waYooJbSJAOI']",
  "event_id=$gZ_e1ws9bh_uKzY4UskhVCO4IFikMi-waYooJbSJAOI,depth=10,body=Historical 52 (batch=0),prevs=['$pOBdCutp83GhxQkP9_tCyYeaN7YvEo0fuf5MiRsu_ew']",
  "event_id=$pOBdCutp83GhxQkP9_tCyYeaN7YvEo0fuf5MiRsu_ew,depth=10,body=Historical 51 (batch=0),prevs=['$HCC02P42Zfcn-vYaZrhQu1IdHOMVQrsc6XsfIGPrWNE']",
  "event_id=$HCC02P42Zfcn-vYaZrhQu1IdHOMVQrsc6XsfIGPrWNE,depth=10,body=Historical 50 (batch=0),prevs=['$R4sIu0EhjycD9c7qHb1Sfdw34PqzIFOiTzCsPs_Z-Fk']",
  "event_id=$R4sIu0EhjycD9c7qHb1Sfdw34PqzIFOiTzCsPs_Z-Fk,depth=10,body=Historical 49 (batch=0),prevs=['$Gdo7IK1ZUK3pHvHvOETByF8J_PLqVP6BDk1qI_3AZ90']",
  "event_id=$Gdo7IK1ZUK3pHvHvOETByF8J_PLqVP6BDk1qI_3AZ90,depth=10,body=Historical 48 (batch=0),prevs=['$VJ5qpulzoG4vkhkWnbwpgtBMHAMQSmhUB76lrI_2KXc']",
  "event_id=$VJ5qpulzoG4vkhkWnbwpgtBMHAMQSmhUB76lrI_2KXc,depth=10,body=Historical 47 (batch=0),prevs=['$-h0DqfdRxhEi6RMiaoKEHqzRDqA5Qy8ZskPGAp5rUI0']",
  "event_id=$-h0DqfdRxhEi6RMiaoKEHqzRDqA5Qy8ZskPGAp5rUI0,depth=10,body=Historical 46 (batch=0),prevs=['$_L-oB9rLJsXOnJkjnUQNDUL1C3jw2-YNbTBDiKG958Q']",
  "event_id=$_L-oB9rLJsXOnJkjnUQNDUL1C3jw2-YNbTBDiKG958Q,depth=10,body=Historical 45 (batch=0),prevs=['$iQeszxPn42bPGUQknk5TCYIWxVqnGDG-h4Bk1JFH_-4']",
  "event_id=$iQeszxPn42bPGUQknk5TCYIWxVqnGDG-h4Bk1JFH_-4,depth=10,body=Historical 44 (batch=0),prevs=['$raFQyAel7PT5FUymMwF3SIOXxuMpX7aJyRFDfsyFTpo']",
  "event_id=$raFQyAel7PT5FUymMwF3SIOXxuMpX7aJyRFDfsyFTpo,depth=10,body=Historical 43 (batch=0),prevs=['$bYfZTSvNiI6nOE7NvnwsaJBlDt8eGifjNWiBoOd-4JQ']",
  "event_id=$bYfZTSvNiI6nOE7NvnwsaJBlDt8eGifjNWiBoOd-4JQ,depth=10,body=Historical 42 (batch=0),prevs=['$iflmqGrv2ERr58y8QWMW8cAYjIwctoOqc_S8Zpxf2LE']",
  "event_id=$iflmqGrv2ERr58y8QWMW8cAYjIwctoOqc_S8Zpxf2LE,depth=10,body=Historical 41 (batch=0),prevs=['$84bhYzG__UwK-Rc1T4UGKzQUcLGOTMB9cWikG6Np_hM']",
  "event_id=$84bhYzG__UwK-Rc1T4UGKzQUcLGOTMB9cWikG6Np_hM,depth=10,body=Historical 40 (batch=0),prevs=['$xqS8HMfkjk_9j2aHTisNv8oeocOq38nxxU8MrepqgMU']",
  "event_id=$xqS8HMfkjk_9j2aHTisNv8oeocOq38nxxU8MrepqgMU,depth=10,body=Historical 39 (batch=0),prevs=['$EChQxibb9iOk8GqNnZRL3dQ0EfcXJpbl942hiEf1uxo']",
  "event_id=$EChQxibb9iOk8GqNnZRL3dQ0EfcXJpbl942hiEf1uxo,depth=10,body=Historical 38 (batch=0),prevs=['$bzBMsp8mhPIhbhv8RBjSP8zwvJ6ix2HYoEeourkeRJE']",
  "event_id=$bzBMsp8mhPIhbhv8RBjSP8zwvJ6ix2HYoEeourkeRJE,depth=10,body=Historical 37 (batch=0),prevs=['$lDNy6ds20jjt077dA_f9nFJrAu6reMkA-a9cZaW6cpI']",
  "event_id=$lDNy6ds20jjt077dA_f9nFJrAu6reMkA-a9cZaW6cpI,depth=10,body=Historical 36 (batch=0),prevs=['$yMQY-EQTPizmc_vOIsWIcY-Acp7dBNJew27NlfDHduo']",
  "event_id=$yMQY-EQTPizmc_vOIsWIcY-Acp7dBNJew27NlfDHduo,depth=10,body=Historical 35 (batch=0),prevs=['$Y9NaQWbjDCwqc8TxXbCduEp2FWGNDwUA_ElYWF148T8']",
  "event_id=$Y9NaQWbjDCwqc8TxXbCduEp2FWGNDwUA_ElYWF148T8,depth=10,body=Historical 34 (batch=0),prevs=['$_if1SSgNjDIT1YefAeoQ0z2aH6pe2jynJUDraBZOCjc']",
  "event_id=$_if1SSgNjDIT1YefAeoQ0z2aH6pe2jynJUDraBZOCjc,depth=10,body=Historical 33 (batch=0),prevs=['$_JgwcI13PE8e2MuebXSLBIyd_MaIorlEZtF_z_uIVHo']",
  "event_id=$_JgwcI13PE8e2MuebXSLBIyd_MaIorlEZtF_z_uIVHo,depth=10,body=Historical 32 (batch=0),prevs=['$CRZ4J3otfULJ7Iw1MaAQvIeakECsoZQs1Ehh-xeaC_k']",
  "event_id=$CRZ4J3otfULJ7Iw1MaAQvIeakECsoZQs1Ehh-xeaC_k,depth=10,body=Historical 31 (batch=0),prevs=['$qaIW_OYBoW7I03J5DnXMEJkRMKtnOOO5ttQjniYzHnI']",
  "event_id=$qaIW_OYBoW7I03J5DnXMEJkRMKtnOOO5ttQjniYzHnI,depth=10,body=Historical 30 (batch=0),prevs=['$9VNFmDvQZiJo50kQf1qPTAQyCPsdHjx23qC0DT34wY0']",
  "event_id=$9VNFmDvQZiJo50kQf1qPTAQyCPsdHjx23qC0DT34wY0,depth=10,body=Historical 29 (batch=0),prevs=['$gFbNkb60KtE_SzR9oi16tpkKE9k4hijC2A6dt66Al4k']",
  "event_id=$gFbNkb60KtE_SzR9oi16tpkKE9k4hijC2A6dt66Al4k,depth=10,body=Historical 28 (batch=0),prevs=['$Mq-dMSTxKEwHS3gPvYwU-Obqs3-Tp3SWjRPUq2VDvJg']",
  "event_id=$Mq-dMSTxKEwHS3gPvYwU-Obqs3-Tp3SWjRPUq2VDvJg,depth=10,body=Historical 27 (batch=0),prevs=['$5EC7XJFkyMLzhujTLI_m9UXNEnn-4NKZj-DCouBgyt8']",
  "event_id=$5EC7XJFkyMLzhujTLI_m9UXNEnn-4NKZj-DCouBgyt8,depth=10,body=Historical 26 (batch=0),prevs=['$UxlPuejLS1jHkUNCMRe2vNmc2tcuut-G2N8q6RFTfVQ']",
  "event_id=$UxlPuejLS1jHkUNCMRe2vNmc2tcuut-G2N8q6RFTfVQ,depth=10,body=Historical 25 (batch=0),prevs=['$BNqTjoMam0rgznru1lqk1U8cCqlJRA4GwORVLBeFYTU']",
  "event_id=$BNqTjoMam0rgznru1lqk1U8cCqlJRA4GwORVLBeFYTU,depth=10,body=Historical 24 (batch=0),prevs=['$7SdwqRy6BHBnNb_sQtj2Rph9era4F68ZM6mGPzjJthA']",
  "event_id=$7SdwqRy6BHBnNb_sQtj2Rph9era4F68ZM6mGPzjJthA,depth=10,body=Historical 23 (batch=0),prevs=['$HiveBkhuxr5NIgx9wGhfrmttnGOi0VpeesmqtjpkqE4']",
  "event_id=$HiveBkhuxr5NIgx9wGhfrmttnGOi0VpeesmqtjpkqE4,depth=10,body=Historical 22 (batch=0),prevs=['$m7iJp9s_Stodzk1I1Lf8egDLbIdyvgJPjAxomFguHb4']",
  "event_id=$m7iJp9s_Stodzk1I1Lf8egDLbIdyvgJPjAxomFguHb4,depth=10,body=Historical 21 (batch=0),prevs=['$KRSKtmQJHl80Hn7UR3emVczq4-QgKycVop3lkSeppIs']",
  "event_id=$KRSKtmQJHl80Hn7UR3emVczq4-QgKycVop3lkSeppIs,depth=10,body=Historical 20 (batch=0),prevs=['$ZCZKQrCeoKEC6oKqwjhK_8EQqwq75b7fEop-GgYCTiU']",
  "event_id=$ZCZKQrCeoKEC6oKqwjhK_8EQqwq75b7fEop-GgYCTiU,depth=10,body=Historical 19 (batch=0),prevs=['$9pZEcOtzeGWbnmRS_zAQAhdCIE8DZ6_cJAR0tm2AG1k']",
  "event_id=$9pZEcOtzeGWbnmRS_zAQAhdCIE8DZ6_cJAR0tm2AG1k,depth=10,body=Historical 18 (batch=0),prevs=['$FFXzuSEKR53_PpJ6EBGp8FEBi5Aig2MIq-F1opufK10']",
  "event_id=$FFXzuSEKR53_PpJ6EBGp8FEBi5Aig2MIq-F1opufK10,depth=10,body=Historical 17 (batch=0),prevs=['$hr0MEvZerIeshwxLzAGbqMXQGOXU_u9H1wUPtzqa6gY']",
  "event_id=$hr0MEvZerIeshwxLzAGbqMXQGOXU_u9H1wUPtzqa6gY,depth=10,body=Historical 16 (batch=0),prevs=['$hjP2ryNx-yqznW3EKrBZE1DaGSbvK8lB_3cWUDDGTVk']",
  "event_id=$hjP2ryNx-yqznW3EKrBZE1DaGSbvK8lB_3cWUDDGTVk,depth=10,body=Historical 15 (batch=0),prevs=['$E24xBJItDTqteFc11s2kk2JuvpQjZ2fd-vEWJ7QS784']",
  "event_id=$E24xBJItDTqteFc11s2kk2JuvpQjZ2fd-vEWJ7QS784,depth=10,body=Historical 14 (batch=0),prevs=['$eOuhBaOKitgobbzyU5XVieMLzgoCITj9eG-ewSMDB9I']",
  "event_id=$eOuhBaOKitgobbzyU5XVieMLzgoCITj9eG-ewSMDB9I,depth=10,body=Historical 13 (batch=0),prevs=['$4F4Xhu7QKpLM-n_aFmCLrKRP6zw7lrDLAYFTPKn8jYo']",
  "event_id=$4F4Xhu7QKpLM-n_aFmCLrKRP6zw7lrDLAYFTPKn8jYo,depth=10,body=Historical 12 (batch=0),prevs=['$e0wqSykOpA8Y0yipT9643SVVQ3DMOnW11yxvqAj9JrE']",
  "event_id=$e0wqSykOpA8Y0yipT9643SVVQ3DMOnW11yxvqAj9JrE,depth=10,body=Historical 11 (batch=0),prevs=['$u5gxt1eo59odS1XqVS-kV2GcaitrzorppcqVj3scevI']",
  "event_id=$u5gxt1eo59odS1XqVS-kV2GcaitrzorppcqVj3scevI,depth=10,body=Historical 10 (batch=0),prevs=['$Z1E79_z_FcAnw4G9NOvx3WhyL2ACyYPgy6zLhomKVks']",
  "event_id=$Z1E79_z_FcAnw4G9NOvx3WhyL2ACyYPgy6zLhomKVks,depth=10,body=Historical 9 (batch=0),prevs=['$USyjgdL7fwX4BvmblVs2cicGO3WDRB-ox56298yZmAA']",
  "event_id=$USyjgdL7fwX4BvmblVs2cicGO3WDRB-ox56298yZmAA,depth=10,body=Historical 8 (batch=0),prevs=['$tggu2uQEL2BWVJiwqsqwCJUca3SEFeylCAbPw5DrIdo']",
  "event_id=$tggu2uQEL2BWVJiwqsqwCJUca3SEFeylCAbPw5DrIdo,depth=10,body=Historical 7 (batch=0),prevs=['$M0iINx5TifxVTJ6pz-lio7xIeSla1RIV_N9WIK5sf4c']",
  "event_id=$M0iINx5TifxVTJ6pz-lio7xIeSla1RIV_N9WIK5sf4c,depth=10,body=Historical 6 (batch=0),prevs=['$hEs6MN2xyjXqHRcWV53_23NeNm--al9b_5BnMYYDPMY']",
  "event_id=$hEs6MN2xyjXqHRcWV53_23NeNm--al9b_5BnMYYDPMY,depth=10,body=Historical 5 (batch=0),prevs=['$L0BJu5ZOx2l8AMivXMBCqrvymVu_nCQM4ZO23uv5tBU']",
  "event_id=$L0BJu5ZOx2l8AMivXMBCqrvymVu_nCQM4ZO23uv5tBU,depth=10,body=Historical 4 (batch=0),prevs=['$V5BZU5LpCbW6XvPLGls0QGRbo98uuf1KpMZ0fg8k_-o']"]

new implementation backfill events(100)=[
  "event_id=$V5BZU5LpCbW6XvPLGls0QGRbo98uuf1KpMZ0fg8k_-o,depth=10,body=Historical 3 (batch=0),prevs=['$lF2XlCU-QmhC0YF8Wn8Q-EfQXqDxF_usgPIZ28rNhII']",
  "event_id=$lF2XlCU-QmhC0YF8Wn8Q-EfQXqDxF_usgPIZ28rNhII,depth=10,body=Historical 2 (batch=0),prevs=['$Ghnwsm4CqaWopAMGjKm8piFuTEqSaWgl1X7esubZOco']",
  "event_id=$Ghnwsm4CqaWopAMGjKm8piFuTEqSaWgl1X7esubZOco,depth=10,body=Historical 1 (batch=0),prevs=['$WRV7zPTr4KknEcYUvPlfxMdxcmlrQSsXAfTTlSBpjko']",
  "event_id=$WRV7zPTr4KknEcYUvPlfxMdxcmlrQSsXAfTTlSBpjko,depth=10,body=Historical 0 (batch=0),prevs=['$30VmrbC0AauqzWQ9F2g_KTPUdJhH_r79olOmfZYe0Pg']",
  "event_id=$30VmrbC0AauqzWQ9F2g_KTPUdJhH_r79olOmfZYe0Pg,depth=10,body=org.matrix.msc2716.insertion,prevs=['$an2lLzRXApWnCuRNrb1jTfMY9lWbz909AxyIRWy5-C0']",
  "event_id=$CmUx1RcXqn9QwVSKFKGmwVXwEFE9sHpnac5xvCgm-Co,depth=10,body=org.matrix.msc2716.batch,prevs=['$Kt2_lR2GlI6YoDY7q1ttaoIUv_s8WsjKgce6PT8utIA']",
  "event_id=$Kt2_lR2GlI6YoDY7q1ttaoIUv_s8WsjKgce6PT8utIA,depth=10,body=Historical 99 (batch=1),prevs=['$eaZav6ZipscONXQMtzaWQrTmiw9yo6_ifeaiI2VTJJM']",
  "event_id=$eaZav6ZipscONXQMtzaWQrTmiw9yo6_ifeaiI2VTJJM,depth=10,body=Historical 98 (batch=1),prevs=['$sSIhVeKMkHlwvQmCdUxouZyc0A7oBK_IcTYYPhQHniY']",
  "event_id=$sSIhVeKMkHlwvQmCdUxouZyc0A7oBK_IcTYYPhQHniY,depth=10,body=Historical 97 (batch=1),prevs=['$tQ6z1P58SRR3qqS6UnjWn85zP6CoU882ehdCkNcwhA8']",
  "event_id=$tQ6z1P58SRR3qqS6UnjWn85zP6CoU882ehdCkNcwhA8,depth=10,body=Historical 96 (batch=1),prevs=['$mh2o2cBnVjU5uQr0UAMPAWPpGsRzIKKpzc54Q2Z9JGY']",
  "event_id=$mh2o2cBnVjU5uQr0UAMPAWPpGsRzIKKpzc54Q2Z9JGY,depth=10,body=Historical 95 (batch=1),prevs=['$O_SPFvep-jjnHsW2484GzRYIDZkJhZYVgmCstDR3Qn0']",
  "event_id=$O_SPFvep-jjnHsW2484GzRYIDZkJhZYVgmCstDR3Qn0,depth=10,body=Historical 94 (batch=1),prevs=['$MkZU6zPnF5L4Mb9Y0AkOlWSG6S5yKhyVsFPm_WPc7EU']",
  "event_id=$MkZU6zPnF5L4Mb9Y0AkOlWSG6S5yKhyVsFPm_WPc7EU,depth=10,body=Historical 93 (batch=1),prevs=['$lts-3h-4-93qb12QQ8PwkvSu4bSJVKulvj_N2UPRhvs']",
  "event_id=$lts-3h-4-93qb12QQ8PwkvSu4bSJVKulvj_N2UPRhvs,depth=10,body=Historical 92 (batch=1),prevs=['$abLL8g1D70BE9EqtgSGoVjtbVK6QKG0KZhgBeJmEAuw']",
  "event_id=$abLL8g1D70BE9EqtgSGoVjtbVK6QKG0KZhgBeJmEAuw,depth=10,body=Historical 91 (batch=1),prevs=['$H2-ZSHXeJII1Jc3XvEjFfmjGUZgpsmlay7YuKKlF7JQ']",
  "event_id=$H2-ZSHXeJII1Jc3XvEjFfmjGUZgpsmlay7YuKKlF7JQ,depth=10,body=Historical 90 (batch=1),prevs=['$VsrkCzgyKUyPT4XpS0pa-K-CwRJ7MeNu9ictxOHPZo4']",
  "event_id=$VsrkCzgyKUyPT4XpS0pa-K-CwRJ7MeNu9ictxOHPZo4,depth=10,body=Historical 89 (batch=1),prevs=['$NSVKzqK5ucXWwWq7j4yfADj5m-8GqWFCWeqo2hvUDWg']",
  "event_id=$NSVKzqK5ucXWwWq7j4yfADj5m-8GqWFCWeqo2hvUDWg,depth=10,body=Historical 88 (batch=1),prevs=['$y5lPUOYBw0g12eIgDUdTn5QnmXD2QeFks61Ivpha1XA']",
  "event_id=$y5lPUOYBw0g12eIgDUdTn5QnmXD2QeFks61Ivpha1XA,depth=10,body=Historical 87 (batch=1),prevs=['$BdehWtfVHIxjlD3-pWh-6kcb1w2oPhAhgCWxNUkKAD8']",
  "event_id=$BdehWtfVHIxjlD3-pWh-6kcb1w2oPhAhgCWxNUkKAD8,depth=10,body=Historical 86 (batch=1),prevs=['$B0afYanjEywl1q1fYru7nL2RmjjZpF0gjTVQ7n9fY6k']",
  "event_id=$B0afYanjEywl1q1fYru7nL2RmjjZpF0gjTVQ7n9fY6k,depth=10,body=Historical 85 (batch=1),prevs=['$F2cOk9oSDKNSlrMoJioWtLi8AdhMkS-QpVwZw2qhWgM']",
  "event_id=$F2cOk9oSDKNSlrMoJioWtLi8AdhMkS-QpVwZw2qhWgM,depth=10,body=Historical 84 (batch=1),prevs=['$QGCvZWFsgW4KTskjaSqxjTenOa3TbbpBDWVyXESXDGI']",
  "event_id=$QGCvZWFsgW4KTskjaSqxjTenOa3TbbpBDWVyXESXDGI,depth=10,body=Historical 83 (batch=1),prevs=['$5evMhA_hLx02z7ydMm_cRfSUsC9OX0MJLayMydLrzN0']",
  "event_id=$5evMhA_hLx02z7ydMm_cRfSUsC9OX0MJLayMydLrzN0,depth=10,body=Historical 82 (batch=1),prevs=['$4jmsKkx-6NRVMCflN_s6sOfC053Zw1wOfCCM0bWlIX8']",
  "event_id=$4jmsKkx-6NRVMCflN_s6sOfC053Zw1wOfCCM0bWlIX8,depth=10,body=Historical 81 (batch=1),prevs=['$sTtp0fKA9BG4GDkJ8CUBQVNqpDDcf_a-PkGhJkGqzrw']",
  "event_id=$sTtp0fKA9BG4GDkJ8CUBQVNqpDDcf_a-PkGhJkGqzrw,depth=10,body=Historical 80 (batch=1),prevs=['$96go7TqZzPq75eOOLiDY0FkylHrmAGly0MTwX4_McMs']",
  "event_id=$96go7TqZzPq75eOOLiDY0FkylHrmAGly0MTwX4_McMs,depth=10,body=Historical 79 (batch=1),prevs=['$kUiJWkPHn1nPrUDzZJvWhTz1T8UBVELMlEdJGOEB_Qo']",
  "event_id=$kUiJWkPHn1nPrUDzZJvWhTz1T8UBVELMlEdJGOEB_Qo,depth=10,body=Historical 78 (batch=1),prevs=['$5GCJZEmchNvy0ikP7K7ZlvFb9FS6aVKRedx-08ZAF3s']",
  "event_id=$5GCJZEmchNvy0ikP7K7ZlvFb9FS6aVKRedx-08ZAF3s,depth=10,body=Historical 77 (batch=1),prevs=['$A3r2LkTKaHxBQh1Ailt4dCtaESyo53Ss1u1s3QiE_k0']",
  "event_id=$A3r2LkTKaHxBQh1Ailt4dCtaESyo53Ss1u1s3QiE_k0,depth=10,body=Historical 76 (batch=1),prevs=['$R7RRyXo6Msc3-368vZpiG84DfqOAiKLiSbV0ECGv6Oo']",
  "event_id=$R7RRyXo6Msc3-368vZpiG84DfqOAiKLiSbV0ECGv6Oo,depth=10,body=Historical 75 (batch=1),prevs=['$jmkYlZN6-oiQOi-bJfs0whVS7kKeJJUN6YJQfubbaEU']",
  "event_id=$jmkYlZN6-oiQOi-bJfs0whVS7kKeJJUN6YJQfubbaEU,depth=10,body=Historical 74 (batch=1),prevs=['$IuGVv6TZj5ZoOP4HHqZsKPWQ9HDTHS4QV7oFHk4kl7o']",
  "event_id=$IuGVv6TZj5ZoOP4HHqZsKPWQ9HDTHS4QV7oFHk4kl7o,depth=10,body=Historical 73 (batch=1),prevs=['$sNds_pVGUFf41AJwHmA1Sy7MUX1N9MtHGfw9ye6EItM']",
  "event_id=$sNds_pVGUFf41AJwHmA1Sy7MUX1N9MtHGfw9ye6EItM,depth=10,body=Historical 72 (batch=1),prevs=['$u0R7zUNs4BpneteeM7PWgGZE4XzQJuAwUtFSQGwTNI0']",
  "event_id=$u0R7zUNs4BpneteeM7PWgGZE4XzQJuAwUtFSQGwTNI0,depth=10,body=Historical 71 (batch=1),prevs=['$VthGtqcIDd04US7pMNTr0gSmR7PIOPLmDOljx1qNleQ']",
  "event_id=$VthGtqcIDd04US7pMNTr0gSmR7PIOPLmDOljx1qNleQ,depth=10,body=Historical 70 (batch=1),prevs=['$qYlzglfNCN1Br-4UkajF4c-SKYI0By5gExUxczeDsg4']",
  "event_id=$qYlzglfNCN1Br-4UkajF4c-SKYI0By5gExUxczeDsg4,depth=10,body=Historical 69 (batch=1),prevs=['$BsxtZqcTIs1aqFRsXtapKXYF6w2GiUEswpl6kH6S_4o']",
  "event_id=$BsxtZqcTIs1aqFRsXtapKXYF6w2GiUEswpl6kH6S_4o,depth=10,body=Historical 68 (batch=1),prevs=['$PmOzgtCDJCyGowxB8gWcYbfA2KUX0MCJ3xTE48FhAyY']",
  "event_id=$PmOzgtCDJCyGowxB8gWcYbfA2KUX0MCJ3xTE48FhAyY,depth=10,body=Historical 67 (batch=1),prevs=['$vZq13XMjZRJdFefxBg0L1U9shZq1TbArtR43YkQG7i8']",
  "event_id=$vZq13XMjZRJdFefxBg0L1U9shZq1TbArtR43YkQG7i8,depth=10,body=Historical 66 (batch=1),prevs=['$526Rm38ribM0XL3HltTbjVCUa3cDUdBh0V-9AMPtsYY']",
  "event_id=$526Rm38ribM0XL3HltTbjVCUa3cDUdBh0V-9AMPtsYY,depth=10,body=Historical 65 (batch=1),prevs=['$-sPB5z3uyN5tr4KezPKt6xu_yoaqgSDKhkzTJ0Um7yI']",
  "event_id=$-sPB5z3uyN5tr4KezPKt6xu_yoaqgSDKhkzTJ0Um7yI,depth=10,body=Historical 64 (batch=1),prevs=['$y5mX5SlScEAyPbUx1UA2-_ml0abbh0nGFYgJzF0XzZ0']",
  "event_id=$y5mX5SlScEAyPbUx1UA2-_ml0abbh0nGFYgJzF0XzZ0,depth=10,body=Historical 63 (batch=1),prevs=['$erT1ogi6tp9tF33_N3ZBLpIMBXOunszhADMYjs8U5AE']",
  "event_id=$erT1ogi6tp9tF33_N3ZBLpIMBXOunszhADMYjs8U5AE,depth=10,body=Historical 62 (batch=1),prevs=['$d9qTo4Ray7GJWJPm-CHkfZmZC0WmD8-K6Z1Riyot3fw']",
  "event_id=$d9qTo4Ray7GJWJPm-CHkfZmZC0WmD8-K6Z1Riyot3fw,depth=10,body=Historical 61 (batch=1),prevs=['$TVgdscs2fWtvx0ak6V0ljabU_4cc6UH18Y1r3OvDLYc']",
  "event_id=$TVgdscs2fWtvx0ak6V0ljabU_4cc6UH18Y1r3OvDLYc,depth=10,body=Historical 60 (batch=1),prevs=['$eMi_3TfsgYw2Fs99AJpr_9KAapZq3uwtv2fUDQ2weTc']",
  "event_id=$eMi_3TfsgYw2Fs99AJpr_9KAapZq3uwtv2fUDQ2weTc,depth=10,body=Historical 59 (batch=1),prevs=['$UBL_4l45DMvVTE12gyRaSDMw_fi_biDx1dL_gWz_v-g']",
  "event_id=$UBL_4l45DMvVTE12gyRaSDMw_fi_biDx1dL_gWz_v-g,depth=10,body=Historical 58 (batch=1),prevs=['$A6-UMuau--ffdnOA0NqWQK9uO0NBCB_rwVCvu_hSAOw']",
  "event_id=$A6-UMuau--ffdnOA0NqWQK9uO0NBCB_rwVCvu_hSAOw,depth=10,body=Historical 57 (batch=1),prevs=['$Eia3O1ptxGtKhINTEVX_FxBU3TjL0MBp1XdeFCStcCg']",
  "event_id=$Eia3O1ptxGtKhINTEVX_FxBU3TjL0MBp1XdeFCStcCg,depth=10,body=Historical 56 (batch=1),prevs=['$Pb5OyRAHNrXDyPt8F1dTWVGi-Zvqkf-HqzCwcu0salY']",
  "event_id=$Pb5OyRAHNrXDyPt8F1dTWVGi-Zvqkf-HqzCwcu0salY,depth=10,body=Historical 55 (batch=1),prevs=['$c1KPN_U0VF8G3jGnD6UpQob_h9X_KQfXv4lciJUb4PQ']",
  "event_id=$c1KPN_U0VF8G3jGnD6UpQob_h9X_KQfXv4lciJUb4PQ,depth=10,body=Historical 54 (batch=1),prevs=['$fwWir53iSGaGEMRt-mQe6hYmkwIQGjDojFO08zlzgX4']",
  "event_id=$fwWir53iSGaGEMRt-mQe6hYmkwIQGjDojFO08zlzgX4,depth=10,body=Historical 53 (batch=1),prevs=['$TmacWSD6J8y8vERZfA-IdRpDeMS_4A1HFNUBg_W5MaA']",
  "event_id=$TmacWSD6J8y8vERZfA-IdRpDeMS_4A1HFNUBg_W5MaA,depth=10,body=Historical 52 (batch=1),prevs=['$_xBvb_mIUJgyvA-rgBppyjXVZ3SXn03isQ0axMOtThc']",
  "event_id=$_xBvb_mIUJgyvA-rgBppyjXVZ3SXn03isQ0axMOtThc,depth=10,body=Historical 51 (batch=1),prevs=['$AYD-cd4lrJLbW35mCK8dKZEZI705gOsA1iiyW3uqDWk']",
  "event_id=$AYD-cd4lrJLbW35mCK8dKZEZI705gOsA1iiyW3uqDWk,depth=10,body=Historical 50 (batch=1),prevs=['$KK1BXAZ3E-U7qKWhO6Y6DfXedQ04BhwuA_bNI5Nee8g']",
  "event_id=$KK1BXAZ3E-U7qKWhO6Y6DfXedQ04BhwuA_bNI5Nee8g,depth=10,body=Historical 49 (batch=1),prevs=['$_oLPTYdg92q_l3c6vxV4tO9ioR9nPrk1lyb2qiLIPOk']",
  "event_id=$_oLPTYdg92q_l3c6vxV4tO9ioR9nPrk1lyb2qiLIPOk,depth=10,body=Historical 48 (batch=1),prevs=['$RZ0Mzvftn99rU6XFl32jKHs0R6YYaMPy_HJy41tK4ig']",
  "event_id=$RZ0Mzvftn99rU6XFl32jKHs0R6YYaMPy_HJy41tK4ig,depth=10,body=Historical 47 (batch=1),prevs=['$mloZAG1tKqfjIBQhJk8Mn7M5ER84cJ7bC0TW3rOsKpU']",
  "event_id=$mloZAG1tKqfjIBQhJk8Mn7M5ER84cJ7bC0TW3rOsKpU,depth=10,body=Historical 46 (batch=1),prevs=['$RY6YWeTkTc8aQP6ZkPq_vL0qs1kg53QeDcKmrLi0ASw']",
  "event_id=$RY6YWeTkTc8aQP6ZkPq_vL0qs1kg53QeDcKmrLi0ASw,depth=10,body=Historical 45 (batch=1),prevs=['$5A2e6qJj9n9ejvUW6TcVkm6-cuHghnRKQDW3T3a4eqI']",
  "event_id=$5A2e6qJj9n9ejvUW6TcVkm6-cuHghnRKQDW3T3a4eqI,depth=10,body=Historical 44 (batch=1),prevs=['$KDMWnrkM1ezjDXlSrXjvhZS-RRuJc6_hrX-Msh0gP7o']",
  "event_id=$KDMWnrkM1ezjDXlSrXjvhZS-RRuJc6_hrX-Msh0gP7o,depth=10,body=Historical 43 (batch=1),prevs=['$9Q6SSr_YMaQ6oc9w7lYs-bRw0cNESLPTE45Fg2_Zzv4']",
  "event_id=$9Q6SSr_YMaQ6oc9w7lYs-bRw0cNESLPTE45Fg2_Zzv4,depth=10,body=Historical 42 (batch=1),prevs=['$b2QYayqOOaCxQqs655Tve2OKeXrT9CJeVIchVUn00hI']",
  "event_id=$b2QYayqOOaCxQqs655Tve2OKeXrT9CJeVIchVUn00hI,depth=10,body=Historical 41 (batch=1),prevs=['$xQQBp4Dwm-3lxCWkJsck9bXuQoCKsdPIortpc79nTtw']",
  "event_id=$xQQBp4Dwm-3lxCWkJsck9bXuQoCKsdPIortpc79nTtw,depth=10,body=Historical 40 (batch=1),prevs=['$RgTwPhV6RzdVBmuqoYBNblHPZ-qpYuTCshDBrfpwvds']",
  "event_id=$RgTwPhV6RzdVBmuqoYBNblHPZ-qpYuTCshDBrfpwvds,depth=10,body=Historical 39 (batch=1),prevs=['$RWyv_RX_EoNzCDnHSCfx6fDnJrhWl46iWYomZSS2GsQ']",
  "event_id=$RWyv_RX_EoNzCDnHSCfx6fDnJrhWl46iWYomZSS2GsQ,depth=10,body=Historical 38 (batch=1),prevs=['$Sj78rQcr9DuCnhtcYuZjWS6o1ZiFDd3ZQxDtJKxjeNg']",
  "event_id=$Sj78rQcr9DuCnhtcYuZjWS6o1ZiFDd3ZQxDtJKxjeNg,depth=10,body=Historical 37 (batch=1),prevs=['$KePkqP3sJgnC8wVMyRFd0v7Fsu-8LiLTZbhx0JsWfnY']",
  "event_id=$KePkqP3sJgnC8wVMyRFd0v7Fsu-8LiLTZbhx0JsWfnY,depth=10,body=Historical 36 (batch=1),prevs=['$QdyPRKCNju-Rzm1u530j3N7AEcRFA5mAO3sVbBZunkQ']",
  "event_id=$QdyPRKCNju-Rzm1u530j3N7AEcRFA5mAO3sVbBZunkQ,depth=10,body=Historical 35 (batch=1),prevs=['$dd3HF7wo7YRhA6e0j5I9eGZtaH1rMK7dAriiNTQd7-Y']",
  "event_id=$dd3HF7wo7YRhA6e0j5I9eGZtaH1rMK7dAriiNTQd7-Y,depth=10,body=Historical 34 (batch=1),prevs=['$l6q9M7E_yWQID_v4YZMYZfNevVqhK0Pm6qIDABcx7ag']",
  "event_id=$l6q9M7E_yWQID_v4YZMYZfNevVqhK0Pm6qIDABcx7ag,depth=10,body=Historical 33 (batch=1),prevs=['$GAdAhlUMns9H8jeNH6yrfNp1c4_-lhQKyBnmAkQNZ5A']",
  "event_id=$GAdAhlUMns9H8jeNH6yrfNp1c4_-lhQKyBnmAkQNZ5A,depth=10,body=Historical 32 (batch=1),prevs=['$6liLB-IPWtTynCd27je2vCoHfDK_S3pwlRe5GoUsLY8']",
  "event_id=$6liLB-IPWtTynCd27je2vCoHfDK_S3pwlRe5GoUsLY8,depth=10,body=Historical 31 (batch=1),prevs=['$9C_ywoPrhy68x4-MurcodrKje7FOw5GuG9yMJ6W2ObM']",
  "event_id=$9C_ywoPrhy68x4-MurcodrKje7FOw5GuG9yMJ6W2ObM,depth=10,body=Historical 30 (batch=1),prevs=['$QyvDk3PQ4dEDxHb0ihFqAwWpqH-NQ86GV_w5njGNOSY']",
  "event_id=$QyvDk3PQ4dEDxHb0ihFqAwWpqH-NQ86GV_w5njGNOSY,depth=10,body=Historical 29 (batch=1),prevs=['$c2ee8wW8tAvev6UB1yOZMA56j-BFcxZJQqPbXxiqmeg']",
  "event_id=$c2ee8wW8tAvev6UB1yOZMA56j-BFcxZJQqPbXxiqmeg,depth=10,body=Historical 28 (batch=1),prevs=['$psirWB7VB7WUVg7IjiG1XX2WMOAiN_FFcdo8crQ3KF0']",
  "event_id=$psirWB7VB7WUVg7IjiG1XX2WMOAiN_FFcdo8crQ3KF0,depth=10,body=Historical 27 (batch=1),prevs=['$AMyblWA-Jny9Nobd-ghaFit6ZDG6OOg3rTxwEIn6zkQ']",
  "event_id=$AMyblWA-Jny9Nobd-ghaFit6ZDG6OOg3rTxwEIn6zkQ,depth=10,body=Historical 26 (batch=1),prevs=['$EtDL8yGxA03ZO5pTeztYOQvHxUaMCCz9Nj7oGgM0CL8']",
  "event_id=$EtDL8yGxA03ZO5pTeztYOQvHxUaMCCz9Nj7oGgM0CL8,depth=10,body=Historical 25 (batch=1),prevs=['$zmqDOTVLDS_khXDmPiPvJ7745wI_yuEAsxS_fyDroIo']",
  "event_id=$zmqDOTVLDS_khXDmPiPvJ7745wI_yuEAsxS_fyDroIo,depth=10,body=Historical 24 (batch=1),prevs=['$lMcNfcLZKCtmrGISzGo3M_lxJapImAmdky9Lezij6TI']",
  "event_id=$lMcNfcLZKCtmrGISzGo3M_lxJapImAmdky9Lezij6TI,depth=10,body=Historical 23 (batch=1),prevs=['$i4vpa73GXD7wZB2r2t-8nV--pGq_GdKCotM2-wpVJ10']",
  "event_id=$i4vpa73GXD7wZB2r2t-8nV--pGq_GdKCotM2-wpVJ10,depth=10,body=Historical 22 (batch=1),prevs=['$9BA_P9uGh-cLjLau5nToTmpsnxdaPkT2RjMf9ZYglNI']",
  "event_id=$9BA_P9uGh-cLjLau5nToTmpsnxdaPkT2RjMf9ZYglNI,depth=10,body=Historical 21 (batch=1),prevs=['$ySO1DPJTl_IiFwv4d6jbNy43a1go9rD_86A2EPIGVZE']",
  "event_id=$ySO1DPJTl_IiFwv4d6jbNy43a1go9rD_86A2EPIGVZE,depth=10,body=Historical 20 (batch=1),prevs=['$XOyDGiQEhlEoT8UGl4HIod5Zb29R3Op2BFIaVeiCRyY']",
  "event_id=$XOyDGiQEhlEoT8UGl4HIod5Zb29R3Op2BFIaVeiCRyY,depth=10,body=Historical 19 (batch=1),prevs=['$XDqJHr3kmi2Z9hpNX2Vz7o524cLEMkjKwk-_V2A3U1U']",
  "event_id=$XDqJHr3kmi2Z9hpNX2Vz7o524cLEMkjKwk-_V2A3U1U,depth=10,body=Historical 18 (batch=1),prevs=['$hlXaBZDCY4XSg-qjuzkAWhYqoBpdX8ehIFZd5M_9rRM']",
  "event_id=$hlXaBZDCY4XSg-qjuzkAWhYqoBpdX8ehIFZd5M_9rRM,depth=10,body=Historical 17 (batch=1),prevs=['$LMLomCl3U7zYXi4sIHHDnfznJkns0-DLRDGMbhJ1AAo']",
  "event_id=$LMLomCl3U7zYXi4sIHHDnfznJkns0-DLRDGMbhJ1AAo,depth=10,body=Historical 16 (batch=1),prevs=['$d0pjrKi7Ws1QM-XyB1_d9Yrg0AGY8MD6JLJqrnaaoJM']",
  "event_id=$d0pjrKi7Ws1QM-XyB1_d9Yrg0AGY8MD6JLJqrnaaoJM,depth=10,body=Historical 15 (batch=1),prevs=['$hx2dBPyOh6KnEzdpw7CnvoSOkiYTKJp82MXaWA_77N0']",
  "event_id=$hx2dBPyOh6KnEzdpw7CnvoSOkiYTKJp82MXaWA_77N0,depth=10,body=Historical 14 (batch=1),prevs=['$-y0R8tPCI4n4L0t2j4Ua3QIfau7n4kqPXgnQjb4e1qk']",
  "event_id=$-y0R8tPCI4n4L0t2j4Ua3QIfau7n4kqPXgnQjb4e1qk,depth=10,body=Historical 13 (batch=1),prevs=['$Af85_d8_UkHp8r8KzOnjCCI0whyMQB7U3G5u8NlMk4I']",
  "event_id=$Af85_d8_UkHp8r8KzOnjCCI0whyMQB7U3G5u8NlMk4I,depth=10,body=Historical 12 (batch=1),prevs=['$1nQJC9ULuCVGYWU_umE58wq5wHmmFx2GdgF6nmQt5VU']",
  "event_id=$1nQJC9ULuCVGYWU_umE58wq5wHmmFx2GdgF6nmQt5VU,depth=10,body=Historical 11 (batch=1),prevs=['$SdAp6h2pjv7aIxDSZJxwQbolYrqzUGf1gHOeBnqAULU']",
  "event_id=$SdAp6h2pjv7aIxDSZJxwQbolYrqzUGf1gHOeBnqAULU,depth=10,body=Historical 10 (batch=1),prevs=['$YDMkSKMoYDtwMd-nKvgAIk38uzc1UejIyBvXNoJ_RFM']",
  "event_id=$YDMkSKMoYDtwMd-nKvgAIk38uzc1UejIyBvXNoJ_RFM,depth=10,body=Historical 9 (batch=1),prevs=['$a8nvUo95cOf-cO4hcgfO1sT0kyWKq8XPUDmJe0DbxNE']",
  "event_id=$a8nvUo95cOf-cO4hcgfO1sT0kyWKq8XPUDmJe0DbxNE,depth=10,body=Historical 8 (batch=1),prevs=['$LunwOlIdMiDjrcQazQ6I3n0T8aLfmSA24fIEeth1hHs']",
  "event_id=$LunwOlIdMiDjrcQazQ6I3n0T8aLfmSA24fIEeth1hHs,depth=10,body=Historical 7 (batch=1),prevs=['$UOvHOp4SKtZqSDWZvD0s3VGdU23-6YqiWy_742cqY-8']",
  "event_id=$UOvHOp4SKtZqSDWZvD0s3VGdU23-6YqiWy_742cqY-8,depth=10,body=Historical 6 (batch=1),prevs=['$W-c9syEfTTufHnNBpuMUIk621ecs9W6bTeLW0OgWFfY']\n"]

new implementation backfill events(17)=[
  "event_id=$W-c9syEfTTufHnNBpuMUIk621ecs9W6bTeLW0OgWFfY,depth=10,body=Historical 5 (batch=1),prevs=['$Gq8-foARgSoEzcEwXhOsiey2pefpCX3vjnCLtsIdvU0']",
  "event_id=$Gq8-foARgSoEzcEwXhOsiey2pefpCX3vjnCLtsIdvU0,depth=10,body=Historical 4 (batch=1),prevs=['$toIu3hsoBDXkVJs91nUG0hodf3nuFkbidY7ztVPrtCM']",
  "event_id=$toIu3hsoBDXkVJs91nUG0hodf3nuFkbidY7ztVPrtCM,depth=10,body=Historical 3 (batch=1),prevs=['$SRcKQ-ggRujFzHbJC44rWvslb4YBKgieWahZakx3zMw']",
  "event_id=$SRcKQ-ggRujFzHbJC44rWvslb4YBKgieWahZakx3zMw,depth=10,body=Historical 2 (batch=1),prevs=['$Pgqvjn_rkQb6eEdQDeKclO-l_ODlnplF7S-rT1i_-64']",
  "event_id=$Pgqvjn_rkQb6eEdQDeKclO-l_ODlnplF7S-rT1i_-64,depth=10,body=Historical 1 (batch=1),prevs=['$jEW2Smmgh5tIh1QMtEqmluy-o3FrcVhNAH5e3R4jMoo']",
  "event_id=$jEW2Smmgh5tIh1QMtEqmluy-o3FrcVhNAH5e3R4jMoo,depth=10,body=Historical 0 (batch=1),prevs=['$1bbHCIrQLToHvum-PGxpIKhCmZ5MYLusAy_gJm7IWTw']",
  "event_id=$1bbHCIrQLToHvum-PGxpIKhCmZ5MYLusAy_gJm7IWTw,depth=10,body=org.matrix.msc2716.insertion,prevs=['$758TrMVR2MlE4EhpRUvCYZFcp11cbE27edQcwKaUSI8']",
  "event_id=$T5-OUXAW3CCYAHskqk5s0V2ZzJoQy5D8lQJ7LZFWvTI,depth=9,body=Message 1 (eventIDsBefore),prevs=['$WYEMee2J5PcH1yM2XDnD2XIhyMS_HHDdYXop6BOL69E']",
  "event_id=$WYEMee2J5PcH1yM2XDnD2XIhyMS_HHDdYXop6BOL69E,depth=8,body=Message 0 (eventIDsBefore),prevs=['$jTD4_ZLQQgXienBOsHMwSqyuuYtXQq4WYj7hnPbRC0o']",
  "event_id=$jTD4_ZLQQgXienBOsHMwSqyuuYtXQq4WYj7hnPbRC0o,depth=7,body=m.room.member,prevs=['$d9EI1fE4ArZLYeQPs3P6dlqTmtskwjJWtlfNEZTdSoA']",
  "event_id=$d9EI1fE4ArZLYeQPs3P6dlqTmtskwjJWtlfNEZTdSoA,depth=6,body=m.room.name,prevs=['$VvA6Hhp67rDLix7Pg9tZs6DDDh0gq_Mw-qr9wd5cO2k']",
  "event_id=$VvA6Hhp67rDLix7Pg9tZs6DDDh0gq_Mw-qr9wd5cO2k,depth=5,body=m.room.history_visibility,prevs=['$7fkTDT3EgbweX381_HInASE-jAzhxcQfkzOWuXxXO7U']",
  "event_id=$7fkTDT3EgbweX381_HInASE-jAzhxcQfkzOWuXxXO7U,depth=4,body=m.room.join_rules,prevs=['$uvO6_KB-CJtOtomqSgSAWLL59mJK0JW2U33ebFTuoK4']",
  "event_id=$uvO6_KB-CJtOtomqSgSAWLL59mJK0JW2U33ebFTuoK4,depth=3,body=m.room.power_levels,prevs=['$HjRQ8RpCPPWO4Ufq6bSCTqa_tSlKez15KvZRMpvvskc']",
  "event_id=$HjRQ8RpCPPWO4Ufq6bSCTqa_tSlKez15KvZRMpvvskc,depth=2,body=m.room.member,prevs=['$yXLeDjVhdX5jmjLVxmsxKhG_rjfA9ujL8Cd8eHNha6U']",
  'event_id=$758TrMVR2MlE4EhpRUvCYZFcp11cbE27edQcwKaUSI8,depth=1,body=m.room.member,prevs=[]',
  'event_id=$yXLeDjVhdX5jmjLVxmsxKhG_rjfA9ujL8Cd8eHNha6U,depth=1,body=m.room.create,prevs=[]'
]
```
This commit is contained in:
Eric Eastwood
2021-10-29 17:27:38 -05:00
parent c772b35a25
commit e0ff66dfd9
2 changed files with 196 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
synapse/federation/federation_client.py
View File

@@ -264,6 +264,62 @@ class FederationClient(FederationBase):
return pdus
async def get_pdu_from_destination_raw(
self,
destination: str,
event_id: str,
room_version: RoomVersion,
outlier: bool = False,
timeout: Optional[int] = None,
) -> Optional[EventBase]:
"""Requests the PDU with given origin and ID from the remote home
server.
Does not have any caching or rate limiting!
Args:
destination: Which homeserver to query
event_id: event to fetch
room_version: version of the room
outlier: Indicates whether the PDU is an `outlier`, i.e. if
it's from an arbitrary point in the context as opposed to part
of the current block of PDUs. Defaults to `False`
timeout: How long to try (in ms) each destination for before
moving to the next destination. None indicates no timeout.
Returns:
The requested PDU, or None if we were unable to find it.
Raises:
SynapseError, NotRetryingDestination, FederationDeniedError
"""
signed_pdu = None
transaction_data = await self.transport_layer.get_event(
destination, event_id, timeout=timeout
)
logger.info(
"retrieved event id %s from %s: %r",
event_id,
destination,
transaction_data,
)
pdu_list: List[EventBase] = [
event_from_pdu_json(p, room_version, outlier=outlier)
for p in transaction_data["pdus"]
]
if pdu_list and pdu_list[0]:
pdu = pdu_list[0]
# Check signatures are correct.
signed_pdu = await self._check_sigs_and_hash(room_version, pdu)
return signed_pdu
async def get_pdu(
self,
destinations: Iterable[str],
@@ -308,30 +364,14 @@ class FederationClient(FederationBase):
continue
try:
transaction_data = await self.transport_layer.get_event(
destination, event_id, timeout=timeout
signed_pdu = await self.get_pdu_from_destination_raw(
destination=destination,
event_id=event_id,
room_version=room_version,
outlier=outlier,
timeout=timeout,
)
logger.debug(
"retrieved event id %s from %s: %r",
event_id,
destination,
transaction_data,
)
pdu_list: List[EventBase] = [
event_from_pdu_json(p, room_version, outlier=outlier)
for p in transaction_data["pdus"]
]
if pdu_list and pdu_list[0]:
pdu = pdu_list[0]
# Check signatures are correct.
signed_pdu = await self._check_sigs_and_hash(room_version, pdu)
break
pdu_attempts[destination] = now
except SynapseError as e:

184
synapse/handlers/federation.py
View File

@@ -62,6 +62,7 @@ from synapse.types import JsonDict, StateMap, get_domain_from_id
from synapse.util.async_helpers import Linearizer
from synapse.util.retryutils import NotRetryingDestination
from synapse.visibility import filter_events_for_server
from synapse.storage.databases.main.event_federation import BackfillQueueNavigationItem
if TYPE_CHECKING:
from synapse.server import HomeServer
@@ -1043,8 +1044,16 @@ class FederationHandler:
return []
async def get_backfill_events(
self, room_id: str, event_id_list: list, limit: int
self, origin: str, room_id: str, event_id_list: list, limit: int
) -> List[EventBase]:
logger.info(
"get_backfill_events(room_id=%s): seeding backfill with event_id_list=%s limit=%s origin=%s",
room_id,
event_id_list,
limit,
origin,
)
event_id_results = set()
# In a PriorityQueue, the lowest valued entries are retrieved first.
@@ -1054,8 +1063,20 @@ class FederationHandler:
# negative depth so that we process the newest-in-time messages first
# going backwards in time. stream_ordering follows the same pattern.
queue = PriorityQueue()
seed_events = await self.store.get_events_as_list(event_id_list)
logger.info(
"get_backfill_events(room_id=%s): seed_events=%s",
room_id,
[
BackfillQueueNavigationItem(
depth=seed_event.depth,
stream_ordering=seed_event.internal_metadata.stream_ordering,
event_id=seed_event.event_id,
type=seed_event.type,
)
for seed_event in seed_events
],
)
for seed_event in seed_events:
# Make sure the seed event actually pertains to this room. We also
# need to make sure the depth is available since our whole DAG
@@ -1079,8 +1100,7 @@ class FederationHandler:
if event_id in event_id_results:
continue
event_id_results.add(event_id)
found_undiscovered_connected_historical_messages = False
if self.hs.config.experimental.msc2716_enabled:
# Try and find any potential historical batches of message history.
#
@@ -1093,8 +1113,10 @@ class FederationHandler:
event_id, limit - len(event_id_results)
)
)
logger.debug(
"_get_backfill_events: connected_insertion_event_backfill_results=%s",
logger.info(
"get_backfill_events(room_id=%s): connected_insertion_event_backfill_results(%s)=%s",
room_id,
event_id,
connected_insertion_event_backfill_results,
)
for (
@@ -1104,15 +1126,63 @@ class FederationHandler:
connected_insertion_event_backfill_item.event_id
not in event_id_results
):
queue.put(
(
-connected_insertion_event_backfill_item.depth,
-connected_insertion_event_backfill_item.stream_ordering,
# Check whether the insertion event is already on the
# federating homeserver we're trying to send backfill
# events to
room_version = await self.store.get_room_version(room_id)
event_exists_on_remote_server = None
try:
# Because of the nature of backfill giving events to
# the federated homeserver in one chunk and then we
# can possibly query about that same event in the
# next chunk, we need to avoid getting a cached
# response. We want to know *now* whether they have
# backfilled the insertion event.
event_exists_on_remote_server = await self.federation_client.get_pdu_from_destination_raw(
origin,
connected_insertion_event_backfill_item.event_id,
connected_insertion_event_backfill_item.type,
room_version=room_version,
outlier=True,
timeout=10000,
)
except Exception as e:
logger.info(
"get_backfill_events(room_id=%s): Failed to fetch insertion event_id=%s from origin=%s but we're just going to assume it's not backfilled there yet. error=%s",
room_id,
connected_insertion_event_backfill_item.event_id,
origin,
e,
)
logger.info(
"get_backfill_events(room_id=%s): checked if insertion event_id=%s exists on federated homeserver(origin=%s) already? event_exists_on_remote_server=%s",
room_id,
connected_insertion_event_backfill_item.event_id,
origin,
event_exists_on_remote_server,
)
# If the event is already on the federated homeserver,
# we don't need to try to branch off onto this
# historical chain of messages. Below, we will instead
# just go up the `prev_events` as normal.
#
# This is important so that the first time we backfill
# the federated homeserver, we jump off and go down the
# historical branch. But after the historical branch is
# exhausted and the event comes up again in backfill, we
# will choose the "live" DAG.
if not event_exists_on_remote_server:
found_undiscovered_connected_historical_messages = True
queue.put(
(
-connected_insertion_event_backfill_item.depth,
-connected_insertion_event_backfill_item.stream_ordering,
connected_insertion_event_backfill_item.event_id,
connected_insertion_event_backfill_item.type,
)
)
# Second, we need to go and try to find any batch events connected
# to a given insertion event (by batch_id). If we find any, we'll
# add them to the queue and navigate up the DAG like normal in the
@@ -1123,8 +1193,10 @@ class FederationHandler:
event_id, limit - len(event_id_results)
)
)
logger.debug(
"_get_backfill_events: connected_batch_event_backfill_results %s",
logger.info(
"get_backfill_events(room_id=%s): connected_batch_event_backfill_results(%s)=%s",
room_id,
event_id,
connected_batch_event_backfill_results,
)
for (
@@ -1143,28 +1215,39 @@ class FederationHandler:
)
)
# Now we just look up the DAG by prev_events as normal
connected_prev_event_backfill_results = (
await self.store.get_connected_prev_event_backfill_results(
event_id, limit - len(event_id_results)
)
)
logger.debug(
"_get_backfill_events: prev_event_ids %s",
connected_prev_event_backfill_results,
)
for (
connected_prev_event_backfill_item
) in connected_prev_event_backfill_results:
if connected_prev_event_backfill_item.event_id not in event_id_results:
queue.put(
(
-connected_prev_event_backfill_item.depth,
-connected_prev_event_backfill_item.stream_ordering,
connected_prev_event_backfill_item.event_id,
connected_prev_event_backfill_item.type,
)
# If we found a historical branch of history off of the message lets
# navigate down that in the next iteration of the loop instead of
# the normal prev_event chain.
if not found_undiscovered_connected_historical_messages:
event_id_results.add(event_id)
# Now we just look up the DAG by prev_events as normal
connected_prev_event_backfill_results = (
await self.store.get_connected_prev_event_backfill_results(
event_id, limit - len(event_id_results)
)
)
logger.info(
"get_backfill_events(room_id=%s): connected_prev_event_backfill_results(%s)=%s",
room_id,
event_id,
connected_prev_event_backfill_results,
)
for (
connected_prev_event_backfill_item
) in connected_prev_event_backfill_results:
if (
connected_prev_event_backfill_item.event_id
not in event_id_results
):
queue.put(
(
-connected_prev_event_backfill_item.depth,
-connected_prev_event_backfill_item.stream_ordering,
connected_prev_event_backfill_item.event_id,
connected_prev_event_backfill_item.type,
)
)
events = await self.store.get_events_as_list(event_id_results)
return sorted(
@@ -1182,24 +1265,25 @@ class FederationHandler:
# Synapse asks for 100 events per backfill request. Do not allow more.
limit = min(limit, 100)
events = await self.store.get_backfill_events(room_id, pdu_list, limit)
logger.info(
"old implementation backfill events=%s",
[
"event_id=%s,depth=%d,body=%s,prevs=%s\n"
% (
event.event_id,
event.depth,
event.content.get("body", event.type),
event.prev_event_ids(),
)
for event in events
],
)
# events = await self.store.get_backfill_events(room_id, pdu_list, limit)
# logger.info(
# "old implementation backfill events=%s",
# [
# "event_id=%s,depth=%d,body=%s,prevs=%s\n"
# % (
# event.event_id,
# event.depth,
# event.content.get("body", event.type),
# event.prev_event_ids(),
# )
# for event in events
# ],
# )
events = await self.get_backfill_events(room_id, pdu_list, limit)
events = await self.get_backfill_events(origin, room_id, pdu_list, limit)
logger.info(
"new implementation backfill events=%s",
"new implementation backfill events(%d)=%s",
len(events),
[
"event_id=%s,depth=%d,body=%s,prevs=%s\n"
% (