Implement MSC3871: Gappy timelines - indicate gaps in /messages

changelog.d/18873.feature

@@ -0,0 +1 @@
+Implement experimental [MSC3871](https://github.com/matrix-org/matrix-spec-proposals/pull/3871) to indicate `gaps` in the `/messages` timeline.

synapse/handlers/federation.py

@@ -259,7 +259,9 @@ async def _maybe_backfill_inner(
             _BackfillPoint(event_id, depth, _BackfillPointType.BACKWARDS_EXTREMITY)
             for event_id, depth in await self.store.get_backfill_points_in_room(
                 room_id=room_id,
-                current_depth=current_depth,
+                # Per the docstring, it's best to pad the `current_depth` by the
+                # number of messages you plan to backfill from these points.
+                current_depth=current_depth + limit,
                 # We only need to end up with 5 extremities combined with the
                 # insertion event extremities to make the `/backfill` request
                 # but fetch an order of magnitude more to make sure there is

synapse/handlers/pagination.py

@@ -414,12 +414,14 @@ async def purge_room(
     @trace
     async def get_messages(
         self,
+        *,
         requester: Requester,
         room_id: str,
         pagin_config: PaginationConfig,
         as_client_event: bool = True,
         event_filter: Optional[Filter] = None,
         use_admin_priviledge: bool = False,
+        backfill: bool = True,
     ) -> JsonDict:
         """Get messages in a room.
 
@@ -432,6 +434,8 @@ async def get_messages(
             use_admin_priviledge: if `True`, return all events, regardless
                 of whether `user` has access to them. To be used **ONLY**
                 from the admin API.
+            backfill: If false, we skip backfill altogether. When true, we backfill as a
+                best effort.
 
         Returns:
             Pagination API results
@@ -522,7 +526,7 @@ async def get_messages(
             event_filter=event_filter,
         )
 
-        if pagin_config.direction == Direction.BACKWARDS:
+        if backfill and pagin_config.direction == Direction.BACKWARDS:
             # We use a `Set` because there can be multiple events at a given depth
             # and we only care about looking at the unique continum of depths to
             # find gaps.
@@ -622,6 +626,7 @@ async def get_messages(
         if not events:
             return {
                 "chunk": [],
+                "gaps": [],
                 "start": await from_token.to_string(self.store),
             }
 
@@ -641,6 +646,7 @@ async def get_messages(
         if not events:
             return {
                 "chunk": [],
+                "gaps": [],
                 "start": await from_token.to_string(self.store),
                 "end": await next_token.to_string(self.store),
             }
@@ -666,6 +672,10 @@ async def get_messages(
             events, user_id
         )
 
+        gaps = await self.store.get_events_next_to_gaps(
+            events=events, direction=pagin_config.direction
+        )
+
         time_now = self.clock.time_msec()
 
         serialize_options = SerializeEventConfig(
@@ -681,6 +691,18 @@ async def get_messages(
                     bundle_aggregations=aggregations,
                 )
             ),
+            "gaps": [
+                {
+                    "prev_pagination_token": await from_token.copy_and_replace(
+                        StreamKeyType.ROOM, gap.prev_token
+                    ).to_string(self.store),
+                    "event_id": gap.event_id,
+                    "next_pagination_token": await from_token.copy_and_replace(
+                        StreamKeyType.ROOM, gap.next_token
+                    ).to_string(self.store),
+                }
+                for gap in gaps
+            ],
             "start": await from_token.to_string(self.store),
             "end": await next_token.to_string(self.store),
         }

synapse/rest/client/room.py

@@ -811,6 +811,17 @@ def __init__(self, hs: "HomeServer"):
     async def on_GET(
         self, request: SynapseRequest, room_id: str
     ) -> Tuple[int, JsonDict]:
+        """
+        Query paremeters:
+            dir
+            from
+            to
+            limit
+            filter
+            backfill: If false, we skip backfill altogether. When true, we backfill as a
+                best effort.
+        """
+
         processing_start_time = self.clock.time_msec()
         # Fire off and hope that we get a result by the end.
         #
@@ -840,12 +851,15 @@ async def on_GET(
         ):
             as_client_event = False
 
+        backfill = parse_boolean(request, "backfill", default=True)
+
         msgs = await self.pagination_handler.get_messages(
             room_id=room_id,
             requester=requester,
             pagin_config=pagination_config,
             as_client_event=as_client_event,
             event_filter=event_filter,
+            backfill=backfill,
         )
 
         processing_end_time = self.clock.time_msec()

synapse/storage/databases/main/event_federation.py

@@ -1212,6 +1212,30 @@ async def get_backfill_points_in_room(
         equal to the `current_depth`. Sorted by depth, highest to lowest (descending)
         so the closest events to the `current_depth` are first in the list.
 
+        Note: We can only do approximate depth comparisons. Backwards extremeties are
+        the oldest events we know of in the room but we only know of them because some
+        other event referenced them by prev_event and aren't persisted in our database
+        yet (meaning we don't know their depth specifically). So we need to look for the
+        approximate depth from the events connected to the current backwards
+        extremeties.
+
+        It's best to pad the `current_depth` by the number of messages you plan to
+        backfill from these points.
+
+        Example:
+
+         - Your pagination token represents a scroll position at `depth` of `100`.
+         - We have a backfill point at an approximate depth of `125`
+         - You plan to backfill `50` events from that backfill point.
+
+        When we pad our `current_depth`, `100` + `50` = `150`, we pick up the backfill
+        point at `125` (because <= `150`, our `current_depth`), backfill `50` events to
+        a depth of `75` in the timeline (exposing new events that we can return `100` ->
+        `75`).
+
+        When we don't pad our `current_depth`, `100` is lower than any of the backfill
+        points so we don't pick any and miss out on backfilling any events.
+
         We ignore extremities that are newer than the user's current scroll position
         (ie, those with depth greater than `current_depth`) as:
             1. we don't really care about getting events that have happened
@@ -1223,7 +1247,7 @@ async def get_backfill_points_in_room(
 
         Args:
             room_id: Room where we want to find the oldest events
-            current_depth: The depth at the user's current scrollback position
+            current_depth: The depth at the user's current scrollback position (see notes above).
             limit: The max number of backfill points to return
 
         Returns: