diff --git a/changelog.d/19748.misc b/changelog.d/19748.misc
new file mode 100644
index 00000000000..eedd4e92a26
--- /dev/null
+++ b/changelog.d/19748.misc
@@ -0,0 +1 @@
+Prefer close backfill points (absolute distance).
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 166a02d7c7e..ba83d4fd268 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -275,11 +275,22 @@ async def _maybe_backfill_inner(
             )
         ]
 
-        # we now have a list of potential places to backpaginate from. We prefer to
-        # start with the most recent (ie, max depth), so let's sort the list.
+        # we now have a list of potential places to backpaginate from. Figure out which
+        # ones we should prefer, so let's sort the list.
         sorted_backfill_points: list[_BackfillPoint] = sorted(
             backwards_extremities,
-            key=lambda e: -int(e.depth),
+            key=lambda e: (
+                # Prefer backfill points that are closer to the `current_depth`
+                # (absolute distance)
+                abs(current_depth - e.depth),
+                # For the tie-break, we care about events that are actually in the past
+                # as they're more likely to reveal history that we can return (something
+                # absolutely in the past is better than something can potentially extend
+                # into the past).
+                #
+                # This sorts ascending so 0 sorts before 1
+                0 if current_depth >= e.depth else 1,
+            ),
         )
 
         logger.debug(
@@ -300,7 +311,7 @@ async def _maybe_backfill_inner(
             str(len(sorted_backfill_points)),
         )
 
-        # If we have no backfill points lower than the `current_depth` then either we
+        # If we have no backfill points lower than the `nearby_depth` then either we
         # can a) bail or b) still attempt to backfill. We opt to try backfilling anyway
         # just in case we do get relevant events. This is good for eventual consistency
         # sake but we don't need to block the client for something that is just as