feat: implement cursor-based pagination across various components and APIs

Author: DrizzleTime

api/response.py

@@ -9,7 +9,25 @@ def success(data: Any = None, msg: str = "ok", code: int = 0):
 def page(items: list[Any], total: int, page: int, page_size: int):
     """统一分页数据结构。"""
     pages = (total + page_size - 1) // page_size if page_size else 0
-    return {"items": items, "total": total, "page": page, "page_size": page_size, "pages": pages}
+    return {"items": items, "total": total, "page": page, "page_size": page_size, "pages": pages, "pagination_mode": "paged"}
+
+
+def cursor_page(
+    items: list[Any],
+    page_size: int,
+    *,
+    cursor: str | None = None,
+    next_cursor: str | None = None,
+):
+    """无总数游标分页结构。"""
+    return {
+        "items": items,
+        "page_size": page_size,
+        "pagination_mode": "cursor",
+        "cursor": cursor,
+        "next_cursor": next_cursor,
+        "has_next": bool(next_cursor),
+    }
 
 
 def error(msg: str, code: int = 1, data: Optional[Any] = None):

domain/adapters/providers/base.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Protocol, runtime_checkable, Tuple, AsyncIterator
+from typing import List, Dict, Protocol, runtime_checkable, Tuple, AsyncIterator, Any
 from models import StorageAdapter
 
 # 约定：任意新适配器模块需定义:
@@ -9,7 +9,7 @@
 @runtime_checkable
 class BaseAdapter(Protocol):
     record: StorageAdapter
-    async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]: ...
+    async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc", cursor: str | None = None) -> Tuple[List[Dict], int] | Dict[str, Any]: ...
     async def read_file(self, root: str, rel: str) -> bytes: ...
     async def write_file(self, root: str, rel: str, data: bytes): ...
     async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]): ... 

domain/adapters/providers/onedrive.py

@@ -4,6 +4,7 @@
 from fastapi.responses import StreamingResponse, Response
 from fastapi import HTTPException
 from models import StorageAdapter
+from api.response import cursor_page
 
 MS_GRAPH_URL = "https://graph.microsoft.com/v1.0"
 MS_OAUTH_URL = "https://login.microsoftonline.com/common/oauth2/v2.0/token"
@@ -114,65 +115,51 @@ def _format_item(self, item: Dict) -> Dict:
             "type": "dir" if is_dir else "file",
         }
 
-    async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
+    async def list_dir(
+        self,
+        root: str,
+        rel: str,
+        page_num: int = 1,
+        page_size: int = 50,
+        sort_by: str = "name",
+        sort_order: str = "asc",
+        cursor: str | None = None,
+    ):
         """
         列出目录内容。
-        由于 Graph API 不支持基于偏移($skip)的分页，此方法将获取所有项目，
+        Graph API 不提供目录总数，使用 nextLink 游标分页。
         :param root: 根路径 (在此适配器中未使用，通过配置的 root 确定)。
         :param rel: 相对路径。
         :param page_num: 页码。
         :param page_size: 每页大小。
         :param sort_by: 排序字段
         :param sort_order: 排序顺序
-        :return: 文件/目录列表和总数。
+        :param cursor: Graph nextLink。
+        :return: 游标分页结果。
         """
-        api_path = self._get_api_path(rel)
-        children_path = f"{api_path}:/children" if api_path else "/children"
-        all_items = []
-        params = {"$top": 999}
-        resp = await self._request("GET", api_path_segment=children_path, params=params)
-
-        while True:
-            if resp.status_code == 404 and not all_items:
-                return [], 0
-            resp.raise_for_status()
-
-            try:
-                data = resp.json()
-            except Exception as e:
-                raise IOError(f"解析 Graph API 响应失败: {e}") from e
-
-            all_items.extend(data.get("value", []))
-            next_link = data.get("@odata.nextLink")
-
-            if not next_link:
-                break
-
-            resp = await self._request("GET", full_url=next_link)
-
-        formatted_items = [self._format_item(item) for item in all_items]
-        
-        # 排序
-        reverse = sort_order.lower() == "desc"
-        def get_sort_key(item):
-            key = (not item["is_dir"],)
-            sort_field = sort_by.lower()
-            if sort_field == "name":
-                key += (item["name"].lower(),)
-            elif sort_field == "size":
-                key += (item["size"],)
-            elif sort_field == "mtime":
-                key += (item["mtime"],)
-            else:
-                key += (item["name"].lower(),)
-            return key
-        formatted_items.sort(key=get_sort_key, reverse=reverse)
+        if cursor:
+            resp = await self._request("GET", full_url=cursor)
+        else:
+            api_path = self._get_api_path(rel)
+            children_path = f"{api_path}:/children" if api_path else "/children"
+            resp = await self._request("GET", api_path_segment=children_path, params={"$top": page_size})
 
-        total_count = len(formatted_items)
-        start_idx = (page_num - 1) * page_size
-        end_idx = start_idx + page_size
+        if resp.status_code == 404:
+            return cursor_page([], page_size, cursor=cursor)
+        resp.raise_for_status()
 
-        return formatted_items[start_idx:end_idx], total_count
+        try:
+            data = resp.json()
+        except Exception as e:
+            raise IOError(f"解析 Graph API 响应失败: {e}") from e
+
+        formatted_items = [self._format_item(item) for item in data.get("value", [])]
+        return cursor_page(
+            formatted_items,
+            page_size,
+            cursor=cursor,
+            next_cursor=data.get("@odata.nextLink"),
+        )
 
     async def read_file(self, root: str, rel: str) -> bytes:
         """

domain/adapters/providers/telegram.py

@@ -6,6 +6,7 @@
 import struct
 import time
 from models import StorageAdapter
+from api.response import cursor_page
 from telethon import TelegramClient, errors, utils
 from telethon.crypto import AuthKey
 from telethon.sessions import StringSession
@@ -280,81 +281,79 @@ def _parse_message_id(rel: str) -> int:
     def get_effective_root(self, sub_path: str | None) -> str:
         return ""
 
-    async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
+    async def list_dir(
+        self,
+        root: str,
+        rel: str,
+        page_num: int = 1,
+        page_size: int = 50,
+        sort_by: str = "name",
+        sort_order: str = "asc",
+        cursor: str | None = None,
+    ):
         if rel:
-            return [], 0
+            return cursor_page([], page_size, cursor=cursor)
 
         client = self._get_client()
         entries = []
+        next_cursor = None
         try:
             await client.connect()
-            messages = await client.get_messages(self.chat_id, limit=200)
-            for message in messages:
-                if not message:
-                    continue
-
-                media = message.document or message.video or message.photo
-                if not media:
-                    continue
-
-                file_meta = message.file
-                if not file_meta:
-                    continue
-
-                filename = file_meta.name
-                if not filename:
-                    if message.text and '.' in message.text and len(message.text) < 256 and '\n' not in message.text:
-                        filename = message.text
-                    else:
-                        filename = f"unknown_{message.id}"
-
-                size = file_meta.size
-                if size is None:
-                    # 兼容缺失 size 的情况
-                    if hasattr(media, "size") and media.size is not None:
-                        size = media.size
-                    elif message.photo and getattr(message.photo, "sizes", None):
-                        photo_size = message.photo.sizes[-1]
-                        size = getattr(photo_size, "size", 0) or 0
-                    else:
-                        size = 0
-
-                entries.append({
-                    "name": f"{message.id}_{filename}",
-                    "is_dir": False,
-                    "size": size,
-                    "mtime": int(message.date.timestamp()),
-                    "type": "file",
-                    "has_thumbnail": False,
-                })
+            offset_id = int(cursor) if cursor else 0
+            batch_limit = min(max(page_size, 50), 200)
+            while len(entries) < page_size:
+                messages = await client.get_messages(self.chat_id, limit=batch_limit, offset_id=offset_id)
+                if not messages:
+                    next_cursor = None
+                    break
+
+                offset_id = messages[-1].id
+                next_cursor = str(offset_id)
+                for message in messages:
+                    if not message:
+                        continue
+
+                    media = message.document or message.video or message.photo
+                    if not media:
+                        continue
+
+                    file_meta = message.file
+                    if not file_meta:
+                        continue
+
+                    filename = file_meta.name
+                    if not filename:
+                        if message.text and '.' in message.text and len(message.text) < 256 and '\n' not in message.text:
+                            filename = message.text
+                        else:
+                            filename = f"unknown_{message.id}"
+
+                    size = file_meta.size
+                    if size is None:
+                        # 兼容缺失 size 的情况
+                        if hasattr(media, "size") and media.size is not None:
+                            size = media.size
+                        elif message.photo and getattr(message.photo, "sizes", None):
+                            photo_size = message.photo.sizes[-1]
+                            size = getattr(photo_size, "size", 0) or 0
+                        else:
+                            size = 0
+
+                    entries.append({
+                        "name": f"{message.id}_{filename}",
+                        "is_dir": False,
+                        "size": size,
+                        "mtime": int(message.date.timestamp()),
+                        "type": "file",
+                        "has_thumbnail": False,
+                    })
+                    if len(entries) >= page_size:
+                        break
         finally:
             if client.is_connected():
                 await client.disconnect()
 
-        # 排序
-        reverse = sort_order.lower() == "desc"
-        def get_sort_key(item):
-            key = (not item["is_dir"],)
-            sort_field = sort_by.lower()
-            if sort_field == "name":
-                key += (item["name"].lower(),)
-            elif sort_field == "size":
-                key += (item["size"],)
-            elif sort_field == "mtime":
-                key += (item["mtime"],)
-            else:
-                key += (item["name"].lower(),)
-            return key
-        entries.sort(key=get_sort_key, reverse=reverse)
-        
-        total_count = len(entries)
-        
-        # 分页
-        start_idx = (page_num - 1) * page_size
-        end_idx = start_idx + page_size
-        page_entries = entries[start_idx:end_idx]
-        
-        return page_entries, total_count
+        return cursor_page(entries, page_size, cursor=cursor, next_cursor=next_cursor)
 
     async def read_file(self, root: str, rel: str) -> bytes:
         message_id = self._parse_message_id(rel)

domain/virtual_fs/api.py

@@ -183,9 +183,10 @@ async def browse_fs(
     page_size: int = Query(50, ge=1, le=500, description="每页条数"),
     sort_by: str = Query("name", description="按字段排序: name, size, mtime"),
     sort_order: str = Query("asc", description="排序顺序: asc, desc"),
+    cursor: str | None = Query(None, description="游标分页位置"),
 ):
     data = await VirtualFSService.list_directory_with_permission(
-        full_path, current_user.id, page_num, page_size, sort_by, sort_order
+        full_path, current_user.id, page_num, page_size, sort_by, sort_order, cursor
     )
     return success(data)
 
@@ -211,9 +212,10 @@ async def root_listing(
     page_size: int = Query(50, ge=1, le=500, description="每页条数"),
     sort_by: str = Query("name", description="按字段排序: name, size, mtime"),
     sort_order: str = Query("asc", description="排序顺序: asc, desc"),
+    cursor: str | None = Query(None, description="游标分页位置"),
 ):
     # 根目录不需要权限检查，但需要过滤无权限的子目录
     data = await VirtualFSService.list_directory_with_permission(
-        "/", current_user.id, page_num, page_size, sort_by, sort_order
+        "/", current_user.id, page_num, page_size, sort_by, sort_order, cursor
     )
     return success(data)

domain/virtual_fs/listing.py

@@ -57,6 +57,7 @@ async def list_virtual_dir(
         page_size: int = 50,
         sort_by: str = "name",
         sort_order: str = "asc",
+        cursor: str | None = None,
     ) -> Dict:
         norm = cls._normalize_path(path).rstrip("/") or "/"
         adapters = await StorageAdapter.filter(enabled=True)
@@ -119,12 +120,28 @@ def annotate_entry(entry: Dict) -> None:
         adapter_entries_for_merge: List[Dict] = []
         adapter_entries_page: List[Dict] | None = None
         adapter_total: int | None = None
+        adapter_listing: Dict[str, Any] | None = None
         if adapter_model and adapter_instance:
             list_dir = getattr(adapter_instance, "list_dir", None)
             if callable(list_dir):
-                adapter_entries_page, adapter_total = await list_dir(
-                    effective_root, rel, page_num, page_size, sort_by, sort_order
-                )
+                try:
+                    parameters = inspect.signature(list_dir).parameters
+                except (TypeError, ValueError):
+                    parameters = {}
+                if "cursor" in parameters:
+                    raw_listing = await list_dir(
+                        effective_root, rel, page_num, page_size, sort_by, sort_order, cursor=cursor
+                    )
+                else:
+                    raw_listing = await list_dir(
+                        effective_root, rel, page_num, page_size, sort_by, sort_order
+                    )
+                if isinstance(raw_listing, dict):
+                    adapter_listing = raw_listing
+                    adapter_entries_page = raw_listing.get("items", [])
+                    adapter_total = raw_listing.get("total")
+                else:
+                    adapter_entries_page, adapter_total = raw_listing
             if rel:
                 parent_rel = cls._parent_rel(rel)
                 if rel:
@@ -189,6 +206,9 @@ def annotate_entry(entry: Dict) -> None:
         annotate_entry_list = adapter_entries_page or []
         for ent in annotate_entry_list:
             annotate_entry(ent)
+        if adapter_listing and adapter_listing.get("pagination_mode") == "cursor":
+            adapter_listing["items"] = annotate_entry_list
+            return adapter_listing
         return page(adapter_entries_page, adapter_total, page_num, page_size)
 
     @classmethod
@@ -296,13 +316,14 @@ async def list_virtual_dir_with_permission(
         page_size: int = 50,
         sort_by: str = "name",
         sort_order: str = "asc",
+        cursor: str | None = None,
     ) -> Dict:
         """
         带权限过滤的目录列表
         
         过滤掉用户没有读取权限的条目
         """
-        result = await cls.list_virtual_dir(path, page_num, page_size, sort_by, sort_order)
+        result = await cls.list_virtual_dir(path, page_num, page_size, sort_by, sort_order, cursor)
         items = result.get("items", [])
         if not items:
             return result