GitHubおよびGitLabクライアントのAPI呼び出しをページング処理に対応させ、ディスカッションおよびノートの取得を改善

notfolder · notfolder · commit 3925f2d88ba7 · 2025-11-30T00:55:46.000+09:00
diff --git a/clients/github_client.py b/clients/github_client.py
@@ -72,7 +72,14 @@ def get_pull_request_labels(self, owner: str, repo: str, pull_number: int) -> li
         return [label["name"] for label in issue.get("labels", [])]
 
     def list_pull_requests_with_label(
-        self, owner: str, repo: str, label: str, state: str = "open",
+        self,
+        owner: str,
+        repo: str,
+        label: str,
+        state: str = "open",
+        *,
+        per_page: int = 100,
+        max_pages: int = 20,
     ) -> list[dict[str, Any]]:
         """指定したラベルが付いているPull Requestの一覧を取得する.
 
@@ -91,12 +98,10 @@ def list_pull_requests_with_label(
         """
         # Pull Request一覧取得のAPIエンドポイント
         url = f"{self.api_url}/repos/{owner}/{repo}/pulls"
-        params = {"state": state, "per_page": 100}
+        params = {"state": state}
 
-        # Pull Request一覧を取得
-        response = requests.get(url, headers=self.headers, params=params, timeout=30)
-        response.raise_for_status()
-        pulls = response.json()
+        # Pull Request一覧をページングしながら取得
+        pulls = self._fetch_paginated_list(url, params, per_page, max_pages)
 
         # 指定されたラベルが付いているPull Requestをフィルタリング
         result = []
@@ -266,11 +271,12 @@ def get_pull_request_comments(
 
         # タイムラインコメント(Issueコメント)を取得
         url_issue = f"{self.api_url}/repos/{owner}/{repo}/issues/{pull_number}/comments"
-        resp_issue = requests.get(
-            url_issue, headers=self.headers, params={"per_page": 200}, timeout=30,
+        issue_comments_raw = self._fetch_paginated_list(
+            url_issue,
+            {},
+            per_page=200,
+            max_pages=20,
         )
-        resp_issue.raise_for_status()
-        issue_comments_raw = resp_issue.json()
 
         # 不要なURLフィールドを削除
         issue_comments = [self.remove_url_fields(c) for c in issue_comments_raw]
@@ -309,20 +315,22 @@ def get_reviews_with_comments(
         """
         # レビュー一覧を取得
         url_reviews = f"{self.api_url}/repos/{owner}/{repo}/pulls/{pull_number}/reviews"
-        resp_reviews = requests.get(
-            url_reviews, headers=self.headers, params={"per_page": 100}, timeout=30,
+        reviews_raw = self._fetch_paginated_list(
+            url_reviews,
+            {},
+            per_page=100,
+            max_pages=20,
         )
-        resp_reviews.raise_for_status()
-        reviews_raw = resp_reviews.json()
         reviews = [self.remove_url_fields(r) for r in reviews_raw]
 
         # レビューコメント一覧を取得
         url_comments = f"{self.api_url}/repos/{owner}/{repo}/pulls/{pull_number}/comments"
-        resp_comments = requests.get(
-            url_comments, headers=self.headers, params={"per_page": 200}, timeout=30,
+        comments_raw = self._fetch_paginated_list(
+            url_comments,
+            {},
+            per_page=200,
+            max_pages=20,
         )
-        resp_comments.raise_for_status()
-        comments_raw = resp_comments.json()
         comments = [self.remove_url_fields(c) for c in comments_raw]
 
         # review_idごとにコメントをまとめる
@@ -425,20 +433,7 @@ def search_issues_and_prs(
         """
         # Search API のエンドポイント
         url = f"{self.api_url}/search/issues"
-        params = {"q": query, "per_page": per_page, "page": page}
-
-        # ソート条件とソート順序を設定
-        if sort:
-            params["sort"] = sort
-        if order:
-            params["order"] = order
-
-        # 検索実行
-        response = requests.get(url, headers=self.headers, params=params, timeout=30)
-        response.raise_for_status()
-        data = response.json()
-
-        return data.get("items", [])
+        return self._fetch_search_results(url, query, sort, order, per_page, page)
 
     def search_pull_requests(
         self,
@@ -497,3 +492,85 @@ def search_issues(
         # 検索実行とIssueのフィルタリング
         items = self.search_issues_and_prs(query, sort, order, per_page, page)
         return [item for item in items if "pull_request" not in item]
+
+    def _fetch_paginated_list(
+        self,
+        url: str,
+        params: dict[str, Any],
+        per_page: int,
+        max_pages: int,
+    ) -> list[dict[str, Any]]:
+        """GitHubの標準REST APIでページングされるリストを全件取得する."""
+        items: list[dict[str, Any]] = []
+        page_number: int = 1
+
+        # Linkヘッダーを使わず、ページ数と件数で終了条件を判定
+        while page_number <= max_pages:
+            page_params = dict(params)
+            page_params["per_page"] = per_page
+            page_params["page"] = page_number
+
+            response = requests.get(url, headers=self.headers, params=page_params, timeout=30)
+            response.raise_for_status()
+            page_items = response.json()
+
+            if not isinstance(page_items, list) or not page_items:
+                break
+
+            items.extend(page_items)
+
+            if len(page_items) < per_page:
+                break
+
+            page_number += 1
+
+        return items
+
+    def _fetch_search_results(
+        self,
+        url: str,
+        query: str,
+        sort: str | None,
+        order: str | None,
+        per_page: int,
+        page: int,
+        *,
+        max_pages: int = 10,
+    ) -> list[dict[str, Any]]:
+        """Search APIで複数ページに跨る結果を収集する."""
+        aggregated: list[dict[str, Any]] = []
+        current_page: int = page
+        pages_fetched: int = 0
+
+        # Search API固有のtotal_countなどを利用してループを制御
+        while pages_fetched < max_pages:
+            params = {"q": query, "per_page": per_page, "page": current_page}
+            if sort:
+                params["sort"] = sort
+            if order:
+                params["order"] = order
+
+            response = requests.get(url, headers=self.headers, params=params, timeout=30)
+            response.raise_for_status()
+            data = response.json()
+
+            page_items = data.get("items", [])
+            if not page_items:
+                break
+
+            aggregated.extend(page_items)
+
+            total_count = data.get("total_count")
+            if isinstance(total_count, int) and total_count <= len(aggregated):
+                break
+
+            if len(page_items) < per_page:
+                break
+
+            if data.get("incomplete_results") is True:
+                break
+
+            current_page += 1
+            pages_fetched += 1
+
+        return aggregated
diff --git a/clients/gitlab_client.py b/clients/gitlab_client.py
@@ -24,23 +24,23 @@ def list_issues(
         labels: list[str] | None = None,
         state: str = "opened",
         per_page: int = 100,
+        max_pages: int = 200,
     ) -> list[dict[str, Any]]:
         url = f"{self.api_url}/projects/{project_id}/issues"
-        params = {"state": state, "per_page": per_page}
+        params: dict[str, Any] = {"state": state}
         if labels:
             params["labels"] = ",".join(labels)
-        resp = requests.get(url, headers=self.headers, params=params, timeout=30)
-        resp.raise_for_status()
-        return resp.json()
+        return self._fetch_paginated_list(url, params, per_page, max_pages)
 
     def list_issue_notes(
-        self, project_id: int | str, issue_iid: int | str, per_page: int = 100,
+        self,
+        project_id: int | str,
+        issue_iid: int | str,
+        per_page: int = 100,
+        max_pages: int = 200,
     ) -> list[dict[str, Any]]:
         url = f"{self.api_url}/projects/{project_id}/issues/{issue_iid}/notes"
-        params = {"per_page": per_page}
-        resp = requests.get(url, headers=self.headers, params=params, timeout=30)
-        resp.raise_for_status()
-        return resp.json()
+        return self._fetch_paginated_list(url, {}, per_page, max_pages)
 
     def add_issue_note(
         self, project_id: int | str, issue_iid: int | str, body: str,
@@ -101,25 +101,25 @@ def list_merge_requests(
         assignee: str | None = None,
         state: str = "opened",
         per_page: int = 100,
+        max_pages: int = 200,
     ) -> list[dict[str, Any]]:
         url = f"{self.api_url}/projects/{project_id}/merge_requests"
-        params = {"state": state, "per_page": per_page}
+        params: dict[str, Any] = {"state": state}
         if labels:
             params["labels"] = ",".join(labels)
         if assignee:
             params["assignee_username"] = assignee
-        resp = requests.get(url, headers=self.headers, params=params, timeout=30)
-        resp.raise_for_status()
-        return resp.json()
+        return self._fetch_paginated_list(url, params, per_page, max_pages)
 
     def list_merge_request_notes(
-        self, project_id: int | str, merge_request_iid: int | str, per_page: int = 100,
+        self,
+        project_id: int | str,
+        merge_request_iid: int | str,
+        per_page: int = 100,
+        max_pages: int = 200,
     ) -> list[dict[str, Any]]:
         url = f"{self.api_url}/projects/{project_id}/merge_requests/{merge_request_iid}/notes"
-        params = {"per_page": per_page}
-        resp = requests.get(url, headers=self.headers, params=params, timeout=30)
-        resp.raise_for_status()
-        return resp.json()
+        return self._fetch_paginated_list(url, {}, per_page, max_pages)
 
     def add_merge_request_note(
         self, project_id: int | str, merge_request_iid: int | str, body: str,
@@ -168,21 +168,79 @@ def get_merge_request(
         return resp.json()
 
     def search_issues(
-        self, query: str, state: str = "opened", per_page: int = 200,
+        self,
+        query: str,
+        state: str = "opened",
+        per_page: int = 200,
+        max_pages: int = 200,
     ) -> list[dict[str, Any]]:
         url = f"{self.api_url}/search"
-        params = {"scope": "issues", "search": query, "state": state, "per_page": per_page}
-        resp = requests.get(url, headers=self.headers, params=params, timeout=30)
-        resp.raise_for_status()
-        return resp.json()
+        params: dict[str, Any] = {"scope": "issues", "search": query, "state": state}
+        return self._fetch_paginated_list(url, params, per_page, max_pages)
 
     def search_merge_requests(
-        self, query: str, state: str | None = None, per_page: int = 200,
+        self,
+        query: str,
+        state: str | None = None,
+        per_page: int = 200,
+        max_pages: int = 200,
     ) -> list[dict[str, Any]]:
         url = f"{self.api_url}/search"
-        params = {"scope": "merge_requests", "search": query, "per_page": per_page}
+        params: dict[str, Any] = {"scope": "merge_requests", "search": query}
         if state:
             params["state"] = state
-        resp = requests.get(url, headers=self.headers, params=params, timeout=30)
-        resp.raise_for_status()
-        return resp.json()
+        return self._fetch_paginated_list(url, params, per_page, max_pages)
+
+    def _fetch_paginated_list(
+        self,
+        url: str,
+        params: dict[str, Any],
+        per_page: int,
+        max_pages: int,
+    ) -> list[dict[str, Any]]:
+        """GitLab APIからページング結果を全件取得するヘルパー."""
+        items: list[dict[str, Any]] = []
+        page: int = 1
+        visited_pages: set[int] = set()
+
+        # X-Next-Pageヘッダーとレスポンス件数を使って次ページを辿る
+        while page not in visited_pages and page <= max_pages:
+            visited_pages.add(page)
+            page_params = dict(params)
+            page_params["per_page"] = per_page
+            page_params["page"] = page
+
+            resp = requests.get(url, headers=self.headers, params=page_params, timeout=30)
+            resp.raise_for_status()
+            payload = resp.json()
+
+            page_items: list[dict[str, Any]]
+            if isinstance(payload, list):
+                page_items = payload
+            elif isinstance(payload, dict) and isinstance(payload.get("items"), list):
+                page_items = payload["items"]
+            else:
+                break
+
+            if not page_items:
+                break
+
+            items.extend(page_items)
+
+            next_page_header = resp.headers.get("X-Next-Page")
+            if next_page_header:
+                try:
+                    next_page = int(next_page_header)
+                except ValueError:
+                    break
+                if next_page <= 0:
+                    break
+                page = next_page
+                continue
+
+            if len(page_items) < per_page:
+                break
+
+            page += 1
+
+        return items
diff --git a/handlers/task_getter_gitlab.py b/handlers/task_getter_gitlab.py
