From d5daaeae6e8700c2ebe2230baa199753727dc577 Mon Sep 17 00:00:00 2001 From: Fede Sanchez Date: Tue, 9 Sep 2025 16:38:30 -0300 Subject: [PATCH 1/3] fix(bitbucket): parse pagination URLs correctly for openapi-fetch --- packages/backend/src/bitbucket.ts | 32 +++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/bitbucket.ts b/packages/backend/src/bitbucket.ts index e204850c0..80dfe365d 100644 --- a/packages/backend/src/bitbucket.ts +++ b/packages/backend/src/bitbucket.ts @@ -170,21 +170,33 @@ const getPaginatedCloud = async ( } return results; } - + +/** + * Parse the url into a path and query parameters to be used with the api client (openapi-fetch) + */ +function parseUrl(url: string, baseUrl: string): { path: string; query: Record; } { + const fullUrl = new URL(url); + const path = fullUrl.pathname; + const query = Object.fromEntries(fullUrl.searchParams); + logger.debug(`Parsed url ${url} into path ${path} and query ${JSON.stringify(query)}`); + return { path, query }; +} + async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{validRepos: CloudRepository[], notFoundWorkspaces: string[]}> { const results = await Promise.allSettled(workspaces.map(async (workspace) => { try { logger.debug(`Fetching all repos for workspace ${workspace}...`); - const path = `/repositories/${workspace}` as CloudGetRequestPath; const { durationMs, data } = await measure(async () => { - const fetchFn = () => getPaginatedCloud(path, async (url) => { - const response = await client.apiClient.GET(url, { + const fetchFn = () => getPaginatedCloud(`/repositories/${workspace}` as CloudGetRequestPath, async (url) => { + const { path, query } = parseUrl(url, client.baseUrl); + const response = await client.apiClient.GET(path, { params: { path: { workspace, - } + }, + query: query, } }); const { data, error } = response; @@ -238,11 +250,15 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin logger.debug(`Fetching all repos for project ${project} for workspace ${workspace}...`); try { - const path = `/repositories/${workspace}` as CloudGetRequestPath; - const repos = await getPaginatedCloud(path, async (url) => { - const response = await client.apiClient.GET(url, { + const repos = await getPaginatedCloud(`/repositories/${workspace}` as CloudGetRequestPath, async (url) => { + const { path, query } = parseUrl(url, client.baseUrl); + const response = await client.apiClient.GET(path, { params: { + path: { + workspace, + }, query: { + ...query, q: `project.key="${project_name}"` } } From 33fa1aa98ea49088da55a7e3776f1d4aa965d8b7 Mon Sep 17 00:00:00 2001 From: Fede Sanchez Date: Sun, 14 Sep 2025 12:48:43 -0300 Subject: [PATCH 2/3] move url parsing into getPaginatedCloud --- packages/backend/src/bitbucket.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/backend/src/bitbucket.ts b/packages/backend/src/bitbucket.ts index 80dfe365d..cfa591cc3 100644 --- a/packages/backend/src/bitbucket.ts +++ b/packages/backend/src/bitbucket.ts @@ -148,13 +148,14 @@ function cloudClient(user: string | undefined, token: string | undefined): Bitbu **/ const getPaginatedCloud = async ( path: CloudGetRequestPath, - get: (url: CloudGetRequestPath) => Promise> + get: (path: CloudGetRequestPath, query?: Record) => Promise> ): Promise => { const results: T[] = []; - let url = path; + let nextPath = path; + let nextQuery = undefined; while (true) { - const response = await get(url); + const response = await get(nextPath, nextQuery); if (!response.values || response.values.length === 0) { break; @@ -166,7 +167,9 @@ const getPaginatedCloud = async ( break; } - url = response.next as CloudGetRequestPath; + const parsedUrl = parseUrl(response.next); + nextPath = parsedUrl.path as CloudGetRequestPath; + nextQuery = parsedUrl.query; } return results; } @@ -174,9 +177,9 @@ const getPaginatedCloud = async ( /** * Parse the url into a path and query parameters to be used with the api client (openapi-fetch) */ -function parseUrl(url: string, baseUrl: string): { path: string; query: Record; } { +function parseUrl(url: string): { path: string; query: Record; } { const fullUrl = new URL(url); - const path = fullUrl.pathname; + const path = fullUrl.pathname.replace(/^\/\d+(\.\d+)*/, ''); // remove version number in the beginning of the path const query = Object.fromEntries(fullUrl.searchParams); logger.debug(`Parsed url ${url} into path ${path} and query ${JSON.stringify(query)}`); return { path, query }; @@ -189,8 +192,7 @@ async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: st logger.debug(`Fetching all repos for workspace ${workspace}...`); const { durationMs, data } = await measure(async () => { - const fetchFn = () => getPaginatedCloud(`/repositories/${workspace}` as CloudGetRequestPath, async (url) => { - const { path, query } = parseUrl(url, client.baseUrl); + const fetchFn = () => getPaginatedCloud(`/repositories/${workspace}` as CloudGetRequestPath, async (path, query) => { const response = await client.apiClient.GET(path, { params: { path: { @@ -250,8 +252,7 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin logger.debug(`Fetching all repos for project ${project} for workspace ${workspace}...`); try { - const repos = await getPaginatedCloud(`/repositories/${workspace}` as CloudGetRequestPath, async (url) => { - const { path, query } = parseUrl(url, client.baseUrl); + const repos = await getPaginatedCloud(`/repositories/${workspace}` as CloudGetRequestPath, async (path, query) => { const response = await client.apiClient.GET(path, { params: { path: { From a42afd88e531817d462d6d0651daebb0b030213d Mon Sep 17 00:00:00 2001 From: Fede Sanchez Date: Mon, 15 Sep 2025 14:20:36 -0300 Subject: [PATCH 3/3] changelog added --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21fefa0bd..ec5963a4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- Fixed Bitbucket Cloud pagination not working beyond first page. [#295](https://github.com/sourcebot-dev/sourcebot/issues/295) + ## [4.6.7] - 2025-09-08 ### Added