fix(bitbucket): migrate off cross-workspace APIs removed by CHANGE-2770 (apache#8916)

* fix(bitbucket): migrate off cross-workspace APIs removed by CHANGE-2770

- listBitbucketWorkspaces: replace GET /user/permissions/workspaces with the supported GET /workspaces (lists the current user's workspaces). Workspace slug/name now parsed from the top level of each value instead of a nested "workspace" object.
- searchBitbucketRepos: replace cross-workspace GET /repositories?role=member with per-workspace GET /repositories/{workspace}, enumerating the user's workspaces first and aggregating up to PageSize matches.
- models: flatten GroupResponse to match the /workspaces response shape.

* fix(bitbucket): list workspaces via supported /user/workspaces (CHANGE-2770)

GET /2.0/workspaces is also deprecated, not just /user/permissions/workspaces.
The supported replacement for enumerating the current user's workspaces is
GET /2.0/user/workspaces, which nests slug/name under a workspace object.

- remote_api.go: listBitbucketWorkspaces / listAllBitbucketWorkspaces now call
  /user/workspaces with workspace.* fields and read via GroupId()/GroupName().
- models/repo.go: re-nest GroupResponse under WorkspaceItem to match the
  /user/workspaces response shape.

Repo listing stays on the supported GET /2.0/repositories/{workspace}.

* fix(bitbucket): drop sort/fields on /user/workspaces (400 Invalid field name)

GET /2.0/user/workspaces rejects sort=workspace.slug with HTTP 400 'Invalid
field name'. The bare paginated call returns the nested workspace objects we
parse, so drop the sort/fields query params in both listBitbucketWorkspaces
and listAllBitbucketWorkspaces.

* fix(bitbucket): ignore 410 Gone on sunset issues API

Bitbucket has sunset the issue tracker/wiki API; the issues endpoint now
returns 410 Gone instead of 404. ignoreHTTPStatus404 only mapped 404 to
ErrIgnoreAndContinue, so a 410 fell through to nil, DoAsync retried 3x and
failed the whole pipeline ("Collect Issues ended unexpectedly").

Treat 410 like 404 (ignore and continue). Shared helper also covers issue
comments and pr commits. Adds a table test for the status-code mapping.

Author: KennethWKZ

backend/plugins/bitbucket/api/remote_api.go

@@ -67,11 +67,15 @@ func listBitbucketWorkspaces(
 	err errors.Error,
 ) {
 	var res *http.Response
+	// /user/permissions/workspaces was removed and /workspaces deprecated by
+	// Bitbucket CHANGE-2770; /user/workspaces lists the current user's workspaces
+	// and is the supported replacement.
 	res, err = apiClient.Get(
-		"/user/permissions/workspaces",
+		"/user/workspaces",
 		url.Values{
-			"sort":    {"workspace.slug"},
-			"fields":  {"values.workspace.slug,values.workspace.name,pagelen,page,size"},
+			// No sort/fields: /user/workspaces rejects sort=workspace.slug with
+			// HTTP 400 "Invalid field name". The bare call returns the nested
+			// workspace objects we need; WorkspaceResponse ignores extra fields.
 			"page":    {fmt.Sprintf("%v", page.Page)},
 			"pagelen": {fmt.Sprintf("%v", page.PageLen)},
 		},
@@ -98,9 +102,9 @@ func listBitbucketWorkspaces(
 	for _, r := range resBody.Values {
 		children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{
 			Type:     api.RAS_ENTRY_TYPE_GROUP,
-			Id:       r.Workspace.Slug,
-			Name:     r.Workspace.Name,
-			FullName: r.Workspace.Name,
+			Id:       r.GroupId(),
+			Name:     r.GroupName(),
+			FullName: r.GroupName(),
 		})
 	}
 	return
@@ -152,46 +156,95 @@ func listBitbucketRepos(
 	return
 }
 
+// searchBitbucketRepos searches repositories by name across the user's workspaces.
+// The cross-workspace GET /repositories?role=member was removed by Bitbucket
+// CHANGE-2770, so we enumerate workspaces and query the workspace-scoped
+// GET /repositories/{workspace} endpoint for each, aggregating up to PageSize hits.
 func searchBitbucketRepos(
 	apiClient plugin.ApiClient,
 	params *dsmodels.DsRemoteApiScopeSearchParams,
 ) (
 	children []dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo],
 	err errors.Error,
 ) {
-	var res *http.Response
-	res, err = apiClient.Get(
-		"/repositories",
-		url.Values{
-			"sort":    {"name"},
-			"fields":  {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"},
-			"role":    {"member"},
-			"q":       {fmt.Sprintf(`full_name~"%s"`, params.Search)},
-			"page":    {fmt.Sprintf("%v", params.Page)},
-			"pagelen": {fmt.Sprintf("%v", params.PageSize)},
-		},
-		nil,
-	)
-	if err != nil {
-		return nil, err
+	pageSize := params.PageSize
+	if pageSize == 0 {
+		pageSize = 100
 	}
-	var resBody models.ReposResponse
-	err = api.UnmarshalResponse(res, &resBody)
+
+	workspaces, err := listAllBitbucketWorkspaces(apiClient)
 	if err != nil {
-		return
+		return nil, err
 	}
-	for _, r := range resBody.Values {
-		children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{
-			Type:     api.RAS_ENTRY_TYPE_SCOPE,
-			Id:       r.FullName,
-			Name:     r.Name,
-			FullName: r.FullName,
-			Data:     r.ConvertApiScope(),
-		})
+
+	for _, workspace := range workspaces {
+		if len(children) >= pageSize {
+			break
+		}
+		var res *http.Response
+		res, err = apiClient.Get(
+			fmt.Sprintf("/repositories/%s", workspace),
+			url.Values{
+				"sort":    {"name"},
+				"fields":  {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"},
+				"q":       {fmt.Sprintf(`name~"%s"`, params.Search)},
+				"pagelen": {fmt.Sprintf("%v", pageSize)},
+			},
+			nil,
+		)
+		if err != nil {
+			return nil, err
+		}
+		var resBody models.ReposResponse
+		err = api.UnmarshalResponse(res, &resBody)
+		if err != nil {
+			return
+		}
+		for _, r := range resBody.Values {
+			children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{
+				Type:     api.RAS_ENTRY_TYPE_SCOPE,
+				Id:       r.FullName,
+				Name:     r.Name,
+				FullName: r.FullName,
+				Data:     r.ConvertApiScope(),
+			})
+		}
 	}
 	return
 }
 
+// listAllBitbucketWorkspaces returns every workspace slug accessible to the
+// authenticated user, following pagination of GET /2.0/user/workspaces.
+func listAllBitbucketWorkspaces(apiClient plugin.ApiClient) ([]string, errors.Error) {
+	var slugs []string
+	for page := 1; ; page++ {
+		res, err := apiClient.Get(
+			"/user/workspaces",
+			url.Values{
+				// No sort/fields (see listBitbucketWorkspaces): /user/workspaces
+				// returns 400 on sort=workspace.slug.
+				"page":    {fmt.Sprintf("%v", page)},
+				"pagelen": {"100"},
+			},
+			nil,
+		)
+		if err != nil {
+			return nil, err
+		}
+		var resBody models.WorkspaceResponse
+		if err = api.UnmarshalResponse(res, &resBody); err != nil {
+			return nil, err
+		}
+		for _, w := range resBody.Values {
+			slugs = append(slugs, w.GroupId())
+		}
+		if len(resBody.Values) == 0 || page*resBody.Pagelen >= resBody.Size {
+			break
+		}
+	}
+	return slugs, nil
+}
+
 // RemoteScopes list all available scopes on the remote server
 // @Summary list all available scopes on the remote server
 // @Description list all available scopes on the remote server

backend/plugins/bitbucket/models/repo.go

@@ -117,17 +117,15 @@ type WorkspaceResponse struct {
 	Values  []GroupResponse `json:"values"`
 }
 
+// GroupResponse maps an entry from GET /2.0/user/workspaces, the supported
+// replacement after Bitbucket CHANGE-2770 removed the cross-workspace
+// GET /2.0/user/permissions/workspaces and deprecated GET /2.0/workspaces.
+// Each value nests the workspace slug/name under a "workspace" object.
 type GroupResponse struct {
-	//Type       string `json:"type"`
-	//Permission string `json:"permission"`
-	//LastAccessed time.Time `json:"last_accessed"`
-	//AddedOn      time.Time `json:"added_on"`
 	Workspace WorkspaceItem `json:"workspace"`
 }
 
 type WorkspaceItem struct {
-	//Type string `json:"type"`
-	//Uuid string `json:"uuid"`
 	Slug string `json:"slug" group:"id"`
 	Name string `json:"name" group:"name"`
 }

backend/plugins/bitbucket/tasks/api_common.go

@@ -238,7 +238,10 @@ func ignoreHTTPStatus404(res *http.Response) errors.Error {
 	if res.StatusCode == http.StatusUnauthorized {
 		return errors.Unauthorized.New("authentication failed, please check your AccessToken")
 	}
-	if res.StatusCode == http.StatusNotFound {
+	// 404: repo has no issue tracker. 410 Gone: Bitbucket has sunset the issue
+	// tracker/wiki API, so the endpoint is permanently removed for the repo.
+	// Both mean "nothing to collect" — skip gracefully instead of retrying.
+	if res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusGone {
 		return api.ErrIgnoreAndContinue
 	}
 	return nil

backend/plugins/bitbucket/tasks/api_common_test.go

@@ -0,0 +1,53 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIgnoreHTTPStatus404(t *testing.T) {
+	cases := []struct {
+		name       string
+		statusCode int
+		wantIgnore bool // expect ErrIgnoreAndContinue (graceful skip, no retry)
+		wantErr    bool // expect a real error
+	}{
+		{"404 no issue tracker -> ignore", http.StatusNotFound, true, false},
+		{"410 issue tracker sunset -> ignore", http.StatusGone, true, false},
+		{"401 unauthorized -> error", http.StatusUnauthorized, false, true},
+		{"200 ok -> continue", http.StatusOK, false, false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := ignoreHTTPStatus404(&http.Response{StatusCode: tc.statusCode})
+			switch {
+			case tc.wantIgnore:
+				assert.Equal(t, api.ErrIgnoreAndContinue, err)
+			case tc.wantErr:
+				assert.Error(t, err)
+			default:
+				assert.NoError(t, err)
+			}
+		})
+	}
+}