fix(providers): normalize JDCloud encoding, surface Volcengine errors

Author: 404tk

pkg/providers/jdcloud/api/client_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"regexp"
 	"strings"
 	"testing"
@@ -113,6 +114,36 @@ func TestClientDoJSONReturnsAPIErrorFromBody(t *testing.T) {
 	}
 }
 
+func TestClientDoJSONUsesPercent20InQuery(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if got := r.URL.RawQuery; got != "name=foo%20bar" {
+			t.Fatalf("unexpected raw query: %s", got)
+		}
+		_, _ = w.Write([]byte(`{"requestId":"req-query","result":{"subUsers":[],"total":0}}`))
+	}))
+	defer server.Close()
+
+	client := newTestClient(server.URL, "", func() string {
+		return "ebf8b26d-c3be-402f-9f10-f8b6573fd823"
+	})
+
+	query := url.Values{}
+	query.Set("name", "foo bar")
+
+	var resp DescribeSubUsersResponse
+	err := client.DoJSON(context.Background(), Request{
+		Service: "iam",
+		Region:  "",
+		Method:  http.MethodGet,
+		Version: "v1",
+		Path:    "/subUsers",
+		Query:   query,
+	}, &resp)
+	if err != nil {
+		t.Fatalf("DoJSON() error = %v", err)
+	}
+}
+
 func TestClientDoJSONHonorsCanceledContext(t *testing.T) {
 	client := newTestClient("http://127.0.0.1:1", "", func() string {
 		return "ebf8b26d-c3be-402f-9f10-f8b6573fd823"

pkg/providers/jdcloud/api/signer.go

@@ -181,7 +181,7 @@ func canonicalQuery(values url.Values) string {
 	if len(values) == 0 {
 		return ""
 	}
-	return values.Encode()
+	return strings.ReplaceAll(values.Encode(), "+", "%20")
 }
 
 func canonicalURI(path string) string {

pkg/providers/jdcloud/iam/users.go

@@ -51,6 +51,8 @@ func (d *Driver) ListUsers(ctx context.Context) ([]schema.User, error) {
 		var resp api.DescribeSubUsersResponse
 		err := d.Client.DoJSON(ctx, api.Request{
 			Service: "iam",
+			// IAM is global. An empty region makes the signer fall back to the
+			// jdcloud-api scope expected by this endpoint.
 			Region:  "",
 			Method:  "GET",
 			Version: "v1",
@@ -106,6 +108,8 @@ func (d *Driver) Validator(_ string) bool {
 	var resp api.DescribeSubUsersResponse
 	err := d.Client.DoJSON(context.Background(), api.Request{
 		Service: "iam",
+		// IAM is global. An empty region makes the signer fall back to the
+		// jdcloud-api scope expected by this endpoint.
 		Region:  "",
 		Method:  "GET",
 		Version: "v1",

pkg/providers/jdcloud/vm/instances.go

@@ -34,6 +34,9 @@ func (d *Driver) GetResource(ctx context.Context) ([]schema.Host, error) {
 		return list, errors.New("jdcloud vm: nil api client")
 	}
 
+	if d.Region == "all" {
+		logger.Warning("JDCloud region=all falls back to cn-north-1")
+	}
 	region := d.requestRegion()
 
 	got, err := paginate.Fetch[schema.Host, pageCursor](ctx, func(ctx context.Context, cursor pageCursor) (paginate.Page[schema.Host, pageCursor], error) {

pkg/providers/volcengine/ecs/instances.go

@@ -3,6 +3,8 @@ package ecs
 import (
 	"context"
 	"errors"
+	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/404tk/cloudtoolkit/pkg/providers/volcengine/api"
@@ -35,7 +37,7 @@ func (d *Driver) GetResource(ctx context.Context) ([]schema.Host, error) {
 	}
 	tracker := processbar.NewRegionTracker()
 	defer tracker.Finish()
-	got, _ := regionrun.ForEach(ctx, regions, 0, tracker, func(ctx context.Context, r string) ([]schema.Host, error) {
+	got, regionErrs := regionrun.ForEach(ctx, regions, 0, tracker, func(ctx context.Context, r string) ([]schema.Host, error) {
 		return paginate.Fetch[schema.Host, string](ctx, func(ctx context.Context, token string) (paginate.Page[schema.Host, string], error) {
 			resp, err := client.DescribeInstances(ctx, r, 100, token)
 			if err != nil {
@@ -68,7 +70,7 @@ func (d *Driver) GetResource(ctx context.Context) ([]schema.Host, error) {
 		})
 	})
 	list = append(list, got...)
-	return list, nil
+	return list, flattenRegionErrors(regionErrs)
 }
 
 func (d *Driver) requireClient() (*api.Client, error) {
@@ -102,3 +104,24 @@ func (d *Driver) requestRegion() string {
 	}
 	return region
 }
+
+func flattenRegionErrors(errs map[string]error) error {
+	if len(errs) == 0 {
+		return nil
+	}
+	regions := make([]string, 0, len(errs))
+	for region := range errs {
+		regions = append(regions, region)
+	}
+	sort.Strings(regions)
+	parts := make([]string, 0, len(regions))
+	for _, region := range regions {
+		if err := errs[region]; err != nil {
+			parts = append(parts, fmt.Sprintf("%s: %v", region, err))
+		}
+	}
+	if len(parts) == 0 {
+		return nil
+	}
+	return fmt.Errorf("partial region errors: %s", strings.Join(parts, "; "))
+}

pkg/providers/volcengine/ecs/instances_test.go

@@ -99,6 +99,54 @@ func TestDriverGetResourceUsesDefaultRegionWhenEmpty(t *testing.T) {
 	}
 }
 
+func TestDriverGetResourceReturnsPartialRegionErrors(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		values, err := url.ParseQuery(r.URL.RawQuery)
+		if err != nil {
+			t.Fatalf("parse query: %v", err)
+		}
+		switch values.Get("Action") {
+		case "DescribeRegions":
+			_, _ = w.Write(mustJSON(t, api.DescribeRegionsResponse{
+				Result: struct {
+					NextToken string          `json:"NextToken"`
+					Regions   []api.ECSRegion `json:"Regions"`
+				}{
+					Regions: []api.ECSRegion{{RegionID: "cn-beijing"}, {RegionID: "cn-shanghai"}},
+				},
+			}))
+		case "DescribeInstances":
+			switch region := scopeRegion(t, r.Header.Get(api.HeaderAuthorization)); region {
+			case "cn-beijing":
+				_, _ = w.Write(mustJSON(t, describeInstancesWithIDs([]string{"bj-001"}, "")))
+			case "cn-shanghai":
+				w.WriteHeader(http.StatusForbidden)
+				_, _ = w.Write([]byte(`{"ResponseMetadata":{"RequestId":"req-sh","Error":{"Code":"Forbidden","Message":"denied"}}}`))
+			default:
+				t.Fatalf("unexpected region: %s", region)
+			}
+		default:
+			t.Fatalf("unexpected action: %s", values.Get("Action"))
+		}
+	}))
+	defer server.Close()
+
+	driver := &Driver{
+		Client: newTestClient(server.URL),
+		Region: "all",
+	}
+	got, err := driver.GetResource(context.Background())
+	if len(got) != 1 || got[0].ID != "bj-001" {
+		t.Fatalf("unexpected hosts: %+v", got)
+	}
+	if err == nil {
+		t.Fatal("expected partial region error")
+	}
+	if !strings.Contains(err.Error(), "cn-shanghai") || !strings.Contains(err.Error(), "Forbidden") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
 func newTestClient(baseURL string) *api.Client {
 	return api.NewClient(
 		auth.New("AKID", "SECRET", ""),

pkg/providers/volcengine/iam/users.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/404tk/cloudtoolkit/pkg/providers/volcengine/api"
 	"github.com/404tk/cloudtoolkit/pkg/schema"
-	"github.com/404tk/cloudtoolkit/utils"
 	"github.com/404tk/cloudtoolkit/utils/logger"
 )
 
@@ -30,7 +29,6 @@ func (d *Driver) ListUsers(ctx context.Context) ([]schema.User, error) {
 	}
 	region := d.requestRegion()
 	var offset int32 = 0
-	//policy_infos = make(map[string]string)
 	for {
 		resp, err := client.ListUsers(ctx, region, 100, offset)
 		if err != nil {
@@ -57,10 +55,6 @@ func (d *Driver) ListUsers(ctx context.Context) ([]schema.User, error) {
 
 			}
 
-			if utils.ListPolicies {
-				// _user.Policies = listPoliciesForUser(client, _user.UserName)
-			}
-
 			list = append(list, _user)
 			select {
 			case <-ctx.Done():