fix: use adaptive retry for AWS API calls to handle rate limiting (#757)

* fix: use adaptive retry for AWS API calls to handle rate limiting

Switch from the default AWS SDK retry strategy (3 attempts, basic
exponential backoff) to adaptive retry mode with up to 10 attempts.

Adaptive mode maintains a client-side token bucket that tracks
throttling responses over time and proactively slows the request
rate. Standard mode just does backoff-and-retry; adaptive mode
avoids hammering a rate-limited API by draining tokens on 429s.

* docs: add comment explaining adaptive retry config

* revert: restore test parallelism to -p=8

Reverts #747 which reduced parallelism from 8 to 6 as a workaround
for AWS rate limiting. The adaptive retry config now handles this
properly, so we can restore the original parallelism.

Author: dangrondahl

Makefile

@@ -121,7 +121,7 @@ setup_test_to_use_staging_server_image:
 
 test_integration: deps vet ensure_network test_setup ## Run tests except the too slow ones
 	@[ -e ~/.kosli.yml ] && mv ~/.kosli.yml ~/.kosli-renamed.yml || true
-	@export KOSLI_TESTS=true $(FAKE_CI_ENV) && $(GOTESTSUM) -- --short -p=6 -coverprofile=cover.out ./...
+	@export KOSLI_TESTS=true $(FAKE_CI_ENV) && $(GOTESTSUM) -- --short -p=8 -coverprofile=cover.out ./...
 	@go tool cover -func=cover.out | grep total:
 	@go tool cover -html=cover.out
 	@[ -e ~/.kosli-renamed.yml ] && mv ~/.kosli-renamed.yml ~/.kosli.yml || true
@@ -130,14 +130,14 @@ test_integration: deps vet ensure_network test_setup ## Run tests except the too
 test_integration_full: deps vet ensure_network test_setup ## Run all tests
 	@[ -e ~/.kosli.yml ] && mv ~/.kosli.yml ~/.kosli-renamed.yml || true
 	@mkdir -p junit-test-results
-	@export KOSLI_TESTS=true $(FAKE_CI_ENV) && $(GOTESTSUM) --junitfile junit-test-results/junit.xml -- -p=6 -coverprofile=cover.out ./...
+	@export KOSLI_TESTS=true $(FAKE_CI_ENV) && $(GOTESTSUM) --junitfile junit-test-results/junit.xml -- -p=8 -coverprofile=cover.out ./...
 	@go tool cover -func=cover.out
 	@[ -e ~/.kosli-renamed.yml ] && mv ~/.kosli-renamed.yml ~/.kosli.yml || true
 
 
 test_integration_restart_server: test_setup_restart_server
 	@[ -e ~/.kosli.yml ] && mv ~/.kosli.yml ~/.kosli-renamed.yml || true
-	@export KOSLI_TESTS=true $(FAKE_CI_ENV) && $(GOTESTSUM) -- --short -p=6 -coverprofile=cover.out ./...
+	@export KOSLI_TESTS=true $(FAKE_CI_ENV) && $(GOTESTSUM) -- --short -p=8 -coverprofile=cover.out ./...
 	@go tool cover -html=cover.out
 	@[ -e ~/.kosli-renamed.yml ] && mv ~/.kosli-renamed.yml ~/.kosli.yml || true
 

internal/aws/aws.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
@@ -95,13 +96,29 @@ func (s *AWSStaticCreds) GetConfigOptFns() []func(*config.LoadOptions) error {
 
 // NewAWSConfigFromEnvOrFlags returns an AWS config that can be used to construct
 // AWS service clients.
-// Credentials for config can be sourced from multiple sources, in this order:
+//
+// Credentials are sourced in this order:
 // 1) static credentials (from CLI flags or KOSLI env vars), if provided
-// 2) AWS Environment variables
-// 3) Shared AWS Configuration/Credentials files (see https://docs.aws.amazon.com/sdkref/latest/guide/file-format.html)
-// more details can be found here: https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#specifying-credentials
+// 2) AWS environment variables
+// 3) shared AWS configuration/credentials files (see https://docs.aws.amazon.com/sdkref/latest/guide/file-format.html)
+//
+// Retry: uses adaptive mode (up to MaxAttempts attempts) with a shared in-memory token
+// bucket. Commands like "kosli snapshot lambda" fetch functions concurrently
+// (see GetLambdaPackageData), which can trigger AWS rate limits (HTTP 429).
+// The shared token bucket slows down the entire batch of goroutines when
+// throttling is detected, rather than each goroutine retrying independently.
+//
+// More details: https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/aws/retry
 func (staticCreds *AWSStaticCreds) NewAWSConfigFromEnvOrFlags() (aws.Config, error) {
-	return config.LoadDefaultConfig(context.TODO(), staticCreds.GetConfigOptFns()...)
+	optFns := staticCreds.GetConfigOptFns()
+	optFns = append(optFns, config.WithRetryer(func() aws.Retryer {
+		return retry.NewAdaptiveMode(func(o *retry.AdaptiveModeOptions) {
+			o.StandardOptions = append(o.StandardOptions, func(so *retry.StandardOptions) {
+				so.MaxAttempts = 10
+			})
+		})
+	}))
+	return config.LoadDefaultConfig(context.TODO(), optFns...)
 }
 
 // NewS3Client returns a new S3 API client