From 17e453fdd22ced475ca50522de456820adf6f4c0 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Wed, 24 Sep 2025 17:34:54 +0530 Subject: [PATCH 1/7] Added helper function for git api limit --- internal/wrappers/github-http.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/internal/wrappers/github-http.go b/internal/wrappers/github-http.go index 6ba2d31e0..2fd165ad4 100644 --- a/internal/wrappers/github-http.go +++ b/internal/wrappers/github-http.go @@ -5,7 +5,9 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" + "time" "github.com/checkmarx/ast-cli/internal/logger" "github.com/checkmarx/ast-cli/internal/params" @@ -248,6 +250,10 @@ func get(client *http.Client, url string, target interface{}, queryParams map[st if err != nil { return nil, err } + resp, err = handleRateLimit(resp, client, req, url, token, tokenFormat, queryParams) + if err != nil { + return nil, err + } defer func() { if err == nil { _ = resp.Body.Close() @@ -276,3 +282,23 @@ func get(client *http.Client, url string, target interface{}, queryParams map[st } return resp, nil } + +func handleRateLimit(resp *http.Response, client *http.Client, req *http.Request, url, token, authFormat string, queryParams map[string]string) (*http.Response, error) { + if resp.StatusCode == http.StatusForbidden { + remaining := resp.Header.Get("X-RateLimit-Remaining") + reset := resp.Header.Get("X-RateLimit-Reset") + if remaining == "0" && reset != "" { + resetUnix, err := strconv.ParseInt(reset, 10, 64) + if err == nil { + waitDuration := time.Until(time.Unix(resetUnix, 0)) + if waitDuration > 0 { + time.Sleep(waitDuration) + return GetWithQueryParamsAndCustomRequest(client, req, url, token, tokenFormat, queryParams) // Indicate to retry + } + } else { + return resp, err + } + } + } + return resp, nil //Not rate limited +} From 08ef49b00907b612405202e1c2da95b53de8659f Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Wed, 24 Sep 2025 18:07:29 +0530 Subject: [PATCH 2/7] fix lint issue --- internal/wrappers/github-http.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/wrappers/github-http.go b/internal/wrappers/github-http.go index 2fd165ad4..266c2117e 100644 --- a/internal/wrappers/github-http.go +++ b/internal/wrappers/github-http.go @@ -250,7 +250,7 @@ func get(client *http.Client, url string, target interface{}, queryParams map[st if err != nil { return nil, err } - resp, err = handleRateLimit(resp, client, req, url, token, tokenFormat, queryParams) + resp, err = handleRateLimit(resp, client, req, url, token, queryParams) if err != nil { return nil, err } @@ -283,7 +283,7 @@ func get(client *http.Client, url string, target interface{}, queryParams map[st return resp, nil } -func handleRateLimit(resp *http.Response, client *http.Client, req *http.Request, url, token, authFormat string, queryParams map[string]string) (*http.Response, error) { +func handleRateLimit(resp *http.Response, client *http.Client, req *http.Request, url, token string, queryParams map[string]string) (*http.Response, error) { if resp.StatusCode == http.StatusForbidden { remaining := resp.Header.Get("X-RateLimit-Remaining") reset := resp.Header.Get("X-RateLimit-Reset") @@ -300,5 +300,5 @@ func handleRateLimit(resp *http.Response, client *http.Client, req *http.Request } } } - return resp, nil //Not rate limited + return resp, nil // Not rate limited } From 0702c86f75d111f70fe672d14743f06f2348fd60 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Wed, 24 Sep 2025 19:30:53 +0530 Subject: [PATCH 3/7] code refactor --- internal/wrappers/github-http.go | 34 +++++++++++++++----------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/internal/wrappers/github-http.go b/internal/wrappers/github-http.go index 266c2117e..cf4345ab6 100644 --- a/internal/wrappers/github-http.go +++ b/internal/wrappers/github-http.go @@ -250,9 +250,11 @@ func get(client *http.Client, url string, target interface{}, queryParams map[st if err != nil { return nil, err } - resp, err = handleRateLimit(resp, client, req, url, token, queryParams) - if err != nil { - return nil, err + if resp.StatusCode == http.StatusForbidden { + resp, err = HandleRateLimit(resp, client, req, url, token, queryParams) + if err != nil { + return nil, err + } } defer func() { if err == nil { @@ -283,22 +285,18 @@ func get(client *http.Client, url string, target interface{}, queryParams map[st return resp, nil } -func handleRateLimit(resp *http.Response, client *http.Client, req *http.Request, url, token string, queryParams map[string]string) (*http.Response, error) { - if resp.StatusCode == http.StatusForbidden { - remaining := resp.Header.Get("X-RateLimit-Remaining") - reset := resp.Header.Get("X-RateLimit-Reset") - if remaining == "0" && reset != "" { - resetUnix, err := strconv.ParseInt(reset, 10, 64) - if err == nil { - waitDuration := time.Until(time.Unix(resetUnix, 0)) - if waitDuration > 0 { - time.Sleep(waitDuration) - return GetWithQueryParamsAndCustomRequest(client, req, url, token, tokenFormat, queryParams) // Indicate to retry - } - } else { - return resp, err +func HandleRateLimit(resp *http.Response, client *http.Client, req *http.Request, url, token string, queryParams map[string]string) (*http.Response, error) { + remaining := resp.Header.Get("X-RateLimit-Remaining") + reset := resp.Header.Get("X-RateLimit-Reset") + if remaining == "0" && reset != "" { + resetUnix, err := strconv.ParseInt(reset, 10, 64) + if err == nil { + waitDuration := time.Until(time.Unix(resetUnix, 0)) + if waitDuration > 0 { + time.Sleep(waitDuration) + return GetWithQueryParamsAndCustomRequest(client, req, url, token, tokenFormat, queryParams) // Indicate to retry } } } - return resp, nil // Not rate limited + return resp, nil } From 599ffd46036e9125427a45f8f229b9c2ea27404e Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Thu, 25 Sep 2025 12:23:31 +0530 Subject: [PATCH 4/7] Added unit test cases --- .../commands/util/usercount/github_test.go | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/internal/commands/util/usercount/github_test.go b/internal/commands/util/usercount/github_test.go index b39bc403d..2638ea32e 100644 --- a/internal/commands/util/usercount/github_test.go +++ b/internal/commands/util/usercount/github_test.go @@ -1,13 +1,17 @@ package usercount import ( - "strings" - "testing" - "github.com/checkmarx/ast-cli/internal/commands/util/printer" "github.com/checkmarx/ast-cli/internal/params" + "github.com/checkmarx/ast-cli/internal/wrappers" "github.com/checkmarx/ast-cli/internal/wrappers/mock" + asserts "github.com/stretchr/testify/assert" "gotest.tools/assert" + "net/http" + "strconv" + "strings" + "testing" + "time" ) func TestGitHubUserCountOrgs(t *testing.T) { @@ -100,3 +104,35 @@ func TestGitHubUserCountManyOrgs(t *testing.T) { err := cmd.Execute() assert.Error(t, err, tooManyOrgs) } + +func TestHandleRateLimit_WaitsAndRetries(t *testing.T) { + resp := &http.Response{ + StatusCode: http.StatusForbidden, + Header: http.Header{}, + } + resp.Header.Set("X-RateLimit-Remaining", "0") + resetTime := time.Now().Add(50 * time.Second).Unix() + resp.Header.Set("X-RateLimit-Reset", strconv.FormatInt(resetTime, 10)) + + client := &http.Client{} + req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil) + + start := time.Now() + _, err := wrappers.HandleRateLimit(resp, client, req, "http://example.com", "token", map[string]string{}) + elapsed := time.Since(start) + + asserts.NoError(t, err) + asserts.GreaterOrEqual(t, elapsed, 20*time.Second) +} + +func TestHandleRateLimit_NoRateLimit(t *testing.T) { + resp := &http.Response{ + StatusCode: http.StatusForbidden, + Header: http.Header{}, + } + client := &http.Client{} + req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil) + outResp, err := wrappers.HandleRateLimit(resp, client, req, "http://example.com", "token", map[string]string{}) + asserts.NoError(t, err) + assert.Equal(t, resp, outResp) +} From dabeb52e3adea3682dd7c70fcdc01d4e67f73de4 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Thu, 25 Sep 2025 12:55:28 +0530 Subject: [PATCH 5/7] fix the lint issue --- .../commands/util/usercount/github_test.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/commands/util/usercount/github_test.go b/internal/commands/util/usercount/github_test.go index 2638ea32e..0c57b9a02 100644 --- a/internal/commands/util/usercount/github_test.go +++ b/internal/commands/util/usercount/github_test.go @@ -7,6 +7,7 @@ import ( "github.com/checkmarx/ast-cli/internal/wrappers/mock" asserts "github.com/stretchr/testify/assert" "gotest.tools/assert" + "io" "net/http" "strconv" "strings" @@ -109,13 +110,18 @@ func TestHandleRateLimit_WaitsAndRetries(t *testing.T) { resp := &http.Response{ StatusCode: http.StatusForbidden, Header: http.Header{}, + Body: io.NopCloser(strings.NewReader("")), } resp.Header.Set("X-RateLimit-Remaining", "0") resetTime := time.Now().Add(50 * time.Second).Unix() resp.Header.Set("X-RateLimit-Reset", strconv.FormatInt(resetTime, 10)) - + defer func() { + if err := resp.Body.Close(); err != nil { + t.Fatal(err) + } + }() client := &http.Client{} - req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil) + req, _ := http.NewRequest(http.MethodGet, "http://example.com", http.NoBody) start := time.Now() _, err := wrappers.HandleRateLimit(resp, client, req, "http://example.com", "token", map[string]string{}) @@ -129,9 +135,15 @@ func TestHandleRateLimit_NoRateLimit(t *testing.T) { resp := &http.Response{ StatusCode: http.StatusForbidden, Header: http.Header{}, + Body: io.NopCloser(strings.NewReader("")), } + defer func() { + if err := resp.Body.Close(); err != nil { + t.Fatal(err) + } + }() client := &http.Client{} - req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil) + req, _ := http.NewRequest(http.MethodGet, "http://example.com", http.NoBody) outResp, err := wrappers.HandleRateLimit(resp, client, req, "http://example.com", "token", map[string]string{}) asserts.NoError(t, err) assert.Equal(t, resp, outResp) From 098200fa786fe8928a8010d6eae0fc193f717520 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Thu, 25 Sep 2025 13:17:03 +0530 Subject: [PATCH 6/7] fix lint issue --- internal/commands/util/usercount/github_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/commands/util/usercount/github_test.go b/internal/commands/util/usercount/github_test.go index 0c57b9a02..be0f8ca81 100644 --- a/internal/commands/util/usercount/github_test.go +++ b/internal/commands/util/usercount/github_test.go @@ -124,7 +124,12 @@ func TestHandleRateLimit_WaitsAndRetries(t *testing.T) { req, _ := http.NewRequest(http.MethodGet, "http://example.com", http.NoBody) start := time.Now() - _, err := wrappers.HandleRateLimit(resp, client, req, "http://example.com", "token", map[string]string{}) + outResp, err := wrappers.HandleRateLimit(resp, client, req, "http://example.com", "token", map[string]string{}) + defer func() { + if err := outResp.Body.Close(); err != nil { + t.Fatal(err) + } + }() elapsed := time.Since(start) asserts.NoError(t, err) @@ -145,6 +150,11 @@ func TestHandleRateLimit_NoRateLimit(t *testing.T) { client := &http.Client{} req, _ := http.NewRequest(http.MethodGet, "http://example.com", http.NoBody) outResp, err := wrappers.HandleRateLimit(resp, client, req, "http://example.com", "token", map[string]string{}) + defer func() { + if err := outResp.Body.Close(); err != nil { + t.Fatal(err) + } + }() asserts.NoError(t, err) assert.Equal(t, resp, outResp) } From 50c04498fee8345257cda693147ba6019b10673a Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Thu, 25 Sep 2025 13:23:39 +0530 Subject: [PATCH 7/7] fix lint issue --- internal/commands/util/usercount/github_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/commands/util/usercount/github_test.go b/internal/commands/util/usercount/github_test.go index be0f8ca81..3e17e1eab 100644 --- a/internal/commands/util/usercount/github_test.go +++ b/internal/commands/util/usercount/github_test.go @@ -1,18 +1,20 @@ package usercount import ( - "github.com/checkmarx/ast-cli/internal/commands/util/printer" - "github.com/checkmarx/ast-cli/internal/params" - "github.com/checkmarx/ast-cli/internal/wrappers" - "github.com/checkmarx/ast-cli/internal/wrappers/mock" - asserts "github.com/stretchr/testify/assert" - "gotest.tools/assert" "io" "net/http" "strconv" "strings" "testing" "time" + + asserts "github.com/stretchr/testify/assert" + "gotest.tools/assert" + + "github.com/checkmarx/ast-cli/internal/commands/util/printer" + "github.com/checkmarx/ast-cli/internal/params" + "github.com/checkmarx/ast-cli/internal/wrappers" + "github.com/checkmarx/ast-cli/internal/wrappers/mock" ) func TestGitHubUserCountOrgs(t *testing.T) {