Tests (#311)

* Refactored Code base and implemented E2E and unit base tests

---------

Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>

Author: frjcomp

.github/workflows/test.yml

@@ -20,9 +20,19 @@ jobs:
         with:
           go-version: stable
 
-      - name: Run tests
+      - name: Run unit tests
         working-directory: ./src/pipeleak
-        run: go test ./... -v -race -coverprofile=coverage.out -covermode=atomic
+        run: go test $(go list ./... | grep -v /tests/e2e) -v -race -coverprofile=coverage.out -covermode=atomic
+      
+      - name: Build pipeleak binary for e2e tests
+        working-directory: ./src/pipeleak
+        run: go build -o pipeleak .
+      
+      - name: Run e2e tests
+        working-directory: ./src/pipeleak
+        run: go test ./tests/e2e/... -v -timeout 10m
+        env:
+          PIPELEAK_BINARY: ./pipeleak
 
       - name: Display coverage
         working-directory: ./src/pipeleak

.gitignore

@@ -5,3 +5,4 @@ renovate-enum-out
 cli-docs
 site
 src/pipeleak/pipeleak
+.vscode

src/pipeleak/cmd/bitbucket/api.go

@@ -16,14 +16,33 @@ import (
 
 // Docs: https://developer.atlassian.com/cloud/bitbucket/rest/intro/
 type BitBucketApiClient struct {
-	Client resty.Client
+	Client          resty.Client
+	BaseURL         string
+	InternalBaseURL string
 }
 
-func NewClient(username string, password string, bitBucketCookie string) BitBucketApiClient {
+func NewClient(username string, password string, bitBucketCookie string, baseURL string) BitBucketApiClient {
+	if baseURL == "" {
+		baseURL = "https://api.bitbucket.org/2.0"
+	}
+	// Derive an internal base URL for BitBucket internal APIs (bitbucket.org/!api/...)
+	// Default behaviour: if API base points to api.bitbucket.org use bitbucket.org for internal endpoints
+	parsedBase, err := url.Parse(baseURL)
+	if err != nil {
+		log.Fatal().Err(err).Msg("Unable to parse baseURL")
+	}
+
+	internalHost := parsedBase.Host
+	if parsedBase.Host == "api.bitbucket.org" || parsedBase.Host == "api.bitbucket.org:443" {
+		internalHost = "bitbucket.org"
+	}
+	internalBase := parsedBase.Scheme + "://" + internalHost + "/!api"
+
 	client := *resty.New().SetBasicAuth(username, password).SetRedirectPolicy(resty.FlexibleRedirectPolicy(5))
 	if len(bitBucketCookie) > 0 {
 		jar, _ := cookiejar.New(nil)
-		targetURL, _ := url.Parse("https://bitbucket.org/")
+		// set cookie on the internal host root so requests to internal endpoints include it
+		targetURL := &url.URL{Scheme: parsedBase.Scheme, Host: internalHost, Path: "/"}
 		jar.SetCookies(targetURL, []*http.Cookie{
 			{
 				Name:  "cloud.session.token",
@@ -35,7 +54,7 @@ func NewClient(username string, password string, bitBucketCookie string) BitBuck
 		log.Debug().Msg("Added cloud.session.token to HTTP client")
 	}
 
-	bbClient := BitBucketApiClient{Client: client}
+	bbClient := BitBucketApiClient{Client: client, BaseURL: baseURL, InternalBaseURL: internalBase}
 	bbClient.Client.AddRetryHooks(
 		func(res *resty.Response, err error) {
 			if res.StatusCode() == 429 {
@@ -50,7 +69,7 @@ func NewClient(username string, password string, bitBucketCookie string) BitBuck
 
 // https://developer.atlassian.com/cloud/bitbucket/rest/api-group-workspaces/#api-workspaces-get
 func (a BitBucketApiClient) ListOwnedWorkspaces(nextPageUrl string) ([]Workspace, string, *resty.Response, error) {
-	url := "https://api.bitbucket.org/2.0/workspaces"
+	url := a.BaseURL + "/workspaces"
 	if nextPageUrl != "" {
 		url = nextPageUrl
 	}
@@ -77,7 +96,7 @@ func (a BitBucketApiClient) ListWorkspaceRepositoires(nextPageUrl string, worksp
 	if nextPageUrl != "" {
 		reqUrl = nextPageUrl
 	} else {
-		u, err := url.Parse("https://api.bitbucket.org/2.0/repositories/")
+		u, err := url.Parse(a.BaseURL + "/repositories/")
 		if err != nil {
 			log.Fatal().Err(err).Msg("Unable to parse ListWorkspaceRepositoires url")
 		}
@@ -108,7 +127,7 @@ func (a BitBucketApiClient) ListPublicRepositories(nextPageUrl string, after tim
 	if nextPageUrl != "" {
 		reqUrl = nextPageUrl
 	} else {
-		u, err := url.Parse("https://api.bitbucket.org/2.0/repositories/")
+		u, err := url.Parse(a.BaseURL + "/repositories/")
 		if err != nil {
 			log.Fatal().Err(err).Msg("Unable to parse ListPublicRepositories url")
 		}
@@ -147,7 +166,7 @@ func (a BitBucketApiClient) ListRepositoryPipelines(nextPageUrl string, workspac
 	if nextPageUrl != "" {
 		reqUrl = nextPageUrl
 	} else {
-		u, err := url.Parse("https://api.bitbucket.org/2.0/repositories/")
+		u, err := url.Parse(a.BaseURL + "/repositories/")
 		if err != nil {
 			log.Fatal().Err(err).Msg("Unable to parse ListRepositoryPipelines url")
 		}
@@ -183,7 +202,7 @@ func (a BitBucketApiClient) ListPipelineSteps(nextPageUrl string, workspaceSlug
 	if nextPageUrl != "" {
 		reqUrl = nextPageUrl
 	} else {
-		u, err := url.Parse("https://api.bitbucket.org/2.0/repositories/")
+		u, err := url.Parse(a.BaseURL + "/repositories/")
 		if err != nil {
 			log.Fatal().Err(err).Msg("Unable to parse ListPipelineSteps url")
 		}
@@ -209,7 +228,7 @@ func (a BitBucketApiClient) ListPipelineSteps(nextPageUrl string, workspaceSlug
 
 // https://developer.atlassian.com/cloud/bitbucket/rest/api-group-pipelines/#api-repositories-workspace-repo-slug-pipelines-pipeline-uuid-steps-step-uuid-log-get
 func (a BitBucketApiClient) GetStepLog(workspaceSlug string, repoSlug string, pipelineUUID string, stepUUID string) ([]byte, *resty.Response, error) {
-	u, err := url.Parse("https://api.bitbucket.org/2.0/repositories/")
+	u, err := url.Parse(a.BaseURL + "/repositories/")
 	if err != nil {
 		log.Fatal().Err(err).Msg("Unable to parse GetStepLog url")
 	}
@@ -240,7 +259,7 @@ func (a BitBucketApiClient) ListDownloadArtifacts(nextPageUrl string, workspaceS
 	if nextPageUrl != "" {
 		reqUrl = nextPageUrl
 	} else {
-		u, err := url.Parse("https://api.bitbucket.org/2.0/repositories/")
+		u, err := url.Parse(a.BaseURL + "/repositories/")
 		if err != nil {
 			log.Fatal().Err(err).Msg("Unable to parse ListDownloadArtifacts url")
 		}
@@ -291,7 +310,7 @@ func (a BitBucketApiClient) ListPipelineArtifacts(nextPageUrl string, workspaceS
 	if nextPageUrl != "" {
 		reqUrl = nextPageUrl
 	} else {
-		u, err := url.Parse("https://bitbucket.org/!api/internal/repositories/")
+		u, err := url.Parse(a.InternalBaseURL + "/internal/repositories/")
 		if err != nil {
 			log.Fatal().Err(err).Msg("Unable to parse ListPipelineArtifacst url")
 		}
@@ -309,8 +328,7 @@ func (a BitBucketApiClient) ListPipelineArtifacts(nextPageUrl string, workspaceS
 
 // Internal API: https://bitbucket.org/!api/internal/repositories/{workspace}/{repo}/pipelines/{buildId}/artifacts/{ArtifactUUID}/content
 func (a BitBucketApiClient) GetPipelineArtifact(workspaceSlug string, repoSlug string, buildNumber int, artifactUUID string) []byte {
-
-	u, err := url.Parse("https://bitbucket.org/!api/internal/repositories/")
+	u, err := url.Parse(a.InternalBaseURL + "/internal/repositories/")
 	if err != nil {
 		log.Fatal().Err(err).Msg("Unable to parse GetPipelineArtifact url")
 	}
@@ -336,7 +354,7 @@ func (a BitBucketApiClient) GetuserInfo() {
 	resp := &UserInfo{}
 	res, err := a.Client.R().
 		SetResult(resp).
-		Get("https://bitbucket.org/!api/2.0/user")
+		Get(a.InternalBaseURL + "/2.0/user")
 
 	if err != nil {
 		log.Error().Err(err).Msg("Failed get user info (network or client error)")

src/pipeleak/cmd/bitbucket/internal/processor/pipeline.go

@@ -0,0 +1,72 @@
+package processor
+
+import (
+	"github.com/CompassSecurity/pipeleak/pkg/scanner"
+	"github.com/h2non/filetype"
+)
+
+type StepLogResult struct {
+	WorkspaceSlug string
+	RepoSlug      string
+	PipelineUUID  string
+	StepUUID      string
+	Findings      []scanner.Finding
+	Error         error
+}
+
+func ProcessStepLogs(logBytes []byte, workspaceSlug, repoSlug, pipelineUUID, stepUUID string, maxGoRoutines int, verifyCredentials bool) (*StepLogResult, error) {
+	result := &StepLogResult{
+		WorkspaceSlug: workspaceSlug,
+		RepoSlug:      repoSlug,
+		PipelineUUID:  pipelineUUID,
+		StepUUID:      stepUUID,
+	}
+
+	findings, err := scanner.DetectHits(logBytes, maxGoRoutines, verifyCredentials)
+	if err != nil {
+		result.Error = err
+		return result, err
+	}
+
+	result.Findings = findings
+	return result, nil
+}
+
+type ArtifactResult struct {
+	Filename      string
+	DownloadURL   string
+	WebURL        string
+	IsArchive     bool
+	ProcessedFile bool
+	Error         error
+}
+
+func ProcessArtifactContent(fileBytes []byte, filename, webURL string, verifyCredentials bool) *ArtifactResult {
+	result := &ArtifactResult{
+		Filename: filename,
+		WebURL:   webURL,
+	}
+
+	if len(fileBytes) == 0 {
+		return result
+	}
+
+	if filetype.IsArchive(fileBytes) {
+		result.IsArchive = true
+		scanner.HandleArchiveArtifact(filename, fileBytes, webURL, "Download Artifact", verifyCredentials)
+		result.ProcessedFile = true
+	} else {
+		result.IsArchive = false
+		scanner.DetectFileHits(fileBytes, webURL, "Download Artifact", filename, "", verifyCredentials)
+		result.ProcessedFile = true
+	}
+
+	return result
+}
+
+func ShouldContinueScanning(currentCount, maxLimit int) bool {
+	if maxLimit <= 0 {
+		return true
+	}
+	return currentCount < maxLimit
+}