Add unauthenticated GitLab public scan (#619)

* Add unauthenticated GitLab public scan

Author: frjcomp

docs/guides/gitlab.md

@@ -25,6 +25,17 @@ See if you can already identify potentially sensitive data e.g. credentials in s
 The next step would be to try to create an account. Head to `https://leakycompany.com/users/sign_up` and try to register a new account.
 Sometimes you can only create an account with an email address managed by the customer, some instances require the admins to accept the register request, and others completely disable it.
 
+## Anonymous Secret Scan
+
+If there is no possibility to register an account and perform an authenticated secrets scan, you can still scan all publicly available CI/CD logs using Pipeleek's unauthenticated mode.
+
+Prefer the authenticated scan over the unauthenticated one whenever possible, as it provides broader coverage.
+
+```bash
+# Scan all publicly accessible CI/CD logs, including artifacts (breadth-first)
+pipeleek gluna scan -g https://leakycompany.com -a --job-limit 10
+```
+
 ## Authenticated Access
 
 Sweet now you have access to the GitLab instance with an account.

docs/introduction/configuration.md

@@ -107,12 +107,20 @@ gitlab:
     privesc: {} # gl renovate privesc (inherits url/token)
 
   register:
-    username: newuser # gl register --username
-    password: secret # gl register --password
-    email: user@example.com # gl register --email
+    username: newuser # gluna register --username
+    password: secret # gluna register --password
+    email: user@example.com # gluna register --email
 
   shodan:
-    json: shodan_data.json # gl shodan --json
+    json: shodan_data.json # gluna shodan --json
+
+  scan_public:
+    search: "" # gluna scan --search
+    repo: "" # gluna scan --repo
+    namespace: "" # gluna scan --namespace
+    job_limit: 0 # gluna scan --job-limit
+    queue: "" # gluna scan --queue
+    artifacts: false # gluna scan --artifacts
 
   scan:
     threads: 10 # gl scan --threads (can override common.threads)

internal/cmd/gitlab/gitlab_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/enum"
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/register"
+	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/scanpublic"
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/shodan"
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/snippets"
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/variables"
@@ -106,3 +107,36 @@ func TestNewSnippetsRootCmd(t *testing.T) {
 	require.NotNil(t, scanCmd)
 	assert.Equal(t, "scan", scanCmd.Name())
 }
+
+func TestNewGitLabRootUnauthenticatedCmd(t *testing.T) {
+	cmd := NewGitLabRootUnauthenticatedCmd()
+
+	require.NotNil(t, cmd)
+	assert.Equal(t, "gluna [command]", cmd.Use)
+	assert.Equal(t, "Helper", cmd.GroupID)
+
+	shodanCmd, _, err := cmd.Find([]string{"shodan"})
+	require.NoError(t, err)
+	assert.NotNil(t, shodanCmd)
+
+	registerCmd, _, err := cmd.Find([]string{"register"})
+	require.NoError(t, err)
+	assert.NotNil(t, registerCmd)
+
+	publicScanCmd, _, err := cmd.Find([]string{"scan"})
+	require.NoError(t, err)
+	assert.NotNil(t, publicScanCmd)
+}
+
+func TestNewScanPublicCmd(t *testing.T) {
+	cmd := scanpublic.NewScanPublicCmd()
+
+	require.NotNil(t, cmd)
+	assert.Equal(t, "scan", cmd.Use)
+	assert.NotEmpty(t, cmd.Short)
+
+	flags := cmd.Flags()
+	assert.NotNil(t, flags.Lookup("repo"), "'repo' flag should be registered")
+	assert.NotNil(t, flags.Lookup("namespace"), "'namespace' flag should be registered")
+	assert.NotNil(t, flags.Lookup("search"), "'search' flag should be registered")
+}

internal/cmd/gitlab/gitlab_unauth.go

@@ -2,6 +2,7 @@ package gitlab
 
 import (
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/register"
+	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/scanpublic"
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/shodan"
 	"github.com/spf13/cobra"
 )
@@ -16,6 +17,7 @@ func NewGitLabRootUnauthenticatedCmd() *cobra.Command {
 
 	glunaCmd.AddCommand(shodan.NewShodanCmd())
 	glunaCmd.AddCommand(register.NewRegisterCmd())
+	glunaCmd.AddCommand(scanpublic.NewScanPublicCmd())
 
 	return glunaCmd
 }

internal/cmd/gitlab/scanpublic/scan_public.go

@@ -0,0 +1,146 @@
+package scanpublic
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/CompassSecurity/pipeleek/internal/cmd/flags"
+	"github.com/CompassSecurity/pipeleek/pkg/config"
+	gitlabscan "github.com/CompassSecurity/pipeleek/pkg/gitlab/scan"
+	"github.com/CompassSecurity/pipeleek/pkg/logging"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+type ScanPublicOptions struct {
+	config.CommonScanOptions
+	ProjectSearchQuery string
+	Repository         string
+	Namespace          string
+	JobLimit           int
+	QueueFolder        string
+}
+
+var options = ScanPublicOptions{
+	CommonScanOptions: config.DefaultCommonScanOptions(),
+}
+
+var maxArtifactSize string
+
+func NewScanPublicCmd() *cobra.Command {
+	scanCmd := &cobra.Command{
+		Use:   "scan",
+		Short: "Scan public GitLab pipelines without an account",
+		Long: `Scan public GitLab project pipelines for secrets in job traces and optionally artifacts.
+
+This command does not require an API token and only covers resources that are publicly accessible.
+Dotenv artifacts are intentionally not scanned in this mode because they require a UI session cookie.`,
+		Example: `
+# Scan public project pipelines and traces
+pipeleek gluna scan --gitlab https://gitlab.example.com
+
+# Scan public pipelines with artifacts and tuned performance
+pipeleek gluna scan --gitlab https://gitlab.example.com --artifacts --job-limit 10 --max-artifact-size 200Mb --threads 8
+
+# Scan one public repository
+pipeleek gluna scan --gitlab https://gitlab.example.com --repo mygroup/myproject
+
+# Scan all public repositories in a namespace
+pipeleek gluna scan --gitlab https://gitlab.example.com --namespace mygroup
+		`,
+		Run: ScanPublic,
+	}
+
+	scanCmd.Flags().StringP("gitlab", "g", "", "GitLab instance URL")
+	flags.AddCommonScanFlagsNoArtifacts(scanCmd, &options.CommonScanOptions)
+	scanCmd.Flags().BoolVarP(&options.Artifacts, "artifacts", "a", false, "Scan artifacts")
+	scanCmd.Flags().StringVarP(&maxArtifactSize, "max-artifact-size", "", "500Mb",
+		"Maximum artifact size to scan. Larger files are skipped. Format: https://pkg.go.dev/github.com/docker/go-units#FromHumanSize")
+	scanCmd.Flags().StringVarP(&options.ProjectSearchQuery, "search", "s", "", "Query string for searching public projects")
+	scanCmd.Flags().StringVarP(&options.Repository, "repo", "r", "", "Single public repository to scan, format: namespace/repo")
+	scanCmd.Flags().StringVarP(&options.Namespace, "namespace", "n", "", "Namespace to scan (all public repos in the namespace will be scanned)")
+	scanCmd.Flags().IntVarP(&options.JobLimit, "job-limit", "j", 0, "Scan a max number of pipeline jobs - trade speed vs coverage. 0 scans all and is the default.")
+	scanCmd.Flags().StringVarP(&options.QueueFolder, "queue", "q", "", "Relative or absolute folderpath where the queue files will be stored. Defaults to system tmp. Non-existing folders will be created.")
+
+	return scanCmd
+}
+
+func ScanPublic(cmd *cobra.Command, args []string) {
+	if err := config.AutoBindFlags(cmd, map[string]string{
+		"gitlab":                   "gitlab.url",
+		"search":                   "gitlab.scan_public.search",
+		"repo":                     "gitlab.scan_public.repo",
+		"namespace":                "gitlab.scan_public.namespace",
+		"job-limit":                "gitlab.scan_public.job_limit",
+		"queue":                    "gitlab.scan_public.queue",
+		"artifacts":                "gitlab.scan_public.artifacts",
+		"threads":                  "common.threads",
+		"truffle-hog-verification": "common.trufflehog_verification",
+		"max-artifact-size":        "common.max_artifact_size",
+		"confidence":               "common.confidence_filter",
+		"hit-timeout":              "common.hit_timeout",
+	}); err != nil {
+		log.Fatal().Err(err).Msg("Failed to bind command flags to configuration keys")
+	}
+
+	if err := config.RequireConfigKeys("gitlab.url"); err != nil {
+		log.Fatal().Err(err).Msg("required configuration missing")
+	}
+
+	gitlabURL := config.GetString("gitlab.url")
+	projectSearchQuery := config.GetString("gitlab.scan_public.search")
+	repository := config.GetString("gitlab.scan_public.repo")
+	namespace := config.GetString("gitlab.scan_public.namespace")
+	jobLimit := config.GetInt("gitlab.scan_public.job_limit")
+	queueFolder := config.GetString("gitlab.scan_public.queue")
+	artifacts := config.GetBool("gitlab.scan_public.artifacts")
+	threads := config.GetInt("common.threads")
+	truffleHogVerification := config.GetBool("common.trufflehog_verification")
+	maxArtifactSize = config.GetString("common.max_artifact_size")
+	confidenceFilter := config.GetStringSlice("common.confidence_filter")
+	hitTimeoutRaw := config.GetString("common.hit_timeout")
+	hitTimeout, err := time.ParseDuration(hitTimeoutRaw)
+	if err != nil {
+		log.Fatal().Err(fmt.Errorf("invalid hit-timeout %q: %w", hitTimeoutRaw, err)).Msg("Invalid hit timeout")
+	}
+
+	if err := config.ValidateURL(gitlabURL, "GitLab URL"); err != nil {
+		log.Fatal().Err(err).Msg("Invalid GitLab URL")
+	}
+	if err := config.ValidateThreadCount(threads); err != nil {
+		log.Fatal().Err(err).Msg("Invalid thread count")
+	}
+
+	scanOpts, err := gitlabscan.InitializeOptions(
+		gitlabURL,
+		"",
+		"",
+		projectSearchQuery,
+		repository,
+		namespace,
+		queueFolder,
+		maxArtifactSize,
+		artifacts,
+		false,
+		false,
+		truffleHogVerification,
+		jobLimit,
+		threads,
+		confidenceFilter,
+		hitTimeout,
+	)
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed initializing public scan options")
+	}
+
+	scanner := gitlabscan.NewScanner(scanOpts)
+	logging.RegisterStatusHook(func() *zerolog.Event {
+		queueLength := scanner.GetQueueStatus()
+		return log.Info().Int("pendingjobs", queueLength)
+	})
+
+	if err := scanner.Scan(); err != nil {
+		log.Fatal().Err(err).Msg("Public scan failed")
+	}
+}

internal/cmd/gitlab/scanpublic/scan_public_test.go

@@ -0,0 +1,46 @@
+package scanpublic
+
+import (
+	"testing"
+
+	"github.com/CompassSecurity/pipeleek/pkg/config"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewScanPublicCmd(t *testing.T) {
+	cmd := NewScanPublicCmd()
+
+	require.NotNil(t, cmd)
+	assert.Equal(t, "scan", cmd.Use)
+	assert.NotEmpty(t, cmd.Short)
+	assert.Contains(t, cmd.Long, "does not require an API token")
+	assert.Contains(t, cmd.Example, "gluna scan --gitlab")
+
+	flags := cmd.Flags()
+	assert.NotNil(t, flags.Lookup("search"))
+	assert.NotNil(t, flags.Lookup("repo"))
+	assert.NotNil(t, flags.Lookup("namespace"))
+	assert.NotNil(t, flags.Lookup("job-limit"))
+	assert.NotNil(t, flags.Lookup("queue"))
+	assert.NotNil(t, flags.Lookup("artifacts"))
+	assert.Nil(t, flags.Lookup("owned"), "'owned' flag must not be present on public scan")
+	assert.NotNil(t, flags.Lookup("threads"))
+	assert.NotNil(t, flags.Lookup("truffle-hog-verification"))
+	assert.NotNil(t, flags.Lookup("confidence"))
+	assert.NotNil(t, flags.Lookup("hit-timeout"))
+
+	assert.Equal(t, "r", flags.Lookup("repo").Shorthand)
+	assert.Equal(t, "n", flags.Lookup("namespace").Shorthand)
+	assert.Equal(t, "s", flags.Lookup("search").Shorthand)
+	assert.Equal(t, "j", flags.Lookup("job-limit").Shorthand)
+	assert.Equal(t, "q", flags.Lookup("queue").Shorthand)
+
+	assert.Equal(t, "0", flags.Lookup("job-limit").DefValue)
+	assert.Equal(t, "", flags.Lookup("repo").DefValue)
+	assert.Equal(t, "", flags.Lookup("namespace").DefValue)
+	assert.Equal(t, "", flags.Lookup("search").DefValue)
+
+	defaults := config.DefaultCommonScanOptions()
+	assert.Equal(t, defaults.TruffleHogVerification, cmd.Flags().Lookup("truffle-hog-verification").DefValue == "true")
+}

pipeleek.example.yaml

@@ -90,16 +90,25 @@ gitlab:
     bots:
       term: "renovate" # Search term for identifying potential renovate bot users
 
-  # register - Register new user account
+  # register - Register new user account (gluna register)
   register:
     username: "newuser"
     password: "securepassword"
     email: "newuser@example.com"
 
-  # shodan - Query Shodan for GitLab instances
+  # shodan - Query Shodan for GitLab instances (gluna shodan)
   shodan:
     json: "shodan_data.json" # Path to Shodan JSON export
 
+  # scan - Scan public pipelines without account or token (gluna scan)
+  scan_public:
+    search: "" # Optional project search query
+    repo: "" # Optional single repository namespace/project
+    namespace: "" # Optional namespace/group to scan
+    job_limit: 0 # Max jobs per project; 0 scans all
+    queue: "" # Optional queue folder path
+    artifacts: false # Scan artifacts in addition to job traces
+
   # scan - Scan CI/CD artifacts for secrets
   scan:
     # Inherits common.* settings, can override per-command