Gitlab refactor (#327)

* Refactor GitLab commands to eliminate duplicate project iteration logic

- Add shared IterateProjects() and IterateGroupProjects() helpers in util package
- Refactor 6 files to use callback-based iteration pattern:
  * cmd/gitlab/variables.go
  * cmd/gitlab/schedule/schedule.go
  * cmd/gitlab/secureFiles/secure_files.go
  * cmd/gitlab/scan/pipeline.go
  * cmd/gitlab/runners/list.go
  * cmd/gitlab/renovate/enum.go
- Eliminate ~200+ lines of duplicated pagination code
- Maintain 100% backward compatibility
- All tests passing (unit + e2e)
- Linter clean (0 issues)

* refactor(gitlab): reorganize package structure into subdirectories

Move single-file commands into their own subdirectories for better organization:
- enum.go -> enum/enum.go
- register.go -> register/register.go
- shodan.go -> shodan/shodan.go
- variables.go -> variables/variables.go
- vuln.go -> vuln/vuln.go

This aligns with the existing structure used by scan/, runners/, renovate/, etc.
All commands now follow a consistent subdirectory pattern.

Updated imports and package references in:
- gitlab.go (command registration)
- gitlab_unauth.go (unauthenticated commands)
- gitlab_test.go (test function calls)

All unit tests, e2e tests, and golangci-lint checks pass.

* Remove explanatory comment from renovate enum.go (#328)

* Initial plan

* fix: remove comment in enum.go as per review feedback

Co-authored-by: frjcomp <107982661+frjcomp@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: frjcomp <107982661+frjcomp@users.noreply.github.com>

---------

Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>

Author: frjcomp

src/pipeleak/cmd/gitlab/enum.go src/pipeleak/cmd/gitlab/enum/enum.gosrc/pipeleak/cmd/gitlab/enum.go renamed to src/pipeleak/cmd/gitlab/enum/enum.go

@@ -1,4 +1,4 @@
-package gitlab
+package enum
 
 import (
 	"net/url"
@@ -16,7 +16,12 @@ import (
 	"resty.dev/v3"
 )
 
-var minAccessLevel int
+var (
+	gitlabApiToken string
+	gitlabUrl      string
+	verbose        bool
+	minAccessLevel int
+)
 
 func NewEnumCmd() *cobra.Command {
 	enumCmd := &cobra.Command{

src/pipeleak/cmd/gitlab/gitlab.go

@@ -2,11 +2,14 @@ package gitlab
 
 import (
 	"github.com/CompassSecurity/pipeleak/cmd/gitlab/cicd"
+	"github.com/CompassSecurity/pipeleak/cmd/gitlab/enum"
 	"github.com/CompassSecurity/pipeleak/cmd/gitlab/renovate"
 	"github.com/CompassSecurity/pipeleak/cmd/gitlab/runners"
 	"github.com/CompassSecurity/pipeleak/cmd/gitlab/scan"
 	"github.com/CompassSecurity/pipeleak/cmd/gitlab/schedule"
 	securefiles "github.com/CompassSecurity/pipeleak/cmd/gitlab/secureFiles"
+	"github.com/CompassSecurity/pipeleak/cmd/gitlab/variables"
+	"github.com/CompassSecurity/pipeleak/cmd/gitlab/vuln"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 )
@@ -39,10 +42,10 @@ For SOCKS5 proxy:
 
 	glCmd.AddCommand(scan.NewScanCmd())
 	glCmd.AddCommand(runners.NewRunnersRootCmd())
-	glCmd.AddCommand(NewVulnCmd())
-	glCmd.AddCommand(NewVariablesCmd())
+	glCmd.AddCommand(vuln.NewVulnCmd())
+	glCmd.AddCommand(variables.NewVariablesCmd())
 	glCmd.AddCommand(securefiles.NewSecureFilesCmd())
-	glCmd.AddCommand(NewEnumCmd())
+	glCmd.AddCommand(enum.NewEnumCmd())
 	glCmd.AddCommand(renovate.NewRenovateRootCmd())
 	glCmd.AddCommand(cicd.NewCiCdCmd())
 	glCmd.AddCommand(schedule.NewScheduleCmd())

src/pipeleak/cmd/gitlab/gitlab_test.go

@@ -2,6 +2,12 @@ package gitlab
 
 import (
 	"testing"
+
+	"github.com/CompassSecurity/pipeleak/cmd/gitlab/enum"
+	"github.com/CompassSecurity/pipeleak/cmd/gitlab/register"
+	"github.com/CompassSecurity/pipeleak/cmd/gitlab/shodan"
+	"github.com/CompassSecurity/pipeleak/cmd/gitlab/variables"
+	"github.com/CompassSecurity/pipeleak/cmd/gitlab/vuln"
 )
 
 func TestNewGitLabRootCmd(t *testing.T) {
@@ -44,7 +50,7 @@ func TestNewGitLabRootCmd(t *testing.T) {
 }
 
 func TestNewVulnCmd(t *testing.T) {
-	cmd := NewVulnCmd()
+	cmd := vuln.NewVulnCmd()
 
 	if cmd == nil {
 		t.Fatal("Expected non-nil command")
@@ -60,7 +66,7 @@ func TestNewVulnCmd(t *testing.T) {
 }
 
 func TestNewVariablesCmd(t *testing.T) {
-	cmd := NewVariablesCmd()
+	cmd := variables.NewVariablesCmd()
 
 	if cmd == nil {
 		t.Fatal("Expected non-nil command")
@@ -81,7 +87,7 @@ func TestNewVariablesCmd(t *testing.T) {
 }
 
 func TestNewEnumCmd(t *testing.T) {
-	cmd := NewEnumCmd()
+	cmd := enum.NewEnumCmd()
 
 	if cmd == nil {
 		t.Fatal("Expected non-nil command")
@@ -97,7 +103,7 @@ func TestNewEnumCmd(t *testing.T) {
 }
 
 func TestNewRegisterCmd(t *testing.T) {
-	cmd := NewRegisterCmd()
+	cmd := register.NewRegisterCmd()
 
 	if cmd == nil {
 		t.Fatal("Expected non-nil command")
@@ -127,7 +133,7 @@ func TestNewRegisterCmd(t *testing.T) {
 }
 
 func TestNewShodanCmd(t *testing.T) {
-	cmd := NewShodanCmd()
+	cmd := shodan.NewShodanCmd()
 
 	if cmd == nil {
 		t.Fatal("Expected non-nil command")

src/pipeleak/cmd/gitlab/gitlab_unauth.go

@@ -1,6 +1,8 @@
 package gitlab
 
 import (
+	"github.com/CompassSecurity/pipeleak/cmd/gitlab/register"
+	"github.com/CompassSecurity/pipeleak/cmd/gitlab/shodan"
 	"github.com/spf13/cobra"
 )
 
@@ -12,8 +14,8 @@ func NewGitLabRootUnauthenticatedCmd() *cobra.Command {
 		GroupID: "Helper",
 	}
 
-	glunaCmd.AddCommand(NewShodanCmd())
-	glunaCmd.AddCommand(NewRegisterCmd())
+	glunaCmd.AddCommand(shodan.NewShodanCmd())
+	glunaCmd.AddCommand(register.NewRegisterCmd())
 
 	glunaCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Verbose logging")
 

src/pipeleak/cmd/gitlab/register.go …pipeleak/cmd/gitlab/register/register.gosrc/pipeleak/cmd/gitlab/register.go renamed to src/pipeleak/cmd/gitlab/register/register.go src/pipeleak/cmd/gitlab/register.go renamed to src/pipeleak/cmd/gitlab/register/register.go

@@ -1,4 +1,4 @@
-package gitlab
+package register
 
 import (
 	"github.com/CompassSecurity/pipeleak/cmd/gitlab/util"
@@ -8,9 +8,11 @@ import (
 )
 
 var (
-	username string
-	password string
-	email    string
+	gitlabUrl string
+	verbose   bool
+	username  string
+	password  string
+	email     string
 )
 
 func NewRegisterCmd() *cobra.Command {

src/pipeleak/cmd/gitlab/renovate/enum.go

@@ -10,7 +10,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
-	"sync"
 
 	"github.com/CompassSecurity/pipeleak/cmd/gitlab/util"
 	"github.com/CompassSecurity/pipeleak/pkg/format"
@@ -132,30 +131,14 @@ func scanNamespace(git *gitlab.Client, namespace string) {
 		IncludeSubGroups: gitlab.Ptr(true),
 	}
 
-	for {
-		projects, resp, err := git.Groups.ListGroupProjects(group.ID, projectOpts)
-		if err != nil {
-			log.Error().Stack().Err(err).Int("page", page).Msg("Failed fetching projects")
-			break
-		}
-
-		var wg sync.WaitGroup
-		for _, project := range projects {
-			wg.Add(1)
-			go func(proj *gitlab.Project) {
-				defer wg.Done()
-				log.Debug().Str("url", proj.WebURL).Msg("Check project")
-				identifyRenovateBotJob(git, proj)
-			}(project)
-		}
-		wg.Wait()
-
-		if resp.NextPage == 0 {
-			break
-		}
-
-		projectOpts.Page = resp.NextPage
-		log.Info().Int("currentPage", projectOpts.Page).Msg("Fetched projects page")
+	err = util.IterateGroupProjects(git, group.ID, projectOpts, func(project *gitlab.Project) error {
+		log.Debug().Str("url", project.WebURL).Msg("Check project")
+		identifyRenovateBotJob(git, project)
+		return nil
+	})
+	if err != nil {
+		log.Error().Stack().Err(err).Msg("Failed iterating group projects")
+		return
 	}
 
 	log.Info().Msg("Fetched all namespace projects")
@@ -175,30 +158,14 @@ func fetchProjects(git *gitlab.Client) {
 		Search:     gitlab.Ptr(projectSearchQuery),
 	}
 
-	for {
-		projects, resp, err := git.Projects.ListProjects(projectOpts)
-		if err != nil {
-			log.Error().Stack().Err(err).Int("page", page).Msg("Failed fetching projects")
-			break
-		}
-
-		var wg sync.WaitGroup
-		for _, project := range projects {
-			wg.Add(1)
-			go func(proj *gitlab.Project) {
-				defer wg.Done()
-				log.Debug().Str("url", proj.WebURL).Msg("Check project")
-				identifyRenovateBotJob(git, proj)
-			}(project)
-		}
-		wg.Wait()
-
-		if resp.NextPage == 0 {
-			break
-		}
-
-		projectOpts.Page = resp.NextPage
-		log.Info().Int("currentPage", projectOpts.Page).Msg("Fetched projects page")
+	err := util.IterateProjects(git, projectOpts, func(project *gitlab.Project) error {
+		log.Debug().Str("url", project.WebURL).Msg("Check project")
+		identifyRenovateBotJob(git, project)
+		return nil
+	})
+	if err != nil {
+		log.Error().Stack().Err(err).Msg("Failed iterating projects")
+		return
 	}
 
 	log.Info().Msg("Fetched all projects")

src/pipeleak/cmd/gitlab/runners/list.go

@@ -86,30 +86,22 @@ func listProjectRunners(git *gitlab.Client, runnerMap map[int]runnerResult) map[
 		MinAccessLevel: gitlab.Ptr(gitlab.MaintainerPermissions),
 	}
 
-	for {
-		projects, resp, err := git.Projects.ListProjects(projectOpts)
-		if err != nil {
-			log.Error().Stack().Err(err).Msg("Failed fetching projects")
+	err := util.IterateProjects(git, projectOpts, func(project *gitlab.Project) error {
+		log.Debug().Str("name", project.Name).Int("id", project.ID).Msg("List runners for")
+		runnerOpts := &gitlab.ListProjectRunnersOptions{
+			ListOptions: gitlab.ListOptions{
+				PerPage: 100,
+				Page:    1,
+			},
 		}
-
-		for _, project := range projects {
-			log.Debug().Str("name", project.Name).Int("id", project.ID).Msg("List runners for")
-			runnerOpts := &gitlab.ListProjectRunnersOptions{
-				ListOptions: gitlab.ListOptions{
-					PerPage: 100,
-					Page:    1,
-				},
-			}
-			runners, _, _ := git.Runners.ListProjectRunners(project.ID, runnerOpts)
-			for _, runner := range runners {
-				runnerMap[runner.ID] = runnerResult{runner: runner, project: project, group: nil}
-			}
+		runners, _, _ := git.Runners.ListProjectRunners(project.ID, runnerOpts)
+		for _, runner := range runners {
+			runnerMap[runner.ID] = runnerResult{runner: runner, project: project, group: nil}
 		}
-
-		if resp.NextPage == 0 {
-			break
-		}
-		projectOpts.Page = resp.NextPage
+		return nil
+	})
+	if err != nil {
+		log.Error().Stack().Err(err).Msg("Failed iterating projects")
 	}
 
 	return runnerMap

src/pipeleak/cmd/gitlab/scan/pipeline.go

@@ -123,24 +123,14 @@ func scanNamespace(git *gitlab.Client, options *ScanOptions, wg *sync.WaitGroup)
 		IncludeSubGroups: gitlab.Ptr(true),
 	}
 
-	for {
-		projects, resp, err := git.Groups.ListGroupProjects(group.ID, projectOpts)
-		if err != nil {
-			log.Error().Stack().Err(err).Msg("Failed fetching projects in namespace")
-			break
-		}
-
-		for _, project := range projects {
-			log.Debug().Str("url", project.WebURL).Msg("Fetch project jobs")
-			getAllJobs(git, project, options)
-		}
-
-		if resp.NextPage == 0 {
-			break
-		}
-
-		projectOpts.Page = resp.NextPage
-		log.Info().Int("currentPage", projectOpts.Page).Msg("Fetched projects page")
+	err = util.IterateGroupProjects(git, group.ID, projectOpts, func(project *gitlab.Project) error {
+		log.Debug().Str("url", project.WebURL).Msg("Fetch project jobs")
+		getAllJobs(git, project, options)
+		return nil
+	})
+	if err != nil {
+		log.Error().Stack().Err(err).Msg("Failed iterating namespace projects")
+		return
 	}
 
 	log.Info().Msg("Fetched all namespace projects")
@@ -183,23 +173,14 @@ func fetchProjects(git *gitlab.Client, options *ScanOptions, wg *sync.WaitGroup)
 		OrderBy:    gitlab.Ptr("last_activity_at"),
 	}
 
-	for {
-		projects, resp, err := git.Projects.ListProjects(projectOpts)
-		if err != nil {
-			log.Error().Stack().Err(err).Msg("Failed fetching projects")
-			break
-		}
-
-		for _, project := range projects {
-			log.Debug().Str("url", project.WebURL).Msg("Fetch project jobs")
-			getAllJobs(git, project, options)
-		}
-
-		if resp.NextPage == 0 {
-			break
-		}
-		projectOpts.Page = resp.NextPage
-		log.Info().Int("total", projectOpts.Page*projectOpts.PerPage).Msg("Fetched projects")
+	err := util.IterateProjects(git, projectOpts, func(project *gitlab.Project) error {
+		log.Debug().Str("url", project.WebURL).Msg("Fetch project jobs")
+		getAllJobs(git, project, options)
+		return nil
+	})
+	if err != nil {
+		log.Error().Stack().Err(err).Msg("Failed iterating projects")
+		return
 	}
 
 	log.Info().Msg("Fetched all projects")

src/pipeleak/cmd/gitlab/schedule/schedule.go

@@ -62,22 +62,13 @@ func FetchSchedules(cmd *cobra.Command, args []string) {
 		OrderBy:        gitlab.Ptr("last_activity_at"),
 	}
 
-	for {
-		projects, resp, err := git.Projects.ListProjects(projectOpts)
-		if err != nil {
-			log.Error().Stack().Err(err).Msg("Failed fetching projects")
-			break
-		}
-
-		for _, project := range projects {
-			log.Debug().Str("project", project.WebURL).Msg("Fetch project schedules")
-			ListPipelineSchedules(git, project)
-		}
-
-		if resp.NextPage == 0 {
-			break
-		}
-		projectOpts.Page = resp.NextPage
+	err = util.IterateProjects(git, projectOpts, func(project *gitlab.Project) error {
+		log.Debug().Str("project", project.WebURL).Msg("Fetch project schedules")
+		ListPipelineSchedules(git, project)
+		return nil
+	})
+	if err != nil {
+		log.Fatal().Stack().Err(err).Msg("Failed iterating projects")
 	}
 
 	log.Info().Msg("Fetched all schedules")