Skip to content

Commit 8b742c5

Browse files
frjcompfrjcomp
authored
GitLab Terraform state scanner command (gl tf) (#480)
*GitLab Terraform state scanner command (gl tf) Add new 'gl tf' subcommand to discover, download, and scan native Terraform/OpenTofu state files stored in GitLab for secrets using TruffleHog integration.
1 parent 1daf313 commit 8b742c5

7 files changed

Lines changed: 532 additions & 0 deletions

File tree

‎docs/introduction/configuration.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,11 @@ gitlab:
113113

114114
scan:
115115
threads: 10 # gl scan --threads (can override common.threads)
116+
117+
tf:
118+
output_dir: ./terraform-states # gl tf --output-dir
119+
threads: 4 # gl tf --threads (can override common.threads)
120+
# Note: artifacts, max_artifact_size, and owned do not apply to gl tf.
116121
```
117122

118123
### GitHub

‎internal/cmd/flags/common.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,14 @@ func AddCommonScanFlags(cmd *cobra.Command, opts *config.CommonScanOptions, maxA
2222
cmd.Flags().DurationVarP(&opts.HitTimeout, "hit-timeout", "", 60*time.Second,
2323
"Maximum time to wait for hit detection per scan item (e.g., 30s, 2m, 1h)")
2424
}
25+
26+
// AddCommonScanFlagsNoArtifacts adds standard scan flags excluding artifact and ownership filters.
27+
func AddCommonScanFlagsNoArtifacts(cmd *cobra.Command, opts *config.CommonScanOptions) {
28+
cmd.Flags().IntVarP(&opts.MaxScanGoRoutines, "threads", "", 4, "Number of concurrent threads for scanning")
29+
cmd.Flags().BoolVarP(&opts.TruffleHogVerification, "truffle-hog-verification", "", true,
30+
"Enable TruffleHog credential verification to actively test found credentials and only report verified ones (enabled by default, disable with --truffle-hog-verification=false)")
31+
cmd.Flags().StringSliceVarP(&opts.ConfidenceFilter, "confidence", "", []string{},
32+
"Filter for confidence level, separate by comma if multiple. See readme for more info.")
33+
cmd.Flags().DurationVarP(&opts.HitTimeout, "hit-timeout", "", 60*time.Second,
34+
"Maximum time to wait for hit detection per scan item (e.g., 30s, 2m, 1h)")
35+
}

‎internal/cmd/gitlab/gitlab.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/scan"
1111
"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/schedule"
1212
securefiles "github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/secureFiles"
13+
"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/tf"
1314
"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/variables"
1415
"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/vuln"
1516
"github.com/spf13/cobra"
@@ -51,6 +52,7 @@ For SOCKS5 proxy:
5152
glCmd.AddCommand(container.NewContainerScanCmd())
5253
glCmd.AddCommand(cicd.NewCiCdCmd())
5354
glCmd.AddCommand(schedule.NewScheduleCmd())
55+
glCmd.AddCommand(tf.NewTFCmd())
5456

5557
glCmd.PersistentFlags().StringVarP(&gitlabUrl, "gitlab", "g", "", "GitLab instance URL")
5658
glCmd.PersistentFlags().StringVarP(&gitlabApiToken, "token", "t", "", "GitLab API Token")

‎internal/cmd/gitlab/tf/tf.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
package tf
2+
3+
import (
4+
"github.com/CompassSecurity/pipeleek/internal/cmd/flags"
5+
"github.com/CompassSecurity/pipeleek/pkg/config"
6+
tfpkg "github.com/CompassSecurity/pipeleek/pkg/gitlab/tf"
7+
"github.com/rs/zerolog/log"
8+
"github.com/spf13/cobra"
9+
)
10+
11+
type TFCommandOptions struct {
12+
config.CommonScanOptions
13+
OutputDir string
14+
}
15+
16+
var options = TFCommandOptions{CommonScanOptions: config.DefaultCommonScanOptions()}
17+
18+
func NewTFCmd() *cobra.Command {
19+
tfCmd := &cobra.Command{
20+
Use: "tf",
21+
Short: "Scan Terraform/OpenTofu state files for secrets",
22+
Long: `Scan GitLab Terraform/OpenTofu state files for secrets
23+
24+
This command iterates through all projects where you have maintainer access,
25+
lists GitLab-managed Terraform states, downloads them locally, and scans them
26+
for secrets using TruffleHog.
27+
28+
GitLab stores Terraform state natively when using the Terraform HTTP backend.
29+
Each project can have multiple named state files.`,
30+
Example: `# Scan all Terraform states in projects with maintainer access
31+
pipeleek gl tf --token glpat-xxxxxxxxxxx --gitlab https://gitlab.example.com
32+
33+
# Save state files to custom directory
34+
pipeleek gl tf --token glpat-xxxxxxxxxxx --gitlab https://gitlab.example.com --output-dir ./tf-states
35+
36+
# Use more threads for TruffleHog scanning
37+
pipeleek gl tf --token glpat-xxxxxxxxxxx --gitlab https://gitlab.example.com --threads 10
38+
39+
# Scan with high confidence filter only
40+
pipeleek gl tf --token glpat-xxxxxxxxxxx --gitlab https://gitlab.example.com --confidence high`,
41+
Run: tfRun,
42+
}
43+
44+
tfCmd.Flags().StringVar(&options.OutputDir, "output-dir", "./terraform-states", "Directory to save downloaded state files")
45+
flags.AddCommonScanFlagsNoArtifacts(tfCmd, &options.CommonScanOptions)
46+
47+
return tfCmd
48+
}
49+
50+
func tfRun(cmd *cobra.Command, args []string) {
51+
if err := config.AutoBindFlags(cmd, map[string]string{
52+
"gitlab": "gitlab.url",
53+
"token": "gitlab.token",
54+
"output-dir": "gitlab.tf.output_dir",
55+
"threads": "common.threads",
56+
"truffle-hog-verification": "common.trufflehog_verification",
57+
"confidence": "common.confidence_filter",
58+
"hit-timeout": "common.hit_timeout",
59+
}); err != nil {
60+
log.Fatal().Err(err).Msg("Failed to bind command flags to configuration keys")
61+
}
62+
63+
if err := config.RequireConfigKeys("gitlab.url", "gitlab.token"); err != nil {
64+
log.Fatal().Err(err).Msg("required configuration missing")
65+
}
66+
67+
gitlabUrl := config.GetString("gitlab.url")
68+
gitlabApiToken := config.GetString("gitlab.token")
69+
options.OutputDir = config.GetString("gitlab.tf.output_dir")
70+
options.MaxScanGoRoutines = config.GetInt("common.threads")
71+
options.ConfidenceFilter = config.GetStringSlice("common.confidence_filter")
72+
options.TruffleHogVerification = config.GetBool("common.trufflehog_verification")
73+
74+
if err := config.ValidateURL(gitlabUrl, "GitLab URL"); err != nil {
75+
log.Fatal().Err(err).Msg("Invalid GitLab URL")
76+
}
77+
if err := config.ValidateToken(gitlabApiToken, "GitLab API Token"); err != nil {
78+
log.Fatal().Err(err).Msg("Invalid GitLab API Token")
79+
}
80+
if err := config.ValidateThreadCount(options.MaxScanGoRoutines); err != nil {
81+
log.Fatal().Err(err).Msg("Invalid thread count")
82+
}
83+
84+
tfOptions := tfpkg.TFOptions{
85+
GitlabUrl: gitlabUrl,
86+
GitlabApiToken: gitlabApiToken,
87+
OutputDir: options.OutputDir,
88+
Threads: options.MaxScanGoRoutines,
89+
ConfidenceFilter: options.ConfidenceFilter,
90+
TruffleHogVerification: options.TruffleHogVerification,
91+
HitTimeout: options.HitTimeout,
92+
}
93+
94+
tfpkg.ScanTerraformStates(tfOptions)
95+
96+
log.Info().Msg("Done, Bye Bye 🏳️‍🌈🔥")
97+
}

‎pipeleek.example.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,12 @@ gitlab:
103103
threads: 15 # Override common.threads for GitLab scans
104104
max_artifact_size: 52428800 # 50MB for GitLab artifacts
105105

106+
# tf - Discover and scan Terraform/OpenTofu state files
107+
tf:
108+
output_dir: ./terraform-states # Directory to save downloaded state files
109+
threads: 4 # Override common.threads for Terraform state scans
110+
# Note: artifacts, max_artifact_size, and owned do not apply to gl tf.
111+
106112
#------------------------------------------------------------------------------
107113
# GitHub Platform Configuration
108114
#------------------------------------------------------------------------------

‎pkg/gitlab/tf/tf.go‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
package tf
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"io"
7+
"net/url"
8+
"os"
9+
"path/filepath"
10+
"time"
11+
12+
"github.com/CompassSecurity/pipeleek/pkg/gitlab/util"
13+
"github.com/CompassSecurity/pipeleek/pkg/logging"
14+
"github.com/CompassSecurity/pipeleek/pkg/scanner"
15+
"github.com/rs/zerolog/log"
16+
gitlab "gitlab.com/gitlab-org/api/client-go"
17+
)
18+
19+
type TFOptions struct {
20+
GitlabUrl string
21+
GitlabApiToken string
22+
GitlabClient *gitlab.Client
23+
OutputDir string
24+
Threads int
25+
ConfidenceFilter []string
26+
TruffleHogVerification bool
27+
HitTimeout time.Duration
28+
}
29+
30+
type terraformState struct {
31+
Name string
32+
ProjectID int
33+
Project *gitlab.Project
34+
}
35+
36+
func ScanTerraformStates(options TFOptions) {
37+
log.Info().Msg("Starting Terraform state scan")
38+
39+
scanner.InitRules(options.ConfidenceFilter)
40+
if !options.TruffleHogVerification {
41+
log.Info().Msg("TruffleHog verification is disabled")
42+
}
43+
44+
if err := os.MkdirAll(options.OutputDir, 0o750); err != nil {
45+
log.Fatal().Err(err).Str("dir", options.OutputDir).Msg("Failed to create output directory")
46+
}
47+
48+
git, err := util.GetGitlabClient(options.GitlabApiToken, options.GitlabUrl)
49+
if err != nil {
50+
log.Fatal().Stack().Err(err).Msg("Failed creating gitlab client")
51+
}
52+
options.GitlabClient = git
53+
54+
states := fetchTerraformStates(git)
55+
log.Info().Int("total", len(states)).Msg("Found Terraform states")
56+
57+
if len(states) == 0 {
58+
log.Warn().Msg("No Terraform states found")
59+
return
60+
}
61+
62+
for _, state := range states {
63+
stateData, filePath, ok := downloadStateFile(state, options)
64+
if !ok {
65+
continue
66+
}
67+
68+
scanStateFile(stateData, filePath, state, options)
69+
}
70+
71+
log.Info().Msg("Terraform state scan complete")
72+
}
73+
74+
func fetchTerraformStates(git *gitlab.Client) []terraformState {
75+
var states []terraformState
76+
77+
projectOpts := &gitlab.ListProjectsOptions{
78+
ListOptions: gitlab.ListOptions{PerPage: 100, Page: 1},
79+
MinAccessLevel: gitlab.Ptr(gitlab.MaintainerPermissions),
80+
OrderBy: gitlab.Ptr("last_activity_at"),
81+
}
82+
83+
log.Info().Msg("Fetching projects with maintainer access")
84+
85+
err := util.IterateProjects(git, projectOpts, func(project *gitlab.Project) error {
86+
log.Debug().Str("project", project.PathWithNamespace).Int64("id", project.ID).Msg("Checking project for Terraform state")
87+
88+
stateList, _, err := git.TerraformStates.List(project.PathWithNamespace)
89+
if err != nil {
90+
if errors.Is(err, gitlab.ErrNotFound) {
91+
return nil
92+
}
93+
log.Error().Err(err).Str("project", project.PathWithNamespace).Msg("Failed to list Terraform states")
94+
return nil
95+
}
96+
97+
if len(stateList) == 0 {
98+
return nil
99+
}
100+
101+
for _, state := range stateList {
102+
if state.Name == "" {
103+
continue
104+
}
105+
states = append(states, terraformState{
106+
Name: state.Name,
107+
ProjectID: int(project.ID),
108+
Project: project,
109+
})
110+
}
111+
112+
log.Info().Str("project", project.PathWithNamespace).Int("states", len(stateList)).Msg("Found Terraform states")
113+
return nil
114+
})
115+
116+
if err != nil {
117+
log.Error().Err(err).Msg("Error iterating projects")
118+
}
119+
120+
return states
121+
}
122+
123+
func downloadStateFile(state terraformState, options TFOptions) ([]byte, string, bool) {
124+
reader, _, err := options.GitlabClient.TerraformStates.DownloadLatest(state.ProjectID, state.Name)
125+
if err != nil {
126+
log.Error().Err(err).Str("project", state.Project.PathWithNamespace).Str("state", state.Name).Msg("Failed to download Terraform state")
127+
return nil, "", false
128+
}
129+
130+
stateData, err := io.ReadAll(reader)
131+
if err != nil {
132+
log.Error().Err(err).Str("project", state.Project.PathWithNamespace).Str("state", state.Name).Msg("Failed to read state data")
133+
return nil, "", false
134+
}
135+
136+
filename := fmt.Sprintf("%d_%s.tfstate", state.ProjectID, url.PathEscape(state.Name))
137+
filePath := filepath.Join(options.OutputDir, filename)
138+
139+
if err := os.WriteFile(filePath, stateData, 0o600); err != nil {
140+
log.Error().Err(err).Str("file", filePath).Msg("Failed to write state file")
141+
return nil, "", false
142+
}
143+
144+
log.Info().Str("project", state.Project.PathWithNamespace).Str("state", state.Name).Str("file", filePath).Msg("Downloaded Terraform state")
145+
146+
return stateData, filePath, true
147+
}
148+
149+
func scanStateFile(content []byte, filePath string, state terraformState, options TFOptions) {
150+
log.Debug().Str("file", filePath).Msg("Scanning Terraform state for secrets")
151+
152+
findings, err := scanner.DetectHits(content, options.Threads, options.TruffleHogVerification, options.HitTimeout)
153+
if err != nil {
154+
log.Debug().Err(err).Str("file", filePath).Msg("Failed detecting secrets")
155+
return
156+
}
157+
158+
if len(findings) > 0 {
159+
log.Warn().Int("findings", len(findings)).Str("project", state.Project.PathWithNamespace).Str("state", state.Name).Str("file", filePath).Msg("Secrets found in Terraform state")
160+
161+
for _, finding := range findings {
162+
logging.Hit().
163+
Str("type", "terraform-state").
164+
Str("project", state.Project.PathWithNamespace).
165+
Str("url", state.Project.WebURL).
166+
Str("state", state.Name).
167+
Str("file", filePath).
168+
Str("ruleName", finding.Pattern.Pattern.Name).
169+
Str("confidence", finding.Pattern.Pattern.Confidence).
170+
Str("value", finding.Text).
171+
Msg("SECRET")
172+
}
173+
}
174+
}

0 commit comments

Comments
 (0)