Skip to content

Commit 762ea8f

Browse files
frjcompCopilot
frjcomp
and
Copilot
authored
refactor code structure (#378)
* refactor code structure --------- Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
1 parent 0bdd144 commit 762ea8f

128 files changed

Lines changed: 4509 additions & 3499 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.devcontainer/devcontainer.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"name": "Pipeleak Development",
3+
"image": "mcr.microsoft.com/devcontainers/go:1.24",
4+
"features": {
5+
"ghcr.io/devcontainers/features/python:1": {
6+
"version": "3.12"
7+
},
8+
"ghcr.io/devcontainers/features/node:1": {
9+
"version": "lts"
10+
},
11+
"ghcr.io/devcontainers/features/docker-in-docker:2": {
12+
"version": "latest",
13+
"moby": true
14+
}
15+
},
16+
"customizations": {
17+
"vscode": {
18+
"extensions": [
19+
"golang.go",
20+
"ms-python.python",
21+
"esbenp.prettier-vscode",
22+
"ms-vscode.makefile-tools",
23+
"streetsidesoftware.code-spell-checker"
24+
],
25+
"settings": {
26+
"go.toolsManagement.autoUpdate": true,
27+
"go.useLanguageServer": true,
28+
"go.lintTool": "golangci-lint",
29+
"go.lintFlags": ["--timeout=10m"],
30+
"editor.formatOnSave": true,
31+
"[go]": {
32+
"editor.defaultFormatter": "golang.go"
33+
}
34+
}
35+
}
36+
},
37+
"portsAttributes": {
38+
"*": {
39+
"onAutoForward": "ignore"
40+
}
41+
},
42+
"postCreateCommand": "bash .devcontainer/post-create.sh",
43+
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}"
44+
}

‎.devcontainer/post-create.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
set -e
3+
4+
echo "=== Pipeleak Development Environment Setup ==="
5+
6+
# Install golangci-lint
7+
echo "Installing golangci-lint..."
8+
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
9+
10+
# Install Python dependencies for documentation
11+
echo "Installing MkDocs and dependencies..."
12+
pip install --user mkdocs mkdocs-material mkdocs-minify-plugin
13+
14+
# Download Go dependencies
15+
echo "Downloading Go dependencies..."
16+
cd src/pipeleak
17+
go mod download
18+
19+
# Build the binary
20+
echo "Building pipeleak binary..."
21+
go build -o pipeleak .
22+
23+
# Setup bash aliases
24+
echo "Setting up bash aliases..."
25+
cat >> ~/.bashrc << 'EOF'
26+
27+
# Custom aliases
28+
alias ll='ls -alh'
29+
alias la='ls -A'
30+
alias l='ls -CF'
31+
32+
# Git shortcuts
33+
alias gs='git status'
34+
alias ga='git add'
35+
alias gc='git commit'
36+
alias gp='git push'
37+
alias gl='git pull'
38+
alias gd='git diff'
39+
alias gco='git checkout'
40+
alias gb='git branch'
41+
alias glog='git log --oneline --graph --decorate'
42+
EOF
43+
44+
echo ""
45+
echo "=== Setup Complete ==="
46+
echo ""
47+
echo "Quick start commands:"
48+
echo " cd src/pipeleak"
49+
echo " make build - Build the binary"
50+
echo " make test-unit - Run unit tests"
51+
echo " make lint - Run linter"
52+
echo " make serve-docs - Generate and serve documentation"
53+
echo ""
54+
echo "Run './pipeleak --help' to see available commands."

‎.github/copilot-instructions.md‎

Lines changed: 107 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,27 +20,36 @@ Pipeleak is a CLI tool designed to scan CI/CD logs and artifacts for secrets acr
2020
pipeleak/
2121
├── src/pipeleak/ # Main Go module
2222
│ ├── cmd/ # CLI commands (using Cobra)
23-
│ │ ├── gitlab/ # GitLab-specific commands
24-
│ │ ├── github/ # GitHub-specific commands
2523
│ │ ├── bitbucket/ # BitBucket-specific commands
2624
│ │ ├── devops/ # Azure DevOps commands
25+
│ │ ├── docs/ # Documentation command
2726
│ │ ├── gitea/ # Gitea commands
28-
│ │ └── docs/ # Documentation command
27+
│ │ ├── github/ # GitHub-specific commands
28+
│ │ ├── gitlab/ # GitLab-specific commands
2929
│ ├── pkg/ # Core business logic packages
3030
│ │ ├── archive/ # Archive handling
31+
│ │ ├── bitbucket/ # BitBucket business logic
3132
│ │ ├── config/ # Configuration management
32-
│ │ ├── gitlab/ # GitLab functionality
33-
│ │ ├── github/ # GitHub functionality
34-
│ │ └── ... # Platform-specific packages
33+
│ │ ├── devops/ # Azure DevOps business logic
34+
│ │ ├── docs/ # Documentation generation
35+
│ │ ├── format/ # Formatting helpers
36+
│ │ ├── gitea/ # Gitea business logic
37+
│ │ ├── github/ # GitHub business logic
38+
│ │ ├── gitlab/ # GitLab business logic
39+
│ │ ├── httpclient/ # HTTP client helpers
40+
│ │ ├── logging/ # Logging helpers
41+
│ │ ├── scan/ # Scan logic
42+
│ │ ├── scanner/ # Scanner engine
43+
│ │ ├── system/ # System helpers
3544
│ ├── tests/ # Test files
36-
│ │ └── e2e/ # End-to-end tests
37-
│ ├── main.go # Application entry point
38-
│ ├── go.mod # Go module definition
39-
│ └── go.sum # Dependency checksums
45+
│ │ └── e2e/ # End-to-end tests
46+
│ ├── main.go # Application entry point
47+
│ ├── go.mod # Go module definition
48+
│ └── go.sum # Dependency checksums
4049
├── docs/ # Documentation (MkDocs)
4150
├── .github/ # GitHub workflows and configs
42-
│ └── workflows/ # CI/CD pipelines
43-
└── goreleaser.yaml # Release configuration
51+
│ └── workflows/ # CI/CD pipelines
52+
└── goreleaser.yaml # Release configuration
4453
```
4554

4655
## Building and Testing
@@ -49,32 +58,116 @@ pipeleak/
4958

5059
```bash
5160
cd src/pipeleak
61+
make build
62+
# Or directly:
5263
go build -o pipeleak .
5364
```
5465

5566
### Running Tests
5667

68+
**Using Makefile (recommended):**
69+
```bash
70+
cd src/pipeleak
71+
make test # Run all tests (unit + e2e)
72+
make test-unit # Run unit tests only
73+
make test-e2e # Run all e2e tests
74+
```
75+
5776
**Unit tests (excluding e2e):**
5877
```bash
5978
cd src/pipeleak
79+
make test-unit
80+
# Or directly:
6081
go test $(go list ./... | grep -v /tests/e2e) -v -race
6182
```
6283

6384
**End-to-end tests:**
85+
86+
E2E tests are organized by platform in a structured folder hierarchy:
87+
```
88+
tests/e2e/
89+
├── gitlab/ # GitLab-specific tests
90+
│ ├── cicd/yaml/ # CICD YAML command tests
91+
│ ├── scan/ # Scan command tests
92+
│ ├── variables/ # Variables command tests
93+
│ ├── schedule/ # Schedule command tests
94+
│ ├── runners/ # Runners list/exploit tests
95+
│ ├── secureFiles/ # Secure files tests
96+
│ ├── vuln/ # Vulnerability check tests
97+
│ ├── renovate/ # Renovate tests
98+
│ └── unauth/ # Unauthenticated commands (shodan)
99+
├── github/ # GitHub Actions tests
100+
├── bitbucket/ # BitBucket tests
101+
├── devops/ # Azure DevOps tests
102+
├── gitea/ # Gitea tests
103+
└── internal/ # Shared test utilities
104+
└── testutil/ # Common helpers (RunCLI, mock servers, etc.)
105+
```
106+
107+
**Using Makefile (recommended):**
108+
```bash
109+
cd src/pipeleak
110+
make test-e2e # Run all e2e tests
111+
make test-e2e-gitlab # Run only GitLab e2e tests
112+
make test-e2e-github # Run only GitHub e2e tests
113+
make test-e2e-bitbucket # Run only BitBucket e2e tests
114+
make test-e2e-devops # Run only Azure DevOps e2e tests
115+
make test-e2e-gitea # Run only Gitea e2e tests
116+
```
117+
118+
**Manual execution:**
119+
To run e2e tests manually, first build the binary and set `PIPELEAK_BINARY`:
64120
```bash
65121
cd src/pipeleak
66122
go build -o pipeleak .
67-
PIPELEAK_BINARY=./pipeleak go test ./tests/e2e/... -v -timeout 10m
123+
PIPELEAK_BINARY=$(pwd)/pipeleak go test ./tests/e2e/... -tags=e2e -v -timeout 10m
68124
```
69125

126+
Run tests for a specific platform:
127+
```bash
128+
# GitLab tests only
129+
PIPELEAK_BINARY=$(pwd)/pipeleak go test ./tests/e2e/gitlab/... -tags=e2e -v
130+
131+
# Specific command tests
132+
PIPELEAK_BINARY=$(pwd)/pipeleak go test ./tests/e2e/gitlab/scan -tags=e2e -v
133+
```
134+
135+
**Important:** E2E tests require the `PIPELEAK_BINARY` environment variable to point to the compiled binary (absolute or relative to module root). Tests use this binary to run commands in isolated subprocesses to avoid Cobra state conflicts.
136+
137+
### Test Coverage
138+
139+
Generate and view test coverage reports:
140+
141+
```bash
142+
cd src/pipeleak
143+
144+
# Generate coverage report with summary
145+
make coverage
146+
147+
# Generate HTML coverage report and open in browser
148+
make coverage-html
149+
```
150+
151+
Coverage reports are stored as workflow artifacts on CI runs (Linux job). Retrieve `coverage.out` from the run's artifacts section for local inspection or HTML generation.
152+
70153
### Linting
71154

72155
The project uses golangci-lint:
73156
```bash
74157
cd src/pipeleak
158+
make lint
159+
# Or directly:
75160
golangci-lint run --timeout=10m
76161
```
77162

163+
### Documentation
164+
165+
Generate and serve CLI documentation locally:
166+
```bash
167+
cd src/pipeleak
168+
make serve-docs # Installs dependencies if needed, generates and serves docs
169+
```
170+
78171
## Code Style and Conventions
79172

80173
### General Guidelines
@@ -85,6 +178,7 @@ golangci-lint run --timeout=10m
85178
4. **Testing**: Write tests for new functionality; maintain existing test coverage
86179
5. **Documentation**: Update documentation when adding or modifying features
87180
6. **Comments**: Only add comments that provide useful context and additional understanding; avoid obvious or redundant comments
181+
7. **File Moves/Copies**: When moving or copying files, always delete any resulting unused or vestigial files to keep the codebase clean and maintainable.
88182

89183
### Command Structure
90184

‎.github/dependabot.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,9 @@ updates:
1616
directory: "/"
1717
schedule:
1818
interval: "daily"
19+
20+
# Devcontainer dependencies (images and features)
21+
- package-ecosystem: "devcontainers"
22+
directory: "/"
23+
schedule:
24+
interval: "weekly"

‎.github/workflows/golangci-lint.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ jobs:
1717
- uses: actions/setup-go@v6
1818
with:
1919
go-version: stable
20-
- name: golangci-lint
21-
uses: golangci/golangci-lint-action@v9
22-
with:
23-
version: latest
24-
working-directory: ./src/pipeleak
25-
args: --timeout=10m
20+
cache: true
21+
- name: Install golangci-lint
22+
run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
23+
- name: Run golangci-lint
24+
working-directory: ./src/pipeleak
25+
run: make lint

‎.github/workflows/test.yml‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,12 @@ jobs:
2727
- name: Run unit tests (Linux)
2828
if: runner.os == 'Linux'
2929
working-directory: ./src/pipeleak
30-
run: go test $(go list ./... | grep -v /tests/e2e) -v -race
30+
run: make test-unit
31+
32+
- name: Generate coverage report (Linux)
33+
if: runner.os == 'Linux'
34+
working-directory: ./src/pipeleak
35+
run: go test $(go list ./... | grep -v /tests/e2e) -coverprofile=coverage.out -covermode=atomic
3136

3237
- name: Run unit tests (Windows)
3338
if: runner.os == 'Windows'
@@ -36,6 +41,13 @@ jobs:
3641
$packages = go list ./... | Where-Object { $_ -notmatch '/tests/e2e' }
3742
go test $packages -v -race
3843
shell: pwsh
44+
45+
- name: Upload coverage artifact (Linux)
46+
if: runner.os == 'Linux'
47+
uses: actions/upload-artifact@v4
48+
with:
49+
name: coverage-linux
50+
path: ./src/pipeleak/coverage.out
3951

4052
e2e-tests:
4153
name: E2E Tests
@@ -51,11 +63,6 @@ jobs:
5163
go-version: stable
5264
cache: true
5365

54-
- name: Build pipeleak binary for e2e tests (Linux)
55-
if: runner.os == 'Linux'
56-
working-directory: ./src/pipeleak
57-
run: go build -o pipeleak .
58-
5966
- name: Build pipeleak binary for e2e tests (Windows)
6067
if: runner.os == 'Windows'
6168
working-directory: ./src/pipeleak
@@ -64,13 +71,12 @@ jobs:
6471
- name: Run e2e tests (Linux)
6572
if: runner.os == 'Linux'
6673
working-directory: ./src/pipeleak
67-
run: go test ./tests/e2e/... -v -timeout 10m
68-
env:
69-
PIPELEAK_BINARY: ./pipeleak
74+
run: make test-e2e
7075

7176
- name: Run e2e tests (Windows)
7277
if: runner.os == 'Windows'
7378
working-directory: ./src/pipeleak
74-
run: go test ./tests/e2e/... -v -timeout 10m
75-
env:
76-
PIPELEAK_BINARY: ./pipeleak.exe
79+
run: |
80+
$env:PIPELEAK_BINARY = "$(Get-Location)\pipeleak.exe"
81+
go test ./tests/e2e/... -tags=e2e -v -timeout 10m
82+
shell: pwsh

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,6 @@ site
77
src/pipeleak/pipeleak
88
src/pipeleak/pipeleak.exe
99
src/pipeleak/coverage.out
10+
src/pipeleak/coverage.html
1011
.vscode
1112
results.json

0 commit comments

Comments
 (0)