Skip to content

Commit efa4995

Browse files
feat: add secret redaction feature to replace sensitive values (#14)
1 parent 1e3a49c commit efa4995

17 files changed

Lines changed: 1210 additions & 602 deletions

File tree

.github/workflows/main.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ jobs:
2525
go install gotest.tools/gotestsum@latest
2626
2727
- name: Install golangci-lint
28-
uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
28+
uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0
2929
with:
30-
version: v1.62.2
30+
version: v2.12.2
3131

3232
github-actions-docs:
3333
runs-on: ubuntu-latest

.golangci.yml

Lines changed: 76 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,82 @@
1-
project_name: ld-find-code-refs
2-
1+
version: "2"
32
run:
4-
deadline: 120s
53
tests: false
6-
74
linters:
8-
enable-all: true
5+
default: all
96
disable:
10-
- golint
11-
- stylecheck
12-
- gochecknoglobals
13-
- lll
14-
- errcheck
15-
- interfacer
16-
- tagliatelle
17-
- varnamelen
18-
- nonamedreturns
19-
- nlreturn # re-enable this after fixing all cases
20-
- gomnd # re-enable this after fixing all cases
21-
- nosnakecase
22-
- ifshort # re-enable
23-
- nilerr #re-enable
24-
- goerr113
25-
- exhaustivestruct
26-
- exhaustruct
27-
- exhaustive
28-
- godot
29-
- wsl #fix
30-
- gocognit
31-
- revive
32-
- errname
33-
- cyclop
34-
- gofumpt # fix
35-
- gci
36-
- nestif # fix
37-
- gosec
38-
- gocritic
39-
- depguard
40-
- tagalign
41-
- nilnil
42-
fast: false
43-
44-
linter-settings:
45-
goimports:
46-
local-prefixes: github.com/launchdarkly,gopkg.in/launchdarkly
47-
7+
- cyclop
8+
- funcorder
9+
- godoclint
10+
- gomodguard
11+
- modernize
12+
- noinlineerr
13+
- wsl_v5
14+
- depguard
15+
- err113
16+
- errcheck
17+
- errname
18+
- exhaustive
19+
- exhaustruct
20+
- gochecknoglobals
21+
- gocognit
22+
- gocritic
23+
- godot
24+
- gosec
25+
- lll
26+
- mnd
27+
- nestif
28+
- nilerr
29+
- nilnil
30+
- nlreturn
31+
- nonamedreturns
32+
- revive
33+
- tagalign
34+
- tagliatelle
35+
- varnamelen
36+
- wsl
37+
settings:
38+
staticcheck:
39+
checks:
40+
- all
41+
- -ST*
42+
- -QF*
43+
exclusions:
44+
generated: lax
45+
rules:
46+
- linters:
47+
- staticcheck
48+
text: SA5011
49+
- path: (.+)\.go$
50+
text: error returned from external package is unwrapped
51+
- path: (.+)\.go$
52+
text: error returned from interface method should be wrapped
53+
- path: (.+)\.go$
54+
text: comparing with == will fail on wrapped errors. Use errors.Is to check for a specific error
55+
- path: (.+)\.go$
56+
text: comparing with != will fail on wrapped errors. Use errors.Is to check for a specific error
57+
- path: (.+)\.go$
58+
text: non-wrapping format verb for fmt.Errorf. Use `%w` to format errors
59+
- path: (.+)\.go$
60+
text: append to slice `projectKeys` with non-zero initialized length
61+
paths:
62+
- third_party$
63+
- builtin$
64+
- examples$
4865
issues:
49-
exclude:
50-
- "error returned from external package is unwrapped"
51-
- "error returned from interface method should be wrapped"
52-
- "comparing with == will fail on wrapped errors. Use errors.Is to check for a specific error" # should fix
53-
- "comparing with != will fail on wrapped errors. Use errors.Is to check for a specific error" # should fix
54-
- "non-wrapping format verb for fmt.Errorf. Use `%w` to format errors" # should fix
55-
- "append to slice `projectKeys` with non-zero initialized length"
56-
exclude-rules:
57-
# Exclude some `staticcheck` messages.
58-
- linters:
59-
- staticcheck
60-
text: "SA5011" # fix
61-
62-
exclude-use-default: false
66+
max-issues-per-linter: 1000
6367
max-same-issues: 1000
64-
max-per-linter: 1000
68+
formatters:
69+
enable:
70+
- gofmt
71+
- goimports
72+
settings:
73+
goimports:
74+
local-prefixes:
75+
- github.com/launchdarkly
76+
- gopkg.in/launchdarkly
77+
exclusions:
78+
generated: lax
79+
paths:
80+
- third_party$
81+
- builtin$
82+
- examples$

README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ This repository provides solutions for configuring [Bucketeer code references](h
1919
- [Using environment variables](docs/CONFIGURATION.md#environment-variables)
2020
- [Using a YAML file](docs/CONFIGURATION.md#YAML)
2121
- [Delimiters](docs/CONFIGURATION.md#delimiters)
22+
- [Secret redaction](docs/CONFIGURATION.md#secret-redaction)
2223
- [Ignoring files and directories](docs/CONFIGURATION.md#ignoring-files-and-directories)
2324

2425
## Execution via CLI

coderefs/coderefs.go

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,8 @@ func fetchExistingReferences(
9999
opts options.Options,
100100
bucketeerApi bucketeer.ApiClient,
101101
refs []bucketeer.ReferenceHunksRep,
102-
) map[string]bucketeer.CodeReference {
103-
existingRefs := make(map[string]bucketeer.CodeReference)
102+
) map[string][]bucketeer.CodeReference {
103+
existingRefs := make(map[string][]bucketeer.CodeReference)
104104
flagCounts := aggregateFeatureFlags(refs)
105105

106106
for flag := range flagCounts {
@@ -113,17 +113,33 @@ func fetchExistingReferences(
113113
}
114114

115115
for _, ref := range codeRefs {
116-
existingRefs[ref.ContentHash] = ref
116+
// the list endpoint is queried by featureId; tolerate responses
117+
// that omit the field in items so keys still match scanned refs
118+
if ref.FeatureID == "" {
119+
ref.FeatureID = flag
120+
}
121+
key := referenceKey(ref)
122+
existingRefs[key] = append(existingRefs[key], ref)
117123
}
118124
}
119125
return existingRefs
120126
}
121127

128+
// referenceKey identifies a code reference during reconciliation. ContentHash
129+
// alone can collide: hunks whose secrets are redacted to identical text, or
130+
// copy-pasted code, hash the same, so the flag key and file path are included.
131+
// References that still share a key (identical snippets of one flag in one
132+
// file, or every hunk when contextLines < 0) are kept as a slice and matched
133+
// one-to-one so duplicates are neither overwritten nor leaked from deletion.
134+
func referenceKey(ref bucketeer.CodeReference) string {
135+
return ref.FeatureID + "\x00" + ref.FilePath + "\x00" + ref.ContentHash
136+
}
137+
122138
func processNewReferences(
123139
opts options.Options,
124140
bucketeerApi bucketeer.ApiClient,
125141
refs []bucketeer.ReferenceHunksRep,
126-
existingRefs map[string]bucketeer.CodeReference,
142+
existingRefs map[string][]bucketeer.CodeReference,
127143
branchName, revision, repoType string,
128144
) {
129145
for _, ref := range refs {
@@ -158,15 +174,21 @@ func createCodeReference(opts options.Options,
158174
func updateOrCreateReference(opts options.Options,
159175
bucketeerApi bucketeer.ApiClient,
160176
codeRef bucketeer.CodeReference,
161-
existingRefs map[string]bucketeer.CodeReference,
177+
existingRefs map[string][]bucketeer.CodeReference,
162178
) {
163-
if existing, exists := existingRefs[codeRef.ContentHash]; exists {
179+
key := referenceKey(codeRef)
180+
if refs := existingRefs[key]; len(refs) > 0 {
181+
existing := refs[0]
164182
log.Info.Printf("updating code reference in Bucketeer: id: %s, content hash: %s", existing.ID, codeRef.ContentHash)
165183
err := bucketeerApi.UpdateCodeReference(context.Background(), opts, existing.ID, codeRef)
166184
if err != nil {
167185
helpers.FatalServiceError(fmt.Errorf("error updating code reference in Bucketeer: %w", err), opts.IgnoreServiceErrors)
168186
}
169-
delete(existingRefs, codeRef.ContentHash)
187+
if len(refs) == 1 {
188+
delete(existingRefs, key)
189+
} else {
190+
existingRefs[key] = refs[1:]
191+
}
170192
} else {
171193
err := bucketeerApi.CreateCodeReference(context.Background(), opts, codeRef)
172194
if err != nil {
@@ -175,14 +197,16 @@ func updateOrCreateReference(opts options.Options,
175197
}
176198
}
177199

178-
func deleteStaleReferences(opts options.Options, bucketeerApi bucketeer.ApiClient, existingRefs map[string]bucketeer.CodeReference) {
179-
for _, ref := range existingRefs {
180-
if ref.RepositoryOwner == opts.RepoOwner && ref.RepositoryName == opts.RepoName {
181-
err := bucketeerApi.DeleteCodeReference(context.Background(), opts, ref.ID)
182-
if err != nil {
183-
helpers.FatalServiceError(fmt.Errorf("error deleting code reference from Bucketeer: %w", err), opts.IgnoreServiceErrors)
200+
func deleteStaleReferences(opts options.Options, bucketeerApi bucketeer.ApiClient, existingRefs map[string][]bucketeer.CodeReference) {
201+
for _, refs := range existingRefs {
202+
for _, ref := range refs {
203+
if ref.RepositoryOwner == opts.RepoOwner && ref.RepositoryName == opts.RepoName {
204+
err := bucketeerApi.DeleteCodeReference(context.Background(), opts, ref.ID)
205+
if err != nil {
206+
helpers.FatalServiceError(fmt.Errorf("error deleting code reference from Bucketeer: %w", err), opts.IgnoreServiceErrors)
207+
}
208+
log.Info.Printf("deleted code reference from Bucketeer: %+v", ref)
184209
}
185-
log.Info.Printf("deleted code reference from Bucketeer: %+v", ref)
186210
}
187211
}
188212
}

coderefs/coderefs_test.go

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
package coderefs
22

33
import (
4+
"context"
45
"os"
56
"testing"
67

8+
"github.com/stretchr/testify/require"
9+
10+
"github.com/bucketeer-io/code-refs/internal/bucketeer"
711
"github.com/bucketeer-io/code-refs/internal/log"
12+
"github.com/bucketeer-io/code-refs/options"
813
)
914

1015
func init() {
@@ -14,6 +19,117 @@ func TestMain(m *testing.M) {
1419
log.Init(true)
1520
os.Exit(m.Run())
1621
}
22+
23+
type fakeApiClient struct {
24+
existing []bucketeer.CodeReference
25+
created []bucketeer.CodeReference
26+
updated map[string]bucketeer.CodeReference
27+
deleted []string
28+
}
29+
30+
func (f *fakeApiClient) GetFlagKeyList(_ context.Context, _ options.Options) ([]string, error) {
31+
return nil, nil
32+
}
33+
34+
func (f *fakeApiClient) CreateCodeReference(_ context.Context, _ options.Options, ref bucketeer.CodeReference) error {
35+
f.created = append(f.created, ref)
36+
return nil
37+
}
38+
39+
func (f *fakeApiClient) UpdateCodeReference(_ context.Context, _ options.Options, id string, ref bucketeer.CodeReference) error {
40+
f.updated[id] = ref
41+
return nil
42+
}
43+
44+
func (f *fakeApiClient) DeleteCodeReference(_ context.Context, _ options.Options, id string) error {
45+
f.deleted = append(f.deleted, id)
46+
return nil
47+
}
48+
49+
func (f *fakeApiClient) ListCodeReferences(_ context.Context, _ options.Options, _ string, _ int64) ([]bucketeer.CodeReference, string, string, error) {
50+
return f.existing, "", "", nil
51+
}
52+
53+
func Test_processCodeReferences_identicalContentHashes(t *testing.T) {
54+
// two refs for the same flag whose snippets are textually identical (e.g.
55+
// different secrets both redacted to the same text) share a ContentHash;
56+
// each must still reconcile with its own scanned hunk instead of one
57+
// overwriting the other and the second being recreated
58+
api := &fakeApiClient{
59+
existing: []bucketeer.CodeReference{
60+
{ID: "ref-a", FeatureID: "my-flag", FilePath: "a.go", ContentHash: "same-hash", RepositoryOwner: "owner", RepositoryName: "repo"},
61+
{ID: "ref-b", FeatureID: "my-flag", FilePath: "b.go", ContentHash: "same-hash", RepositoryOwner: "owner", RepositoryName: "repo"},
62+
},
63+
updated: map[string]bucketeer.CodeReference{},
64+
}
65+
opts := options.Options{RepoOwner: "owner", RepoName: "repo"}
66+
refs := []bucketeer.ReferenceHunksRep{
67+
{Path: "a.go", Hunks: []bucketeer.HunkRep{{FlagKey: "my-flag", ContentHash: "same-hash", Lines: `key = "[REDACTED]"`}}},
68+
{Path: "b.go", Hunks: []bucketeer.HunkRep{{FlagKey: "my-flag", ContentHash: "same-hash", Lines: `key = "[REDACTED]"`}}},
69+
}
70+
71+
processCodeReferences(opts, api, refs, "main", "abc123", "GITHUB")
72+
73+
require.Empty(t, api.created)
74+
require.Empty(t, api.deleted)
75+
require.Len(t, api.updated, 2)
76+
require.Equal(t, "a.go", api.updated["ref-a"].FilePath)
77+
require.Equal(t, "b.go", api.updated["ref-b"].FilePath)
78+
}
79+
80+
func Test_processCodeReferences_duplicateReferenceKeys(t *testing.T) {
81+
// references that share even the composite key (identical snippets of one
82+
// flag in one file, or every hunk when contextLines < 0 hashes the empty
83+
// string) must reconcile one-to-one instead of the duplicate row being
84+
// dropped from the map and accumulating on the server run after run
85+
api := &fakeApiClient{
86+
existing: []bucketeer.CodeReference{
87+
{ID: "ref-a", FeatureID: "my-flag", FilePath: "a.go", ContentHash: "h", RepositoryOwner: "owner", RepositoryName: "repo"},
88+
{ID: "ref-b", FeatureID: "my-flag", FilePath: "a.go", ContentHash: "h", RepositoryOwner: "owner", RepositoryName: "repo"},
89+
},
90+
updated: map[string]bucketeer.CodeReference{},
91+
}
92+
opts := options.Options{RepoOwner: "owner", RepoName: "repo"}
93+
refs := []bucketeer.ReferenceHunksRep{
94+
{Path: "a.go", Hunks: []bucketeer.HunkRep{
95+
{FlagKey: "my-flag", ContentHash: "h"},
96+
{FlagKey: "my-flag", ContentHash: "h"},
97+
}},
98+
}
99+
100+
processCodeReferences(opts, api, refs, "main", "abc123", "GITHUB")
101+
102+
require.Empty(t, api.created)
103+
require.Empty(t, api.deleted)
104+
require.Len(t, api.updated, 2)
105+
require.Contains(t, api.updated, "ref-a")
106+
require.Contains(t, api.updated, "ref-b")
107+
}
108+
109+
func Test_processCodeReferences_backfillsMissingFeatureID(t *testing.T) {
110+
// the list endpoint is queried by featureId; if the server omits the
111+
// field in response items, the queried flag must be backfilled so the
112+
// reconciliation key still matches instead of every ref being recreated
113+
// and the old rows deleted
114+
api := &fakeApiClient{
115+
existing: []bucketeer.CodeReference{
116+
{ID: "ref-a", FilePath: "a.go", ContentHash: "h", RepositoryOwner: "owner", RepositoryName: "repo"},
117+
},
118+
updated: map[string]bucketeer.CodeReference{},
119+
}
120+
opts := options.Options{RepoOwner: "owner", RepoName: "repo"}
121+
refs := []bucketeer.ReferenceHunksRep{
122+
{Path: "a.go", Hunks: []bucketeer.HunkRep{{FlagKey: "my-flag", ContentHash: "h"}}},
123+
}
124+
125+
processCodeReferences(opts, api, refs, "main", "abc123", "GITHUB")
126+
127+
require.Empty(t, api.created)
128+
require.Empty(t, api.deleted)
129+
require.Len(t, api.updated, 1)
130+
require.Contains(t, api.updated, "ref-a")
131+
}
132+
17133
// func Test_calculateStaleBranches(t *testing.T) {
18134
// specs := []struct {
19135
// name string

0 commit comments

Comments
 (0)