feat: add secret redaction feature to replace sensitive values

.github/workflows/main.yml

@@ -25,9 +25,9 @@ jobs:
           go install gotest.tools/gotestsum@latest
 
       - name: Install golangci-lint
-        uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
+        uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0
         with:
-          version: v1.62.2
+          version: v2.12.2
 
   github-actions-docs:
     runs-on: ubuntu-latest

.golangci.yml

@@ -1,64 +1,82 @@
-project_name: ld-find-code-refs
-
+version: "2"
 run:
-  deadline: 120s
   tests: false
-
 linters:
-  enable-all: true
+  default: all
   disable:
-      - golint
-      - stylecheck
-      - gochecknoglobals
-      - lll
-      - errcheck
-      - interfacer
-      - tagliatelle
-      - varnamelen
-      - nonamedreturns
-      - nlreturn # re-enable this after fixing all cases
-      - gomnd # re-enable this after fixing all cases
-      - nosnakecase
-      - ifshort # re-enable
-      - nilerr #re-enable
-      - goerr113
-      - exhaustivestruct
-      - exhaustruct
-      - exhaustive
-      - godot
-      - wsl #fix
-      - gocognit
-      - revive
-      - errname
-      - cyclop
-      - gofumpt # fix
-      - gci
-      - nestif # fix
-      - gosec
-      - gocritic
-      - depguard
-      - tagalign
-      - nilnil
-  fast: false
-
-linter-settings:
-  goimports:
-    local-prefixes: github.com/launchdarkly,gopkg.in/launchdarkly
-
+    - cyclop
+    - funcorder
+    - godoclint
+    - gomodguard
+    - modernize
+    - noinlineerr
+    - wsl_v5
+    - depguard
+    - err113
+    - errcheck
+    - errname
+    - exhaustive
+    - exhaustruct
+    - gochecknoglobals
+    - gocognit
+    - gocritic
+    - godot
+    - gosec
+    - lll
+    - mnd
+    - nestif
+    - nilerr
+    - nilnil
+    - nlreturn
+    - nonamedreturns
+    - revive
+    - tagalign
+    - tagliatelle
+    - varnamelen
+    - wsl
+  settings:
+    staticcheck:
+      checks:
+        - all
+        - -ST*
+        - -QF*
+  exclusions:
+    generated: lax
+    rules:
+      - linters:
+          - staticcheck
+        text: SA5011
+      - path: (.+)\.go$
+        text: error returned from external package is unwrapped
+      - path: (.+)\.go$
+        text: error returned from interface method should be wrapped
+      - path: (.+)\.go$
+        text: comparing with == will fail on wrapped errors. Use errors.Is to check for a specific error
+      - path: (.+)\.go$
+        text: comparing with != will fail on wrapped errors. Use errors.Is to check for a specific error
+      - path: (.+)\.go$
+        text: non-wrapping format verb for fmt.Errorf. Use `%w` to format errors
+      - path: (.+)\.go$
+        text: append to slice `projectKeys` with non-zero initialized length
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
 issues:
-  exclude:
-    - "error returned from external package is unwrapped"
-    - "error returned from interface method should be wrapped"
-    - "comparing with == will fail on wrapped errors. Use errors.Is to check for a specific error" # should fix
-    - "comparing with != will fail on wrapped errors. Use errors.Is to check for a specific error" # should fix
-    - "non-wrapping format verb for fmt.Errorf. Use `%w` to format errors" # should fix
-    - "append to slice `projectKeys` with non-zero initialized length"
-  exclude-rules:
-    # Exclude some `staticcheck` messages.
-    - linters:
-        - staticcheck
-      text: "SA5011" # fix
-
-  exclude-use-default: false
+  max-issues-per-linter: 1000
   max-same-issues: 1000
-  max-per-linter: 1000
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  settings:
+    goimports:
+      local-prefixes:
+        - github.com/launchdarkly
+        - gopkg.in/launchdarkly
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

README.md

@@ -19,6 +19,7 @@ This repository provides solutions for configuring [Bucketeer code references](h
   - [Using environment variables](docs/CONFIGURATION.md#environment-variables)
   - [Using a YAML file](docs/CONFIGURATION.md#YAML)
   - [Delimiters](docs/CONFIGURATION.md#delimiters)
+  - [Secret redaction](docs/CONFIGURATION.md#secret-redaction)
   - [Ignoring files and directories](docs/CONFIGURATION.md#ignoring-files-and-directories)
 
 ## Execution via CLI

coderefs/coderefs.go

@@ -99,8 +99,8 @@ func fetchExistingReferences(
 	opts options.Options,
 	bucketeerApi bucketeer.ApiClient,
 	refs []bucketeer.ReferenceHunksRep,
-) map[string]bucketeer.CodeReference {
-	existingRefs := make(map[string]bucketeer.CodeReference)
+) map[string][]bucketeer.CodeReference {
+	existingRefs := make(map[string][]bucketeer.CodeReference)
 	flagCounts := aggregateFeatureFlags(refs)
 
 	for flag := range flagCounts {
@@ -113,17 +113,33 @@ func fetchExistingReferences(
 		}
 
 		for _, ref := range codeRefs {
-			existingRefs[ref.ContentHash] = ref
+			// the list endpoint is queried by featureId; tolerate responses
+			// that omit the field in items so keys still match scanned refs
+			if ref.FeatureID == "" {
+				ref.FeatureID = flag
+			}
+			key := referenceKey(ref)
+			existingRefs[key] = append(existingRefs[key], ref)
 		}
 	}
 	return existingRefs
 }
 
+// referenceKey identifies a code reference during reconciliation. ContentHash
+// alone can collide: hunks whose secrets are redacted to identical text, or
+// copy-pasted code, hash the same, so the flag key and file path are included.
+// References that still share a key (identical snippets of one flag in one
+// file, or every hunk when contextLines < 0) are kept as a slice and matched
+// one-to-one so duplicates are neither overwritten nor leaked from deletion.
+func referenceKey(ref bucketeer.CodeReference) string {
+	return ref.FeatureID + "\x00" + ref.FilePath + "\x00" + ref.ContentHash
+}
+
 func processNewReferences(
 	opts options.Options,
 	bucketeerApi bucketeer.ApiClient,
 	refs []bucketeer.ReferenceHunksRep,
-	existingRefs map[string]bucketeer.CodeReference,
+	existingRefs map[string][]bucketeer.CodeReference,
 	branchName, revision, repoType string,
 ) {
 	for _, ref := range refs {
@@ -158,15 +174,21 @@ func createCodeReference(opts options.Options,
 func updateOrCreateReference(opts options.Options,
 	bucketeerApi bucketeer.ApiClient,
 	codeRef bucketeer.CodeReference,
-	existingRefs map[string]bucketeer.CodeReference,
+	existingRefs map[string][]bucketeer.CodeReference,
 ) {
-	if existing, exists := existingRefs[codeRef.ContentHash]; exists {
+	key := referenceKey(codeRef)
+	if refs := existingRefs[key]; len(refs) > 0 {
+		existing := refs[0]
 		log.Info.Printf("updating code reference in Bucketeer: id: %s, content hash: %s", existing.ID, codeRef.ContentHash)
 		err := bucketeerApi.UpdateCodeReference(context.Background(), opts, existing.ID, codeRef)
 		if err != nil {
 			helpers.FatalServiceError(fmt.Errorf("error updating code reference in Bucketeer: %w", err), opts.IgnoreServiceErrors)
 		}
-		delete(existingRefs, codeRef.ContentHash)
+		if len(refs) == 1 {
+			delete(existingRefs, key)
+		} else {
+			existingRefs[key] = refs[1:]
+		}
 	} else {
 		err := bucketeerApi.CreateCodeReference(context.Background(), opts, codeRef)
 		if err != nil {
@@ -175,14 +197,16 @@ func updateOrCreateReference(opts options.Options,
 	}
 }
 
-func deleteStaleReferences(opts options.Options, bucketeerApi bucketeer.ApiClient, existingRefs map[string]bucketeer.CodeReference) {
-	for _, ref := range existingRefs {
-		if ref.RepositoryOwner == opts.RepoOwner && ref.RepositoryName == opts.RepoName {
-			err := bucketeerApi.DeleteCodeReference(context.Background(), opts, ref.ID)
-			if err != nil {
-				helpers.FatalServiceError(fmt.Errorf("error deleting code reference from Bucketeer: %w", err), opts.IgnoreServiceErrors)
+func deleteStaleReferences(opts options.Options, bucketeerApi bucketeer.ApiClient, existingRefs map[string][]bucketeer.CodeReference) {
+	for _, refs := range existingRefs {
+		for _, ref := range refs {
+			if ref.RepositoryOwner == opts.RepoOwner && ref.RepositoryName == opts.RepoName {
+				err := bucketeerApi.DeleteCodeReference(context.Background(), opts, ref.ID)
+				if err != nil {
+					helpers.FatalServiceError(fmt.Errorf("error deleting code reference from Bucketeer: %w", err), opts.IgnoreServiceErrors)
+				}
+				log.Info.Printf("deleted code reference from Bucketeer: %+v", ref)
 			}
-			log.Info.Printf("deleted code reference from Bucketeer: %+v", ref)
 		}
 	}
 }

coderefs/coderefs_test.go

@@ -1,10 +1,15 @@
 package coderefs
 
 import (
+	"context"
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
+	"github.com/bucketeer-io/code-refs/internal/bucketeer"
 	"github.com/bucketeer-io/code-refs/internal/log"
+	"github.com/bucketeer-io/code-refs/options"
 )
 
 func init() {
@@ -14,6 +19,117 @@ func TestMain(m *testing.M) {
 	log.Init(true)
 	os.Exit(m.Run())
 }
+
+type fakeApiClient struct {
+	existing []bucketeer.CodeReference
+	created  []bucketeer.CodeReference
+	updated  map[string]bucketeer.CodeReference
+	deleted  []string
+}
+
+func (f *fakeApiClient) GetFlagKeyList(_ context.Context, _ options.Options) ([]string, error) {
+	return nil, nil
+}
+
+func (f *fakeApiClient) CreateCodeReference(_ context.Context, _ options.Options, ref bucketeer.CodeReference) error {
+	f.created = append(f.created, ref)
+	return nil
+}
+
+func (f *fakeApiClient) UpdateCodeReference(_ context.Context, _ options.Options, id string, ref bucketeer.CodeReference) error {
+	f.updated[id] = ref
+	return nil
+}
+
+func (f *fakeApiClient) DeleteCodeReference(_ context.Context, _ options.Options, id string) error {
+	f.deleted = append(f.deleted, id)
+	return nil
+}
+
+func (f *fakeApiClient) ListCodeReferences(_ context.Context, _ options.Options, _ string, _ int64) ([]bucketeer.CodeReference, string, string, error) {
+	return f.existing, "", "", nil
+}
+
+func Test_processCodeReferences_identicalContentHashes(t *testing.T) {
+	// two refs for the same flag whose snippets are textually identical (e.g.
+	// different secrets both redacted to the same text) share a ContentHash;
+	// each must still reconcile with its own scanned hunk instead of one
+	// overwriting the other and the second being recreated
+	api := &fakeApiClient{
+		existing: []bucketeer.CodeReference{
+			{ID: "ref-a", FeatureID: "my-flag", FilePath: "a.go", ContentHash: "same-hash", RepositoryOwner: "owner", RepositoryName: "repo"},
+			{ID: "ref-b", FeatureID: "my-flag", FilePath: "b.go", ContentHash: "same-hash", RepositoryOwner: "owner", RepositoryName: "repo"},
+		},
+		updated: map[string]bucketeer.CodeReference{},
+	}
+	opts := options.Options{RepoOwner: "owner", RepoName: "repo"}
+	refs := []bucketeer.ReferenceHunksRep{
+		{Path: "a.go", Hunks: []bucketeer.HunkRep{{FlagKey: "my-flag", ContentHash: "same-hash", Lines: `key = "[REDACTED]"`}}},
+		{Path: "b.go", Hunks: []bucketeer.HunkRep{{FlagKey: "my-flag", ContentHash: "same-hash", Lines: `key = "[REDACTED]"`}}},
+	}
+
+	processCodeReferences(opts, api, refs, "main", "abc123", "GITHUB")
+
+	require.Empty(t, api.created)
+	require.Empty(t, api.deleted)
+	require.Len(t, api.updated, 2)
+	require.Equal(t, "a.go", api.updated["ref-a"].FilePath)
+	require.Equal(t, "b.go", api.updated["ref-b"].FilePath)
+}
+
+func Test_processCodeReferences_duplicateReferenceKeys(t *testing.T) {
+	// references that share even the composite key (identical snippets of one
+	// flag in one file, or every hunk when contextLines < 0 hashes the empty
+	// string) must reconcile one-to-one instead of the duplicate row being
+	// dropped from the map and accumulating on the server run after run
+	api := &fakeApiClient{
+		existing: []bucketeer.CodeReference{
+			{ID: "ref-a", FeatureID: "my-flag", FilePath: "a.go", ContentHash: "h", RepositoryOwner: "owner", RepositoryName: "repo"},
+			{ID: "ref-b", FeatureID: "my-flag", FilePath: "a.go", ContentHash: "h", RepositoryOwner: "owner", RepositoryName: "repo"},
+		},
+		updated: map[string]bucketeer.CodeReference{},
+	}
+	opts := options.Options{RepoOwner: "owner", RepoName: "repo"}
+	refs := []bucketeer.ReferenceHunksRep{
+		{Path: "a.go", Hunks: []bucketeer.HunkRep{
+			{FlagKey: "my-flag", ContentHash: "h"},
+			{FlagKey: "my-flag", ContentHash: "h"},
+		}},
+	}
+
+	processCodeReferences(opts, api, refs, "main", "abc123", "GITHUB")
+
+	require.Empty(t, api.created)
+	require.Empty(t, api.deleted)
+	require.Len(t, api.updated, 2)
+	require.Contains(t, api.updated, "ref-a")
+	require.Contains(t, api.updated, "ref-b")
+}
+
+func Test_processCodeReferences_backfillsMissingFeatureID(t *testing.T) {
+	// the list endpoint is queried by featureId; if the server omits the
+	// field in response items, the queried flag must be backfilled so the
+	// reconciliation key still matches instead of every ref being recreated
+	// and the old rows deleted
+	api := &fakeApiClient{
+		existing: []bucketeer.CodeReference{
+			{ID: "ref-a", FilePath: "a.go", ContentHash: "h", RepositoryOwner: "owner", RepositoryName: "repo"},
+		},
+		updated: map[string]bucketeer.CodeReference{},
+	}
+	opts := options.Options{RepoOwner: "owner", RepoName: "repo"}
+	refs := []bucketeer.ReferenceHunksRep{
+		{Path: "a.go", Hunks: []bucketeer.HunkRep{{FlagKey: "my-flag", ContentHash: "h"}}},
+	}
+
+	processCodeReferences(opts, api, refs, "main", "abc123", "GITHUB")
+
+	require.Empty(t, api.created)
+	require.Empty(t, api.deleted)
+	require.Len(t, api.updated, 1)
+	require.Contains(t, api.updated, "ref-a")
+}
+
 // func Test_calculateStaleBranches(t *testing.T) {
 // 	specs := []struct {
 // 		name           string