feat: merge upstream improvements

aliases/alias.go

@@ -7,34 +7,50 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/bmatcuk/doublestar/v4"
 	"github.com/iancoleman/strcase"
+	regexp "github.com/wasilibs/go-re2"
 
 	"github.com/bucketeer-io/code-refs/internal/helpers"
 	"github.com/bucketeer-io/code-refs/internal/log"
 	"github.com/bucketeer-io/code-refs/internal/validation"
 	"github.com/bucketeer-io/code-refs/options"
 )
 
+var globCache = make(map[string][]string)
+var regexCache = make(map[string]*regexp.Regexp)
+
 // GenerateAliases returns a map of flag keys to aliases based on config.
 func GenerateAliases(flags []string, aliases []options.Alias, dir string) (map[string][]string, error) {
 	allFileContents, err := processFileContent(aliases, dir)
 	if err != nil {
 		return nil, err
 	}
 
+	// Filepattern contents concatenated once per alias; rebuilding them for
+	// every flag key multiplies peak memory by the number of flags.
+	patternContents := make(map[int]string, len(aliases))
+
 	ret := make(map[string][]string, len(flags))
 	for _, flag := range flags {
 		for i, a := range aliases {
 			if a.Name == "" {
 				a.Name = strconv.Itoa(i)
 			}
-			flagAliases, err := generateAlias(a, flag, dir, allFileContents)
+			if a.Type.Canonical() == options.FilePattern {
+				if _, ok := patternContents[i]; !ok {
+					contents, err := concatFilePatternContents(a, dir, allFileContents)
+					if err != nil {
+						return nil, err
+					}
+					patternContents[i] = contents
+				}
+			}
+			flagAliases, err := generateAlias(a, flag, dir, patternContents[i])
 			if err != nil {
 				return nil, err
 			}
@@ -45,12 +61,12 @@ func GenerateAliases(flags []string, aliases []options.Alias, dir string) (map[s
 	return ret, nil
 }
 
-func generateAlias(a options.Alias, flag, dir string, allFileContents FileContentsMap) (ret []string, err error) {
+func generateAlias(a options.Alias, flag, dir string, patternContents string) (ret []string, err error) {
 	switch a.Type.Canonical() {
 	case options.Literal:
 		ret = a.Flags[flag]
 	case options.FilePattern:
-		ret, err = GenerateAliasesFromFilePattern(a, flag, dir, allFileContents)
+		ret = matchFilePatternAliases(a, flag, patternContents)
 	case options.Command:
 		ret, err = GenerateAliasesFromCommand(a, flag, dir)
 	default:
@@ -84,33 +100,48 @@ func GenerateNamingConventionAlias(a options.Alias, flag string) (alias string,
 }
 
 func GenerateAliasesFromFilePattern(a options.Alias, flag, dir string, allFileContents FileContentsMap) ([]string, error) {
-	ret := []string{}
-	// Concatenate the contents of all files into a single byte array to be matched by specified patterns
-	fileContents := []byte{}
+	fileContents, err := concatFilePatternContents(a, dir, allFileContents)
+	if err != nil {
+		return nil, err
+	}
+	return matchFilePatternAliases(a, flag, fileContents), nil
+}
+
+// concatFilePatternContents concatenates the contents of all files matched by
+// the alias paths into a single string to be matched by specified patterns
+func concatFilePatternContents(a options.Alias, dir string, allFileContents FileContentsMap) (string, error) {
+	var sb strings.Builder
 	for _, path := range a.Paths {
-		absGlob := filepath.Join(dir, path)
-		matches, err := doublestar.FilepathGlob(absGlob)
+		matches, err := cacheFilepathGlob(dir, path)
 		if err != nil {
-			return nil, fmt.Errorf("filepattern '%s': could not process path glob '%s'", a.Name, absGlob)
+			return "", fmt.Errorf("filepattern '%s': could not process path glob '%s'", a.Name, path)
 		}
 		for _, match := range matches {
 			if pathFileContents := allFileContents[match]; len(pathFileContents) > 0 {
-				fileContents = append(fileContents, pathFileContents...)
+				sb.Write(pathFileContents)
 			}
 		}
 	}
+	return sb.String(), nil
+}
 
+func matchFilePatternAliases(a options.Alias, flag, fileContents string) []string {
+	ret := []string{}
 	for _, p := range a.Patterns {
-		pattern := regexp.MustCompile(strings.ReplaceAll(p, "FLAG_KEY", flag))
-		results := pattern.FindAllStringSubmatch(string(fileContents), -1)
+		patternStr := strings.ReplaceAll(p, "FLAG_KEY", flag)
+		pattern, ok := regexCache[patternStr]
+		if !ok {
+			pattern = regexp.MustCompile(patternStr)
+			regexCache[patternStr] = pattern
+		}
+		results := pattern.FindAllStringSubmatch(fileContents, -1)
 		for _, res := range results {
 			if len(res) > 1 {
 				ret = append(ret, res[1:]...)
 			}
 		}
 	}
-
-	return ret, nil
+	return ret
 }
 
 func GenerateAliasesFromCommand(a options.Alias, flag, dir string) ([]string, error) {
@@ -157,13 +188,12 @@ func processFileContent(aliases []options.Alias, dir string) (FileContentsMap, e
 
 		paths := []string{}
 		for _, glob := range a.Paths {
-			absGlob := filepath.Join(dir, glob)
-			matches, err := doublestar.FilepathGlob(absGlob)
+			matches, err := cacheFilepathGlob(dir, glob)
 			if err != nil {
-				return nil, fmt.Errorf("filepattern '%s': could not process path glob '%s'", aliasId, absGlob)
+				return nil, fmt.Errorf("filepattern '%s': could not process path glob '%s'", aliasId, glob)
 			}
 			if matches == nil {
-				log.Info.Printf("filepattern '%s': no matching files found for alias path glob '%s'", aliasId, absGlob)
+				log.Info.Printf("filepattern '%s': no matching files found for alias path glob '%s'", aliasId, glob)
 			}
 			paths = append(paths, matches...)
 		}
@@ -188,3 +218,20 @@ func processFileContent(aliases []options.Alias, dir string) (FileContentsMap, e
 	}
 	return allFileContents, nil
 }
+
+func cacheFilepathGlob(dir, glob string) ([]string, error) {
+	absGlob := filepath.Join(dir, glob)
+	if cachedMatches, ok := globCache[absGlob]; ok {
+		return cachedMatches, nil
+	}
+
+	matches, err := doublestar.FilepathGlob(absGlob)
+	if err != nil {
+		// Don't cache failures: a cache hit always returns a nil error, which
+		// would silently mask the original glob error on any later call.
+		return nil, err
+	}
+	globCache[absGlob] = matches
+
+	return matches, nil
+}

aliases/alias_test.go

@@ -93,17 +93,17 @@ func Test_GenerateAliases(t *testing.T) {
 			name:  "file exact pattern",
 			flags: slice(testFlagKey),
 			aliases: []o.Alias{
-				fileExactPattern(testFlagKey),
+				fileExactPattern(),
 			},
 			want: map[string][]string{testFlagKey: slice("SOME_FLAG")},
 		},
 		{
 			name:  "file wildcard pattern",
 			flags: slice(testFlagKey, testWildFlagKey),
 			aliases: []o.Alias{
-				fileWildPattern(testFlagKey),
+				fileWildPattern(),
 			},
-			want: map[string][]string{testWildFlagKey: slice("WILD_FLAG", "WILD_FLAG_SECOND_ALIAS"), testFlagKey: slice("SOME_FLAG")},
+			want: map[string][]string{testWildFlagKey: slice("WILD_FLAG", "WILD_FLAG_SECOND_ALIAS", "ABSOLUTELY_WILD"), testFlagKey: slice("SOME_FLAG")},
 		},
 		{
 			name:  "command",
@@ -198,6 +198,24 @@ func Test_processFileContent(t *testing.T) {
 	}
 }
 
+func Test_GenerateAliasesFromFilePattern(t *testing.T) {
+	expectedAliases := []string{"WILD_FLAG", "WILD_FLAG_SECOND_ALIAS", "ABSOLUTELY_WILD"}
+
+	fileContents := map[string][]byte{
+		"testdata/alias_test.txt":                                  []byte("SOME_FLAG = 'someFlag'"),
+		"testdata/wild/alias_test.txt":                             []byte("WILD_FLAG = 'wildFlag'"),
+		"testdata/wild/nested-wild/alias_test.txt":                 []byte("WILD_FLAG_SECOND_ALIAS = 'wildFlag'"),
+		"testdata/wild/nested-wild/another/another/alias_test.txt": []byte("ABSOLUTELY_WILD = 'wildFlag'"),
+	}
+
+	alias := fileWildPattern()
+
+	aliases, err := GenerateAliasesFromFilePattern(alias, testWildFlagKey, "", fileContents)
+	require.NoError(t, err)
+
+	assert.ElementsMatch(t, expectedAliases, aliases)
+}
+
 func slice(args ...string) []string {
 	return args
 }
@@ -222,15 +240,15 @@ func literal(flags []string) o.Alias {
 	return a
 }
 
-func fileExactPattern(flag string) o.Alias {
+func fileExactPattern() o.Alias {
 	a := alias(o.FilePattern)
 	pattern := "(\\w+)\\s= 'FLAG_KEY'"
 	a.Paths = []string{"testdata/alias_test.txt"}
 	a.Patterns = []string{pattern}
 	return a
 }
 
-func fileWildPattern(flag string) o.Alias {
+func fileWildPattern() o.Alias {
 	a := alias(o.FilePattern)
 	pattern := "(\\w+)\\s= 'FLAG_KEY'"
 	a.Paths = []string{"testdata/**/*.txt"}

aliases/testdata/wild/nested-wild/another/another/alias_test.txt

@@ -0,0 +1 @@
+ABSOLUTELY_WILD = 'wildFlag'

docs/CONFIGURATION.md

@@ -57,6 +57,15 @@ Flags:
 
 All command line flags are available as environment variables following the "upper snake case" format, with a prefix of `BUCKETEER_`. For example, the command line option `apiKey` may be set as an environment variable e.g. `export BUCKETEER_APIKEY = 'myTestToken'`. For multiple API keys, you can use comma-separated values: `export BUCKETEER_APIKEY = 'key1,key2,key3'`.
 
+### GitHub Actions
+
+When running inside GitHub Actions (`GITHUB_ACTIONS=true`), remote git operations used by branch pruning authenticate with the workflow's `GITHUB_TOKEN` environment variable if it is set. Provide it when scanning a private repository with `prune` enabled (the default), otherwise listing remote branches will fail:
+
+```yaml
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+```
+
 ## YAML
 
 A YAML file may be used to specify most command line arguments, as well as a number of additional options for advanced usage of `bucketeer-find-code-refs`. The configuration YAML file should be stored as `${dir}/.bucketeer/coderefs.yaml`.

go.mod

@@ -3,20 +3,20 @@ module github.com/bucketeer-io/code-refs
 go 1.25.0
 
 require (
-	github.com/bmatcuk/doublestar/v4 v4.7.1
-	github.com/go-git/go-git/v5 v5.13.1
+	github.com/bmatcuk/doublestar/v4 v4.9.1
+	github.com/go-git/go-git/v5 v5.16.3
 	github.com/iancoleman/strcase v0.3.0
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00
 	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9
 	github.com/spf13/cobra v1.10.1
 	github.com/spf13/pflag v1.0.10
-	github.com/spf13/viper v1.19.0
+	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
-	golang.org/x/tools v0.45.0 // indirect
 )
 
 require (
 	github.com/cenkalti/backoff/v4 v4.3.0
+	github.com/wasilibs/go-re2 v1.10.0
 	github.com/zricethezav/gitleaks/v8 v8.30.1
 )
 
@@ -26,84 +26,77 @@ require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/Masterminds/sprig/v3 v3.3.0 // indirect
-	github.com/Microsoft/go-winio v0.6.1 // indirect
-	github.com/ProtonMail/go-crypto v1.1.3 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/ProtonMail/go-crypto v1.1.6 // indirect
 	github.com/STARRY-S/zip v0.2.3 // indirect
 	github.com/andybalholm/brotli v1.2.1 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/bodgit/plumbing v1.3.0 // indirect
 	github.com/bodgit/sevenzip v1.6.2 // indirect
 	github.com/bodgit/windows v1.0.1 // indirect
 	github.com/charmbracelet/lipgloss v0.5.0 // indirect
-	github.com/cloudflare/circl v1.3.7 // indirect
-	github.com/cyphar/filepath-securejoin v0.3.6 // indirect
+	github.com/cloudflare/circl v1.6.1 // indirect
+	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fatih/semgroup v1.2.0 // indirect
-	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/gitleaks/go-gitdiff v0.9.1 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
-	github.com/go-git/go-billy/v5 v5.6.1 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/go-git/go-billy/v5 v5.6.2 // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/h2non/filetype v1.1.3 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.18.6 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
-	github.com/magiconair/properties v1.8.9 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mholt/archives v0.1.6-0.20260429171216-ef71b7a32fae // indirect
 	github.com/mikelolasagasti/xz v1.0.1 // indirect
 	github.com/minio/minlz v1.1.1 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 // indirect
 	github.com/muesli/termenv v0.15.1 // indirect
 	github.com/nwaples/rardecode/v2 v2.2.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.3.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.26 // indirect
-	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/pjbgf/sha1cd v0.3.2 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/rs/zerolog v1.33.0 // indirect
-	github.com/sagikazarmark/locafero v0.7.0 // indirect
-	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sagikazarmark/locafero v0.11.0 // indirect
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
-	github.com/skeema/knownhosts v1.3.0 // indirect
+	github.com/skeema/knownhosts v1.3.1 // indirect
 	github.com/sorairolake/lzip-go v0.3.8 // indirect
-	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/stretchr/objx v0.5.3 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/tetratelabs/wazero v1.12.0 // indirect
 	github.com/ulikunitz/xz v0.5.15 // indirect
-	github.com/wasilibs/go-re2 v1.9.0 // indirect
 	github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	go4.org v0.0.0-20260112195520-a5071408f32f // indirect
 	golang.org/x/crypto v0.53.0 // indirect
 	golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa // indirect
-	golang.org/x/mod v0.36.0 // indirect
 	golang.org/x/net v0.55.0 // indirect
 	golang.org/x/sync v0.21.0 // indirect
 	golang.org/x/sys v0.46.0 // indirect
 	golang.org/x/text v0.38.0 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )