Skip to content

Commit 197adae

Browse files
committed
Add support for mapped repository names in use_repo_add
Implement support for three types of repository specifications: - Simple names: foo -> use_repo(ext, "foo") - Mapped names with valid identifiers: bar=baz -> use_repo(ext, bar = "baz") - Invalid identifiers use dict unpacking: foo.2=foo -> use_repo(ext, **{"foo.2": "foo"}) This follows the Bazel documentation recommendations for handling repository names that are not valid Starlark identifiers by using dictionary unpacking syntax.
1 parent 6e81bc3 commit 197adae

3 files changed

Lines changed: 521 additions & 13 deletions

File tree

buildozer/README.md

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,11 @@ The following commands only apply to `MODULE.bazel` files (e.g. the target
172172
* `use_repo_add <use_extension variable name> <repo(s)>`:
173173
Ensures that the given repositories are imported via `use_repo` for the
174174
extension for which the given top-level variable contains the return value
175-
of a `use_extension` call.
175+
of a `use_extension` call. Repositories can be specified as:
176+
- Simple names: `foo` generates `use_repo(ext, "foo")`
177+
- Mapped names with valid identifiers: `bar=baz` generates `use_repo(ext, bar = "baz")`
178+
- Mapped names with invalid identifiers (containing dots, hyphens, etc.):
179+
`foo.2=foo` generates `use_repo(ext, **{"foo.2": "foo"})`
176180
* `use_repo_remove <use_extension variable name> <repo(s)>`:
177181
Ensures that the given repositories are *not* imported via `use_repo` for
178182
the extension for which the given top-level variable contains the return
@@ -181,7 +185,9 @@ The following commands only apply to `MODULE.bazel` files (e.g. the target
181185
Ensures that the given repositories generated by the given extension are
182186
imported via `use_repo`. If the `dev` argument is given, extension usages
183187
with `dev_dependency = True` will be considered instead. Extension usages
184-
with `isolated = True` are ignored.
188+
with `isolated = True` are ignored. Supports the same repository name
189+
formats as the first variant (simple, mapped with valid identifiers, and
190+
mapped with invalid identifiers).
185191
* `use_repo_remove [dev] <extension .bzl file> <extension name> <repo(s)>`:
186192
Ensures that the given repositories generated by the given extension are
187193
*not* imported via `use_repo`. If the `dev` argument is given, extension

edit/bzlmod/bzlmod.go

Lines changed: 262 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package bzlmod
1919

2020
import (
2121
"path"
22+
"strings"
2223

2324
"github.com/bazelbuild/buildtools/build"
2425
"github.com/bazelbuild/buildtools/labels"
@@ -118,10 +119,20 @@ func NewUseRepo(f *build.File, proxies []string) (*build.File, *build.CallExpr)
118119
return &build.File{Path: f.Path, Comments: f.Comments, Stmt: stmt, Type: build.TypeModule}, useRepo
119120
}
120121

122+
// repoToAdd represents a repository to be added to a use_repo call.
123+
type repoToAdd struct {
124+
key string
125+
value string
126+
isMapping bool
127+
isValidIdent bool
128+
}
129+
121130
// AddRepoUsages adds the given repos to the given use_repo calls without introducing duplicate
122131
// arguments.
123132
// useRepos must not be empty.
124-
// Keyword arguments are preserved but adding them is currently not supported.
133+
// Keyword arguments are preserved and can be added using the syntax "key=value".
134+
// For invalid identifiers (e.g., containing dots), dict unpacking syntax is used.
135+
// If a mapping conflicts with an existing one (same key or same value), the existing one is replaced.
125136
func AddRepoUsages(useRepos []*build.CallExpr, repos ...string) {
126137
if len(repos) == 0 {
127138
return
@@ -130,25 +141,178 @@ func AddRepoUsages(useRepos []*build.CallExpr, repos ...string) {
130141
panic("useRepos must not be empty")
131142
}
132143

133-
seen := make(map[string]struct{})
144+
// Parse all repos to add and track which keys/values are being added
145+
var reposToAdd []repoToAdd
146+
mappingValuesToAdd := make(map[string]struct{}) // Values being added as mappings
147+
keysToAdd := make(map[string]struct{})
148+
149+
for _, repo := range repos {
150+
key, value, isMapping := parseRepoMapping(repo)
151+
152+
// Simplify foo=foo to just foo (positional argument)
153+
if isMapping && key == value {
154+
isMapping = false
155+
}
156+
157+
validIdent := isValidIdentifier(key)
158+
159+
reposToAdd = append(reposToAdd, repoToAdd{
160+
key: key,
161+
value: value,
162+
isMapping: isMapping,
163+
isValidIdent: validIdent,
164+
})
165+
166+
if isMapping {
167+
mappingValuesToAdd[value] = struct{}{}
168+
keysToAdd[key] = struct{}{}
169+
}
170+
}
171+
172+
// Track existing values to detect duplicates when adding positional args
173+
existingValues := collectExistingValues(useRepos)
174+
175+
// Remove conflicting arguments from all use_repo calls
176+
removeConflictingRepos(useRepos, mappingValuesToAdd, keysToAdd)
177+
178+
// Add new repos to the last use_repo call
179+
lastUseRepo := getLastUseRepo(useRepos)
180+
addNewRepos(lastUseRepo, reposToAdd, existingValues)
181+
}
182+
183+
// collectExistingValues returns a set of all repository values currently referenced in use_repo calls.
184+
func collectExistingValues(useRepos []*build.CallExpr) map[string]struct{} {
185+
existingValues := make(map[string]struct{})
134186
for _, useRepo := range useRepos {
135187
if len(useRepo.List) == 0 {
136-
// Invalid use_repo call, skip.
137188
continue
138189
}
139190
for _, arg := range useRepo.List[1:] {
140-
seen[repoFromUseRepoArg(arg)] = struct{}{}
191+
if val := repoFromUseRepoArg(arg); val != "" {
192+
existingValues[val] = struct{}{}
193+
}
141194
}
142195
}
196+
return existingValues
197+
}
143198

144-
lastUseRepo := getLastUseRepo(useRepos)
145-
for _, repo := range repos {
146-
if _, ok := seen[repo]; ok {
199+
// removeConflictingRepos removes arguments from use_repo calls that conflict with repos being added.
200+
func removeConflictingRepos(useRepos []*build.CallExpr, mappingValuesToAdd, keysToAdd map[string]struct{}) {
201+
for _, useRepo := range useRepos {
202+
if len(useRepo.List) == 0 {
147203
continue
148204
}
149-
// Sorting of use_repo arguments is handled by Buildify.
150-
// TODO: Add a keyword argument instead if repo is of the form "key=value".
151-
lastUseRepo.List = append(lastUseRepo.List, &build.StringExpr{Value: repo})
205+
206+
var newArgs []build.Expr
207+
var dictToUpdate *build.DictExpr
208+
209+
for _, arg := range useRepo.List[1:] {
210+
shouldKeep := true
211+
212+
switch arg := arg.(type) {
213+
case *build.StringExpr:
214+
// Positional argument: only remove if a mapping is being added with this value
215+
if _, conflicts := mappingValuesToAdd[arg.Value]; conflicts {
216+
shouldKeep = false
217+
}
218+
case *build.AssignExpr:
219+
// Keyword argument: remove if same key is being added
220+
if ident, ok := arg.LHS.(*build.Ident); ok {
221+
if _, conflicts := keysToAdd[ident.Name]; conflicts {
222+
shouldKeep = false
223+
}
224+
}
225+
// Or if same value is being added as a mapping (replacement)
226+
if str, ok := arg.RHS.(*build.StringExpr); ok {
227+
if _, conflicts := mappingValuesToAdd[str.Value]; conflicts {
228+
shouldKeep = false
229+
}
230+
}
231+
case *build.UnaryExpr:
232+
// Dict unpacking: we'll handle this separately
233+
if arg.Op == "**" {
234+
if dict, ok := arg.X.(*build.DictExpr); ok {
235+
dictToUpdate = dict
236+
shouldKeep = false // Remove from list, we'll re-add after filtering
237+
}
238+
}
239+
}
240+
241+
if shouldKeep {
242+
newArgs = append(newArgs, arg)
243+
}
244+
}
245+
246+
// Filter dict entries if we have a dict to update
247+
if dictToUpdate != nil {
248+
filteredDictEntries := filterDictEntries(dictToUpdate, mappingValuesToAdd, keysToAdd)
249+
// Only keep dict if it still has entries after filtering
250+
if len(filteredDictEntries) > 0 {
251+
dictToUpdate.List = filteredDictEntries
252+
newArgs = append(newArgs, &build.UnaryExpr{Op: "**", X: dictToUpdate})
253+
}
254+
}
255+
256+
useRepo.List = append(useRepo.List[:1], newArgs...)
257+
}
258+
}
259+
260+
// filterDictEntries removes dict entries that conflict with repos being added.
261+
// Filters in-place to reuse the underlying array for better memory efficiency.
262+
func filterDictEntries(dict *build.DictExpr, mappingValuesToAdd, keysToAdd map[string]struct{}) []*build.KeyValueExpr {
263+
n := 0
264+
for _, kv := range dict.List {
265+
shouldKeep := true
266+
// Check if key conflicts
267+
if keyStr, ok := kv.Key.(*build.StringExpr); ok {
268+
if _, conflicts := keysToAdd[keyStr.Value]; conflicts {
269+
shouldKeep = false
270+
}
271+
}
272+
// Check if value conflicts with a mapping being added
273+
if valStr, ok := kv.Value.(*build.StringExpr); ok {
274+
if _, conflicts := mappingValuesToAdd[valStr.Value]; conflicts {
275+
shouldKeep = false
276+
}
277+
}
278+
if shouldKeep {
279+
dict.List[n] = kv
280+
n++
281+
}
282+
}
283+
return dict.List[:n]
284+
}
285+
286+
// addNewRepos adds new repository arguments to the given use_repo call.
287+
func addNewRepos(useRepo *build.CallExpr, reposToAdd []repoToAdd, existingValues map[string]struct{}) {
288+
var dictEntries []*build.KeyValueExpr
289+
290+
for _, repo := range reposToAdd {
291+
if !repo.isMapping {
292+
// Simple positional argument: use_repo(ext, "foo")
293+
// Skip if already exists (to preserve existing mappings)
294+
if _, exists := existingValues[repo.value]; !exists {
295+
useRepo.List = append(useRepo.List, &build.StringExpr{Value: repo.value})
296+
}
297+
} else if repo.isValidIdent {
298+
// Valid identifier as keyword argument: use_repo(ext, bar = "baz")
299+
useRepo.List = append(useRepo.List, &build.AssignExpr{
300+
LHS: &build.Ident{Name: repo.key},
301+
Op: "=",
302+
RHS: &build.StringExpr{Value: repo.value},
303+
})
304+
} else {
305+
// Invalid identifier, need dict unpacking: use_repo(ext, **{"foo.2": "foo"})
306+
dictEntries = append(dictEntries, &build.KeyValueExpr{
307+
Key: &build.StringExpr{Value: repo.key},
308+
Value: &build.StringExpr{Value: repo.value},
309+
})
310+
}
311+
}
312+
313+
// If we have dict entries, add or extend the **kwargs dict
314+
if len(dictEntries) > 0 {
315+
addOrExtendDictUnpack(useRepo, dictEntries)
152316
}
153317
}
154318

@@ -432,3 +596,91 @@ func collectApparentNamesAndIncludes(f *build.File) (map[string]string, []string
432596

433597
return apparentNames, includeLabels
434598
}
599+
600+
// parseRepoMapping parses a repo string which may be in the form "key=value" or just "value".
601+
// Returns (key, value, isMapping) where isMapping indicates if it was a mapping.
602+
func parseRepoMapping(repo string) (key, value string, isMapping bool) {
603+
key, value, found := strings.Cut(repo, "=")
604+
if found {
605+
return key, value, true
606+
}
607+
return "", repo, false
608+
}
609+
610+
// starlarkReservedKeywords contains Starlark reserved keywords that cannot be used as identifiers.
611+
var starlarkReservedKeywords = map[string]bool{
612+
"and": true,
613+
"break": true,
614+
"continue": true,
615+
"def": true,
616+
"elif": true,
617+
"else": true,
618+
"for": true,
619+
"if": true,
620+
"in": true,
621+
"lambda": true,
622+
"load": true,
623+
"not": true,
624+
"or": true,
625+
"pass": true,
626+
"return": true,
627+
"while": true,
628+
"False": true,
629+
"None": true,
630+
"True": true,
631+
}
632+
633+
// isValidIdentifier checks if a string is a valid Starlark identifier.
634+
// Per Starlark rules: must start with ASCII letter (a-z, A-Z) or underscore,
635+
// followed by ASCII letters, digits (0-9), or underscores, and must not be a reserved keyword.
636+
// This matches Bazel's Lexer.scanIdentifier behavior.
637+
func isValidIdentifier(s string) bool {
638+
if len(s) == 0 {
639+
return false
640+
}
641+
642+
// Check if it's a reserved keyword
643+
if starlarkReservedKeywords[s] {
644+
return false
645+
}
646+
647+
for i := 0; i < len(s); i++ {
648+
c := s[i]
649+
if i == 0 {
650+
// First character must be ASCII letter or underscore
651+
if !(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '_') {
652+
return false
653+
}
654+
} else {
655+
// Subsequent characters must be ASCII letter, digit, or underscore
656+
if !(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9') || c == '_') {
657+
return false
658+
}
659+
}
660+
}
661+
662+
return true
663+
}
664+
665+
// addOrExtendDictUnpack adds dict entries to an existing **kwargs dict unpacking expression,
666+
// or creates a new one if none exists.
667+
func addOrExtendDictUnpack(useRepo *build.CallExpr, newEntries []*build.KeyValueExpr) {
668+
// Look for an existing **dict unpacking expression
669+
for _, arg := range useRepo.List {
670+
if unary, ok := arg.(*build.UnaryExpr); ok && unary.Op == "**" {
671+
if dict, ok := unary.X.(*build.DictExpr); ok {
672+
// Found existing dict unpacking, add new entries to it
673+
dict.List = append(dict.List, newEntries...)
674+
return
675+
}
676+
}
677+
}
678+
679+
// No existing **dict found, create a new one
680+
useRepo.List = append(useRepo.List, &build.UnaryExpr{
681+
Op: "**",
682+
X: &build.DictExpr{
683+
List: newEntries,
684+
},
685+
})
686+
}

0 commit comments

Comments
 (0)