[pull] main from bazel-contrib:main

CHANGELOG.md

@@ -87,6 +87,7 @@ END_UNRELEASED_TEMPLATE
   ({gh-issue}`3043`).
 * (pypi) The pipstar `defaults` configuration now supports any custom platform
   name.
+* Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle.
 
 {#v0-0-0-added}
 ### Added
@@ -105,6 +106,8 @@ END_UNRELEASED_TEMPLATE
     * 3.12.11
     * 3.13.5
     * 3.14.0b3
+* (gazelle) New directive `gazelle:python_generate_proto`; when `true`,
+  Gazelle generates `py_proto_library` rules for `proto_library`. `false` by default.
 
 {#v0-0-0-removed}
 ### Removed

examples/bzlmod/py_proto_library/BUILD.bazel

@@ -6,15 +6,15 @@ py_test(
     srcs = ["test.py"],
     main = "test.py",
     deps = [
-        "//py_proto_library/example.com/proto:pricetag_proto_py_pb2",
+        "//py_proto_library/example.com/proto:pricetag_py_pb2",
     ],
 )
 
 py_test(
     name = "message_test",
     srcs = ["message_test.py"],
     deps = [
-        "//py_proto_library/example.com/another_proto:message_proto_py_pb2",
+        "//py_proto_library/example.com/another_proto:message_py_pb2",
     ],
 )
 

examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel

@@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
 load("@rules_python//python:proto.bzl", "py_proto_library")
 
 py_proto_library(
-    name = "message_proto_py_pb2",
+    name = "message_py_pb2",
     visibility = ["//visibility:public"],
     deps = [":message_proto"],
 )

examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel

@@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
 load("@rules_python//python:proto.bzl", "py_proto_library")
 
 py_proto_library(
-    name = "pricetag_proto_py_pb2",
+    name = "pricetag_py_pb2",
     visibility = ["//visibility:public"],
     deps = [":pricetag_proto"],
 )

examples/py_proto_library/BUILD.bazel

@@ -5,14 +5,14 @@ py_test(
     srcs = ["test.py"],
     main = "test.py",
     deps = [
-        "//example.com/proto:pricetag_proto_py_pb2",
+        "//example.com/proto:pricetag_py_pb2",
     ],
 )
 
 py_test(
     name = "message_test",
     srcs = ["message_test.py"],
     deps = [
-        "//example.com/another_proto:message_proto_py_pb2",
+        "//example.com/another_proto:message_py_pb2",
     ],
 )

examples/py_proto_library/example.com/another_proto/BUILD.bazel

@@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
 load("@rules_python//python:proto.bzl", "py_proto_library")
 
 py_proto_library(
-    name = "message_proto_py_pb2",
+    name = "message_py_pb2",
     visibility = ["//visibility:public"],
     deps = [":message_proto"],
 )

examples/py_proto_library/example.com/proto/BUILD.bazel

@@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
 load("@rules_python//python:proto.bzl", "py_proto_library")
 
 py_proto_library(
-    name = "pricetag_proto_py_pb2",
+    name = "pricetag_py_pb2",
     visibility = ["//visibility:public"],
     deps = [":pricetag_proto"],
 )

gazelle/README.md

@@ -224,6 +224,8 @@ Python-specific directives are as follows:
 | Controls whether Gazelle resolves dependencies for import statements that use paths relative to the current package. Can be "true" or "false".|
 | `# gazelle:python_generate_pyi_deps`                                                                                                                                                                                                                                                            | `false` |
 | Controls whether to generate a separate `pyi_deps` attribute for type-checking dependencies or merge them into the regular `deps` attribute. When `false` (default), type-checking dependencies are merged into `deps` for backward compatibility. When `true`, generates separate `pyi_deps`. Imports in blocks with the format `if typing.TYPE_CHECKING:`/`if TYPE_CHECKING:` and type-only stub packages (eg. boto3-stubs) are recognized as type-checking dependencies. |
+| [`# gazelle:python_generate_proto`](#directive-python_generate_proto)                                                                                                                                                                                                                                                                | `false` |
+| Controls whether to generate a `py_proto_library` for each `proto_library` in the package. By default we load this rule from the `@protobuf` repository; use `gazelle:map_kind` if you need to load this from somewhere else. |
 
 #### Directive: `python_root`:
 
@@ -484,6 +486,41 @@ def py_test(name, main=None, **kwargs):
 )
 ```
 
+#### Directive: `python_generate_proto`:
+
+When `# gazelle:python_generate_proto true`, Gazelle will generate one
+`py_proto_library` for each `proto_library`, generating Python clients for
+protobuf in each package. By default this is turned off. Gazelle will also
+generate a load statement for the `py_proto_library` - attempting to detect
+the configured name for the `@protobuf` / `@com_google_protobuf` repo in your
+`MODULE.bazel`, and otherwise falling back to `@com_google_protobuf` for
+compatibility with `WORKSPACE`.
+
+For example, in a package with `# gazelle:python_generate_proto true` and a
+`foo.proto`, if you have both the proto extension and the Python extension
+loaded into Gazelle, you'll get something like:
+
+```starlark
+load("@protobuf//bazel:py_proto_library.bzl", "py_proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+# gazelle:python_generate_proto true
+
+proto_library(
+    name = "foo_proto",
+    srcs = ["foo.proto"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_proto_library(
+    name = "foo_py_pb2",
+    visibility = ["//:__subpackages__"],
+    deps = [":foo_proto"],
+)
+```
+
+When `false`, Gazelle will ignore any `py_proto_library`, including previously-generated or hand-created rules.
+
 ### Annotations
 
 *Annotations* refer to comments found _within Python files_ that configure how

gazelle/python/BUILD.bazel

@@ -34,6 +34,7 @@ go_library(
         "@bazel_gazelle//config:go_default_library",
         "@bazel_gazelle//label:go_default_library",
         "@bazel_gazelle//language:go_default_library",
+        "@bazel_gazelle//language/proto:go_default_library",
         "@bazel_gazelle//repo:go_default_library",
         "@bazel_gazelle//resolve:go_default_library",
         "@bazel_gazelle//rule:go_default_library",
@@ -91,7 +92,10 @@ gazelle_test(
 
 gazelle_binary(
     name = "gazelle_binary",
-    languages = [":python"],
+    languages = [
+        "@bazel_gazelle//language/proto",
+        ":python",
+    ],
     visibility = ["//visibility:public"],
 )
 

gazelle/python/configure.go

@@ -70,6 +70,7 @@ func (py *Configurer) KnownDirectives() []string {
 		pythonconfig.LabelNormalization,
 		pythonconfig.GeneratePyiDeps,
 		pythonconfig.ExperimentalAllowRelativeImports,
+		pythonconfig.GenerateProto,
 	}
 }
 
@@ -237,6 +238,12 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) {
 				log.Fatal(err)
 			}
 			config.SetGeneratePyiDeps(v)
+		case pythonconfig.GenerateProto:
+			v, err := strconv.ParseBool(strings.TrimSpace(d.Value))
+			if err != nil {
+				log.Fatal(err)
+			}
+			config.SetGenerateProto(v)
 		}
 	}
 

gazelle/python/file_parser.go

@@ -144,6 +144,16 @@ func parseImportStatement(node *sitter.Node, code []byte) (Module, bool) {
 	return Module{}, false
 }
 
+// cleanImportString removes backslashes and all whitespace from the string.
+func cleanImportString(s string) string {
+	s = strings.ReplaceAll(s, "\r\n", "")
+	s = strings.ReplaceAll(s, "\\", "")
+	s = strings.ReplaceAll(s, " ", "")
+	s = strings.ReplaceAll(s, "\n", "")
+	s = strings.ReplaceAll(s, "\t", "")
+	return s
+}
+
 // parseImportStatements parses a node for import statements, returning true if the node is
 // an import statement. It updates FileParser.output.Modules with the `module` that the
 // import represents.
@@ -154,6 +164,8 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool {
 			if !ok {
 				continue
 			}
+			m.From = cleanImportString(m.From)
+			m.Name = cleanImportString(m.Name)
 			m.Filepath = p.relFilepath
 			m.TypeCheckingOnly = p.inTypeCheckingBlock
 			if strings.HasPrefix(m.Name, ".") {
@@ -163,6 +175,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool {
 		}
 	} else if node.Type() == sitterNodeTypeImportFromStatement {
 		from := node.Child(1).Content(p.code)
+		from = cleanImportString(from)
 		// If the import is from the current package, we don't need to add it to the modules i.e. from . import Class1.
 		// If the import is from a different relative package i.e. from .package1 import foo, we need to add it to the modules.
 		if from == "." {
@@ -175,6 +188,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool {
 			}
 			m.Filepath = p.relFilepath
 			m.From = from
+			m.Name = cleanImportString(m.Name)
 			m.Name = fmt.Sprintf("%s.%s", from, m.Name)
 			m.TypeCheckingOnly = p.inTypeCheckingBlock
 			p.output.Modules = append(p.output.Modules, m)

gazelle/python/file_parser_test.go

@@ -291,3 +291,95 @@ def example_function():
 		}
 	}
 }
+
+func TestParseImportStatements_MultilineWithBackslashAndWhitespace(t *testing.T) {
+	t.Parallel()
+	t.Run("multiline from import", func(t *testing.T) {
+		p := NewFileParser()
+		code := []byte(`from foo.bar.\
+    baz import (
+    Something,
+    AnotherThing
+)
+
+from foo\
+	.test import (
+    Foo,
+    Bar
+)
+`)
+		p.SetCodeAndFile(code, "", "test.py")
+		output, err := p.Parse(context.Background())
+		assert.NoError(t, err)
+		// Updated expected to match parser output
+		expected := []Module{
+			{
+				Name:       "foo.bar.baz.Something",
+				LineNumber: 3,
+				Filepath:   "test.py",
+				From:       "foo.bar.baz",
+			},
+			{
+				Name:       "foo.bar.baz.AnotherThing",
+				LineNumber: 4,
+				Filepath:   "test.py",
+				From:       "foo.bar.baz",
+			},
+			{
+				Name:       "foo.test.Foo",
+				LineNumber: 9,
+				Filepath:   "test.py",
+				From:       "foo.test",
+			},
+			{
+				Name:       "foo.test.Bar",
+				LineNumber: 10,
+				Filepath:   "test.py",
+				From:       "foo.test",
+			},
+		}
+		assert.ElementsMatch(t, expected, output.Modules)
+	})
+	t.Run("multiline import", func(t *testing.T) {
+		p := NewFileParser()
+		code := []byte(`import foo.bar.\
+    baz
+`)
+		p.SetCodeAndFile(code, "", "test.py")
+		output, err := p.Parse(context.Background())
+		assert.NoError(t, err)
+		// Updated expected to match parser output
+		expected := []Module{
+			{
+				Name:       "foo.bar.baz",
+				LineNumber: 1,
+				Filepath:   "test.py",
+				From:       "",
+			},
+		}
+		assert.ElementsMatch(t, expected, output.Modules)
+	})
+	t.Run("windows line endings", func(t *testing.T) {
+		p := NewFileParser()
+		code := []byte("from foo.bar.\r\n baz import (\r\n    Something,\r\n    AnotherThing\r\n)\r\n")
+		p.SetCodeAndFile(code, "", "test.py")
+		output, err := p.Parse(context.Background())
+		assert.NoError(t, err)
+		// Updated expected to match parser output
+		expected := []Module{
+			{
+				Name:       "foo.bar.baz.Something",
+				LineNumber: 3,
+				Filepath:   "test.py",
+				From:       "foo.bar.baz",
+			},
+			{
+				Name:       "foo.bar.baz.AnotherThing",
+				LineNumber: 4,
+				Filepath:   "test.py",
+				From:       "foo.bar.baz",
+			},
+		}
+		assert.ElementsMatch(t, expected, output.Modules)
+	})
+}

gazelle/python/generate.go

@@ -226,6 +226,10 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 	var result language.GenerateResult
 	result.Gen = make([]*rule.Rule, 0)
 
+	if cfg.GenerateProto() {
+		generateProtoLibraries(args, pythonProjectRoot, visibility, &result)
+	}
+
 	collisionErrors := singlylinkedlist.New()
 
 	appendPyLibrary := func(srcs *treeset.Set, pyLibraryTargetName string) {
@@ -551,3 +555,51 @@ func ensureNoCollision(file *rule.File, targetName, kind string) error {
 	}
 	return nil
 }
+
+func generateProtoLibraries(args language.GenerateArgs, pythonProjectRoot string, visibility []string, res *language.GenerateResult) {
+	// First, enumerate all the proto_library in this package.
+	var protoRuleNames []string
+	for _, r := range args.OtherGen {
+		if r.Kind() != "proto_library" {
+			continue
+		}
+		protoRuleNames = append(protoRuleNames, r.Name())
+	}
+	sort.Strings(protoRuleNames)
+
+	// Next, enumerate all the pre-existing py_proto_library in this package, so we can delete unnecessary rules later.
+	pyProtoRules := map[string]bool{}
+	if args.File != nil {
+		for _, r := range args.File.Rules {
+			if r.Kind() == "py_proto_library" {
+				pyProtoRules[r.Name()] = false
+			}
+		}
+	}
+
+	emptySiblings := treeset.Set{}
+	// Generate a py_proto_library for each proto_library.
+	for _, protoRuleName := range protoRuleNames {
+		pyProtoLibraryName := strings.TrimSuffix(protoRuleName, "_proto") + "_py_pb2"
+		pyProtoLibrary := newTargetBuilder(pyProtoLibraryKind, pyProtoLibraryName, pythonProjectRoot, args.Rel, &emptySiblings).
+			addVisibility(visibility).
+			addResolvedDependency(":" + protoRuleName).
+			generateImportsAttribute().build()
+
+		res.Gen = append(res.Gen, pyProtoLibrary)
+		res.Imports = append(res.Imports, pyProtoLibrary.PrivateAttr(config.GazelleImportsKey))
+		pyProtoRules[pyProtoLibrary.Name()] = true
+
+	}
+
+	// Finally, emit an empty rule for each pre-existing py_proto_library that we didn't already generate.
+	for ruleName, generated := range pyProtoRules {
+		if generated {
+			continue
+		}
+
+		emptyRule := newTargetBuilder(pyProtoLibraryKind, ruleName, pythonProjectRoot, args.Rel, &emptySiblings).build()
+		res.Empty = append(res.Empty, emptyRule)
+	}
+
+}