Add mypy support (#288)

* Add mypy support

* Add explicit version for mypy

* newer mypy version default

* align names with Ruby's .rbi

* add test

* declare Dockerfile variable in outer scope

* --mypy_grpc_out

* add to mypy to renovate config

* remove pip from final container

* Update .github/renovate.json5

Co-authored-by: Ido David <36866853+ido-namely@users.noreply.github.com>

* add file assertion

* add test for consistency of language and --with-pyi

* remove extra whitespace

Co-authored-by: Ido David <36866853+ido-namely@users.noreply.github.com>

Author: Max Jakob

.github/renovate.json5

@@ -180,6 +180,16 @@
       depNameTemplate: "debian",
       datasourceTemplate: "docker",
       versioningTemplate: "debian",
+    },
+    {
+      fileMatch: [
+        "variables.sh$"
+      ],
+      matchStrings: [
+        "MYPY_VERSION=\\$\\{MYPY_VERSION:-(?<currentValue>.*?)\\}\\n"
+      ],
+      depNameTemplate: "mypy-protobuf",
+      datasourceTemplate: "pypi",
     }
   ],
   packageRules: [

Dockerfile

@@ -14,6 +14,7 @@ ARG go_envoyproxy_pgv_version
 ARG go_mwitkow_gpv_version
 ARG go_protoc_gen_go_version
 ARG go_protoc_gen_go_grpc_version
+ARG mypy_version
 
 FROM golang:$go_version-$debian_version AS build
 
@@ -28,6 +29,7 @@ ARG go_mwitkow_gpv_version
 ARG uber_prototool_version
 ARG go_protoc_gen_go_version
 ARG go_protoc_gen_go_grpc_version
+ARG mypy_version
 
 
 RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
@@ -42,7 +44,8 @@ RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
     autoconf \
     zlib1g-dev \
     libssl-dev \
-    clang
+    clang \
+    python3-pip
 
 WORKDIR /tmp
 RUN git clone --depth 1 --shallow-submodules -b v$grpc_version.x --recursive https://github.com/grpc/grpc && \
@@ -110,6 +113,9 @@ RUN curl -fsSL "https://github.com/grpc/grpc-web/releases/download/${grpc_web_ve
     -o /tmp/grpc_web_plugin && \
     chmod +x /tmp/grpc_web_plugin
 
+# Add mypy support
+RUN pip3 install -t /opt/mypy-protobuf mypy-protobuf==${mypy_version}
+
 FROM debian:$debian_version-slim AS protoc-all
 
 ARG grpc_version
@@ -173,6 +179,11 @@ COPY --from=build /go/pkg/mod/github.com/envoyproxy/protoc-gen-validate@v${go_en
 
 COPY --from=build /go/pkg/mod/github.com/mwitkow/go-proto-validators@v${go_mwitkow_gpv_version}/ /opt/include/github.com/mwitkow/go-proto-validators/
 
+# Copy mypy
+COPY --from=build /opt/mypy-protobuf/ /opt/mypy-protobuf/
+RUN mv /opt/mypy-protobuf/bin/* /usr/local/bin/
+ENV PYTHONPATH="${PYTHONPATH}:/opt/mypy-protobuf/"
+
 ADD all/entrypoint.sh /usr/local/bin
 RUN chmod +x /usr/local/bin/entrypoint.sh
 

all/entrypoint.sh

@@ -16,6 +16,7 @@ printUsage() {
     echo " --lint CHECKS                                    Enable linting protoc-lint (CHECKS are optional - see https://github.com/ckaznocha/protoc-gen-lint#optional-checks)"
     echo " --with-gateway                                   Generate grpc-gateway files (experimental)."
     echo " --with-docs FORMAT                               Generate documentation (FORMAT is optional - see https://github.com/pseudomuto/protoc-gen-doc#invoking-the-plugin)"
+    echo " --with-pyi                                       Generate mypy stub files (.pyi files) - see https://github.com/nipunn1313/mypy-protobuf"
     echo " --with-rbi                                       Generate Sorbet type declaration files (.rbi files) - see https://github.com/coinbase/protoc-gen-rbi"
     echo " --with-typescript                                Generate TypeScript declaration files (.d.ts files) - see https://github.com/improbable-eng/ts-protoc-gen#readme"
     echo " --with-validator                                 Generate validations for (${VALIDATOR_SUPPORTED_LANGUAGES[@]}) - see https://github.com/envoyproxy/protoc-gen-validate"
@@ -50,6 +51,7 @@ GEN_VALIDATOR=false
 VALIDATOR_SUPPORTED_LANGUAGES=("go" "gogo" "cpp" "java" "python")
 DOCS_FORMAT="html,index.html"
 GEN_RBI=false
+GEN_PYI=false
 GEN_TYPESCRIPT=false
 LINT=false
 LINT_CHECKS=""
@@ -134,6 +136,10 @@ while test $# -gt 0; do
             GEN_RBI=true
             shift
             ;;
+        --with-pyi)
+            GEN_PYI=true
+            shift
+            ;;
         --with-typescript)
             GEN_TYPESCRIPT=true
             shift
@@ -296,6 +302,11 @@ if [[ "$GEN_RBI" == true && "$GEN_LANG" != "ruby" ]]; then
     exit 1
 fi
 
+if [[ "$GEN_PYI" == true && "$GEN_LANG" != "python" ]]; then
+    echo "Generating .pyi mypy stub files is a Python specific option."
+    exit 1
+fi
+
 if [[ "$GEN_TYPESCRIPT" == true && "$GEN_LANG" != "node" ]]; then
     echo "Generating TypeScript declaration files is Node specific."
     exit 1
@@ -432,6 +443,10 @@ if [[ $GEN_RBI == true ]]; then
     GEN_STRING="$GEN_STRING --rbi_out=$OUT_DIR"
 fi
 
+if [[ $GEN_PYI == true ]]; then
+    GEN_STRING="$GEN_STRING --mypy_out=$OUT_DIR --mypy_grpc_out=$OUT_DIR"
+fi
+
 if [[ $GEN_TYPESCRIPT == true ]]; then
     GEN_STRING="$GEN_STRING --plugin=protoc-gen-ts=$(which protoc-gen-ts) --ts_out=$GRPC_OUT:$OUT_DIR"
 fi

all/test/all_test.go

@@ -140,6 +140,27 @@ func (s *TestSuite) TestAllCases() {
 				{fileName: "/all/test/test_pb2_grpc.py"},
 			},
 		},
+		"python pyi": {
+			lang:              "python",
+			protofileName:     "all/test/test.proto",
+			expectedOutputDir: "gen/pb_python",
+			fileExpectations: []FileExpectation{
+				{fileName: "/__init__.py"},
+				{fileName: "/all/__init__.py"},
+				{fileName: "/all/test/__init__.py"},
+				{fileName: "/all/test/test_pb2.py"},
+				{fileName: "/all/test/test_pb2.pyi"},
+				{fileName: "/all/test/test_pb2_grpc.py"},
+				{fileName: "/all/test/test_pb2_grpc.pyi"},
+			},
+			extraArgs: []string{"--with-pyi"},
+		},
+		"python pyi with inconsistent language parameter": {
+			lang:             "ruby",
+			protofileName:    "all/test/test.proto",
+			expectedExitCode: 1,
+			extraArgs:        []string{"--with-pyi"},
+		},
 		"python with alternative output dir": {
 			lang:              "python",
 			protofileName:     "all/test/test.proto",

build.sh

@@ -16,6 +16,7 @@ for build in ${BUILDS[@]}; do
         --build-arg go_version="${GO_VERSION}" \
         --build-arg uber_prototool_version="${UBER_PROTOTOOL_VERSION}" \
         --build-arg scala_pb_version="${SCALA_PB_VERSION}" \
+        --build-arg mypy_version="${MYPY_VERSION}" \
         --build-arg node_version="${NODE_VERSION}" \
         --build-arg node_grpc_tools_node_protoc_ts_version="${NODE_GRPC_TOOLS_NODE_PROTOC_TS_VERSION}" \
         --build-arg node_grpc_tools_version="${NODE_GRPC_TOOLS_VERSION}" \

variables.sh

@@ -14,6 +14,7 @@ GRPC_WEB_VERSION=${GRPC_WEB_VERSION:-1.3.1}
 GRPC_GATEWAY_VERSION=${GRPC_GATEWAY_VERSION:-v2.0.1}
 UBER_PROTOTOOL_VERSION=${UBER_PROTOTOOL_VERSION:-1.10.0}
 SCALA_PB_VERSION=${SCALA_PB_VERSION:-0.11.0}
+MYPY_VERSION=${MYPY_VERSION:-3.3.0}
 NODE_VERSION=${NODE_VERSION:-14}
 NODE_GRPC_TOOLS_NODE_PROTOC_TS_VERSION=${NODE_GRPC_TOOLS_NODE_PROTOC_TS_VERSION:-5.3.2}
 NODE_GRPC_TOOLS_VERSION=${NODE_GRPC_TOOLS_VERSION:-1.11.2}