Merge pull request #45 from boschglobal/fix/proto_gen

Fix proto generation from setup.py

Author: wba2hi

.github/workflows/kuksa-client.yml

@@ -141,7 +141,8 @@ jobs:
         run: |
           cd kuksa-client
           pip install -r requirements.txt -r test-requirements.txt
-          python3 -m proto
+          python3 -m prototagandcopy
+          python3 -m protobuild
           pip install -e .
       - name: Run tests
         run: |

docs/building.md

@@ -21,12 +21,20 @@ python3 -m venv ~/.venv/kuksa-client
 source ~/.venv/kuksa-client/bin/activate  # Run this every time you want to activate kuksa-client's virtual environment
 ```
 
-To use the right api interfaces of databroker run the following:
+To use the right api interfaces of databroker run the following
+
 ```console
-python3 -m proto
+python3 -m prototagandcopy
 ```
+
 This should copy the corresponding proto files to the kuksa-client directory.
 
+You still might need to compile the proto files, to do so do
+
+```console
+python3 -m protobuild
+```
+
 Your prompt should change to somehting indicating you are in the virutal environment now, e.g.
 
 ```console

kuksa-client/Dockerfile

@@ -9,7 +9,7 @@
 # Note: This dockerfile needs to be executed one level above in the root folder
 
 
-FROM python:3.10-slim-bookworm as build
+FROM python:3.10-slim-bookworm AS build
 # binutils is required by pyinstaller to strip any .so libs that are collected
 # git is used to determine & embed version information during build time
 RUN apt update && apt -yy install binutils git
@@ -19,8 +19,6 @@ RUN pip install --upgrade pip build pyinstaller
 COPY . /kuksa-python-sdk/
 WORKDIR /kuksa-python-sdk/kuksa-client
 RUN git submodule update --recursive --remote --init
-# install files from submodules to kuksa-client repo to generate protos out of it
-RUN python3 -m proto
 
 RUN python3 -m build
 # We install globally on build container, so pyinstaller can easily gather all files
@@ -29,7 +27,7 @@ RUN pip install --no-cache-dir dist/*.whl
 WORKDIR /
 RUN rm -rf dist
 # Letting pyinstaller collect everything that is required
-RUN pyinstaller --collect-data kuksa_client --clean  -s  /usr/local/bin/kuksa-client
+RUN pyinstaller --collect-data kuksa_client --add-data=/kuksa-python-sdk/kuksa-client/kuksa:kuksa --add-data=/kuksa-python-sdk/kuksa-client/sdv:sdv --clean  -s  /usr/local/bin/kuksa-client
 
 
 # Debian 12 is bookworm, so the glibc version matches. Distroless is a lot smaller than

kuksa-client/MANIFEST.in

@@ -1,2 +1,6 @@
 include  *.py
+recursive-include kuksa *.py
+recursive-include sdv *.py
 global-exclude *.pyc
+global-exclude protobuild.py
+global-exclude prototagandcopy.py

kuksa-client/protobuild.py

@@ -0,0 +1,25 @@
+# /********************************************************************************
+# * Copyright (c) 2025 Contributors to the Eclipse Foundation
+# *
+# * See the NOTICE file(s) distributed with this work for additional
+# * information regarding copyright ownership.
+# *
+# * This program and the accompanying materials are made available under the
+# * terms of the Apache License 2.0 which is available at
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * SPDX-License-Identifier: Apache-2.0
+# ********************************************************************************/
+
+from grpc_tools import command
+
+# Helper to compile all protos in cwd
+
+
+def main():
+    print("Compiling protobuf files...")
+    command.build_package_protos(".", strict_mode=True)
+
+
+if __name__ == "__main__":
+    main()

kuksa-client/proto.py kuksa-client/prototagandcopy.pykuksa-client/proto.py renamed to kuksa-client/prototagandcopy.py

@@ -19,6 +19,10 @@
 
 
 def main():
+    '''
+    This will tag all proto folders as Python packages by creating an __init__.py file
+    in each subdirectory and then copy the proto files to the current working directory
+    '''
     for root, dirs, files in os.walk(PROTO_PATH):
         for directory in dirs:
             # Create an __init__.py file in each subdirectory

kuksa-client/setup.py

@@ -21,6 +21,52 @@
 from setuptools.command.develop import develop as _develop
 
 
+class BuildGenerateProtos(setuptools.Command):
+    def run(self):
+        self.run_command('generate_proto')
+        return super().run()
+
+
+class GenerateProtosCommand(setuptools.Command):
+    """Command to run prototagandcopy.py script."""
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        import subprocess  # pylint: disable=import-outside-toplevel
+        import os
+        from pathlib import Path
+
+        # Check if we're in an sdist build (proto files should already exist)
+        proto_files_exist = any(Path('.').glob('**/*_pb2.py'))
+        proto_dir_exists = Path("../submodules/kuksa-proto/proto/").exists()
+
+        if proto_files_exist:
+            print("Proto files already exist, skipping package tagging")
+            # List files in the current directory
+            print("Existing proto files:")
+            for file in Path('.').glob('**/*_pb2.py'):
+                print(f" - {file}")
+            return
+
+        if not proto_dir_exists:
+            print("Warning: Proto directory not found, skipping package tagging")
+            return
+
+        print(f"Generating package from proto: {os.getcwd()}")
+        result = subprocess.call(['python', 'prototagandcopy.py'])
+        if result != 0:
+            print(f"Warning: prototagandcopy.py failed with exit code {result}")
+        # print("This is how it looks like:")
+        # for file in Path('./kuksa').glob('**/*'):
+        #    print(f" - {file}")
+
+
 class BuildPackageProtos(setuptools.Command):
     def run(self):
         self.run_command('build_pb2')
@@ -37,32 +83,110 @@ def finalize_options(self):
         pass
 
     def run(self):
-        from grpc_tools import command  # pylint: disable=import-outside-toplevel
+        import subprocess
+        import sys
+        from pathlib import Path
+
+        # Check if proto files already exist (generated by previous step)
+        proto_files_exist = any(Path('.').glob('**/*_pb2.py'))
+        if proto_files_exist:
+            print("Proto files already exist, skipping protobuf compilation")
+            # Recursively list every file in folder "kuksa"
+            for file in Path('./kuksa').glob('**/*'):
+                print(f" - {file}")
+            return
+
+        # Check if we have .proto files to compile
+        proto_source_files = list(Path('.').glob('**/*.proto'))
+        if not proto_source_files:
+            print("No .proto files found, skipping protobuf compilation")
+            return
+
+        print(f"Found {len(proto_source_files)} .proto files to compile")
+
+        try:
+            from grpc_tools import command
+            print("Compiling protobuf files...")
+            command.build_package_protos(".", strict_mode=True)
+        except ImportError:
+            # Fallback to direct protoc call
+            print("grpc_tools not found, using protoc directly.")
+            if proto_source_files:
+                subprocess.check_call([
+                    sys.executable, "-m", "grpc_tools.protoc",
+                    "--proto_path=.",
+                    "--python_out=.",
+                    "--grpc_python_out=.",
+                    *[str(f) for f in proto_source_files]
+                ])
+
+
+class BuildCommand(BuildGenerateProtos, BuildPackageProtos, build.build):
+    ...
 
-        command.build_package_protos(".", strict_mode=True)
 
+class BuildPyCommand(BuildGenerateProtos, BuildPackageProtos, build_py.build_py):  # pylint: disable=too-many-ancestors
+    def finalize_options(self):
+        # First run the parent finalize_options which includes proto generation
+        super().finalize_options()
 
-class BuildCommand(BuildPackageProtos, build.build):
-    ...
+        # After proto generation, ensure packages are properly set BEFORE build_py runs
+        self.ensure_proto_packages()
+
+    def run(self):
+        # Just run the normal build_py
+        build_py.build_py.run(self)
 
+    def ensure_proto_packages(self):
+        """Ensure generated proto packages are included in the distribution."""
+        import os
 
-class BuildPyCommand(BuildPackageProtos, build_py.build_py):  # pylint: disable=too-many-ancestors
-    ...
+        # Find all proto packages
+        print("Searching and adding dynamically built proto packages...")
+        proto_packages = []
+
+        for root, dirs, files in os.walk('.'):
+            if '__init__.py' in files:
+                package = root.replace('./', '').replace('/', '.').lstrip('.')
+                print(f"Found package: {package}")
+                if package and any(proto_name in package for proto_name in ['kuksa', 'sdv']):
+                    # Skip build directories
+                    if not package.startswith('build.'):
+                        proto_packages.append(package)
+                        print(f"Adding proto package: {package}")
+
+        # Get current packages from distribution or setup.cfg
+        current_packages = []
+        if hasattr(self.distribution, 'packages') and self.distribution.packages:
+            current_packages = list(self.distribution.packages)
+
+        # Add proto packages
+        all_packages = current_packages + [pkg for pkg in proto_packages if pkg not in current_packages]
+
+        # Update the distribution packages
+        self.distribution.packages = all_packages
+        print(f"Final package list: {all_packages}")
 
 
-class SDistCommand(BuildPackageProtos, sdist.sdist):
+class SDistCommand(BuildGenerateProtos, BuildPackageProtos, sdist.sdist):
     ...
 
 
-class DevelopCommand(BuildPackageProtos, _develop):
+class DevelopCommand(BuildGenerateProtos, BuildPackageProtos, _develop):
 
     def run(self):
-        self.run_command("build_pb2")
+        try:
+            self.run_command("generate_proto")
+            self.run_command("build_pb2")
+        except Exception as e:
+            print(f"Warning: Proto generation failed: {e}")
+            print("Continuing with development install...")
         super().run()
 
 
 setuptools.setup(
     cmdclass={
+        "generate_proto": GenerateProtosCommand,
         "build": BuildCommand,
         "build_pb2": BuildPackageProtosCommand,
         "build_py": BuildPyCommand,  # Used for editable installs but also for building wheels