feat: protobuf gen dynamically instead of static inclusion

- Update CI workflow to use simplified protobuf generation approach
- Modify .gitignore to exclude generated protobuf code from version
control
- Update compile_proto.sh to be more flexible and handle multiple proto
files
- Fix all import paths for new directory structure
(resolver_athena_client)
- Update AGENTS.md documentation to reflect new workflow
- Change package name in pyproject.toml to use underscores

This brings the protobuf compilation process in line with modern
practices
where generated code is built dynamically rather than checked into the
repo.

Author: snus-kin

.github/workflows/ci.yml

@@ -52,31 +52,11 @@ jobs:
       - name: Install the project
         run: uv sync --locked --all-extras --dev
 
-      - name: Ensure no differences in generated code
-        if: runner.os != 'Windows'
+      - name: Generate protobuf code
         shell: bash
         run: |
           source .venv/bin/activate
-          GENERATED_DIR="src/resolver_athena_client/generated"
-          BACKUP_DIR="src/resolver_athena_client/generated_backup"
-
-          cp -r $GENERATED_DIR $BACKUP_DIR
-
-          ./scripts/compile_proto.sh || (echo "Protobuf compilation failed. Ensure submodules are initialized and the proto file exists." && exit 1)
-
-          # Fix imports in generated files
-          if [[ -f "$GENERATED_DIR/athena/athena_pb2_grpc.py" && -f "$GENERATED_DIR/athena/athena_pb2.py" ]]; then
-            sed -i.bak 's/^from athena /from resolver_athena_client.generated.athena /' "$GENERATED_DIR/athena/athena_pb2_grpc.py"
-            sed -i.bak 's/^from athena /from resolver_athena_client.generated.athena /' "$GENERATED_DIR/athena/athena_pb2.py"
-            rm -f "$GENERATED_DIR/athena/athena_pb2_grpc.py.bak" "$GENERATED_DIR/athena/athena_pb2.py.bak"
-          else
-            echo "Error: Expected files not found in $GENERATED_DIR/athena"
-            exit 1
-          fi
-
-          diff -r $GENERATED_DIR $BACKUP_DIR || (echo "Generated code differs. Please commit the changes after running compile_proto.sh." && exit 1)
-
-          rm -rf $BACKUP_DIR
+          ./scripts/compile_proto.sh || (echo "Protobuf compilation failed. Ensure submodules are initialized and proto files exist." && exit 1)
 
       - name: Run linter
         run: |
@@ -123,6 +103,17 @@ jobs:
         with:
           enable-cache: true
 
+      - name: Set up Python
+        run: uv python install
+
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
+
+      - name: Generate protobuf code
+        run: |
+          source .venv/bin/activate
+          ./scripts/compile_proto.sh || (echo "Protobuf compilation failed. Ensure submodules are initialized and proto files exist." && exit 1)
+
       - name: Set version from tag
         run: |
           if [[ "$GITHUB_REF" == refs/tags/* ]]; then

.gitignore

@@ -11,3 +11,6 @@ wheels/
 .env
 
 docs/_build/
+
+# Generated protobuf code
+src/resolver_athena_client/generated/

AGENTS.md

@@ -8,7 +8,7 @@
 - Format code: `ruff format`
 - Lint code: `ruff check`
 - Install git hooks: `pre-commit install`
-- Compile protobufs: `bash scripts/compile_proto.sh` (run from root)
+- Compile protobufs: `bash scripts/compile_proto.sh` (run from root, required for local development)
 
 ## Code style
 - Use Python type hints throughout
@@ -30,7 +30,7 @@
 
 ## PR instructions
 - Title format: [component] Description
-- Run `ruff check`, `pyright`, and `pytest` before committing
+- Run `bash scripts/compile_proto.sh`, `ruff check`, `pyright`, and `pytest` before committing
 - Keep PRs focused on a single change
 - Add tests for new functionality
 - Update documentation for API changes
@@ -40,7 +40,8 @@
 - Use `uv` package manager instead of pip
 - Don't use `uv pip` commands, just the base `uv` commands
 - Run formatters before committing
-- Check generated code in `src/athena_client/generated/`
+- Generated code is built automatically in CI, but run `bash scripts/compile_proto.sh` locally for development
+- Generated code in `src/resolver_athena_client/generated/` is not committed to the repo
 - Add error handling at each pipeline stage
 - Use correlation IDs for request tracing
 

pyproject.toml

@@ -1,5 +1,5 @@
 [project]
-name = "resolver-athena-client"
+name = "resolver_athena_client"
 version = "0.0.0"
 description = "Python client library for Athena API - CSAM detection and content classification"
 readme = "README.md"

scripts/compile_proto.sh

@@ -7,33 +7,50 @@ OUT_DIR="src/resolver_athena_client/generated"
 # Ensure output directory exists
 mkdir -p "$OUT_DIR"
 
+# Create __init__.py files for Python packages
+touch "$OUT_DIR/__init__.py"
+mkdir -p "$OUT_DIR/athena"
+touch "$OUT_DIR/athena/__init__.py"
+
 # Compile .proto files
-if [ ! -f "$PROTO_DIR/athena/athena.proto" ]; then
-  echo "Error: Protobuf file not found at $PROTO_DIR/athena/athena.proto. Ensure the submodule is initialized and updated."
+if [ ! -d "$PROTO_DIR/athena" ]; then
+  echo "Error: Athena protobuf directory not found at $PROTO_DIR/athena. Ensure the submodule is initialized and updated."
   exit 1
 fi
 
-python -m grpc_tools.protoc \
+# Find all .proto files in the athena directory
+PROTO_FILES=$(find "$PROTO_DIR/athena" -name "*.proto")
+if [ -z "$PROTO_FILES" ]; then
+  echo "Error: No .proto files found in $PROTO_DIR/athena."
+  exit 1
+fi
+
+echo "Found proto files: $PROTO_FILES"
+
+uv run python -m grpc_tools.protoc \
   --proto_path="$PROTO_DIR" \
-  --proto_path="$(python -c "import grpc_tools; print(grpc_tools.__path__[0])")" \
   --python_out="$OUT_DIR" \
   --grpc_python_out="$OUT_DIR" \
   --mypy_out="$OUT_DIR" \
-  "$PROTO_DIR/athena/models.proto" \
-  "$PROTO_DIR/athena/athena.proto"
+  $PROTO_FILES
 
 # Fix imports in generated files
-if [[ "$OSTYPE" == "darwin"* ]]; then
-  sed -i '' -e 's/^from athena /from resolver_athena_client.generated.athena /' "$OUT_DIR/athena/athena_pb2_grpc.py"
-  sed -i '' -e 's/^from athena /from resolver_athena_client.generated.athena /' "$OUT_DIR/athena/athena_pb2.py"
-  sed -i '' -e 's/^from athena /from resolver_athena_client.generated.athena /' "$OUT_DIR/athena/models_pb2.py"
+GENERATED_FILES=$(find "$OUT_DIR/athena" -name "*_pb2*.py" 2>/dev/null)
+if [ -n "$GENERATED_FILES" ]; then
+  for file in $GENERATED_FILES; do
+    echo "Fixing imports in $file"
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+      sed -i '' -e 's/^from athena /from resolver_athena_client.generated.athena /' "$file"
+    else
+      sed -i -e 's/^from athena /from resolver_athena_client.generated.athena /' "$file"
+    fi
+  done
 else
-  sed -i -e 's/^from athena /from resolver_athena_client.generated.athena /' "$OUT_DIR/athena/athena_pb2_grpc.py"
-  sed -i -e 's/^from athena /from resolver_athena_client.generated.athena /' "$OUT_DIR/athena/athena_pb2.py"
-  sed -i -e 's/^from athena /from resolver_athena_client.generated.athena /' "$OUT_DIR/athena/models_pb2.py"
+  echo "Warning: No generated Python files found to fix imports."
 fi
 
-if [ ! -f "$OUT_DIR/athena/athena_pb2.py" ] || [ ! -f "$OUT_DIR/athena/athena_pb2_grpc.py" ] || [ ! -f "$OUT_DIR/athena/models_pb2.py" ]; then
+# Verify at least some files were generated
+if [ ! -d "$OUT_DIR/athena" ] || [ -z "$(find "$OUT_DIR/athena" -name "*_pb2*.py" 2>/dev/null)" ]; then
   echo "Error: Protobuf files were not generated successfully in $OUT_DIR."
   exit 1
 fi