add missing files

davidgamez · davidgamez · commit e5797ed2de83 · 2025-10-23T16:59:14.000-04:00
diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
@@ -0,0 +1,110 @@
+# Mobility Feed API - AI Coding Assistant Instructions
+
+## Project Architecture
+
+This is a **mobility data API service** built with FastAPI, serving open mobility data from across the world. The architecture follows a **code-generation pattern** with clear separation between generated and implementation code.
+
+### Core Components
+
+- **`api/`**: Main FastAPI application with spec-first development using OpenAPI Generator
+- **`functions-python/`**: Google Cloud Functions for data processing (batch jobs, validation, analytics)
+- **`web-app/`**: Frontend application
+- **PostgreSQL + PostGIS**: Database with geospatial support for mobility data
+
+### Key Generated vs Implementation Split
+
+- **Generated code** (never edit): `api/src/feeds_gen/` and `api/src/shared/database_gen/`
+- **Implementation code**: `api/src/feeds/impl/` contains actual business logic
+- **Schema source**: `docs/DatabaseCatalogAPI.yaml` drives code generation
+
+## Critical Development Workflows
+
+### Initial Setup
+```bash
+# One-time OpenAPI setup
+scripts/setup-openapi-generator.sh
+
+# Install dependencies
+cd api && pip3 install -r requirements.txt -r requirements_dev.txt
+
+# Start local database
+docker-compose --env-file ./config/.env.local up -d --force-recreate
+
+# Generate API stubs (run after schema changes)
+scripts/api-gen.sh
+scripts/db-gen.sh
+```
+
+### Common Development Commands
+```bash
+# Start API server (includes Swagger UI at http://localhost:8080/docs/)
+scripts/api-start.sh
+
+# Run tests with coverage
+scripts/api-tests.sh
+# Run specific test file
+scripts/api-tests.sh my_test_file.py
+
+# Lint checks (Flake8 + Black)
+scripts/lint-tests.sh
+
+# Reset and populate local database
+./scripts/docker-localdb-rebuild-data.sh --populate-db
+# Include test datasets
+./scripts/docker-localdb-rebuild-data.sh --populate-db --populate-test-data
+```
+
+## Project-Specific Patterns
+
+### Error Handling Convention
+- Use `shared.common.error_handling.InternalHTTPException` for internal errors
+- Convert to FastAPI HTTPException using `feeds.impl.error_handling.convert_exception()`
+- Store error messages as Finals in `api/src/feeds/impl/error_handling.py`
+- Error responses follow: `{"details": "The error message"}`
+
+### Database Patterns
+- **Polymorphic inheritance**: `Feed` base class with `GtfsFeed`, `GbfsFeed`, `GtfsRTFeed` subclasses
+- **SQLAlchemy ORM**: Models in `shared/database_gen/sqlacodegen_models.py` (generated)
+- **Session management**: Use `@with_db_session` decorator for database operations
+- **Unique IDs**: Generate with `generate_unique_id()` (36-char UUID4)
+
+### API Implementation Structure
+- Endpoints in `feeds/impl/*_api_impl.py` extend generated base classes from `feeds_gen/`
+- Filter classes in `shared/feed_filters/` for query parameter handling
+- Model implementations in `feeds/impl/models/` extend generated models
+
+### Code Generation Workflow
+1. Modify `docs/DatabaseCatalogAPI.yaml` for API changes
+2. Run `scripts/api-gen.sh` to regenerate FastAPI stubs
+3. Run `scripts/db-gen.sh` for database schema changes
+4. Implement business logic in `feeds/impl/` classes
+
+### Testing Patterns
+- Tests use empty local test DB (reset with `--use-test-db` flag)
+- Coverage reports in `scripts/coverage_reports/`
+- Python path configured to `src/` in `pyproject.toml`
+
+### Functions Architecture
+- **Google Cloud Functions** in `functions-python/` for background processing
+- Shared database models via `database_gen/` symlink
+- Each function has its own deployment configuration
+- Tasks include: validation reports, batch datasets, GBFS validation, BigQuery ingestion
+
+### Authentication
+- **OAuth2 Bearer tokens** for API access
+- Refresh tokens from mobilitydatabase.org account
+- Access tokens valid for 1 hour
+- Test endpoint: `/v1/metadata` with Bearer token
+
+## Integration Points
+
+- **BigQuery**: Analytics data pipeline via `big_query_ingestion/` function  
+- **PostGIS**: Geospatial queries for location-based feed filtering
+- **Liquibase**: Database schema migrations in `liquibase/` directory
+- **Docker**: Multi-service setup with PostgreSQL, test DB, and schema documentation
+
+## File Exclusions for AI Context
+- Skip `src/feeds_gen/*` and `src/shared/database_gen/*` (generated code)
+- Skip `data/` and `data-test/` (database volumes)
+- Skip `htmlcov/` (coverage reports)
+- Black formatter excludes these paths automatically
diff --git a/api/src/shared/db_models/model_utils.py b/api/src/shared/db_models/model_utils.py
@@ -0,0 +1,26 @@
+from packaging.version import Version
+
+
+def compare_java_versions(v1: str | None, v2: str | None):
+    """
+    Compare two version strings v1 and v2.
+    Returns 1 if v1 > v2, -1 if v1 < v2,
+    otherwise 0.
+    The version strings are expected to be in the format of
+    major.minor.patch[-SNAPSHOT]
+    """
+    if v1 is None and v2 is None:
+        return 0
+    if v1 is None:
+        return -1
+    if v2 is None:
+        return 1
+    # clean version strings replacing the SNAPSHOT suffix with .dev0
+    v1 = v1.replace("-SNAPSHOT", ".dev0")
+    v2 = v2.replace("-SNAPSHOT", ".dev0")
+    if Version(v1) > Version(v2):
+        return 1
+    elif Version(v1) < Version(v2):
+        return -1
+    else:
+        return 0
diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_feed_impl.py
@@ -0,0 +1,37 @@
+from feeds_gen.models.operation_gtfs_feed import OperationGtfsFeed
+from shared.database_gen.sqlacodegen_models import Gtfsfeed
+from shared.db_models.gtfs_feed_impl import GtfsFeedImpl
+
+
+class OperationGtfsFeedImpl(GtfsFeedImpl, OperationGtfsFeed):
+    """Base implementation of the feeds models."""
+
+    class Config:
+        """Pydantic configuration.
+        Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object.
+        """
+
+        from_attributes = True
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        self.locations = self.locations or []
+        self.redirects = self.redirects or []
+
+    @classmethod
+    def from_orm(cls, feed: Gtfsfeed | None) -> OperationGtfsFeed | None:
+        """Convert a SQLAlchemy row object to a Pydantic model."""
+        if not feed:
+            return None
+        operation_gtfs_feed = super().from_orm(feed)
+        if not operation_gtfs_feed:
+            return None
+
+        data = dict(operation_gtfs_feed.__dict__)
+        # Override id and add stable_id
+        data["id"] = feed.id
+        data["stable_id"] = feed.stable_id
+        # Add missing fields from public API model
+        data["operational_status"] = feed.operational_status
+
+        return cls.model_construct(**data)
diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_rt_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_rt_feed_impl.py
@@ -0,0 +1,39 @@
+from feeds_gen.models.operation_gtfs_rt_feed import OperationGtfsRtFeed
+from shared.database_gen.sqlacodegen_models import Gtfsrealtimefeed
+from shared.db_models.gtfs_rt_feed_impl import GtfsRTFeedImpl
+
+
+class OperationGtfsRtFeedImpl(GtfsRTFeedImpl, OperationGtfsRtFeed):
+    """Base implementation of the feeds models."""
+
+    class Config:
+        """Pydantic configuration.
+        Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object.
+        """
+
+        from_attributes = True
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        self.entity_types = self.entity_types or []
+        self.locations = self.locations or []
+        self.redirects = self.redirects or []
+        self.feed_references = self.feed_references or []
+
+    @classmethod
+    def from_orm(cls, feed: Gtfsrealtimefeed | None) -> OperationGtfsRtFeed | None:
+        """Convert a SQLAlchemy row object to a Pydantic model."""
+        if not feed:
+            return None
+        operation_gtfs_feed = super().from_orm(feed)
+        if not operation_gtfs_feed:
+            return None
+
+        data = dict(operation_gtfs_feed.__dict__)
+        # Override id and add stable_id
+        data["id"] = feed.id
+        data["stable_id"] = feed.stable_id
+        # Add missing fields from public API model
+        data["operational_status"] = feed.operational_status
+
+        return cls.model_construct(**data)
diff --git a/scripts/api-operations-update-schema.sh b/scripts/api-operations-update-schema.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Usage:
+#   ./scripts/api-operations-update-schema.sh \
+#     [--source ./docs/DatabaseCatalogAPI.yaml] \
+#     [--dest ./docs/OperationsAPI.yaml]
+#
+# Behavior:
+# - Replaces components.schemas in Operations with those from Catalog.
+# - Preserves only schemas in Operations that have x-operation: true at the schema root (these override source).
+# - Removes any non-operation schemas that exist only in Operations.
+
+SCRIPT_DIR="$(cd "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+SOURCE=""
+DEST=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --source|-s) SOURCE="${2:-}"; shift 2 ;;
+    --dest|-d) DEST="${2:-}"; shift 2 ;;
+    -h|--help) echo "Usage: $0 [--source <CatalogYAML>] [--dest <OperationsYAML>]"; exit 0 ;;
+    *) echo "Unknown arg: $1" >&2; exit 1 ;;
+  esac
+done
+
+: "${SOURCE:=${REPO_ROOT}/docs/DatabaseCatalogAPI.yaml}"
+: "${DEST:=${REPO_ROOT}/docs/OperationsAPI.yaml}"
+
+if ! command -v yq >/dev/null 2>&1; then
+  echo "yq not found. Install with: brew install yq" >&2
+  exit 1
+fi
+YQ_MAJOR="$(yq --version 2>/dev/null | sed -n 's/.*version v\([0-9][0-9]*\).*/\1/p')"
+if [[ -z "${YQ_MAJOR:-}" || "${YQ_MAJOR}" -lt 4 ]]; then
+  echo "yq v4+ required. Current: $(yq --version 2>/dev/null)" >&2
+  exit 1
+fi
+
+[[ -f "${SOURCE}" ]] || { echo "Source not found: ${SOURCE}" >&2; exit 1; }
+[[ -f "${DEST}" ]]   || { echo "Dest not found: ${DEST}"   >&2; exit 1; }
+
+cp -f "${DEST}" "${DEST}.bak"
+
+# Merge strategy:
+# - Start from source schemas (Catalog): ensures Operations aligns with source by default
+# - Overlay ONLY the destination schemas that are explicitly marked with x-operation: true
+#   (these are preserved and override the source)
+# - Any non-operation schemas that exist only in Operations are DROPPED
+SRC_ABS="$(cd "$(dirname "${SOURCE}")" && pwd)/$(basename "${SOURCE}")"
+export SRC="${SRC_ABS}"
+
+yq -i '
+  (.components.schemas // {}) as $dst
+  | (load(strenv(SRC)).components.schemas // {}) as $src
+  | .components.schemas = (
+      $src
+      * (
+          $dst
+          | with_entries( select(.value."x-operation" == true) )
+        )
+    )
+' "${DEST}"
+
+echo "Synced schemas from ${SOURCE} -> ${DEST} (${DEST}.bak created)."
+echo "Note: Schemas in Operations with x-operation: true were preserved."
