feat: update ingestor-api runtime to use uv

chore: deal with pydantic deprecation warnings

Author: hrodmn

.github/workflows/deploy.yaml

@@ -130,16 +130,23 @@ jobs:
             select(.key | test("tipgapioutput"; "i")) |
             .value' | head -1)
 
+          INGESTOR_API_URL=$(cat stack_outputs.json | jq -r '
+            to_entries[] |
+            select(.key | test("ingestorapioutput"; "i")) |
+            .value' | head -1)
+
           echo "Extracted URLs:"
           echo "STAC_API_URL: $STAC_API_URL"
           echo "TITILER_PGSTAC_API_URL: $TITILER_PGSTAC_API_URL"
           echo "TIPG_API_URL: $TIPG_API_URL"
+          echo "INGESTOR_API_URL: $INGESTOR_API_URL"
 
           # Array of API URLs to check
           declare -a API_HEALTH_ENDPOINTS=(
             "STAC_API_URL:${STAC_API_URL}_mgmt/health"
             "TITILER_PGSTAC_API_URL:${TITILER_PGSTAC_API_URL}healthz"
             "TIPG_API_URL:${TIPG_API_URL}healthz"
+            "INGESTOR_API_URL:${INGESTOR_API_URL}ingestions"
           )
 
           # Check each API
@@ -166,6 +173,127 @@ jobs:
             fi
           done
 
+          echo "=== Ingestor Integration Test ==="
+
+          if [ -n "$INGESTOR_API_URL" ] && [ "$INGESTOR_API_URL" != "null" ]; then
+            COLLECTION_ID="integration-test-collection"
+            ITEM_ID="integration-test-item-001"
+
+            # Use a public URL that responds to HEAD requests for asset validation
+            PUBLIC_ASSET_URL="https://raw.githubusercontent.com/radiantearth/stac-spec/v1.0.0/examples/simple-item.json"
+
+            # No JWKS_URL configured, so identity is provided via ?provided_by query param
+            INGESTOR_USER="ci-test"
+
+            echo "--- Posting collection $COLLECTION_ID ---"
+            COLLECTION_STATUS=$(curl -s -o /tmp/collection_response.json -w "%{http_code}" \
+              --max-time 30 \
+              -X POST "${INGESTOR_API_URL}collections?provided_by=${INGESTOR_USER}" \
+              -H "Content-Type: application/json" \
+              -d "{
+                \"id\": \"$COLLECTION_ID\",
+                \"type\": \"Collection\",
+                \"stac_version\": \"1.0.0\",
+                \"description\": \"Integration test collection\",
+                \"title\": \"Integration Test Collection\",
+                \"license\": \"proprietary\",
+                \"item_assets\": {
+                  \"data\": {\"type\": \"application/json\", \"roles\": [\"data\"]}
+                },
+                \"extent\": {
+                  \"spatial\": {\"bbox\": [[-180, -90, 180, 90]]},
+                  \"temporal\": {\"interval\": [[\"2020-01-01T00:00:00Z\", null]]}
+                },
+                \"links\": [],
+                \"stac_extensions\": []
+              }")
+
+            echo "Collection POST status: $COLLECTION_STATUS"
+            cat /tmp/collection_response.json
+
+            if [ "$COLLECTION_STATUS" != "201" ]; then
+              echo "❌ Failed to post collection (status $COLLECTION_STATUS)"
+              exit 1
+            fi
+            echo "✅ Collection posted successfully"
+
+            echo "--- Posting item $ITEM_ID ---"
+            ITEM_STATUS=$(curl -s -o /tmp/item_response.json -w "%{http_code}" \
+              --max-time 30 \
+              -X POST "${INGESTOR_API_URL}ingestions?provided_by=${INGESTOR_USER}" \
+              -H "Content-Type: application/json" \
+              -d "{
+                \"stac_version\": \"1.0.0\",
+                \"stac_extensions\": [],
+                \"type\": \"Feature\",
+                \"id\": \"$ITEM_ID\",
+                \"collection\": \"$COLLECTION_ID\",
+                \"bbox\": [-180, -90, 180, 90],
+                \"geometry\": {
+                  \"type\": \"Polygon\",
+                  \"coordinates\": [[[-180, -90], [180, -90], [180, 90], [-180, 90], [-180, -90]]]
+                },
+                \"properties\": {\"datetime\": \"2020-01-01T00:00:00Z\"},
+                \"assets\": {
+                  \"data\": {
+                    \"href\": \"$PUBLIC_ASSET_URL\",
+                    \"type\": \"application/json\",
+                    \"roles\": [\"data\"]
+                  }
+                },
+                \"links\": []
+              }")
+
+            echo "Item POST status: $ITEM_STATUS"
+            cat /tmp/item_response.json
+
+            if [ "$ITEM_STATUS" != "201" ]; then
+              echo "❌ Failed to post item (status $ITEM_STATUS)"
+              exit 1
+            fi
+            echo "✅ Item queued for ingestion"
+
+            echo "--- Polling ingestion status for $ITEM_ID ---"
+            INGESTION_STATUS="queued"
+            POLL_ATTEMPTS=0
+            MAX_POLL_ATTEMPTS=20
+
+            while [ "$INGESTION_STATUS" = "queued" ] || [ "$INGESTION_STATUS" = "started" ]; do
+              if [ "$POLL_ATTEMPTS" -ge "$MAX_POLL_ATTEMPTS" ]; then
+                echo "❌ Timed out waiting for ingestion of $ITEM_ID (status: $INGESTION_STATUS)"
+                exit 1
+              fi
+
+              sleep 15
+              POLL_ATTEMPTS=$((POLL_ATTEMPTS + 1))
+
+              INGESTION_RESPONSE=$(curl -s --max-time 30 "${INGESTOR_API_URL}ingestions/${ITEM_ID}?provided_by=${INGESTOR_USER}")
+              INGESTION_STATUS=$(echo "$INGESTION_RESPONSE" | jq -r '.status // "unknown"')
+              echo "Attempt $POLL_ATTEMPTS: ingestion status = $INGESTION_STATUS"
+            done
+
+            if [ "$INGESTION_STATUS" != "succeeded" ]; then
+              echo "❌ Ingestion failed with status: $INGESTION_STATUS"
+              echo "$INGESTION_RESPONSE" | jq .
+              exit 1
+            fi
+            echo "✅ Item ingested successfully"
+
+            echo "--- Verifying item in STAC API ---"
+            STAC_ITEM_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
+              --max-time 30 \
+              "${STAC_API_URL}collections/$COLLECTION_ID/items/$ITEM_ID")
+
+            if [ "$STAC_ITEM_STATUS" = "200" ]; then
+              echo "✅ Item $ITEM_ID is accessible in STAC API"
+            else
+              echo "❌ Item $ITEM_ID not found in STAC API (status $STAC_ITEM_STATUS)"
+              exit 1
+            fi
+          else
+            echo "⚠️ INGESTOR_API_URL not found in stack outputs, skipping ingestor test"
+          fi
+
           echo "=== Operational Checks Complete ==="
           cd -
 

.github/workflows/tox.yaml

@@ -1,6 +1,7 @@
 import {
   aws_apigateway as apigateway,
   aws_logs,
+  CfnOutput,
   Duration,
   aws_dynamodb as dynamodb,
   aws_ec2 as ec2,
@@ -23,6 +24,11 @@ export class StacIngestor extends Construct {
   table: dynamodb.Table;
   public handlerRole: iam.Role;
 
+  /**
+   * URL of the Ingestor API.
+   */
+  public readonly url: string;
+
   constructor(scope: Construct, id: string, props: StacIngestorProps) {
     super(scope, id);
 
@@ -64,14 +70,21 @@ export class StacIngestor extends Construct {
       pgstacVersion: props.pgstacVersion,
     });
 
-    this.buildApiEndpoint({
+    const api = this.buildApiEndpoint({
       handler,
       stage: props.stage,
       endpointConfiguration: props.apiEndpointConfiguration,
       policy: props.apiPolicy,
       ingestorDomainNameOptions: props.ingestorDomainNameOptions,
     });
 
+    this.url = api.url;
+
+    new CfnOutput(this, "ingestor-api-output", {
+      exportName: `${Stack.of(this).stackName}-ingestor-url`,
+      value: this.url,
+    });
+
     this.buildIngestor({
       table: this.table,
       env: env,

lib/ingestor-api/index.ts

@@ -1,18 +1,22 @@
-ARG PYTHON_VERSION
-
+ARG PYTHON_VERSION=3.12
 FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
+COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /uvx /bin/
 
 WORKDIR /tmp
 
-RUN dnf install -y git
-
-RUN python -m pip install pip -U
-
-COPY runtime/requirements.txt requirements.txt
+COPY runtime/pyproject.toml runtime/uv.lock ./
 
 ARG PGSTAC_VERSION
-RUN echo "pypgstac==${PGSTAC_VERSION}" > constraints.txt
-RUN python -m pip install -r requirements.txt -c constraints.txt "mangum>=0.14,<0.15" -t /asset --no-binary pydantic
+RUN <<EOF
+uv add --no-sync pypgstac==${PGSTAC_VERSION}
+uv export --locked --no-editable --no-dev --format requirements.txt -o requirements.txt
+uv pip install \
+  --compile-bytecode \
+  --target /asset \
+  --no-cache-dir \
+  --disable-pip-version-check \
+  -r requirements.txt
+EOF
 
 RUN mkdir -p /asset/src
 COPY runtime/src/*.py /asset/src/

lib/ingestor-api/runtime/Dockerfile

@@ -0,0 +1,38 @@
+[project]
+name = "ingestor-api"
+version = "0.0.0"
+description = "ingestor-api runtime"
+requires-python = ">=3.12"
+dependencies = [
+    "authlib>=1.3",
+    "boto3",
+    "cachetools>=5.0",
+    "fastapi>=0.110",
+    "mangum==0.19",
+    "orjson>=3.9",
+    "psycopg[binary,pool]>=3.0",
+    "pydantic>=2.0",
+    "pydantic-settings>=2.0",
+    "pydantic-ssm-settings>=1.0,<2.0",
+    "pypgstac",
+    "requests>=2.27",
+    "stac-pydantic>=3.0,<4.0",
+]
+
+[dependency-groups]
+dev = [
+    "boto3",
+    "httpx>=0.24",
+    "moto[dynamodb,ssm]>=4.0,<5.0",
+    "pytest>=7.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

lib/ingestor-api/runtime/dev_requirements.txt

@@ -7,6 +7,7 @@
     AnyHttpUrl,
     Field,
     constr,
+    field_validator,
 )
 from pydantic_settings import BaseSettings
 from pydantic_ssm_settings.settings import SsmBaseSettings
@@ -26,6 +27,14 @@ class Settings(BaseSettings):
         description="URL of JWKS, e.g. https://cognito-idp.{region}.amazonaws.com/{userpool_id}/.well-known/jwks.json",  # noqa
     )
 
+    @field_validator("jwks_url", mode="before")
+    @classmethod
+    def empty_str_to_none(cls, v):
+        """Treat empty string as None so callers can disable auth by setting JWKS_URL=''."""
+        if v == "":
+            return None
+        return v
+
     stac_url: HttpUrlString = Field(description="URL of STAC API")
 
     data_access_role: AwsArn = Field(

lib/ingestor-api/runtime/pyproject.toml

@@ -42,7 +42,7 @@ def update_dynamodb(
     with table.batch_writer(overwrite_by_pkeys=["created_by", "id"]) as batch:
         for ingestion in ingestions:
             batch.put_item(
-                Item=ingestion.copy(
+                Item=ingestion.model_copy(
                     update={
                         "status": status,
                         "message": message,

lib/ingestor-api/runtime/requirements.txt

@@ -3,17 +3,15 @@
 import enum
 import json
 from datetime import datetime
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 from urllib.parse import urlparse
 
 from fastapi.encoders import jsonable_encoder
-from fastapi.exceptions import RequestValidationError
 from pydantic import (
     BaseModel,
     Json,
     PositiveInt,
     dataclasses,
-    error_wrappers,
     field_validator,
 )
 from stac_pydantic import Collection, Item, shared
@@ -107,25 +105,19 @@ def dynamodb_dict(self):
 class ListIngestionRequest:
     status: Status = Status.queued
     limit: Optional[PositiveInt] = None
-    next: Optional[str] = None
-
-    def __post_init_post_parse__(self) -> None:
-        # https://github.com/tiangolo/fastapi/issues/1474#issuecomment-1049987786
-        if self.next is None:
-            return
+    next: Optional[Any] = None
 
+    @field_validator("next", mode="before")
+    @classmethod
+    def decode_next_token(cls, v: Optional[str]) -> Optional[Any]:
+        """Decode the base64-encoded JSON pagination token supplied as a query param."""
+        if v is None:
+            return None
         try:
-            self.next = json.loads(base64.b64decode(self.next))
+            return json.loads(base64.b64decode(v))
         except (UnicodeDecodeError, binascii.Error) as e:
-            raise RequestValidationError(
-                [
-                    error_wrappers.ErrorWrapper(
-                        ValueError(
-                            "Unable to decode next token. Should be base64 encoded JSON"
-                        ),
-                        "query.next",
-                    )
-                ]
+            raise ValueError(
+                "Unable to decode next token. Should be base64 encoded JSON"
             ) from e