Merge pull request #1239 from CodeForPhilly/staging

PR from staging to main

Author: nlebovits

.github/workflows/pr_checks_backend.yml

@@ -6,8 +6,6 @@ on:
       - staging
     paths:
       - 'data/**'
-      - 'Dockerfile-pg'
-      - 'init_pg.sql'
       - 'docker compose.yml'
   workflow_dispatch:
 
@@ -24,53 +22,17 @@ jobs:
       - name: Set up Docker Compose
         uses: docker/setup-qemu-action@v3
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11.4'
-
   test:
     runs-on: ubuntu-latest
     needs: setup
-    defaults:
-      run:
-        working-directory: data/src
-    env:
-      VACANT_LOTS_DB: 'postgresql://postgres:temp-CI-only@localhost:5433/vacantlotdb'
-    services:
-      postgres:
-        image: postgis/postgis:16-3.4
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: temp-CI-only # CI-only, safe to hardcode for temporary container
-          POSTGRES_DB: vacantlotdb
-        ports:
-          - 5433:5432
-        # Set health checks to wait until postgres is ready
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11.4'
-
-      - name: Install and configure pipenv
-        run: |
-          python -m pip install --upgrade pip
-          pip install pipenv
-          echo "Using Python: $(which python)"
-          pipenv --python $(which python) install --dev
-
       - name: Run Pytest
-        working-directory: data/src
-        run: PYTHONPATH=$PYTHONPATH:. pipenv run pytest
+        run: |
+          cd data
+          docker compose run --rm vacant-lots-proj sh -c "pytest"
 
   run-formatter:
     runs-on: ubuntu-latest
@@ -84,7 +46,7 @@ jobs:
       - name: Run Ruff Formatter in Docker
         run: |
           cd data
-          docker compose run --rm formatter sh -c "pip install ruff && ruff format --check --exclude '/usr/src/app/awkde/'"
+          docker compose run --rm vacant-lots-proj sh -c "ruff format --check"
 
   run-linter:
     runs-on: ubuntu-latest
@@ -98,7 +60,7 @@ jobs:
       - name: Run Ruff Linter in Docker
         run: |
           cd data
-          docker compose run --rm linter
+          docker compose run --rm vacant-lots-proj sh -c "ruff check"
 
   build-project:
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -25,7 +25,7 @@ jobs:
         run: npm ci
 
       - name: Create Tag
-        uses: cycjimmy/semantic-release-action@v4.2.0
+        uses: cycjimmy/semantic-release-action@v4.2.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.gitignore

@@ -60,7 +60,8 @@ data/src/app/service-account-key.json
 # compiled python files
 *.pyc
 
-tmp/
-
 # Local python development files
 .python-version
+
+# Cached and temporary data files from pipeline
+storage/

.pre-commit-config.yaml

@@ -18,13 +18,6 @@ repos:
         types_or: [python, pyi, jupyter]
         files: \.py$
 
-      # Dependency checks for Python
-      - id: check-python-dependencies
-        name: Check Python dependencies
-        entry: bash -c 'cd data/src && pipenv check --bare --quiet || (echo "Python dependencies check failed" && exit 1)'
-        language: system
-        files: ^data/src/Pipfile$
-
       # Radon MI and Halstead checks
       - id: radon-check
         name: Radon Maintainability and Halstead Checks

data/Dockerfile

@@ -1,10 +1,32 @@
-# Use an official Python 3.11 runtime as a parent image
-FROM python:3.11.4
+FROM python:3.11.4 AS tippecanoe
+RUN apt-get update && apt-get -y install \
+    make \
+    gcc \
+    g++ \
+    libsqlite3-dev \
+    zlib1g-dev
+# TODO: Pin Tippecanoe to a specific version
+RUN curl -L https://github.com/felt/tippecanoe/tarball/master -o tippecanoe.tar.gz \
+    && mkdir tippecanoe && tar -xzf tippecanoe.tar.gz -C tippecanoe --strip-components=1 \
+    && tar -xzf tippecanoe.tar.gz \
+    && cd tippecanoe \
+    && make \
+    && make install
+
+FROM python:3.11.4 AS dependencies
+RUN curl -LsSf https://astral.sh/uv/0.7.12/install.sh | sh
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=0 PATH="/root/.local/bin/:$PATH"
+WORKDIR /app
+COPY uv.lock pyproject.toml ./
+#TODO: in the future, remove development dependencies from final container
+RUN uv sync --locked
+
 
-# Set the working directory in the container
-WORKDIR /usr/src/app
+FROM python:3.11.4
 
 # Install system dependencies for GDAL and Tippecanoe
+# TODO: Figure out which Tippecanoe dependencies we can remove since we've moved this to a
+# separate stage
 RUN apt-get update && apt-get install -y \
     libgdal-dev \
     gcc \
@@ -15,38 +37,17 @@ RUN apt-get update && apt-get install -y \
     lsb-release \
     && rm -rf /var/lib/apt/lists/*
 
-# install postgres client 16 for psql and pg_dump executables for backups.
-# should match the version used in the other docker file for the postgres install
-RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc |  gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg
-RUN apt update
-RUN apt install -y postgresql-client-16
-
-# Set GDAL environment variables
-ENV GDAL_VERSION=3.6.2
-ENV GDAL_CONFIG=/usr/bin/gdal-config
-
-# Install Pipenv
-RUN pip install pipenv
-
-# Copy the Pipfile and Pipfile.lock from the src directory
-COPY src/Pipfile src/Pipfile.lock ./
+COPY --from=tippecanoe /usr/local/bin/tippecanoe /usr/local/bin/tippecanoe
+COPY --from=dependencies /app /app
 
-# update pipfile
-RUN pipenv lock
+ENV GDAL_VERSION=3.6.2 \
+    GDAL_CONFIG=/usr/bin/gdal-config \
+    # place .venv binaries copied from dependencies stage to front of PATH.
+    # This includes python, pip, and individual package binaries
+    PATH="/app/.venv/bin:$PATH"
 
-# Install the dependencies from Pipfile
-RUN pipenv install --deploy --ignore-pipfile
-
-# Clone and build Tippecanoe
-RUN git clone https://github.com/felt/tippecanoe.git \
-    && cd tippecanoe \
-    && make \
-    && make install
+WORKDIR /app
+COPY src ./src
 
-# Copy the src directory
-COPY src/ .
 
-# Use Pipenv to run the script
-# Adjust the path to your main Python script if needed
-CMD ["pipenv", "run", "python", "./script.py"]
+CMD ["python", "-m", "src.main"]

data/Dockerfile-pg

@@ -6,94 +6,16 @@ services:
     image: vacant-lots-proj:latest
     environment:
       - GOOGLE_APPLICATION_CREDENTIALS=/app/service-account-key.json
-      - VACANT_LOTS_DB
-      - POSTGRES_PASSWORD
-      - POSTGRES_PORT=5434
       - CLEAN_GREEN_GOOGLE_KEY
       - PYTHONUNBUFFERED=1
       - GOOGLE_CLOUD_BUCKET_NAME
       - GOOGLE_CLOUD_PROJECT
       - CAGP_SLACK_API_TOKEN
     volumes:
-      - ./src:/usr/src/app
+      - ./src:/app/src
       - ~/.config/gcloud/application_default_credentials.json:/app/service-account-key.json
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
     extra_hosts:
       - host.docker.internal:host-gateway
     network_mode: 'host'
-
-  formatter:
-    build:
-      context: .
-    volumes:
-      - ./src:/usr/src/app
-    command: sh -c "pip install ruff && ruff format"
-    network_mode: 'host'
-
-  linter:
-    build:
-      context: .
-    volumes:
-      - ./src:/usr/src/app
-    command: sh -c "pip install ruff && ruff check --fix"
-    network_mode: 'host'
-
-  streetview:
-    build:
-      context: .
-    environment:
-      - GOOGLE_APPLICATION_CREDENTIALS=/app/service-account-key.json
-      - CLEAN_GREEN_GOOGLE_KEY
-      - VACANT_LOTS_DB
-      - GOOGLE_CLOUD_BUCKET_NAME
-    volumes:
-      - ./src:/usr/src/app
-      - ~/.config/gcloud/application_default_credentials.json:/app/service-account-key.json
-      - /etc/timezone:/etc/timezone:ro
-      - /etc/localtime:/etc/localtime:ro
-    command: sh -c "pipenv run python streetview.py"
-    extra_hosts:
-      - host.docker.internal:host-gateway
-    network_mode: 'host'
-
-  postgres:
-    container_name: cagp-postgres
-    build:
-      context: .
-      dockerfile: Dockerfile-pg
-    environment:
-      PGPORT: 5433
-      POSTGRES_PASSWORD:
-    restart: always
-    ports:
-      - '5433:5433'
-    volumes:
-      - database_volume:/var/lib/postgresql/data
-      - ./init_pg.sql:/docker-entrypoint-initdb.d/init_pg.sql
-      - /etc/timezone:/etc/timezone:ro
-      - /etc/localtime:/etc/localtime:ro
-    extra_hosts:
-      - host.docker.internal:host-gateway
-
-  postgres-timescale:
-    container_name: cagp-postgres-timescale
-    build:
-      context: .
-      dockerfile: Dockerfile-timescale
-    environment:
-      PGPORT: 5434
-      POSTGRES_PASSWORD:
-    restart: always
-    ports:
-      - '5434:5434'
-    volumes:
-      - timescale_database_volume:/var/lib/postgresql/data
-      - ./init_pg_timescale.sql:/docker-entrypoint-initdb.d/init_pg.sql
-      - /etc/timezone:/etc/timezone:ro
-      - /etc/localtime:/etc/localtime:ro
-    extra_hosts:
-      - host.docker.internal:host-gateway
-volumes:
-  database_volume:
-  timescale_database_volume: