Merge pull request #33 from Sonnet-Scripts/mf-simplify-linux-python-build

Mf simplify linux python build

Author: markfreeman-dot-dev

.devcontainer.json

@@ -2,15 +2,16 @@
 	"name": "Sonnet Scripts Dev Container",
 	"dockerComposeFile": "docker-compose.yml",
 	"service": "pythonbase",
+	"workspaceFolder": "/sonnet-scripts",
 	"initializeCommand": [
-		"/bin/sh", "-c", "docker compose build linuxbase && docker compose build pythonbase"
+		"/bin/sh", "-c", "docker compose build pythonbase"
 	],
-	"workspaceFolder": "/workspaces/sonnet-scripts",
-	"mounts": [
-		"source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
-	],
-	 "features": {
-		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
+	"features": {
+		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {
+			"moby": false
+		},
+		"ghcr.io/devcontainers/features/git:1": {},
+		"ghcr.io/devcontainers/features/common-utils:2": {}
 	},
 	"customizations": {
 		"vscode": {
@@ -19,4 +20,4 @@
 			]
 		}
 	}
-}
+}

.github/workflows/ci_pipeline.yml

@@ -27,7 +27,6 @@ jobs:
 
       - name: Build Docker Images
         run: |
-          docker build -t linuxbase linuxbase
           docker build -t pythonbase pythonbase
 
       - name: Start Docker Compose

Makefile

@@ -1,10 +1,7 @@
 # Set up the project with Docker Compose
 setup:
-	@docker compose build linuxbase
-	@docker compose build pythonbase
-	@docker compose build pipelinebase
-	@docker compose build dbtbase
-	@docker compose build
+	@docker compose ps pythonbase --format json | grep -q pythonbase || docker compose build pythonbase
+	@docker compose build $$(docker compose config --services | grep -v pythonbase)
 	@docker compose up -d
 
 # Install the sonnet CLI tool
@@ -21,7 +18,6 @@ rebuild:
 # Completely clean up Docker environment and rebuild containers from scratch
 rebuild-clean:
 	@docker compose down -v --remove-orphans --rmi all
-	@docker compose build --no-cache linuxbase
 	@docker compose build --no-cache pythonbase
 	@docker compose build --no-cache pipelinebase
 	@docker compose build --no-cache dbtbase
@@ -66,10 +62,6 @@ else
 	xdg-open "http://localhost:8978"
 endif
 
-# Execute a shell inside the linuxbase container
-exec-linuxbase:
-	@docker compose exec linuxbase bash
-
 # Load data into PostgreSQL
 load-db:
 	@docker compose exec -e PYTHONPATH=/apps pipelinebase /venv/bin/python -m ingest_claims.load_claims_to_db

dbtbase/Dockerfile

@@ -1,4 +1,4 @@
-FROM pythonbase:latest
+FROM pythonbase
 
 # Set the Working Directory
 WORKDIR /apps

docker-compose.yml

@@ -5,20 +5,29 @@ services:
       context: ./pythonbase
     container_name: pythonbase
     image: pythonbase
-    depends_on:
-      linuxbase:
-        condition: service_completed_successfully
+    stdin_open: true
+    tty: true
     ports:
       - "4213:4213"
-      
+    volumes:
+      - .:/sonnet-scripts
+      - /var/run/docker.sock:/var/run/docker.sock
+    working_dir: /sonnet-scripts
+    healthcheck:
+      test: ["CMD", "python", "--version"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 15s
+
   jupyterbase:
     build:
       context: ./jupyterbase
     container_name: jupyterbase
     image: jupyterbase
     depends_on:
       pythonbase:
-        condition: service_started
+        condition: service_healthy
       pgduckdb:
         condition: service_healthy
     ports:
@@ -31,7 +40,7 @@ services:
     image: pipelinebase
     depends_on:
       pythonbase:
-        condition: service_started
+        condition: service_healthy
       pgduckdb:
         condition: service_healthy
       minio:
@@ -44,7 +53,7 @@ services:
     image: dbtbase
     depends_on:
       pythonbase:
-        condition: service_started
+        condition: service_healthy
       pgduckdb:
         condition: service_healthy
     environment:
@@ -56,12 +65,6 @@ services:
     volumes:
       - dbt_data:/apps/data
 
-  linuxbase:
-    build:
-      context: ./linuxbase
-    container_name: linuxbase
-    image: linuxbase
-
   pgduckdb:
     image: pgduckdb/pgduckdb:17-v0.1.0
     container_name: pgduckdb
@@ -76,6 +79,8 @@ services:
     healthcheck:
       test: ["CMD", "pg_isready", "-U", "postgres"]
       retries: 5
+      interval: 10s
+      timeout: 5s
 
   pgadmin:
     image: dpage/pgadmin4:9.3.0
@@ -121,24 +126,10 @@ services:
       - 9000:9000
     command: ["server", "/data", "--console-address", ":9001"]
 
-  cloudbeaver:
-    image: dbeaver/cloudbeaver:25.0.3
-    container_name: cloudbeaver
-    ports:
-      - "8978:8978"
-    volumes:
-      - cloudbeaver-data:/opt/cloudbeaver/workspace
-      - ./cloudbeaver/conf/.cloudbeaver.auto.conf:/opt/cloudbeaver/conf/.cloudbeaver.auto.conf
-      - ./cloudbeaver/conf/.cloudbeaver.runtime.conf:/opt/cloudbeaver/workspace/.data/.cloudbeaver.runtime.conf
-    depends_on:
-      - pgduckdb
-
 volumes:
   pgduckdb_data:
    driver: local
   data:
    driver: local
-  cloudbeaver-data:
-    driver: local
   dbt_data:
     driver: local

jupyterbase/Dockerfile

@@ -1,4 +1,4 @@
-FROM pythonbase:latest
+FROM pythonbase
 
 # Set the Working Directory
 WORKDIR /apps

linuxbase/Dockerfile

@@ -1,4 +1,4 @@
-FROM pythonbase:latest
+FROM pythonbase
 
 # Set the Working Directory
 WORKDIR /apps

pipelinebase/Dockerfile

@@ -1,67 +1,60 @@
-FROM linuxbase AS pythonbase
+FROM python:3.12
+
+# Set environment variables for localization and timezone settings
+ENV LANG=en_US.UTF-8 \
+    LANGUAGE=en_US:en \
+    LC_ALL=en_US.UTF-8 \
+    TZ="America/Chicago" \
+    DEBIAN_FRONTEND=noninteractive
 
 RUN mkdir /apps
 
 # Set build arguments for cross-compilation
 ARG TARGETARCH
+ARG DUCKDB_VERSION="v1.2.2"
 
 # Set environment variables
-ENV PATH=/venv/bin:$PATH \
-    PATH=/root/.cargo/bin:$PATH \
-    PATH="/root/.local/bin/:$PATH"
+ENV PATH=/root/.local/bin:/venv/bin:$PATH
 
-# Install uv, curl, and the PostgreSQL client (`psql`)
-RUN apt-get update && apt-get install -y ca-certificates curl postgresql-client unzip
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    postgresql-client gzip curl && \
+    rm -rf /var/lib/apt/lists/*
 
-# Download UV and MinIO Client (mc) based on architecture
+# Install UV (Python package manager)
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Install MinIO client (architecture-specific)
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
-        curl -LsSf https://astral.sh/uv/install.sh | sh \
-        && curl -O https://dl.min.io/client/mc/release/linux-amd64/mc \
-        && chmod +x mc \
-        && mv mc /usr/local/bin/; \
+        curl -O https://dl.min.io/client/mc/release/linux-amd64/mc; \
     elif [ "$TARGETARCH" = "arm64" ]; then \
-        curl -LsSf https://astral.sh/uv/install.sh | sh \
-        && curl -O https://dl.min.io/client/mc/release/linux-arm64/mc \
-        && chmod +x mc \
-        && mv mc /usr/local/bin/; \
+        curl -O https://dl.min.io/client/mc/release/linux-arm64/mc; \
     else \
         echo "Unsupported architecture: $TARGETARCH" && exit 1; \
-    fi
-
-# Install DuckDB
-ARG DUCKDB_VERSION="v1.2.2"
+    fi && \
+    chmod +x mc && \
+    mv mc /usr/local/bin/mc
 
-RUN set -eux; \
-    apt-get update && apt-get install -y --no-install-recommends \
-        curl ca-certificates gzip && \
-    rm -rf /var/lib/apt/lists/* && \
-    # pick the correct pre‑built binary for the platform
-    if [ "$TARGETARCH" = "amd64" ]; then \
+# Install DuckDB CLI (architecture-specific)
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
         DUCKDB_DIST="linux-amd64"; \
     elif [ "$TARGETARCH" = "arm64" ]; then \
         DUCKDB_DIST="linux-arm64"; \
     else \
         echo "Unsupported architecture: $TARGETARCH" && exit 1; \
     fi && \
     curl -fsSL "https://github.com/duckdb/duckdb/releases/download/${DUCKDB_VERSION}/duckdb_cli-${DUCKDB_DIST}.gz" \
-      | gunzip -c > /usr/local/bin/duckdb && \
+        | gunzip -c > /usr/local/bin/duckdb && \
     chmod +x /usr/local/bin/duckdb && \
-    duckdb --version   # quick sanity check
+    duckdb --version
 
-# Set up a virtual env to use for whatever app is destined for this container.
+# Create Python virtual environment
 RUN uv venv --python 3.12.6 /venv && \
-    echo "\nsource /venv/bin/activate\n" >> /root/.zshrc && \
-    uv --version
-
-# Copy application-specific files
-COPY requirements.txt /apps/requirements.txt
+    /venv/bin/python --version
 
 # Run install of requirements
-RUN . /venv/bin/activate && \
-    uv pip install --upgrade -r /apps/requirements.txt && \
-    uv pip install --upgrade granian[pname]
-
-RUN echo "We're good:" && \
-    /venv/bin/python --version
+COPY requirements.txt /apps/requirements.txt
+RUN uv pip install --upgrade -r /apps/requirements.txt
 
-CMD ["tail", "-f", "/dev/null"]
+# Keep container running for interactive use
+CMD ["tail", "-f", "/dev/null"]