refactor: Add local dependencies for rag-core-api and rag-core-lib in prod-local group for development

Author: a-klos

.github/workflows/lint-and-test.yml

@@ -8,7 +8,7 @@ on:
 
 env:
   NODE_VERSION: '18'
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: '3.13'
 
 jobs:
   changes:
@@ -81,7 +81,7 @@ jobs:
 
       - name: Build Docker image
         run: |
-          docker build -t $IMAGE_NAME --build-arg dev=1 -f services/${{ matrix.service }}/Dockerfile .
+          docker build -t $IMAGE_NAME -f services/${{ matrix.service }}/Dockerfile.dev .
 
       - name: Run linting
         run: |
@@ -104,15 +104,16 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Set Docker Image Names
+      - name: Set Docker Image Name
         run: |
           echo "LINT_IMAGE_NAME=${{ matrix.library }}-lint:${{ needs.sanitize-branch-name.outputs.sanitized_ref }}-${{ github.run_number }}" >> $GITHUB_ENV
           echo "TEST_IMAGE_NAME=${{ matrix.library }}-test:${{ needs.sanitize-branch-name.outputs.sanitized_ref }}-${{ github.run_number }}" >> $GITHUB_ENV
         shell: bash
 
       - name: Build lint image
         run: |
-          docker build -t $LINT_IMAGE_NAME --build-arg TEST=0 -f libs/Dockerfile libs
+          docker build -t $LINT_IMAGE_NAME --build-arg DIRECTORY=${{ matrix.library }} --build-arg TEST=0 -f libs/Dockerfile libs
+
 
       - name: Run linting
         run: |

.github/workflows/semantic-release.yml

@@ -90,27 +90,21 @@ jobs:
           - name: rag-backend
             dockerfile: services/rag-backend/Dockerfile
             image: rag-backend
-            build-args: "dev=0"
           - name: admin-backend
             dockerfile: services/admin-backend/Dockerfile
             image: admin-backend
-            build-args: "dev=0"
           - name: document-extractor
             dockerfile: services/document-extractor/Dockerfile
             image: document-extractor
-            build-args: "dev=0"
           - name: mcp-server
             dockerfile: services/mcp-server/Dockerfile
             image: mcp-server
-            build-args: "dev=0"
           - name: frontend
             dockerfile: services/frontend/apps/chat-app/Dockerfile
             image: frontend
-            build-args: ""
           - name: admin-frontend
             dockerfile: services/frontend/apps/admin-app/Dockerfile
             image: admin-frontend
-            build-args: ""
     steps:
       - name: debug-job-inputs
         run: |

Tiltfile

@@ -7,17 +7,33 @@ if os.path.exists(".env"):
     dotenv(fn=".env")
 
 config.define_bool("debug")
+config.define_bool("dev")
 cfg = config.parse()
 backend_debug = cfg.get("debug", False)
+dev_mode = cfg.get("dev", False)
+
+# Print mode information
+if dev_mode:
+    print("🚀 Running in DEVELOPMENT mode (using Dockerfile.dev)")
+    print("   - Live code updates enabled")
+    print("   - Fast iteration optimized builds")
+else:
+    print("🏭 Running in PRODUCTION mode (using Dockerfile)")
+    print("   - Production-optimized builds")
+    print("   - No live updates")
 
 core_library_context = "./libs"
 
 
 def create_linter_command(folder_name, name):
-    # Build with Dockerfile.dev for services, Dockerfile for libs
-    dockerfile = folder_name + ("/Dockerfile" if folder_name == "./libs" else "/Dockerfile.dev")
-    # Only libs use TEST build arg; services use no build args
-    build_args = " --build-arg TEST=1" if folder_name == "./libs" else ""
+    # Build with appropriate Dockerfile based on context
+    if folder_name == "./libs":
+        dockerfile = folder_name + "/Dockerfile"
+        build_args = " --build-arg TEST=0"
+    else:
+        # For services, always use .dev for testing/linting (dev tools needed)
+        dockerfile = folder_name + "/Dockerfile.dev"
+        build_args = ""
     return (
         "docker build -t "
         + name
@@ -31,9 +47,14 @@ def create_linter_command(folder_name, name):
 
 
 def create_test_command(folder_name, name):
-    # Build with Dockerfile.dev for services, Dockerfile for libs
-    dockerfile = folder_name + ("/Dockerfile" if folder_name == "./libs" else "/Dockerfile.dev")
-    build_args = " --build-arg TEST=1" if folder_name == "./libs" else ""
+    # Build with appropriate Dockerfile based on context
+    if folder_name == "./libs":
+        dockerfile = folder_name + "/Dockerfile"
+        build_args = " --build-arg TEST=1"
+    else:
+        # For services, always use .dev for testing/linting (dev tools needed)
+        dockerfile = folder_name + "/Dockerfile.dev"
+        build_args = ""
     return (
         "docker build -t "
         + name
@@ -152,16 +173,15 @@ rag_api_image_name = "rag-backend"
 
 backend_context = "./services/rag-backend"
 rag_api_full_image_name = "%s/%s" % (registry, rag_api_image_name)
-docker_build(
-    rag_api_full_image_name,
-    ".",
-    live_update=[
-        sync(backend_context, "/app/services/rag-backend"),
-        sync(core_library_context+"/rag-core-api", "/app/libs/rag-core-api"),
-        sync(core_library_context+"/rag-core-lib", "/app/libs/rag-core-lib"),
-    ],
-    dockerfile=backend_context + "/Dockerfile.dev",
-    ignore=[
+
+# Choose dockerfile based on dev mode
+backend_dockerfile = backend_context + ("/Dockerfile.dev" if dev_mode else "/Dockerfile")
+
+docker_build_config = {
+    "ref": rag_api_full_image_name,
+    "context": ".",
+    "dockerfile": backend_dockerfile,
+    "ignore": [
         "infrastructure/",
         "libs/admin-api-lib/",
         "libs/extractor-api-lib/",
@@ -170,7 +190,22 @@ docker_build(
         "services/mcp-server/",
         "services/frontend/",
     ],
-)
+}
+
+# Add build args and live_update based on dev mode
+if dev_mode:
+    docker_build_config["live_update"] = [
+        sync(backend_context, "/app/services/rag-backend"),
+        sync(core_library_context+"/rag-core-api", "/app/libs/rag-core-api"),
+        sync(core_library_context+"/rag-core-lib", "/app/libs/rag-core-lib"),
+    ]
+else:
+    # Use prod-local for Tilt with production Dockerfile
+    docker_build_config["build_args"] = {
+        "DEPENDENCY_GROUP": "prod-local"
+    }
+
+docker_build(**docker_build_config)
 
 # Add linter trigger
 local_resource(
@@ -199,22 +234,31 @@ mcp_image_name = "mcp-server"
 
 mcp_context = "./services/mcp-server"
 mcp_full_image_name = "%s/%s" % (registry, mcp_image_name)
-docker_build(
-    mcp_full_image_name,
-    ".",
-    live_update=[
-        sync(mcp_context, "/app/services/mcp-server"),
-    ],
-    dockerfile=mcp_context + "/Dockerfile.dev",
-    ignore=[
+
+# Choose dockerfile based on dev mode
+mcp_dockerfile = mcp_context + ("/Dockerfile.dev" if dev_mode else "/Dockerfile")
+
+mcp_docker_build_config = {
+    "ref": mcp_full_image_name,
+    "context": ".",
+    "dockerfile": mcp_dockerfile,
+    "ignore": [
         "infrastructure/",
         "libs/",
         "services/admin-backend/",
         "services/document-extractor/",
         "services/rag-backend/",
         "services/frontend/",
     ],
-)
+}
+
+# Add live_update only in dev mode
+if dev_mode:
+    mcp_docker_build_config["live_update"] = [
+        sync(mcp_context, "/app/services/mcp-server"),
+    ]
+
+docker_build(**mcp_docker_build_config)
 
 # Add linter trigger
 local_resource(
@@ -236,16 +280,15 @@ admin_api_image_name = "admin-backend"
 
 admin_backend_context = "./services/admin-backend"
 admin_api_full_image_name = "%s/%s" % (registry, admin_api_image_name)
-docker_build(
-    admin_api_full_image_name,
-    ".",
-    live_update=[
-        sync(admin_backend_context, "/app/services/admin-backend"),
-        sync(core_library_context + "/rag-core-lib", "/app/libs/rag-core-lib"),
-        sync(core_library_context + "/admin-api-lib", "/app/libs/admin-api-lib"),
-    ],
-    dockerfile=admin_backend_context + "/Dockerfile.dev",
-    ignore=[
+
+# Choose dockerfile based on dev mode
+admin_dockerfile = admin_backend_context + ("/Dockerfile.dev" if dev_mode else "/Dockerfile")
+
+admin_docker_build_config = {
+    "ref": admin_api_full_image_name,
+    "context": ".",
+    "dockerfile": admin_dockerfile,
+    "ignore": [
         "infrastructure/",
         "libs/rag-core-api/",
         "libs/extractor-api-lib/",
@@ -254,7 +297,22 @@ docker_build(
         "services/mcp-server/",
         "services/frontend/",
     ],
-)
+}
+
+# Add build args and live_update based on dev mode
+if dev_mode:
+    admin_docker_build_config["live_update"] = [
+        sync(admin_backend_context, "/app/services/admin-backend"),
+        sync(core_library_context + "/rag-core-lib", "/app/libs/rag-core-lib"),
+        sync(core_library_context + "/admin-api-lib", "/app/libs/admin-api-lib"),
+    ]
+else:
+    # Use prod-local for Tilt with production Dockerfile
+    admin_docker_build_config["build_args"] = {
+        "DEPENDENCY_GROUP": "prod-local"
+    }
+
+docker_build(**admin_docker_build_config)
 
 # Add linter trigger
 local_resource(
@@ -286,15 +344,15 @@ document_extractor_image_name = "document-extractor"
 
 extractor_context = "./services/document-extractor"
 document_extractor_full_image_name = "%s/%s" % (registry, document_extractor_image_name)
-docker_build(
-    document_extractor_full_image_name,
-    ".",
-    live_update=[
-        sync(extractor_context, "/app/services/document-extractor"),
-        sync(core_library_context +"/extractor-api-lib", "/app/libs/extractor-api-lib"),
-        ],
-    dockerfile=extractor_context + "/Dockerfile.dev",
-    ignore=[
+
+# Choose dockerfile based on dev mode
+extractor_dockerfile = extractor_context + ("/Dockerfile.dev" if dev_mode else "/Dockerfile")
+
+extractor_docker_build_config = {
+    "ref": document_extractor_full_image_name,
+    "context": ".",
+    "dockerfile": extractor_dockerfile,
+    "ignore": [
         "infrastructure/",
         "libs/rag-core-api/",
         "libs/rag-core-lib/",
@@ -305,7 +363,21 @@ docker_build(
         "services/frontend/",
         "scripts/",
     ],
-)
+}
+
+# Add build args and live_update based on dev mode
+if dev_mode:
+    extractor_docker_build_config["live_update"] = [
+        sync(extractor_context, "/app/services/document-extractor"),
+        sync(core_library_context +"/extractor-api-lib", "/app/libs/extractor-api-lib"),
+    ]
+else:
+    # Use prod-local for Tilt with production Dockerfile
+    extractor_docker_build_config["build_args"] = {
+        "DEPENDENCY_GROUP": "prod-local"
+    }
+
+docker_build(**extractor_docker_build_config)
 
 # Add linter trigger
 local_resource(

services/admin-backend/Dockerfile

@@ -4,6 +4,9 @@ ENV POETRY_VIRTUALENVS_PATH=/app/services/admin-backend/.venv
 ENV VIRTUAL_ENV="${POETRY_VIRTUALENVS_PATH}"
 ENV POETRY_VERSION=2.1.3
 
+# Build argument to control which dependency group to install
+ARG DEPENDENCY_GROUP=prod
+
 RUN DEBIAN_FRONTEND=noninteractive apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential --no-install-recommends make \
     && python3 -m venv "${POETRY_VIRTUALENVS_PATH}" \
@@ -13,12 +16,18 @@ ENV PATH="${POETRY_VIRTUALENVS_PATH}/bin:$PATH"
 WORKDIR /app/services/admin-backend
 COPY services/admin-backend/pyproject.toml services/admin-backend/poetry.lock ./
 
+# Copy the required lib dependencies
+COPY libs/admin-api-lib/pyproject.toml libs/admin-api-lib/poetry.lock /app/libs/admin-api-lib/
+COPY libs/admin-api-lib/src /app/libs/admin-api-lib/src
+COPY libs/rag-core-lib/pyproject.toml libs/rag-core-lib/poetry.lock /app/libs/rag-core-lib/
+COPY libs/rag-core-lib/src /app/libs/rag-core-lib/src
+
 RUN mkdir -p log && chmod 700 log \
     && touch /app/services/admin-backend/log/logfile.log && chmod 600 /app/services/admin-backend/log/logfile.log
 
 RUN poetry config virtualenvs.create false \
     && cd /app/services/admin-backend \
-    && poetry install --no-interaction --no-ansi --no-root --with prod
+    && poetry install --no-interaction --no-ansi --no-root --with ${DEPENDENCY_GROUP}
 
 FROM --platform=linux/amd64 python:3.13-bookworm
 WORKDIR /app/services/admin-backend
@@ -31,6 +40,23 @@ COPY --from=build /usr/local/bin/ /usr/local/bin/
 COPY --from=build /usr/bin/make /usr/bin/make
 COPY --from=build /usr/local/lib/ /usr/local/lib/
 
+# Copy the library source code to the final stage only for local path dependencies
+ARG DEPENDENCY_GROUP=prod
+COPY --from=build --chown=nonroot:nonroot \
+    /app/libs/admin-api-lib \
+    /tmp/conditional-libs/admin-api-lib
+COPY --from=build --chown=nonroot:nonroot \
+    /app/libs/rag-core-lib \
+    /tmp/conditional-libs/rag-core-lib
+
+# Move libraries to final location only if using prod-local
+RUN if [ "$DEPENDENCY_GROUP" = "prod-local" ]; then \
+        mkdir -p /app/libs && \
+        mv /tmp/conditional-libs/* /app/libs/ && \
+        chown -R nonroot:nonroot /app/libs; \
+    fi && \
+    rm -rf /tmp/conditional-libs
+
 # cleanup
 RUN apt-get clean autoclean && apt-get autoremove --yes \
     && while read -r shell; do rm -f "$shell"; done < /etc/shells || true \
@@ -43,3 +69,5 @@ USER nonroot
 
 COPY --chown=nonroot:nonroot services/admin-backend .
 COPY --from=build --chown=nonroot:nonroot  /app/services/admin-backend/log /app/services/admin-backend/log
+
+