Make eval task worker/orchestrator use the same image, build in github action (#1626)

nishu-builder · web-flow · commit 259e859c1e0a · 2025-07-23T19:57:17.000Z
- Rename (confusing) build-docker-img and Dockerfile.local to policy-evaluator-specific names - Add github action for pushing new image - Update policy evaluator helm charts to only one use image type - Remove kubectl install from app_backend image since we are not using it anymore for orchestrator [Asana Task](https://app.asana.com/1/1209016784099267/project/1210348820405981/task/1210872207013045)
diff --git a/.github/workflows/build-metta-policy-evaluator-docker.yml b/.github/workflows/build-metta-policy-evaluator-docker.yml
@@ -0,0 +1,60 @@
+name: "Build Policy Evaluator Docker Image"
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "app_backend/**"
+      - ".github/workflows/**"
+  workflow_dispatch: {}
+
+env:
+  IMAGE_NAME: metta-policy-evaluator
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE }}
+          aws-region: us-east-1
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ steps.login-ecr.outputs.registry }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha
+            type=sha,format=long
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./devops/docker/Dockerfile.policy_evaluator
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/app_backend/Dockerfile b/app_backend/Dockerfile
@@ -1,17 +1,10 @@
 # Use Python 3.11 slim image as base
 FROM python:3.11-slim
 
-# Install system dependencies and add Kubernetes apt repository
+# Install system dependencies
 RUN apt-get update && apt-get install -y \
     curl \
     git \
-    apt-transport-https \
-    ca-certificates \
-    gnupg \
-    && curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg \
-    && echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /' > /etc/apt/sources.list.d/kubernetes.list \
-    && apt-get update \
-    && apt-get install -y kubectl \
     && rm -rf /var/lib/apt/lists/*
 
 # Install uv
diff --git a/app_backend/README.md b/app_backend/README.md
@@ -60,7 +60,7 @@ Ensure that you have app_backend running. If it is running anywhere except for y
 #### Local docker
 
 Getting the service running
-- `metta local build-docker-img` to build the `metta-local:latest` image. This will serve as the base for both the orchestrator and the workers
+- `metta local build-policy-evaluator-img` to build the `metta-policy-evaluator-local:latest` image. This will serve as the base for both the orchestrator and the workers
 - `WANDB_API_KEY=your-key-here docker compose -f app_backend/src/metta/app_backend/docker-compose.yml up`
 
 Viewing logs
@@ -74,7 +74,7 @@ Kind is a tool for running local Kubernetes clusters using Docker container node
 - Ensure `metta status` shows that you are connected to `observatory-local-key` and `wandb`
 - Run your local app_backend server, which should be serving on `localhost:8000`
 - Run your local frontend from `observatory/`
-- Build orchestrator and worker images: `metta local build-docker-img`
+- Build orchestrator and worker images: `metta local build-policy-evaluator-img`
 - Set up helm: `metta local kind build`
 - Launch: `metta local kind up`
 - Monitor: `metta local kind get-pods` and `metta local kind pods {pod-name}`
diff --git a/app_backend/src/metta/app_backend/docker-compose.yml b/app_backend/src/metta/app_backend/docker-compose.yml
@@ -1,16 +1,16 @@
 services:
   orchestrator:
-    image: metta-local:latest # metta local build-app-backend-img
+    image: metta-policy-evaluator-local:latest # metta local build-policy-evaluator-img
     environment:
       - BACKEND_URL=${BACKEND_URL:-http://host.docker.internal:8000}
       - POLL_INTERVAL=${POLL_INTERVAL:-20}
       - CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-docker}
       - WANDB_API_KEY=${WANDB_API_KEY}
-      - DOCKER_IMAGE=${DOCKER_IMAGE:-metta-local:latest}
+      - DOCKER_IMAGE=${DOCKER_IMAGE:-metta-policy-evaluator-local:latest}
     volumes:
       - ../../../:/workspace/metta/app_backend
       - /var/run/docker.sock:/var/run/docker.sock
-    command: ["uv", "run", "python", "-m", "metta.app_backend.eval_task_orchestrator"]
+    command: [ "uv", "run", "python", "-m", "metta.app_backend.eval_task_orchestrator" ]
     restart: unless-stopped
     # Run as root to access Docker socket - in production, use a proper docker group
     user: root
diff --git a/app_backend/src/metta/app_backend/eval_task_orchestrator.py b/app_backend/src/metta/app_backend/eval_task_orchestrator.py
@@ -34,7 +34,7 @@ def __init__(
         self,
         backend_url: str,
         machine_token: str,
-        docker_image: str = "metta-local:latest",
+        docker_image: str = "metta-policy-evaluator-local:latest",
         poll_interval: float = 5.0,
         worker_idle_timeout: float = 600.0,
         container_manager: AbstractContainerManager | None = None,
@@ -179,7 +179,7 @@ async def main() -> None:
     logger = logging.getLogger(__name__)
 
     backend_url = os.environ.get("BACKEND_URL", "http://localhost:8000")
-    docker_image = os.environ.get("DOCKER_IMAGE", "metta-local:latest")
+    docker_image = os.environ.get("DOCKER_IMAGE", "metta-policy-evaluator-local:latest")
     poll_interval = float(os.environ.get("POLL_INTERVAL", "5"))
     worker_idle_timeout = float(os.environ.get("WORKER_IDLE_TIMEOUT", "600"))
     machine_token = os.environ["MACHINE_TOKEN"]
diff --git a/app_backend/tests/test_docker_integration.py b/app_backend/tests/test_docker_integration.py
@@ -231,12 +231,6 @@ def test_protected_endpoint_with_auth(self, app_backend_container):
         # but not fail due to authentication (401)
         assert response.status_code != 401
 
-    def test_kubectl_available(self, app_backend_container):
-        """Test that kubectl is available in the container."""
-        container, _, __ = app_backend_container
-        exit_code, output = container.exec("kubectl version --client")
-        assert exit_code == 0, f"kubectl command failed with exit code {exit_code}. Output: {output}"
-
 
 if __name__ == "__main__":
     # Simple test runner for debugging
diff --git a/devops/charts/orchestrator/environments/kind.yaml b/devops/charts/orchestrator/environments/kind.yaml
@@ -1,10 +1,5 @@
 image:
-  repository: metta-local
-  tag: "latest"
-  pullPolicy: Never
-
-workerImage:
-  repository: metta-local
+  repository: metta-policy-evaluator-local
   tag: "latest"
   pullPolicy: Never
 
diff --git a/devops/charts/orchestrator/templates/deployment.yaml b/devops/charts/orchestrator/templates/deployment.yaml
@@ -30,7 +30,7 @@ spec:
         - name: WORKER_IDLE_TIMEOUT
           value: {{ .Values.env.WORKER_IDLE_TIMEOUT | quote }}
         - name: DOCKER_IMAGE
-          value: "{{ .Values.workerImage.repository }}:{{ .Values.workerImage.tag }}"
+          value: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         - name: KUBERNETES_NAMESPACE
           valueFrom:
             fieldRef:
diff --git a/devops/charts/orchestrator/values.yaml b/devops/charts/orchestrator/values.yaml
@@ -2,17 +2,12 @@ replicaCount: 1
 
 image:
   registry: 751442549699.dkr.ecr.us-east-1.amazonaws.com
-  name: metta-app-backend
+  name: metta-policy-evaluator
   tag: ""  # set by ci/cd
 
 serviceAccount:
   create: true
 
-workerImage:
-  repository: 751442549699.dkr.ecr.us-east-1.amazonaws.com
-  name: metta
-  tag: "latest"
-
 env:
   BACKEND_URL: "https://api.observatory.softmax-research.net"
   CONTAINER_RUNTIME: "k8s"
diff --git a/devops/docker/Dockerfile.policy_evaluator b/devops/docker/Dockerfile.policy_evaluator
diff --git a/metta/setup/local_commands.py b/metta/setup/local_commands.py
@@ -24,8 +24,10 @@ def _build_img(self, tag: str, dockerfile_path: Path) -> None:
     def build_app_backend_img(self) -> None:
         self._build_img("metta-app-backend:latest", self.repo_root / "app_backend" / "Dockerfile")
 
-    def build_docker_img(self) -> None:
-        self._build_img("metta-local:latest", self.repo_root / "devops" / "docker" / "Dockerfile.local")
+    def build_policy_evaluator_img(self) -> None:
+        self._build_img(
+            "metta-policy-evaluator-local:latest", self.repo_root / "devops" / "docker" / "Dockerfile.policy_evaluator"
+        )
 
     def load_policies(self, unknown_args) -> None:
         """Load W&B artifacts as policies into stats database."""
diff --git a/metta/setup/metta_cli.py b/metta/setup/metta_cli.py
@@ -236,8 +236,8 @@ def cmd_shell(self) -> None:
     def cmd_local(self, args, unknown_args=None) -> None:
         """Handle local development commands."""
         if hasattr(args, "local_command") and args.local_command:
-            if args.local_command == "build-docker-img":
-                self.local_commands.build_docker_img()
+            if args.local_command == "build-policy-evaluator-img":
+                self.local_commands.build_policy_evaluator_img()
             elif args.local_command == "build-app-backend-img":
                 self.local_commands.build_app_backend_img()
             elif args.local_command == "load-policies":
@@ -573,7 +573,9 @@ def main(self) -> None:
         local_subparsers = local_parser.add_subparsers(dest="local_command", help="Available local commands")
 
         # Local subcommands
-        local_subparsers.add_parser("build-docker-img", help="Build local development Docker image")
+        local_subparsers.add_parser(
+            "build-policy-evaluator-img", help="Build local development policy evaluator Docker image"
+        )
         local_subparsers.add_parser("build-app-backend-img", help="Build local development app_backend Docker image")
 
         # Add load-policies command
diff --git a/metta/setup/tools/local/kind.py b/metta/setup/tools/local/kind.py
@@ -68,7 +68,9 @@ def build(self) -> None:
 
         from metta.setup.local_commands import LocalCommands
 
-        self._ensure_docker_img_built("metta-local:latest", LocalCommands(self.repo_root).build_docker_img)
+        self._ensure_docker_img_built(
+            "metta-policy-evaluator-local:latest", LocalCommands(self.repo_root).build_policy_evaluator_img
+        )
 
         # No need to create RBAC manually - Helm chart will handle it
         success("Kind cluster ready!")
