GPU pass through container ollama service added

Author: Himmat Rai

agent/app/agentic_loop.py

@@ -12,7 +12,7 @@
 
 load_dotenv()
 
-OLLAMA_HOST = "http://localhost:11434"
+OLLAMA_HOST = "http://ollama:11434"
 OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3.2:3b")
 
 

docker-compose.yaml

@@ -4,8 +4,6 @@ services:
     container_name: ollama
     ports:
       - 11434:11434
-    # expose:
-    #   - 11434
     volumes:
       - ollama_data:/root/.ollama
       - ./scripts/start_ollama.sh:/start_ollama.sh:ro
@@ -20,8 +18,21 @@ services:
       - .env
     environment:
       - OLLAMA_HOST=0.0.0.0
+        # - CUDA_VISIBLE_DEVICES=0  # Prioritizes GPU 0 exclusively for container
+      - OLLAMA_CONTEXT_LENGTH=2048  # Safe for 4GB
+        # - OLLAMA_FLASH_ATTENTION=false  # Avoids allocation crashes
+      - OLLAMA_NUM_PARALLEL=1  # Single model load
+      - OLLAMA_MAX_LOADED_MODELS=1
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              device_ids: ["0"]  # Lock to GPU 0
+              capabilities: [gpu]
     networks:
       - local_code_network
+    
     entrypoint: ["/start_ollama.sh"]
 
   agent:
@@ -58,4 +69,4 @@ volumes:
 
 networks:
   local_code_network:
-    driver: bridge
+    driver: bridge