LLM-Langchain-on-NVIDIA-Jetson/docker-compose.yml at main · Advantech-Containers/LLM-Langchain-on-NVIDIA-Jetson · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
# ==========================================================================
# LLM-Langchain-on-NVIDIA-Jetson Docker Compose File
# ==========================================================================
# Version:      1.5.0-Ubuntu20.04-ARM
# Author:       Samir Singh <samir.singh@advantech.com>
# Created:      March 25, 2025
# Last Updated: August 28, 2025
#
# Description:
#   This Docker Compose file deploys the Advantech Jetson™ LLM-LangChain
#   Edge AI development environment with full NVIDIA GPU acceleration
#   enabled for Jetson platforms.
#
#   The stack includes:
#     - LangChain Service: Runs the Advantech Jetson™ LLM-LangChain container
#       with CUDA, Tensor, and NVIDIA Jetson multimedia API integration for
#       optimized large language model workloads at the edge.
#     - OpenWebUI Service: Provides a browser-based user interface for prompt
#       management, visualization, and RAG pipeline testing with seamless
#       integration to the LangChain backend.
#
#   The configuration ensures direct hardware access to NVIDIA GPU, CUDA,
#   Tensor cores, and multimedia accelerators on Jetson devices, while also
#   integrating Advantech benchmarking tools (`wise-bench.sh`) for performance
#   profiling.
#
# Terms and Conditions:
#   1. This software is provided by Advantech Corporation "as is" and any
#      express or implied warranties, including, but not limited to, the implied
#      warranties of merchantability and fitness for a particular purpose are
#      disclaimed.
#   2. In no event shall Advantech Corporation be liable for any direct, indirect,
#      incidental, special, exemplary, or consequential damages arising in any way
#      out of the use of this software.
#   3. Redistribution and use in source and binary forms, with or without
#      modification, are permitted provided that the above copyright notice and
#      this permission notice appear in all copies.
#
# Copyright (c) 2025 Advantech Corporation. All rights reserved.
# ==========================================================================

services:
  langchain-service:
    image: edgesync.azurecr.io/advantech/jetson-llm-langchain:1.5.0-Ubuntu20.04-ARM
    container_name: LLM-Langchain-on-NVIDIA-Jetson
    network_mode: host
    privileged: true
    runtime: nvidia
    env_file: .env
    tty: true
    stdin_open: true
    entrypoint: ["/bin/bash"]
    labels:
      maintainer: "Samir Singh <samir.singh@advantech.com>"
      vendor: "Advantech"
      version: "1.2"
      description: "Jetson™ LLM Langchain Edge AI Development Container"
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - NVIDIA_DRIVER_CAPABILITIES=all,compute,video,utility,graphics
    volumes:
      - ./langchain-service:/workspace/langchain-service/
      - ./wise-bench.sh:/workspace/wise-bench.sh
      - ollama-models:/root/.ollama
      - /etc/nv_tegra_release:/etc/nv_tegra_release
      - /usr/lib/aarch64-linux-gnu/tegra:/usr/lib/aarch64-linux-gnu/tegra
      - /usr/src/jetson_multimedia_api:/usr/src/jetson_multimedia_api
      - /usr/lib/aarch64-linux-gnu/gstreamer-1.0:/usr/lib/aarch64-linux-gnu/gstreamer-1.0
      - /usr/local/cuda:/usr/local/cuda
    restart: always
    devices:
      - /dev/nvhost-ctrl
      - /dev/nvhost-ctrl-gpu
      - /dev/nvhost-prof-gpu
      - /dev/nvmap
      - /dev/nvhost-gpu
      - /dev/nvhost-as-gpu
      - /dev/nvhost-vic
      - /dev/nvhost-msenc
      - /dev/nvhost-nvdec
      - /dev/nvhost-nvjpg
      - /dev/nvgpu/igpu0

  openweb-ui-service:
    image: ghcr.io/open-webui/open-webui:0.6.5
    container_name: openweb-ui-service
    env_file: .env
    network_mode: host
    environment:
      OPENAI_API_BASE_URL: ${OPENAI_API_LANGCHAIN_BASE}
      PORT: ${OPENWEBUI_PORT}
      ENABLE_TAGS_GENERATION: ${ENABLE_TAGS_GENERATION}
      ENABLE_TITLE_GENERATION: ${ENABLE_TITLE_GENERATION}
    volumes:
      - open-webui:/app/backend/data
    restart: always

volumes:
  open-webui:
    driver: local
  ollama-models:
    driver: local