Enable docsum to connect with external llm endpoints via helm (#1141)

* Enable docsum to connect with external llm endpoints via helm

Signed-off-by: devpramod <pramod.pai@intel.com>

* modify values yaml name

Signed-off-by: Ubuntu <azureuser@denvr-inf.kifxisxbiwme5gt4kkwqsfdjuh.dx.internal.cloudapp.net>

* addressed PR comment

Signed-off-by: Ubuntu <azureuser@denvr-inf.kifxisxbiwme5gt4kkwqsfdjuh.dx.internal.cloudapp.net>

* change host and port names for external llm env variables

Signed-off-by: devpramod <pramod.pai@intel.com>

* enable llm-uservice to point to external llm endpoints

Signed-off-by: devpramod <pramod.pai@intel.com>

* add env variable for api key in llm-uservice configmap

Signed-off-by: devpramod <pramod.pai@intel.com>

---------

Signed-off-by: devpramod <pramod.pai@intel.com>
Signed-off-by: Ubuntu <azureuser@denvr-inf.kifxisxbiwme5gt4kkwqsfdjuh.dx.internal.cloudapp.net>
Co-authored-by: Ubuntu <azureuser@denvr-inf.kifxisxbiwme5gt4kkwqsfdjuh.dx.internal.cloudapp.net>

Author: devpramod

helm-charts/common/llm-uservice/templates/configmap.yaml

@@ -70,6 +70,9 @@ data:
   {{- if .Values.LLM_MODEL_ID }}
   LLM_MODEL_ID: {{ .Values.LLM_MODEL_ID | quote }}
   {{- end }}
+  {{- if .Values.OPENAI_API_KEY }}
+  OPENAI_API_KEY: {{ .Values.OPENAI_API_KEY | quote }}
+  {{- end }}
   HF_HOME: "/tmp/.cache/huggingface"
   {{- if not .Values.global.offline }}
   HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | default .Values.global.HF_TOKEN | quote }}

helm-charts/docsum/Chart.yaml

@@ -21,13 +21,16 @@ dependencies:
   - name: llm-uservice
     version: 0-latest
     repository: "file://../common/llm-uservice"
+    condition: llm-uservice.enabled
   - name: whisper
     version: 0-latest
     repository: "file://../common/whisper"
+    condition: whisper.enabled
   - name: ui
     version: 0-latest
     repository: "file://../common/ui"
     alias: docsum-ui
+    condition: docsum-ui.enabled
   - name: nginx
     version: 0-latest
     repository: "file://../common/nginx"

helm-charts/docsum/README.md

@@ -26,7 +26,12 @@ helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --set global.modelUs
 # helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/rocm-values.yaml
 # To use AMD ROCm device with TGI
 # helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/rocm-tgi-values.yaml
-
+# To use with external OpenAI-compatible LLM endpoints (OpenAI, vLLM, TGI, etc.)
+# This configures the llm-uservice to connect to external LLM providers while maintaining DocSum compatibility
+# For OpenAI:
+# helm install docsum docsum --values docsum/variant_external-llm-values.yaml --set llm-uservice.OPENAI_API_KEY="your-api-key" --set llm-uservice.LLM_ENDPOINT="https://api.openai.com" --set llm-uservice.LLM_MODEL_ID="gpt-4-turbo"
+# For vLLM/TGI or other OpenAI-compatible endpoints:
+# helm install docsum docsum --values docsum/variant_external-llm-values.yaml --set llm-uservice.LLM_ENDPOINT="http://your-server-url" --set llm-uservice.LLM_MODEL_ID="your-model"
 ```
 
 ## Verify

helm-charts/docsum/templates/deployment.yaml

@@ -38,10 +38,12 @@ spec:
               value: {{ include "llm-uservice.fullname" (index .Subcharts "llm-uservice") }}
             - name: LLM_SERVICE_PORT
               value: {{ index .Values "llm-uservice" "service" "port" | quote }}
+            {{- if .Values.whisper.enabled }}
             - name: ASR_SERVICE_HOST_IP
               value: {{ include "whisper.fullname" (index .Subcharts "whisper") }}
             - name: ASR_SERVICE_PORT
               value: {{ index .Values "whisper" "service" "port" | quote }}
+            {{- end }}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"

helm-charts/docsum/values.yaml

@@ -57,7 +57,14 @@ tolerations: []
 affinity: {}
 
 # To override values in subchart llm-uservice
+# For external LLM configuration, you can override these values here or use variant_external-llm-values.yaml
+# Example for external OpenAI-compatible endpoints:
+#   DOCSUM_BACKEND: "vLLM"  # Use vLLM backend for OpenAI-compatible APIs
+#   LLM_ENDPOINT: "https://api.openai.com"  # External endpoint (omit /v1 suffix)
+#   LLM_MODEL_ID: "gpt-4-turbo"  # Model to use
+#   OPENAI_API_KEY: "your-api-key"  # API key for authentication
 llm-uservice:
+  enabled: true
   image:
     repository: opea/llm-docsum
   DOCSUM_BACKEND: "vLLM"
@@ -79,6 +86,8 @@ vllm:
 nginx:
   enabled: false
 docsum-ui:
+  # if false, set also nginx.enabled=false
+  enabled: true
   image:
     repository: opea/docsum-gradio-ui
     tag: "latest"
@@ -101,8 +110,12 @@ docsum-ui:
 #     type: ClusterIP
 
 dashboard:
+  enabled: true
   prefix: "OPEA DocSum"
 
+whisper:
+  enabled: true
+
 global:
   http_proxy: ""
   https_proxy: ""

helm-charts/docsum/variant_external-llm-values.yaml

@@ -0,0 +1,17 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# External LLM configuration - configures llm-uservice to use external LLM providers
+# This keeps the llm-uservice wrapper (required for /v1/docsum endpoint) but connects it to external LLMs
+llm-uservice:
+  enabled: true  # Keep the wrapper service for DocSum compatibility
+  DOCSUM_BACKEND: "vLLM"                      # Use vLLM backend for OpenAI-compatible APIs
+  LLM_ENDPOINT: "https://api.openai.com"      # External LLM API endpoint (omit /v1 suffix)
+  OPENAI_API_KEY: "${OPENAI_API_KEY}"         # API key for authentication
+  LLM_MODEL_ID: "gpt-4-turbo"                 # Model to use
+
+# Disable local inference services since we're using external LLMs
+vllm:
+  enabled: false
+tgi:
+  enabled: false