diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..c6c344432 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "spellright.language": ["en"], + "spellright.documentTypes": ["markdown", "latex", "plaintext", "yaml"] +} diff --git a/helm-charts/common/data-prep/Chart.yaml b/helm-charts/common/data-prep/Chart.yaml index 52b99ab25..454c6f747 100644 --- a/helm-charts/common/data-prep/Chart.yaml +++ b/helm-charts/common/data-prep/Chart.yaml @@ -17,6 +17,10 @@ dependencies: version: 1.0.0 repository: file://../redis-vector-db condition: redis-vector-db.enabled + - name: vdms-vector-db + version: 1.0.0 + repository: file://../vdms-vector-db + condition: vdms-vector-db.enabled - name: milvus version: 4.2.12 repository: https://zilliztech.github.io/milvus-helm/ diff --git a/helm-charts/common/data-prep/README.md b/helm-charts/common/data-prep/README.md index 4a05e2f34..50c6a9fbc 100644 --- a/helm-charts/common/data-prep/README.md +++ b/helm-charts/common/data-prep/README.md @@ -1,14 +1,18 @@ -# data-prep +# Data-Prep Microservice -Helm chart for deploying data-prep microservice. +Helm chart for deploying data-prep microservice. Data-Prep is consumed by several reference applications present in [GenAIExample](https://github.com/opea-project/GenAIExamples/tree/main). -data-prep will use redis and tei service, please specify the endpoints. +There are 2 versions of Data-Prep microservice. First version is unimodal based on redis-vector-db and TEI. It performs data preparation for textual data. An alternative multimodal version based on `videoqna-values.yaml` file, performs data preparation for visual data input. Follow along to select and install the version which suites your use case. -## (Option1): Installing the chart separately +Data-Prep uses redis-vector-db and tei. The multimodal version uses vdms-vector-db service. Endpoints for these dependencies should be set properly before installing the chart. + +## Install the chart for data preparation using Redis Vector DB + +### (Option1): Installing the chart separately First, you need to install the tei and redis-vector-db chart, please refer to the [tei](../tei/README.md) and [redis-vector-db](../redis-vector-db/README.md) for more information. -After you've deployted the tei and redis-vector-db chart successfully, please run `kubectl get svc` to get the service endpoint and URL respectively, i.e. `http://tei`, `redis://redis-vector-db:6379`. +After you've deployed the tei and redis-vector-db chart successfully, please run `kubectl get svc` to get the service endpoint and URL respectively, i.e. `http://tei`, `redis://redis-vector-db:6379`. To install data-prep chart, run the following: @@ -20,7 +24,7 @@ helm dependency update helm install data-prep . --set REDIS_URL=${REDIS_URL} --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} ``` -## (Option2): Installing the chart with dependencies automatically +### (Option2): Installing the chart with dependencies automatically ```console cd GenAIInfra/helm-charts/common/data-prep @@ -29,6 +33,52 @@ helm install data-prep . --set redis-vector-db.enabled=true --set tei.enabled=tr ``` +## Install the chart for multimodal data preparation using VDMS Vector DB + +### (Option1): Installing the chart separately + +First, you need to install the `vdms-vector-db` chart. Please refer to the [vdms-vector-db](../vdms-vector-db/README.md) for more information. + +After you've deployed the `vdms-vector-db` chart successfully, please run `kubectl get svc` to get the service host and port respectively, for example: `http://vdms-vector-db:8001`. + +Next, Run the following commands to install data-prep chart: + +```bash +cd GenAIInfra/helm-charts/common/data-prep + +# Use the host and port received in previous step as VDMS_HOST and VDMS_PORT. +export VDMS_HOST="vdms-vector-db" +export VDMS_PORT="8001" +export INDEX_NAME="mega-videoqna" +export HFTOKEN= +# Set a directory to cache emdedding models +export MODELDIR=/mnt/opea-models + +# Export the proxy variables. Assign empty string if no proxy setup required. +export https_proxy="your_http_proxy" +export http_proxy="your_https_proxy" + +helm dependency update +helm install data-prep . -f ../variant_videoqna-values.yaml --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set INDEX_NAME=${INDEX_NAME} --set global.modelUseHostPath=${MODELDIR} --set vdmsHost=${VDMS_HOST} --set vdmsPort=${VDMS_PORT} --set global.https_proxy=${https_proxy} --set global.http_proxy=${http_proxy} +``` + +### (Option2): Installing the chart with dependencies automatically + +```bash +cd GenAIInfra/helm-charts/common/data-prep +export INDEX_NAME="mega-videoqna" +export HFTOKEN= +# Set a directory to cache emdedding models +export MODELDIR=/mnt/opea-models + +# Export the proxy variables. Assign empty string if no proxy setup required. +export https_proxy="your_http_proxy" +export http_proxy="your_https_proxy" + +helm dependency update +helm install data-prep . -f ./variant_videoqna-values.yaml --set vdms-vector-db.enabled=true --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set INDEX_NAME=${INDEX_NAME} --set global.modelUseHostPath=${MODELDIR} --set global.https_proxy=${https_proxy} --set global.http_proxy=${http_proxy} +``` + ## Verify To verify the installation, run the command `kubectl get pod` to make sure all pods are running. @@ -37,21 +87,40 @@ Then run the command `kubectl port-forward svc/data-prep 6007:6007` to expose th Open another terminal and run the following command to verify the service if working: -```console +### 1. For Data-prep service using redis-vector-db: + +```bash + curl http://localhost:6007/v1/dataprep \ -X POST \ -H "Content-Type: multipart/form-data" \ -F "files=@./README.md" ``` +### 2. For multimodal data prep service using vdms-vector-db: + +```bash +# 1) Download a sample video in current dir: +curl -svLO "https://github.com/opea-project/GenAIExamples/raw/refs/heads/main/VideoQnA/docker_compose/intel/cpu/xeon/data/op_1_0320241830.mp4" + +# 2) Verify using above video +curl -X POST http://localhost:6007/v1/dataprep \ + -H "Content-Type: multipart/form-data" \ + -F "files=@./op_1_0320241830.mp4" +``` + ## Values -| Key | Type | Default | Description | -| ---------------------- | ------ | ----------------------- | ----------- | -| image.repository | string | `"opea/dataprep-redis"` | | -| service.port | string | `"6007"` | | -| REDIS_URL | string | `""` | | -| TEI_EMBEDDING_ENDPOINT | string | `""` | | +| Key | Type | Default | Description | +| -------------------------------- | ------ | --------------------------------- | ----------- | +| image.repository | string | `"opea/dataprep-redis"` | | +| service.port | string | `"6007"` | | +| REDIS_URL | string | `""` | | +| TEI_EMBEDDING_ENDPOINT | string | `""` | | +| videoqna-values:image.repository | string | `"opea/dataprep-multimodal-vdms"` | | +| videoqna-values:vdmsHost | string | `""` | | +| videoqna-values:vdmsPort | string | `"8001"` | | +| videoqna-values:INDEX_NAME | string | `"mega-videoqna"` | | ## Milvus support diff --git a/helm-charts/common/data-prep/ci-videoqna-values.yaml b/helm-charts/common/data-prep/ci-videoqna-values.yaml new file mode 100644 index 000000000..7b2620937 --- /dev/null +++ b/helm-charts/common/data-prep/ci-videoqna-values.yaml @@ -0,0 +1,21 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +vdms-vector-db: + enabled: true + +image: + repository: opea/dataprep-multimodal-vdms + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +INDEX_NAME: "mega-videoqna" +vdmsHost: "" +vdmsPort: "8001" +entryCommand: ["/bin/sh"] +extraArgs: ["-c", "sleep 15 && python ingest_videos.py"] + +# Set cacheUseHostPath to for caching encoding/embedding models and other related data +global: + cacheUseHostPath: "" diff --git a/helm-charts/common/data-prep/templates/configmap.yaml b/helm-charts/common/data-prep/templates/configmap.yaml index e0306c0b1..299d3b54a 100644 --- a/helm-charts/common/data-prep/templates/configmap.yaml +++ b/helm-charts/common/data-prep/templates/configmap.yaml @@ -8,6 +8,14 @@ metadata: labels: {{- include "data-prep.labels" . | nindent 4 }} data: + {{- if .Values.vdmsHost }} + VDMS_HOST: {{ .Values.vdmsHost | quote }} + {{- else }} + VDMS_HOST: "{{ .Release.Name }}-vdms-vector-db" + {{- end }} + {{- if .Values.vdmsPort }} + VDMS_PORT: {{ .Values.vdmsPort | quote }} + {{- end }} {{- if .Values.MOSEC_EMBEDDING_ENDPOINT }} MOSEC_EMBEDDING_ENDPOINT: {{ .Values.MOSEC_EMBEDDING_ENDPOINT | quote}} MOSEC_EMBEDDING_MODEL: {{ .Values.MOSEC_EMBEDDING_MODEL | quote}} @@ -47,7 +55,7 @@ data: http_proxy: {{ .Values.global.http_proxy | quote }} https_proxy: {{ .Values.global.https_proxy | quote }} {{- if and (not .Values.REDIS_URL) (and (not .Values.TEI_EMBEDDING_ENDPOINT) (or .Values.global.http_proxy .Values.global.https_proxy)) }} - no_proxy: "{{ .Release.Name }}-tei,{{ .Release.Name }}-redis-vector-db,{{ .Values.global.no_proxy }}" + no_proxy: "{{ .Release.Name }}-tei,{{ .Release.Name }}-redis-vector-db,{{ .Release.Name }}-vdms-vector-db,{{ .Values.global.no_proxy }}" {{- else }} no_proxy: {{ .Values.global.no_proxy | quote }} {{- end }} diff --git a/helm-charts/common/data-prep/templates/deployment.yaml b/helm-charts/common/data-prep/templates/deployment.yaml index 84d6bd273..32b453580 100644 --- a/helm-charts/common/data-prep/templates/deployment.yaml +++ b/helm-charts/common/data-prep/templates/deployment.yaml @@ -41,11 +41,28 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.entryCommand }} + command: {{ .Values.entryCommand }} + {{- end }} + {{- if .Values.extraArgs }} + args: + {{- range .Values.extraArgs }} + - {{ . | quote }} + {{- end }} + {{- end }} ports: - name: data-prep - containerPort: {{ .Values.port }} + containerPort: {{ .Values.service.containerPort }} protocol: TCP volumeMounts: + {{- if .Values.global.modelUseHostPath }} + - mountPath: /home/user/.cache/clip + name: model-volume + subPath: clip + - mountPath: /home/user/.cache/huggingface/hub + name: model-volume + subPath: huggingface/hub + {{- end }} - mountPath: /tmp name: tmp {{- if .Values.livenessProbe }} @@ -63,6 +80,16 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumes: + {{- if .Values.global.modelUseHostPath }} + - name: model-volume + hostPath: + path: {{ .Values.global.modelUseHostPath }} + type: Directory + {{- else if .Values.global.modelUsePVC }} + - name: model-volume + persistentVolumeClaim: + claimName: {{ .Values.global.modelUsePVC }} + {{- end }} - name: tmp emptyDir: {} {{- with .Values.nodeSelector }} diff --git a/helm-charts/common/data-prep/templates/service.yaml b/helm-charts/common/data-prep/templates/service.yaml index afeff3ecf..793ba2ae4 100644 --- a/helm-charts/common/data-prep/templates/service.yaml +++ b/helm-charts/common/data-prep/templates/service.yaml @@ -11,7 +11,7 @@ spec: type: {{ .Values.service.type }} ports: - port: {{ .Values.service.port }} - targetPort: {{ .Values.port }} + targetPort: {{ .Values.service.containerPort }} protocol: TCP name: data-prep selector: diff --git a/helm-charts/common/data-prep/templates/tests/test-pod.yaml b/helm-charts/common/data-prep/templates/tests/test-pod.yaml index 127fa1167..0148fda4e 100644 --- a/helm-charts/common/data-prep/templates/tests/test-pod.yaml +++ b/helm-charts/common/data-prep/templates/tests/test-pod.yaml @@ -15,6 +15,21 @@ spec: - name: curl image: python:3.10.14 command: ['bash', '-c'] + {{- if contains "dataprep-multimodal-vdms" .Values.image.repository }} + args: + - | + https_proxy={{ .Values.global.https_proxy }} curl -svLO "https://github.com/opea-project/GenAIExamples/raw/refs/heads/main/VideoQnA/docker_compose/intel/cpu/xeon/data/op_1_0320241830.mp4" + max_retry=5 + for ((i=1; i<=max_retry; i++)); do + curl http://{{ include "data-prep.fullname" . }}:{{ .Values.service.port }}/v1/dataprep -sS --fail-with-body \ + -X POST \ + -H "Content-Type: multipart/form-data" \ + -F "files=@./op_1_0320241830.mp4" && break; + curlcode=$? + if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi; + done; + if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi + {{- else }} args: - | echo "test file" > /tmp/file1.txt; @@ -32,4 +47,5 @@ spec: -H "Content-Type: application/json" \ -d '{"file_path": "file1.txt"}'; if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi + {{- end }} restartPolicy: Never diff --git a/helm-charts/common/data-prep/values.yaml b/helm-charts/common/data-prep/values.yaml index ac6f6b8e3..56ad6788c 100644 --- a/helm-charts/common/data-prep/values.yaml +++ b/helm-charts/common/data-prep/values.yaml @@ -11,6 +11,8 @@ milvus: enabled: false redis-vector-db: enabled: false +vdms-vector-db: + enabled: false replicaCount: 1 @@ -44,6 +46,7 @@ port: 6007 service: type: ClusterIP port: 6007 + containerPort: 6007 resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/helm-charts/common/data-prep/variant_videoqna-values.yaml b/helm-charts/common/data-prep/variant_videoqna-values.yaml new file mode 100644 index 000000000..b25e159f8 --- /dev/null +++ b/helm-charts/common/data-prep/variant_videoqna-values.yaml @@ -0,0 +1,19 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +image: + repository: opea/dataprep-multimodal-vdms + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +INDEX_NAME: "mega-videoqna" +vdmsHost: "" +vdmsPort: "8001" +entryCommand: ["/bin/sh"] +extraArgs: ["-c", "sleep 15 && python ingest_videos.py"] + +# Set cacheUseHostPath to for caching encoding/embedding models and other related data +global: + modelUseHostPath: "" + modelUsePVC: "" diff --git a/helm-charts/common/embedding-usvc/README.md b/helm-charts/common/embedding-usvc/README.md index 1734fef93..bfd8e7c2f 100644 --- a/helm-charts/common/embedding-usvc/README.md +++ b/helm-charts/common/embedding-usvc/README.md @@ -1,10 +1,16 @@ -# embedding-usvc +# Embedding Microservice (embedding-usvc) -Helm chart for deploying embedding microservice. +**Helm chart for deploying Embedding Microservice.** -embedding-usvc depends on TEI, set TEI_EMBEDDING_ENDPOINT. +Embedding microservice is consumed by several reference applications present in [GenAIExample](https://github.com/opea-project/GenAIExamples/tree/main). -## (Option1): Installing the chart separately +There are 2 versions of embedding microservice. First version is unimodal based on TEI creating embeddings for textual data. An alternative version based on `vdms-values.yaml` file, uses multimodal embedding models for creating embedding for visual data. Follow along to select and install the version which suites your use case. + +embedding-usvc depends on TEI, and TEI_EMBEDDING_ENDPOINT should be set properly. Multimodal version has no dependencies. + +## Install Embedding microservice chart based on TEI + +### (Option1): Installing the chart separately First, you need to install the tei chart, please refer to the [tei](../tei) chart for more information. @@ -19,7 +25,7 @@ helm dependency update helm install embedding-usvc . --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} ``` -## (Option2): Installing the chart with dependencies automatically +### (Option2): Installing the chart with dependencies automatically ```console cd GenAIInfra/helm-charts/common/embedding-usvc @@ -27,6 +33,24 @@ helm dependency update helm install embedding-usvc . --set tei.enabled=true ``` +## Install Multimodal Embedding Microservice chart + +To install the multimodal embedding-usvc chart, run the following: + +```bash +cd GenAIInfra/helm-charts/common/embedding-usvc + +# Set a directory to cache emdedding models +export MODELDIR=/mnt/opea-models + +# Export the proxy variables. Assign empty string if no proxy setup required. +export https_proxy="your_http_proxy" +export http_proxy="your_https_proxy" + +helm dependency update +helm install embedding-usvc . -f ./variant_videoqna-values.yaml --set global.modelUseHostPath=${MODELDIR} --set global.https_proxy=${https_proxy} --set global.http_proxy=${http_proxy} +``` + ## Verify To verify the installation, run the command `kubectl get pod` to make sure all pods are running. @@ -35,7 +59,7 @@ Then run the command `kubectl port-forward svc/embedding-usvc 6000:6000` to expo Open another terminal and run the following command to verify the service if working: -```console +```bash curl http://localhost:6000/v1/embeddings \ -X POST \ -d '{"text":"hello"}' \ @@ -44,9 +68,10 @@ curl http://localhost:6000/v1/embeddings \ ## Values -| Key | Type | Default | Description | -| ---------------------- | ------ | ---------------------- | ----------- | -| image.repository | string | `"opea/embedding-tei"` | | -| service.port | string | `"6000"` | | -| TEI_EMBEDDING_ENDPOINT | string | `""` | | -| global.monitoring | bop; | false | | +| Key | Type | Default | Description | +| ---------------------------- | ------ | ---------------------------------- | ----------- | +| image.repository | string | `"opea/embedding-tei"` | | +| service.port | string | `"6000"` | | +| TEI_EMBEDDING_ENDPOINT | string | `""` | | +| global.monitoring | bop; | `false` | | +| vdms-values:image.repository | string | `"opea/embedding-multimodal-clip"` | | diff --git a/helm-charts/common/embedding-usvc/ci-videoqna-values.yaml b/helm-charts/common/embedding-usvc/ci-videoqna-values.yaml new file mode 120000 index 000000000..313991379 --- /dev/null +++ b/helm-charts/common/embedding-usvc/ci-videoqna-values.yaml @@ -0,0 +1 @@ +variant_videoqna-values.yaml \ No newline at end of file diff --git a/helm-charts/common/embedding-usvc/templates/deployment.yaml b/helm-charts/common/embedding-usvc/templates/deployment.yaml index 2fff1e6d5..1cacbd863 100644 --- a/helm-charts/common/embedding-usvc/templates/deployment.yaml +++ b/helm-charts/common/embedding-usvc/templates/deployment.yaml @@ -43,9 +43,14 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: embedding-usvc - containerPort: 6000 + containerPort: {{ .Values.service.containerPort }} protocol: TCP volumeMounts: + {{- if .Values.global.modelUseHostPath }} + - mountPath: /home/user/.cache/huggingface/hub + name: model-volume + subPath: huggingface/hub + {{- end }} - mountPath: /tmp name: tmp {{- if .Values.livenessProbe }} @@ -63,6 +68,16 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumes: + {{- if .Values.global.modelUseHostPath }} + - name: model-volume + hostPath: + path: {{ .Values.global.modelUseHostPath }} + type: Directory + {{- else if .Values.global.modelUsePVC }} + - name: model-volume + persistentVolumeClaim: + claimName: {{ .Values.global.modelUsePVC }} + {{- end }} - name: tmp emptyDir: {} {{- with .Values.nodeSelector }} diff --git a/helm-charts/common/embedding-usvc/templates/service.yaml b/helm-charts/common/embedding-usvc/templates/service.yaml index 2aff873f2..2614b270d 100644 --- a/helm-charts/common/embedding-usvc/templates/service.yaml +++ b/helm-charts/common/embedding-usvc/templates/service.yaml @@ -11,7 +11,7 @@ spec: type: {{ .Values.service.type }} ports: - port: {{ .Values.service.port }} - targetPort: 6000 + targetPort: {{ .Values.service.containerPort }} protocol: TCP name: embedding-usvc selector: diff --git a/helm-charts/common/embedding-usvc/values.yaml b/helm-charts/common/embedding-usvc/values.yaml index ca3d30791..d5ae0786d 100644 --- a/helm-charts/common/embedding-usvc/values.yaml +++ b/helm-charts/common/embedding-usvc/values.yaml @@ -45,6 +45,7 @@ service: type: ClusterIP # The default port for embedding service is 9000 port: 6000 + containerPort: 6000 resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/helm-charts/common/embedding-usvc/variant_videoqna-values.yaml b/helm-charts/common/embedding-usvc/variant_videoqna-values.yaml new file mode 100644 index 000000000..68091ccf2 --- /dev/null +++ b/helm-charts/common/embedding-usvc/variant_videoqna-values.yaml @@ -0,0 +1,16 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +image: + repository: opea/embedding-multimodal-clip + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +securityContext: + readOnlyRootFilesystem: false + +# Set cacheUseHostPath to for caching encoding/embedding models and other related data +global: + modelUseHostPath: "" + modelUsePVC: "" diff --git a/helm-charts/common/lvm-uservice/Chart.yaml b/helm-charts/common/lvm-uservice/Chart.yaml index 46e4fdb20..43cfdb5e2 100644 --- a/helm-charts/common/lvm-uservice/Chart.yaml +++ b/helm-charts/common/lvm-uservice/Chart.yaml @@ -13,3 +13,7 @@ dependencies: version: 1.0.0 repository: file://../tgi condition: tgi.enabled + - name: video-llama-lvm + version: 1.0.0 + repository: file://../video-llama-lvm + condition: video-llama-lvm.enabled diff --git a/helm-charts/common/lvm-uservice/README.md b/helm-charts/common/lvm-uservice/README.md index d28a49ed1..83a2d9ec4 100644 --- a/helm-charts/common/lvm-uservice/README.md +++ b/helm-charts/common/lvm-uservice/README.md @@ -1,18 +1,22 @@ -# lvm-uservice +# LVM Microservice -Helm chart for deploying LVM microservice. +**Helm chart for deploying lvm-uservice microservice.** -lvm-uservice depends on TGI, you should set LVM_ENDPOINT as tgi endpoint. +There are two versions of `lvm-uservice`. First version runs with `tgi` service and another one runs with `video-llama-lvm` service. We will try to learn both setups in following sections. -## (Option1): Installing the chart separately +## 1. Installing lvm-uservice to be used with tgi microservice + +In this setup, lvm-uservice depends on TGI, you should set LVM_ENDPOINT as tgi endpoint. + +### (Option1): Installing the chart separately First, you need to install the tgi chart, please refer to the [tgi](../tgi) chart for more information. -After you've deployted the tgi chart successfully, please run `kubectl get svc` to get the tgi service endpoint, i.e. `http://tgi`. +After you've deployed the tgi chart successfully, please run `kubectl get svc` to get the tgi service endpoint, i.e. `http://tgi`. To install the chart, run the following: -```console +```bash cd GenAIInfra/helm-charts/common/lvm-uservice export HFTOKEN="insert-your-huggingface-token-here" export LVM_ENDPOINT="http://tgi" @@ -20,25 +24,85 @@ helm dependency update helm install lvm-uservice . --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set LVM_ENDPOINT=${LVM_ENDPOINT} --wait ``` -## (Option2): Installing the chart with dependencies automatically +### (Option2): Installing the chart with dependencies automatically (with auto-installing tgi) -```console +```bash cd GenAIInfra/helm-charts/common/lvm-uservice export HFTOKEN="insert-your-huggingface-token-here" helm dependency update helm install lvm-uservice . --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set tgi.enabled=true --wait ``` +## 2. Installing lvm-uservice to be used with video-llama-lvm microservice (serving VideoLlama-7B) + +This setup of `lvm-uservice` is utilized in some of the examples like [VideoQnA](https://github.com/opea-project/GenAIExamples/tree/main/VideoQnA). Here, `lvm-uservice` helps communicate to `video-llama-lvm` microservice. It facilitates sending queries and receiving response from `video-llama-lvm` microservice. Hence, it depends on video-llama-lvm microservice and you should make sure that `lvmEndpoint` value is set properly. + +### (Option1): Installing the chart separately + +First, you need to install the `video-llama-lvm` chart. Please refer to the [video-llama-lvm](../video-llama-lvm) chart for more information. + +After you've deployed the `video-llama-lvm` chart successfully, please run `kubectl get svc` to get `video-llama-lvm` service host and port. The endpoint url for `video-llama-lvm` will be formed using the host and port. For example, default value would be `http://video-llama-lvm:80`. + +To install the chart, run the following: + +```bash +cd GenAIInfra/helm-charts/common/lvm-uservice +export LVM_ENDPOINT="http://video-llama-lvm:80" + +# Export the proxy variables. Assign empty string if no proxy setup required. +export https_proxy="your_http_proxy" +export http_proxy="your_https_proxy" + +helm dependency update +helm install lvm-uservice . -f ./variant_videoqna-values.yaml --set lvmEndpoint=${LVM_ENDPOINT} --set global.https_proxy=${https_proxy} --set global.http_proxy=${http_proxy} --wait +``` + +### (Option2): Installing the chart with dependencies automatically (with auto-installing video-llama-lvm dependency) + +```bash +cd GenAIInfra/helm-charts/common/lvm-uservice + +export HFTOKEN="insert-your-huggingface-token-here" +# Set a dir to cache downloaded Video-Llama Model, other embedding models and clips +export MODELDIR=/mnt/opea-models +# When setting up for first time, model needs to be downloaded. Set LLM_DOWNLOAD flag to true to download models. Please note, when redeploying we should set this value to false, otherwise model download will restart. +export LLM_DOWNLOAD=true + +# Set the proxy variables. Assign empty string if no proxy setup required. +export https_proxy="your_http_proxy" +export http_proxy="your_https_proxy" + +helm dependency update +helm install lvm-uservice . -f ./variant_videoqna-values.yaml --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set video-llama-lvm.enabled=true --set video-llama-lvm.llmDownload=${LLM_DOWNLOAD} --set global.modelUseHostPath=${MODELDIR} --set global.https_proxy=${https_proxy} --set global.http_proxy=${http_proxy} --wait +``` + ## Verify To verify the installation, run the command `kubectl get pod` to make sure all pods are running. -Then run the command `kubectl port-forward svc/lvm-uservice 9000:9000` to expose the lvm-uservice service for access. +### For TGI based lvm-uservice + +Run the command `kubectl port-forward svc/lvm-uservice 9399:9399` to expose the lvm-uservice service for access. + +### For video-llama-lvm based lvm-uservice + +Run the command `kubectl port-forward svc/lvm-uservice 9000:9000` to expose the lvm-uservice service for access. Open another terminal and run the following command to verify the service if working: -```console -curl http://localhost:9000/v1/chat/completions \ +### Verify lvm-uservice running with video-llama-lvm (Video-Llama 7B) service + +```bash +curl http://localhost:9000/v1/lvm \ + -X POST \ + -d '{"video_url":"https://github.com/DAMO-NLP-SG/Video-LLaMA/raw/main/examples/silence_girl.mp4","chunk_start": 0,"chunk_duration": 7,"prompt":"What is the person doing?","max_new_tokens": 50}' \ + -H 'Content-Type: application/json' +``` + +### Verify lvm-uservice running with TGI service + +```bash +curl http://localhost:9399/v1/chat/completions \ -X POST \ -d '{"query":"What is Deep Learning?","max_tokens":17,"top_k":10,"top_p":0.95,"typical_p":0.95,"temperature":0.01,"repetition_penalty":1.03,"streaming":true}' \ -H 'Content-Type: application/json' @@ -46,10 +110,11 @@ curl http://localhost:9000/v1/chat/completions \ ## Values -| Key | Type | Default | Description | -| ------------------------------- | ------ | ---------------- | ------------------------------- | -| global.HUGGINGFACEHUB_API_TOKEN | string | `""` | Your own Hugging Face API token | -| image.repository | string | `"opea/lvm-tgi"` | | -| service.port | string | `"9000"` | | -| LVM_ENDPOINT | string | `""` | LVM endpoint | -| global.monitoring | bop; | false | Service usage metrics | +| Key | Type | Default | Description | +| ------------------------------- | ------ | ------------------------ | ------------------------------- | +| global.HUGGINGFACEHUB_API_TOKEN | string | `""` | Your own Hugging Face API token | +| image.repository | string | `"opea/lvm-tgi"` | | +| videoqna: image.repository | string | `"opea/lvm-video-llama"` | | +| service.port | string | `"9000"` | | +| LVM_ENDPOINT | string | `""` | LVM endpoint | +| global.monitoring | bop; | false | Service usage metrics | diff --git a/helm-charts/common/lvm-uservice/ci-values.yaml b/helm-charts/common/lvm-uservice/ci-values.yaml index a4c378251..64508ed40 100644 --- a/helm-charts/common/lvm-uservice/ci-values.yaml +++ b/helm-charts/common/lvm-uservice/ci-values.yaml @@ -2,8 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # Default values for lvm-uservice. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. tgi: enabled: true diff --git a/helm-charts/common/lvm-uservice/ci-videoqna-values.yaml b/helm-charts/common/lvm-uservice/ci-videoqna-values.yaml new file mode 100644 index 000000000..698cbe5c9 --- /dev/null +++ b/helm-charts/common/lvm-uservice/ci-videoqna-values.yaml @@ -0,0 +1,18 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +video-llama-lvm: + enabled: true + llmDownload: true + +image: + repository: opea/lvm-video-llama + # This sets the pull policy for images. + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +service: + type: ClusterIP + port: 9000 + containerPort: 9000 diff --git a/helm-charts/common/lvm-uservice/templates/configmap.yaml b/helm-charts/common/lvm-uservice/templates/configmap.yaml index 62f75d323..00eec23a4 100644 --- a/helm-charts/common/lvm-uservice/templates/configmap.yaml +++ b/helm-charts/common/lvm-uservice/templates/configmap.yaml @@ -9,9 +9,11 @@ metadata: {{- include "lvm-uservice.labels" . | nindent 4 }} data: {{- if .Values.LVM_ENDPOINT }} - LVM_ENDPOINT: {{ .Values.LVM_ENDPOINT | quote}} - {{- else }} + LVM_ENDPOINT: {{ .Values.LVM_ENDPOINT }} + {{- else if contains "lvm-tgi" .Values.image.repository }} LVM_ENDPOINT: "http://{{ .Release.Name }}-tgi" + {{- else if contains "lvm-video-llama" .Values.image.repository }} + LVM_ENDPOINT: "http://{{ .Release.Name }}-video-llama-lvm" {{- end }} HUGGINGFACEHUB_API_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}} HF_HOME: "/tmp/.cache/huggingface" @@ -21,7 +23,7 @@ data: http_proxy: {{ .Values.global.http_proxy | quote }} https_proxy: {{ .Values.global.https_proxy | quote }} {{- if and (not .Values.LVM_ENDPOINT) (or .Values.global.http_proxy .Values.global.https_proxy) }} - no_proxy: "{{ .Release.Name }}-tgi,{{ .Values.global.no_proxy }}" + no_proxy: "{{ .Release.Name }}-tgi,{{ .Release.Name }}-video-llama-lvm,{{ .Values.global.no_proxy }}" {{- else }} no_proxy: {{ .Values.global.no_proxy | quote }} {{- end }} diff --git a/helm-charts/common/lvm-uservice/templates/deployment.yaml b/helm-charts/common/lvm-uservice/templates/deployment.yaml index 308158a95..93e95a97e 100644 --- a/helm-charts/common/lvm-uservice/templates/deployment.yaml +++ b/helm-charts/common/lvm-uservice/templates/deployment.yaml @@ -43,7 +43,7 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: lvm-uservice - containerPort: 9399 + containerPort: {{ .Values.service.containerPort }} protocol: TCP volumeMounts: - mountPath: /tmp diff --git a/helm-charts/common/lvm-uservice/templates/service.yaml b/helm-charts/common/lvm-uservice/templates/service.yaml index 37e454dec..28f4744a1 100644 --- a/helm-charts/common/lvm-uservice/templates/service.yaml +++ b/helm-charts/common/lvm-uservice/templates/service.yaml @@ -11,7 +11,7 @@ spec: type: {{ .Values.service.type }} ports: - port: {{ .Values.service.port }} - targetPort: 9399 + targetPort: {{ .Values.service.containerPort }} protocol: TCP name: lvm-uservice selector: diff --git a/helm-charts/common/lvm-uservice/templates/tests/test-pod.yaml b/helm-charts/common/lvm-uservice/templates/tests/test-pod.yaml index 7782cdb1e..c9dea1850 100644 --- a/helm-charts/common/lvm-uservice/templates/tests/test-pod.yaml +++ b/helm-charts/common/lvm-uservice/templates/tests/test-pod.yaml @@ -15,6 +15,20 @@ spec: - name: curl image: python:3.10.14 command: ['bash', '-c'] + {{- if contains "lvm-video-llama" .Values.image.repository }} + args: + - | + max_retry=20 + for ((i=1; i<=max_retry; i++)); do + curl http://{{ include "lvm-uservice.fullname" . }}:{{ .Values.service.port }}/v1/lvm -sS --fail-with-body \ + -X POST \ + -d '{"video_url":"https://github.com/DAMO-NLP-SG/Video-LLaMA/raw/main/examples/silence_girl.mp4","chunk_start": 0,"chunk_duration": 7,"prompt":"What is the person doing?","max_new_tokens": 50}' \ + -H 'Content-Type: application/json' && break; + curlcode=$? + if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi; + done; + if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi + {{- else if contains "lvm-tgi" .Values.image.repository }} args: - | max_retry=20; @@ -27,4 +41,5 @@ spec: if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi; done; if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi + {{- end }} restartPolicy: Never diff --git a/helm-charts/common/lvm-uservice/values.yaml b/helm-charts/common/lvm-uservice/values.yaml index fdc1be187..c523534cf 100644 --- a/helm-charts/common/lvm-uservice/values.yaml +++ b/helm-charts/common/lvm-uservice/values.yaml @@ -10,6 +10,9 @@ tgi: LLM_MODEL_ID: llava-hf/llava-v1.6-mistral-7b-hf MAX_INPUT_LENGTH: "4096" MAX_TOTAL_TOKENS: "8192" +video-llama-lvm: + enabled: false + llmDownload: true replicaCount: 1 LVM_ENDPOINT: "" @@ -48,6 +51,7 @@ service: type: ClusterIP # The default port for lvm service is 9399 port: 9399 + containerPort: 9399 resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/helm-charts/common/lvm-uservice/variant_videoqna-values.yaml b/helm-charts/common/lvm-uservice/variant_videoqna-values.yaml new file mode 100644 index 000000000..3268202dd --- /dev/null +++ b/helm-charts/common/lvm-uservice/variant_videoqna-values.yaml @@ -0,0 +1,19 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# Values file for VideoQnA Application for lvm-uservice + +video-llama-lvm: + llmDownload: true + +image: + repository: opea/lvm-video-llama + # This sets the pull policy for images. + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +service: + type: ClusterIP + port: 9000 + containerPort: 9000 diff --git a/helm-charts/common/reranking-usvc/Chart.yaml b/helm-charts/common/reranking-usvc/Chart.yaml index 06c2dd7fe..8b8b5e8be 100644 --- a/helm-charts/common/reranking-usvc/Chart.yaml +++ b/helm-charts/common/reranking-usvc/Chart.yaml @@ -13,3 +13,11 @@ dependencies: version: 1.0.0 repository: file://../teirerank condition: teirerank.enabled + - name: data-prep + version: 1.0.0 + repository: file://../data-prep + condition: data-prep.enabled + - name: vdms-vector-db + version: 1.0.0 + repository: file://../vdms-vector-db + condition: data-prep.enabled diff --git a/helm-charts/common/reranking-usvc/README.md b/helm-charts/common/reranking-usvc/README.md index d2e6ef25e..79671aaf6 100644 --- a/helm-charts/common/reranking-usvc/README.md +++ b/helm-charts/common/reranking-usvc/README.md @@ -1,10 +1,14 @@ -# reranking-usvc +# Re-ranking Microservice (reranking-usvc) -Helm chart for deploying reranking microservice. +**Helm chart for deploying reranking microservice.** -reranking-usvc depends on teirerank, set the TEI_RERANKING_ENDPOINT as teirerank endpoint. +There are two versions of Reranking microservice. First one is based on TEI model and does reranking for textual data. An alternative multimodal version based on `vdms-values.yaml` file, performs reranking for visual data. Follow along to select and install the version which suites your use case. -## (Option1): Installing the chart separately +reranking-usvc depends on teirerank. Set the TEI_RERANKING_ENDPOINT as teirerank endpoint. Multimodal version depends on data-prep and vdms-vector-db service. Endpoints for these dependencies should be set properly before installing the chart. + +## Install Re-ranking Microservice based on TEI + +### (Option1): Installing the chart separately First, you need to install the teirerank chart, please refer to the [teirerank](../teirerank) chart for more information. @@ -19,7 +23,7 @@ helm dependency update helm install reranking-usvc . --set TEI_RERANKING_ENDPOINT=${TEI_RERANKING_ENDPOINT} ``` -## (Option2): Installing the chart with dependencies automatically +### (Option2): Installing the chart with dependencies automatically ```console cd GenAIInfra/helm-charts/common/reranking-usvc @@ -27,6 +31,46 @@ helm dependency update helm install reranking-usvc . --set teirerank.enabled=true ``` +## Install Re-ranking Microservice for visual data + +### (Option1): Installing the chart separately + +First, you need to install the multimodal data-prep chart with vdms based values file and `vdms-vector-db` chart. `vdms-vector-db` is used by multimodal version of data-prep service. Please refer to the [data-prep](../data-prep) and [vdms-vector-db](../vdms-vector-db) charts guide for more information. + +After you've deployed the data-prep chart successfully, please run `kubectl get svc` to get the data-prep service endpoint and port, i.e. `http://data-prep:6007`. + +To install the reranking-usvc chart, run the following: + +```bash +cd GenAIInfra/helm-charts/common/reranking-usvc + +# Use the host and port returned in first step for setting the following environment variables. +export FILE_SERVER_ENDPOINT="http://data-prep:6007/v1/dataprep/get_file" +export GET_VIDEO_LIST_ENDPOINT="http://data-prep:6007/v1/dataprep/get_videos" + +# Export the proxy variables if you are behind a proxy. Assign empty string if no proxy setup required. +export https_proxy="your_http_proxy" +export http_proxy="your_https_proxy" + +helm dependency update +helm install reranking-usvc . -f ./variant_videoqna-values.yaml --set fileServerEndpoint=${FILE_SERVER_ENDPOINT} --set getVideoListEndpoint=${GET_VIDEO_LIST_ENDPOINT} --set global.https_proxy=${https_proxy} --set global.http_proxy=${http_proxy} +``` + +### (Option2): Installing the chart with dependencies automatically + +Multimodal Reranking microservice depends on Multimodal Data-Prep Microservice. Multimodal Data-Prep microservice in turn depends on vdms-vector-db microservice. + +```bash +cd GenAIInfra/helm-charts/common/reranking-usvc + +# Export the proxy variables. Assign empty string if no proxy setup required. +export https_proxy="your_http_proxy" +export http_proxy="your_https_proxy" + +helm dependency update +helm install reranking-usvc . -f ./variant_videoqna-values.yaml --set data-prep.enabled=true --set global.https_proxy=${https_proxy} --set global.http_proxy=${http_proxy} +``` + ## Verify To verify the installation, run the command `kubectl get pod` to make sure all pods are running. @@ -35,18 +79,41 @@ Then run the command `kubectl port-forward svc/reranking-usvc 8000:8000` to expo Open another terminal and run the following command to verify the service if working: -```console +### 1. For Reranking Service based on TEI + +````bash + curl http://localhost:8000/v1/reranking \ -X POST \ -d '{"initial_query":"What is Deep Learning?", "retrieved_docs": [{"text":"Deep Learning is not..."}, {"text":"Deep learning is..."}]}' \ - -H 'Content-Type: application/json' + -```H 'Content-Type: application/json' +```` + +### 2. For visual data Reranking service + +```bash +curl http://localhost:8000/v1/reranking \ + -X 'POST' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "retrieved_docs": [{"doc": [{"text": "this is the retrieved text"}]}], + "initial_query": "this is the query", + "top_n": 1, + "metadata": [ + {"other_key": "value", "video":"top_video_name", "timestamp":"20"} + ] + }' ``` ## Values -| Key | Type | Default | Description | -| ---------------------- | ------ | ---------------------- | ----------- | -| image.repository | string | `"opea/reranking-tgi"` | | -| TEI_RERANKING_ENDPOINT | string | `""` | | -| service.port | string | `"8000"` | | -| global.monitoring | bop; | false | | +| Key | Type | Default | Description | +| -------------------------------- | ------ | --------------------------- | ----------- | +| image.repository | string | `"opea/reranking-tgi"` | | +| TEI_RERANKING_ENDPOINT | string | `""` | | +| service.port | string | `"8000"` | | +| global.monitoring | bop; | `false` | | +| vdms-values:image.repository | string | `"opea/reranking-videoqna"` | | +| vdms-values:fileServerEndpoint | string | `""` | | +| vdms-values:getVideoListEndpoint | string | `""` | | diff --git a/helm-charts/common/reranking-usvc/ci-videoqna-values.yaml b/helm-charts/common/reranking-usvc/ci-videoqna-values.yaml new file mode 100644 index 000000000..32d9a760e --- /dev/null +++ b/helm-charts/common/reranking-usvc/ci-videoqna-values.yaml @@ -0,0 +1,31 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +image: + repository: opea/reranking-videoqna + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +chunkDuration: "" + +# Replace with actual Endpoint URLs +fileServerEndpoint: "" +getVideoListEndpoint: "" + +# As reranking depends on Data-prep service, enable data-prep dependency and +# override the values for multimodal setup. + +data-prep: + enabled: true + + image: + repository: opea/dataprep-multimodal-vdms + pullPolicy: IfNotPresent + tag: "latest" + + INDEX_NAME: "mega-videoqna" + vdmsHost: "" + vdmsPort: "8001" + entryCommand: ["/bin/sh"] + extraArgs: ["-c", "sleep 15 && python ingest_videos.py"] diff --git a/helm-charts/common/reranking-usvc/templates/configmap.yaml b/helm-charts/common/reranking-usvc/templates/configmap.yaml index 69b1fcd91..70ecd5a54 100644 --- a/helm-charts/common/reranking-usvc/templates/configmap.yaml +++ b/helm-charts/common/reranking-usvc/templates/configmap.yaml @@ -8,6 +8,19 @@ metadata: labels: {{- include "reranking-usvc.labels" . | nindent 4 }} data: + {{- if .Values.chunkDuration }} + CHUNK_DURATION: {{ .Values.chunkDuration | quote }} + {{- end }} + {{- if .Values.fileServerEndpoint }} + FILE_SERVER_ENDPOINT: {{ tpl .Values.fileServerEndpoint . | quote }} + {{- else }} + FILE_SERVER_ENDPOINT: http://{{ .Release.Name }}-data-prep:6007/v1/dataprep/get_file" + {{- end }} + {{- if .Values.getVideoListEndpoint }} + DATAPREP_GET_VIDEO_LIST_ENDPOINT: {{ tpl .Values.getVideoListEndpoint . | quote }} + {{- else }} + DATAPREP_GET_VIDEO_LIST_ENDPOINT: http://{{ .Release.Name }}-data-prep:6007/v1/dataprep/get_videos" + {{- end }} {{- if .Values.TEI_RERANKING_ENDPOINT }} TEI_RERANKING_ENDPOINT: {{ .Values.TEI_RERANKING_ENDPOINT | quote }} {{- else }} diff --git a/helm-charts/common/reranking-usvc/templates/deployment.yaml b/helm-charts/common/reranking-usvc/templates/deployment.yaml index 319026554..e329f29d2 100644 --- a/helm-charts/common/reranking-usvc/templates/deployment.yaml +++ b/helm-charts/common/reranking-usvc/templates/deployment.yaml @@ -43,7 +43,7 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: reranking-usvc - containerPort: 8000 + containerPort: {{ .Values.service.containerPort }} protocol: TCP volumeMounts: - mountPath: /tmp diff --git a/helm-charts/common/reranking-usvc/templates/service.yaml b/helm-charts/common/reranking-usvc/templates/service.yaml index 2b1bb7afa..4abca325d 100644 --- a/helm-charts/common/reranking-usvc/templates/service.yaml +++ b/helm-charts/common/reranking-usvc/templates/service.yaml @@ -11,7 +11,7 @@ spec: type: {{ .Values.service.type }} ports: - port: {{ .Values.service.port }} - targetPort: 8000 + targetPort: {{ .Values.service.containerPort }} protocol: TCP name: reranking-usvc selector: diff --git a/helm-charts/common/reranking-usvc/templates/tests/test-pod.yaml b/helm-charts/common/reranking-usvc/templates/tests/test-pod.yaml index 4bd7ac745..16af468bd 100644 --- a/helm-charts/common/reranking-usvc/templates/tests/test-pod.yaml +++ b/helm-charts/common/reranking-usvc/templates/tests/test-pod.yaml @@ -15,6 +15,28 @@ spec: - name: curl image: python:3.10.14 command: ['bash', '-c'] + {{- if contains "reranking-videoqna" .Values.image.repository }} + args: + - | + max_retry=20; + for ((i=1; i<=max_retry; i++)); do + curl http://{{ include "reranking-usvc.fullname" . }}:{{ .Values.service.port }}/v1/reranking -sS --fail-with-body \ + -X 'POST' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "retrieved_docs": [{"doc": [{"text": "this is the retrieved text"}]}], + "initial_query": "this is the query", + "top_n": 1, + "metadata": [ + {"other_key": "value", "video":"top_video_name", "timestamp":"20"} + ] + }' && break; + curlcode=$? + if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi; + done; + if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi + {{- else }} args: - | max_retry=20; @@ -27,4 +49,5 @@ spec: if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi; done; if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi + {{- end }} restartPolicy: Never diff --git a/helm-charts/common/reranking-usvc/values.yaml b/helm-charts/common/reranking-usvc/values.yaml index 5ce273f63..cbeb44eac 100644 --- a/helm-charts/common/reranking-usvc/values.yaml +++ b/helm-charts/common/reranking-usvc/values.yaml @@ -8,6 +8,9 @@ teirerank: enabled: false +data-prep: + enabled: false + replicaCount: 1 # Set it as a non-null string, such as true, if you want to enable logging facility, @@ -45,6 +48,7 @@ service: type: ClusterIP # The default port for reranking service is 9000 port: 8000 + containerPort: 8000 resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/helm-charts/common/reranking-usvc/variant_videoqna-values.yaml b/helm-charts/common/reranking-usvc/variant_videoqna-values.yaml new file mode 100644 index 000000000..7d0f6ad0e --- /dev/null +++ b/helm-charts/common/reranking-usvc/variant_videoqna-values.yaml @@ -0,0 +1,28 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +image: + repository: opea/reranking-videoqna + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +chunkDuration: "" + +# Replace with actual Endpoint URLs +fileServerEndpoint: "" +getVideoListEndpoint: "" + +# As reranking depends on Data-prep service, enable data-prep dependency and +# override the values for multimodal setup. +data-prep: + image: + repository: opea/dataprep-multimodal-vdms + pullPolicy: IfNotPresent + tag: "latest" + + INDEX_NAME: "mega-videoqna" + vdmsHost: "" + vdmsPort: "8001" + entryCommand: ["/bin/sh"] + extraArgs: ["-c", "sleep 15 && python ingest_videos.py"] diff --git a/helm-charts/common/retriever-usvc/Chart.yaml b/helm-charts/common/retriever-usvc/Chart.yaml index f7ba209f1..76fc5d0b5 100644 --- a/helm-charts/common/retriever-usvc/Chart.yaml +++ b/helm-charts/common/retriever-usvc/Chart.yaml @@ -17,6 +17,10 @@ dependencies: version: 1.0.0 repository: file://../redis-vector-db condition: redis-vector-db.enabled + - name: vdms-vector-db + version: 1.0.0 + repository: file://../vdms-vector-db + condition: vdms-vector-db.enabled - name: milvus version: 4.2.12 repository: https://zilliztech.github.io/milvus-helm/ diff --git a/helm-charts/common/retriever-usvc/README.md b/helm-charts/common/retriever-usvc/README.md index a64e4f292..9cd01ceb1 100644 --- a/helm-charts/common/retriever-usvc/README.md +++ b/helm-charts/common/retriever-usvc/README.md @@ -1,10 +1,14 @@ -# retriever-usvc +# Retriever Microservice (retriever-usvc) -Helm chart for deploying Retriever microservice. +**Helm chart for deploying Retriever microservice.** -retriever-usvc depends on redis and tei, you should set these endpoints before start. +There are two versions of Retriever microservice. First one is based on redis-vector-db and TEI. It does retrieval for textual data. An alternative multimodal version based on `vdms-values.yaml` file, performs retrieval for visual data. Follow along to select and install the version which suites your use case. -## (Option1): Installing the chart separately +retriever-usvc depends on redis-vector-db, tei. The multimodal version depends on vdms-vector-db. Endpoints for these dependencies should be set properly before installing the chart. + +## Install Retriever Microservice based on redis-vector-db and TEI + +### (Option1): Installing the chart separately First, you need to install the tei and redis-vector-db chart, refer to the [tei](../tei/README.md) and [redis-vector-db](../redis-vector-db/README.md) for more information. @@ -20,7 +24,7 @@ helm dependency update helm install retriever-usvc . --set REDIS_URL=${REDIS_URL} --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} ``` -## (Option2): Installing the chart with dependencies automatically +### (Option2): Installing the chart with dependencies automatically ```console cd GenAIInfra/helm-charts/common/retriever-usvc @@ -28,6 +32,53 @@ helm dependency update helm install retriever-usvc . --set tei.enabled=true --set redis-vector-db.enabled=true ``` +## Install Retriever microservice based on vdms-vector-db + +### (Option1): Installing the chart separately + +First, you need to install the `vdms-vector-db`. Refer to [vdms-vector-db](../vdms-vector-db) chart guide for more information. + +After you've deployed `vdms-vector-db` chart successfully, run `kubectl get svc` to get the service endpoint and port for vdms-vector-db service. + +To install retriever-usvc chart, run the following: + +```bash +cd GenAIInfra/helm-charts/common/retriever-usvc + +# Use the host and port received in previous step as VDMS_HOST and VDMS_PORT. +export VDMS_HOST="vdms-vector-db" +export VDMS_PORT="8001" +export INDEX_NAME="mega-videoqna" +export HFTOKEN= +# Set a directory to cache emdedding models +export MODELDIR=/mnt/opea-models + +# Export the proxy variables. Assign empty string if no proxy setup required. +export https_proxy="your_http_proxy" +export http_proxy="your_https_proxy" + +helm dependency update +helm install retriever-usvc . -f ./variant_videoqna-values.yaml --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set indexName=${INDEX_NAME} --set global.modelUseHostPath=${MODELDIR} --set global.https_proxy=${https_proxy} --set global.http_proxy=${http_proxy} + +``` + +### (Option2): Installing the chart with dependencies automatically + +```bash +cd GenAIInfra/helm-charts/common/retriever-usvc +export INDEX_NAME="mega-videoqna" +export HFTOKEN= +# Set a directory to cache emdedding models +export MODELDIR=/mnt/opea-models + +# Export the proxy variables. Assign empty string if no proxy setup required. +export https_proxy="your_http_proxy" +export http_proxy="your_https_proxy" + +helm dependency update +helm install retriever-usvc . -f ./variant_videoqna-values.yaml --set vdms-vector-db.enabled=true --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set indexName=${INDEX_NAME} --set global.modelUseHostPath=${MODELDIR} --set global.https_proxy=${https_proxy} --set global.http_proxy=${http_proxy} +``` + ## Verify To verify the installation, run the command `kubectl get pod` to make sure all pods are running. @@ -36,7 +87,7 @@ Then run the command `kubectl port-forward svc/retriever-usvc 7000:7000` to expo Open another terminal and run the following command to verify the service if working: -```console +```bash export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") curl http://localhost:7000/v1/retrieval \ -X POST \ @@ -46,13 +97,14 @@ curl http://localhost:7000/v1/retrieval \ ## Values -| Key | Type | Default | Description | -| ---------------------- | ------ | ---------------------- | ----------- | -| image.repository | string | `"opea/retriever-tgi"` | | -| service.port | string | `"7000"` | | -| REDIS_URL | string | `""` | | -| TEI_EMBEDDING_ENDPOINT | string | `""` | | -| global.monitoring | bop; | false | | +| Key | Type | Default | Description | +| ---------------------------- | ------ | ----------------------- | ----------- | +| image.repository | string | `"opea/retriever-tgi"` | | +| service.port | string | `"7000"` | | +| REDIS_URL | string | `""` | | +| TEI_EMBEDDING_ENDPOINT | string | `""` | | +| global.monitoring | bop; | `false` | | +| vdms-values:image.repository | string | `"opea/retriever-vdms"` | | ## Milvus support diff --git a/helm-charts/common/retriever-usvc/ci-values.yaml b/helm-charts/common/retriever-usvc/ci-values.yaml index cbc29c7ee..bda05801e 100644 --- a/helm-charts/common/retriever-usvc/ci-values.yaml +++ b/helm-charts/common/retriever-usvc/ci-values.yaml @@ -7,7 +7,7 @@ tei: enabled: true -redis-vector-db: - enabled: true milvus: enabled: false +redis-vector-db: + enabled: true diff --git a/helm-charts/common/retriever-usvc/ci-videoqna-values.yaml b/helm-charts/common/retriever-usvc/ci-videoqna-values.yaml new file mode 100644 index 000000000..561009cf8 --- /dev/null +++ b/helm-charts/common/retriever-usvc/ci-videoqna-values.yaml @@ -0,0 +1,22 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +vdms-vector-db: + enabled: true + +image: + repository: opea/retriever-vdms + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +INDEX_NAME: "mega-videoqna" +vdmsHost: "" +vdmsPort: "8001" +useClip: "1" +entryCommand: ["/bin/sh"] +extraArgs: ["-c", "sleep 30 && python retriever_vdms.py"] + +# Set cacheUseHostPath to for caching encoding/embedding models and other related data +global: + cacheUseHostPath: "" diff --git a/helm-charts/common/retriever-usvc/templates/configmap.yaml b/helm-charts/common/retriever-usvc/templates/configmap.yaml index 6ada51029..15521db82 100644 --- a/helm-charts/common/retriever-usvc/templates/configmap.yaml +++ b/helm-charts/common/retriever-usvc/templates/configmap.yaml @@ -8,6 +8,17 @@ metadata: labels: {{- include "retriever-usvc.labels" . | nindent 4 }} data: + {{- if .Values.vdmsHost }} + VDMS_HOST: {{ .Values.vdmsHost | quote }} + {{- else }} + VDMS_HOST: "{{ .Release.Name }}-vdms-vector-db" + {{- end }} + {{- if .Values.vdmsPort }} + VDMS_PORT: {{ .Values.vdmsPort | quote }} + {{- end }} + {{- if .Values.useClip }} + USECLIP: {{ .Values.useClip | quote }} + {{- end }} {{- if .Values.MOSEC_EMBEDDING_ENDPOINT }} MOSEC_EMBEDDING_ENDPOINT: {{ .Values.MOSEC_EMBEDDING_ENDPOINT | quote}} MOSEC_EMBEDDING_MODEL: {{ .Values.MOSEC_EMBEDDING_MODEL | quote}} @@ -21,7 +32,7 @@ data: LOCAL_EMBEDDING_MODEL: {{ .Values.LOCAL_EMBEDDING_MODEL | quote }} {{- end }} {{- if .Values.REDIS_URL }} - REDIS_URL: {{ .Values.REDIS_URL | quote}} + REDIS_URL: {{ .Values.REDIS_URL | quote }} {{- else }} REDIS_URL: "redis://{{ .Release.Name }}-redis-vector-db:6379" {{- end }} @@ -40,7 +51,7 @@ data: http_proxy: {{ .Values.global.http_proxy | quote }} https_proxy: {{ .Values.global.https_proxy | quote }} {{- if and (not .Values.REDIS_URL) (and (not .Values.TEI_EMBEDDING_ENDPOINT) (or .Values.global.http_proxy .Values.global.https_proxy)) }} - no_proxy: "{{ .Release.Name }}-tei,{{ .Release.Name }}-redis-vector-db,{{ .Values.global.no_proxy }}" + no_proxy: "{{ .Release.Name }}-tei,{{ .Release.Name }}-redis-vector-db,{{ .Release.Name }}-vdms-vector-db,{{ .Values.global.no_proxy }}" {{- else }} no_proxy: {{ .Values.global.no_proxy | quote }} {{- end }} diff --git a/helm-charts/common/retriever-usvc/templates/deployment.yaml b/helm-charts/common/retriever-usvc/templates/deployment.yaml index e42cd0980..bbe3bf997 100644 --- a/helm-charts/common/retriever-usvc/templates/deployment.yaml +++ b/helm-charts/common/retriever-usvc/templates/deployment.yaml @@ -41,11 +41,25 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.entryCommand }} + command: {{ .Values.entryCommand }} + {{- end }} + {{- if .Values.extraArgs }} + args: + {{- range .Values.extraArgs }} + - {{ . | quote }} + {{- end }} + {{- end }} ports: - name: retriever-usvc - containerPort: {{ .Values.port }} + containerPort: {{ .Values.service.containerPort }} protocol: TCP volumeMounts: + {{- if .Values.global.modelUseHostPath }} + - mountPath: /home/user/.cache/huggingface/hub + name: model-volume + subPath: huggingface/hub + {{- end }} - mountPath: /tmp name: tmp {{- if .Values.livenessProbe }} @@ -63,6 +77,16 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumes: + {{- if .Values.global.modelUseHostPath }} + - name: model-volume + hostPath: + path: {{ .Values.global.modelUseHostPath }} + type: Directory + {{- else if .Values.global.modelUsePVC }} + - name: model-volume + persistentVolumeClaim: + claimName: {{ .Values.global.modelUsePVC }} + {{- end }} - name: tmp emptyDir: {} {{- with .Values.nodeSelector }} diff --git a/helm-charts/common/retriever-usvc/templates/service.yaml b/helm-charts/common/retriever-usvc/templates/service.yaml index e127bf858..e6cf33d3d 100644 --- a/helm-charts/common/retriever-usvc/templates/service.yaml +++ b/helm-charts/common/retriever-usvc/templates/service.yaml @@ -11,7 +11,7 @@ spec: type: {{ .Values.service.type }} ports: - port: {{ .Values.service.port }} - targetPort: {{ .Values.port }} + targetPort: {{ .Values.service.containerPort }} protocol: TCP name: retriever-usvc selector: diff --git a/helm-charts/common/retriever-usvc/values.yaml b/helm-charts/common/retriever-usvc/values.yaml index a71cac068..5dde5ab37 100644 --- a/helm-charts/common/retriever-usvc/values.yaml +++ b/helm-charts/common/retriever-usvc/values.yaml @@ -11,6 +11,8 @@ milvus: enabled: false redis-vector-db: enabled: false +vdms-vector-db: + enabled: false replicaCount: 1 @@ -55,6 +57,7 @@ service: type: ClusterIP # The default port for retriever service is 7000 port: 7000 + containerPort: 7000 resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/helm-charts/common/retriever-usvc/variant_videoqna-values.yaml b/helm-charts/common/retriever-usvc/variant_videoqna-values.yaml new file mode 100644 index 000000000..770427eeb --- /dev/null +++ b/helm-charts/common/retriever-usvc/variant_videoqna-values.yaml @@ -0,0 +1,20 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +image: + repository: opea/retriever-vdms + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +INDEX_NAME: "mega-videoqna" +vdmsHost: "" +vdmsPort: "8001" +useClip: "1" +entryCommand: ["/bin/sh"] +extraArgs: ["-c", "sleep 30 && python retriever_vdms.py"] + +# Set cacheUseHostPath to for caching encoding/embedding models and other related data +global: + modelUseHostPath: "" + modelUsePVC: "" diff --git a/helm-charts/common/ui/ci-variant_videoqna_streamlit-values.yaml b/helm-charts/common/ui/ci-variant_videoqna_streamlit-values.yaml new file mode 120000 index 000000000..5a6e67ecd --- /dev/null +++ b/helm-charts/common/ui/ci-variant_videoqna_streamlit-values.yaml @@ -0,0 +1 @@ +variant_videoqna_streamlit-values.yaml \ No newline at end of file diff --git a/helm-charts/common/ui/templates/_helpers.tpl b/helm-charts/common/ui/templates/_helpers.tpl index 07e404849..e8e367ff2 100644 --- a/helm-charts/common/ui/templates/_helpers.tpl +++ b/helm-charts/common/ui/templates/_helpers.tpl @@ -59,4 +59,4 @@ Create the name of the service account to use {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-charts/common/ui/templates/configmap.yaml b/helm-charts/common/ui/templates/configmap.yaml index ce22b014b..291c91c24 100644 --- a/helm-charts/common/ui/templates/configmap.yaml +++ b/helm-charts/common/ui/templates/configmap.yaml @@ -34,6 +34,9 @@ data: CHAT_URL: {{ .Values.BACKEND_SERVICE_ENDPOINT | quote }} {{- else if contains "faqgen-react-ui" .Values.image.repository }} VITE_FAQ_GEN_URL: {{ .Values.BACKEND_SERVICE_ENDPOINT | quote }} + {{- else if contains "videoqna-ui" .Values.image.repository }} + BACKEND_SERVICE_ENDPOINT: {{ tpl .Values.BACKEND_SERVICE_ENDPOINT . | quote }} + BACKEND_HEALTH_CHECK_ENDPOINT: {{ tpl .Values.BACKEND_SERVICE_ENDPOINT . | replace "videoqna" "health_check" | quote }} {{- else }} {{- fail "Unsupported ui image: " .Values.image.repository }} {{- end }} diff --git a/helm-charts/common/ui/variant_videoqna_streamlit-values.yaml b/helm-charts/common/ui/variant_videoqna_streamlit-values.yaml new file mode 100644 index 000000000..759c1bd20 --- /dev/null +++ b/helm-charts/common/ui/variant_videoqna_streamlit-values.yaml @@ -0,0 +1,15 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +image: + repository: opea/videoqna-ui + pullPolicy: IfNotPresent + tag: "latest" + +# Endpoint for VideoQnA Backend. Test Value - Override it with actual Backend endpoint +BACKEND_SERVICE_ENDPOINT: "http://localhost:8888/v1/videoqna" + +containerPort: 5173 +service: + type: ClusterIP + port: 5173 diff --git a/helm-charts/common/vdms-vector-db/.helmignore b/helm-charts/common/vdms-vector-db/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/helm-charts/common/vdms-vector-db/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm-charts/common/vdms-vector-db/Chart.yaml b/helm-charts/common/vdms-vector-db/Chart.yaml new file mode 100644 index 000000000..8cf33fa90 --- /dev/null +++ b/helm-charts/common/vdms-vector-db/Chart.yaml @@ -0,0 +1,9 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v2 +name: vdms-vector-db +description: A Helm chart for Intel VDMS Vector DB +type: application +version: 1.0.0 +appVersion: "v2.8.0" diff --git a/helm-charts/common/vdms-vector-db/README.md b/helm-charts/common/vdms-vector-db/README.md new file mode 100644 index 000000000..608c229b1 --- /dev/null +++ b/helm-charts/common/vdms-vector-db/README.md @@ -0,0 +1,32 @@ +# vdms-vector-db + +Helm chart for deploying Intel VDMS Vector DB service. + +## Install the Chart + +To install the chart, run the following: + +```bash +cd GenAIInfra/helm-charts/common +helm install vdms-vector-db vdms-vector-db +``` + +## Verify + +To verify the installation, run the command `kubectl get pod` to make sure that vdms-vector-db pod is running. + +Then run the command `kubectl port-forward svc/vdms-vector-db 8001:8001` to expose the vdms-vector-db service for access on current host. + +Next, verify whether you can reach the `vdms-vector-db` pod. As there are no http services running in `vdms-vector-db` pod, we will verify sanity by making a tcp connection request to the pod and check whether `8001` port is open. + +We will use `netcat` command utility for this. + +Try running `nc -zv vdms-vector-db 8001`. The command will succeed if vdms-vector-db can accept TCP request and the required port (8001) is open. + +## Values + +| Key | Type | Default | Description | +| ---------------- | ------ | ------------------ | ------------------------------- | +| image.repository | string | `"intellabs/vdms"` | | +| image.tag | string | `"v2.8.0"` | | +| service.port | string | `"8001"` | The vdms-vector-db service port | diff --git a/helm-charts/common/vdms-vector-db/ci-values.yaml b/helm-charts/common/vdms-vector-db/ci-values.yaml new file mode 120000 index 000000000..7d1010096 --- /dev/null +++ b/helm-charts/common/vdms-vector-db/ci-values.yaml @@ -0,0 +1 @@ +values.yaml \ No newline at end of file diff --git a/helm-charts/common/vdms-vector-db/templates/_helpers.tpl b/helm-charts/common/vdms-vector-db/templates/_helpers.tpl new file mode 100644 index 000000000..fa206e852 --- /dev/null +++ b/helm-charts/common/vdms-vector-db/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "vdms-vector-db.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "vdms-vector-db.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "vdms-vector-db.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "vdms-vector-db.labels" -}} +helm.sh/chart: {{ include "vdms-vector-db.chart" . }} +{{ include "vdms-vector-db.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "vdms-vector-db.selectorLabels" -}} +app.kubernetes.io/name: {{ include "vdms-vector-db.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "vdms-vector-db.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "vdms-vector-db.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm-charts/common/vdms-vector-db/templates/deployment.yaml b/helm-charts/common/vdms-vector-db/templates/deployment.yaml new file mode 100644 index 000000000..41af73b0a --- /dev/null +++ b/helm-charts/common/vdms-vector-db/templates/deployment.yaml @@ -0,0 +1,70 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "vdms-vector-db.fullname" . }} + labels: + {{- include "vdms-vector-db.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "vdms-vector-db.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "vdms-vector-db.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Release.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.containerPort }} + protocol: TCP + startupProbe: + tcpSocket: + port: {{ .Values.service.containerPort }} + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 120 + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm-charts/common/vdms-vector-db/templates/service.yaml b/helm-charts/common/vdms-vector-db/templates/service.yaml new file mode 100644 index 000000000..6569cb7b0 --- /dev/null +++ b/helm-charts/common/vdms-vector-db/templates/service.yaml @@ -0,0 +1,18 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "vdms-vector-db.fullname" . }} + labels: + {{- include "vdms-vector-db.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.containerPort }} + protocol: TCP + name: http + selector: + {{- include "vdms-vector-db.selectorLabels" . | nindent 4 }} diff --git a/helm-charts/common/vdms-vector-db/templates/tests/test-pod.yaml b/helm-charts/common/vdms-vector-db/templates/tests/test-pod.yaml new file mode 100644 index 000000000..fb178e6f8 --- /dev/null +++ b/helm-charts/common/vdms-vector-db/templates/tests/test-pod.yaml @@ -0,0 +1,19 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "vdms-vector-db.fullname" . }}-testpod" + labels: + {{- include "vdms-vector-db.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['nc'] + args: ['-vw', '20', '{{ include "vdms-vector-db.fullname" . }}:{{ .Values.service.port }}'] + + restartPolicy: Never diff --git a/helm-charts/common/vdms-vector-db/values.yaml b/helm-charts/common/vdms-vector-db/values.yaml new file mode 100644 index 000000000..9cfb552d5 --- /dev/null +++ b/helm-charts/common/vdms-vector-db/values.yaml @@ -0,0 +1,60 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# Default values for vdms-vector-db. + +replicaCount: 1 + +image: + repository: intellabs/vdms + # This sets the pull policy for images. + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "v2.8.0" + +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "" +fullnameOverride: "" + + +podAnnotations: {} +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: false + runAsUser: 0 + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + +service: + type: ClusterIP + port: 8001 + containerPort: 55555 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/helm-charts/common/video-llama-lvm/.helmignore b/helm-charts/common/video-llama-lvm/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/helm-charts/common/video-llama-lvm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm-charts/common/video-llama-lvm/Chart.yaml b/helm-charts/common/video-llama-lvm/Chart.yaml new file mode 100644 index 000000000..ac8dd8bec --- /dev/null +++ b/helm-charts/common/video-llama-lvm/Chart.yaml @@ -0,0 +1,9 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v2 +name: video-llama-lvm +description: A Helm chart for serving LVM +type: application +version: 1.0.0 +appVersion: "v1.0" diff --git a/helm-charts/common/video-llama-lvm/README.md b/helm-charts/common/video-llama-lvm/README.md new file mode 100644 index 000000000..7253ec7b1 --- /dev/null +++ b/helm-charts/common/video-llama-lvm/README.md @@ -0,0 +1,54 @@ +# LVM Serving Microservice (video-llama-lvm) + +Helm chart for deploying video-llama-lvm microservice. + +`video-llama-lvm` is a microservice which provides inference from Video-Llama-2-7b. + +## Installing the Chart + +To install the chart, run the following: + +```bash +cd GenAIInfra/helm-charts/common +export HFTOKEN="insert-your-huggingface-token-here" + +# Set a dir to cache downloaded Models and clips +export MODELDIR=/mnt/opea-models + +# When setting up for first time, model needs to be downloaded. Set LLM_DOWNLOAD flag to true to download models. Please note, when redeploying we should set this value to false, otherwise model download will restart. +export LLM_DOWNLOAD=true + +# Export the proxy variables. Assign empty string if no proxy setup required. +export https_proxy="your_http_proxy" +export http_proxy="your_https_proxy" + +helm install video-llama-lvm video-llama-lvm --set global.modelUseHostPath=${MODELDIR} --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set llmDownload=${LLM_DOWNLOAD} --set global.https_proxy=${https_proxy} --set global.http_proxy=${http_proxy} +``` + +> \***\*NOTE:\*\*** **Model download may take up to 1.5 Hours.** When installing chart for the first time we should set `llmDownload` value to be **true**. This helps download model for the first run. Afterwards, for re-installing charts set `llmDownload` value in the chart to be **false**. If not set to **false**, model download will again start and service will again long time to be ready. + +## Verify + +To verify the installation, run the command `kubectl get pod` to make sure all pods are running. + +Then run the command `kubectl port-forward svc/video-llama-lvm 2080:80` to expose the video-llama-lvm service for access on the host machine. + +Open another terminal and run the following command to verify the service if working: + +```bash +curl -X POST \ + "http://localhost:2080/generate?video_url=silence_girl.mp4&start=0.0&duration=9&prompt=What%20is%20the%20person%20doing%3F&max_new_tokens=150" \ + -H "accept: */*" \ + -d '' +``` + +## Values + +| Key | Type | Default | Description | +| ------------------------------- | ------- | ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| global.cacheUseHostPath | string | "/home/$USER/.cache" | Cache the embedding model and related data. | +| global.HUGGINGFACEHUB_API_TOKEN | string | `insert-your-huggingface-token-here` | Hugging Face API token | +| global.modelUseHostPath | string | `"/mnt/opea-models"` | Cached models directory. The host path "modelUseHostPath" will be mounted to container as /home/user/model directory. | +| image.repository | string | `"opea/video-llama-lvm-server"` | | +| image.tag | string | `"latest"` | | +| llmDownload | boolean | `true` | This value when true, makes video-llama-lvm download a model. Change it to false for stopping video-llama-lvm from re-downloading model. | diff --git a/helm-charts/common/video-llama-lvm/ci-values.yaml b/helm-charts/common/video-llama-lvm/ci-values.yaml new file mode 120000 index 000000000..7d1010096 --- /dev/null +++ b/helm-charts/common/video-llama-lvm/ci-values.yaml @@ -0,0 +1 @@ +values.yaml \ No newline at end of file diff --git a/helm-charts/common/video-llama-lvm/templates/_helpers.tpl b/helm-charts/common/video-llama-lvm/templates/_helpers.tpl new file mode 100644 index 000000000..316304428 --- /dev/null +++ b/helm-charts/common/video-llama-lvm/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "video-llama-lvm.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "video-llama-lvm.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "video-llama-lvm.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "video-llama-lvm.labels" -}} +helm.sh/chart: {{ include "video-llama-lvm.chart" . }} +{{ include "video-llama-lvm.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "video-llama-lvm.selectorLabels" -}} +app.kubernetes.io/name: {{ include "video-llama-lvm.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "video-llama-lvm.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "video-llama-lvm.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm-charts/common/video-llama-lvm/templates/configmap.yaml b/helm-charts/common/video-llama-lvm/templates/configmap.yaml new file mode 100644 index 000000000..9e7d1e618 --- /dev/null +++ b/helm-charts/common/video-llama-lvm/templates/configmap.yaml @@ -0,0 +1,14 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "video-llama-lvm.fullname" . }}-config + labels: + {{- include "video-llama-lvm.labels" . | nindent 4 }} +data: + llm_download: {{ .Values.llmDownload | quote | title }} + http_proxy: {{ .Values.global.http_proxy | quote }} + https_proxy: {{ .Values.global.https_proxy | quote }} + no_proxy: {{ .Values.global.no_proxy | quote }} diff --git a/helm-charts/common/video-llama-lvm/templates/deployment.yaml b/helm-charts/common/video-llama-lvm/templates/deployment.yaml new file mode 100644 index 000000000..3fd3a50bb --- /dev/null +++ b/helm-charts/common/video-llama-lvm/templates/deployment.yaml @@ -0,0 +1,103 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "video-llama-lvm.fullname" . }} + labels: + {{- include "video-llama-lvm.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "video-llama-lvm.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "video-llama-lvm.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Release.Name }} + envFrom: + - configMapRef: + name: {{ include "video-llama-lvm.fullname" . }}-config + {{- if .Values.global.extraEnvConfig }} + - configMapRef: + name: {{ .Values.global.extraEnvConfig }} + optional: true + {{- end }} + securityContext: + {{- if .Values.global.modelUseHostPath }} + {} + {{- else }} + {{- toYaml .Values.securityContext | nindent 12 }} + {{- end }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.containerPort }} + protocol: TCP + volumeMounts: + - mountPath: /home/user/model + name: model-volume + - mountPath: /home/user/.cache/huggingface/hub + name: model-volume + subPath: huggingface/hub + - mountPath: /home/user/.cache/torch/hub + name: model-volume + subPath: torch/hub + - mountPath: /tmp + name: tmp + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + startupProbe: + {{- if .Values.llmDownload }} + {{- .Values.startupProbeWithLlmDownload | toYaml | nindent 12 }} + {{- else }} + {{- .Values.startupProbe | toYaml | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumes: + - name: model-volume + {{- if .Values.global.modelUsePVC }} + persistentVolumeClaim: + claimName: {{ .Values.global.modelUsePVC }} + {{- else if .Values.global.modelUseHostPath }} + hostPath: + path: {{ .Values.global.modelUseHostPath }} + type: Directory + {{- else }} + emptyDir: {} + {{- end }} + - name: tmp + emptyDir: {} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm-charts/common/video-llama-lvm/templates/service.yaml b/helm-charts/common/video-llama-lvm/templates/service.yaml new file mode 100644 index 000000000..01b9e01bb --- /dev/null +++ b/helm-charts/common/video-llama-lvm/templates/service.yaml @@ -0,0 +1,18 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "video-llama-lvm.fullname" . }} + labels: + {{- include "video-llama-lvm.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.containerPort }} + protocol: TCP + name: http + selector: + {{- include "video-llama-lvm.selectorLabels" . | nindent 4 }} diff --git a/helm-charts/common/video-llama-lvm/templates/tests/test-pod.yaml b/helm-charts/common/video-llama-lvm/templates/tests/test-pod.yaml new file mode 100644 index 000000000..c7bb2d927 --- /dev/null +++ b/helm-charts/common/video-llama-lvm/templates/tests/test-pod.yaml @@ -0,0 +1,29 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "video-llama-lvm.fullname" . }}-testpod" + labels: + {{- include "video-llama-lvm.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: curl + image: python:3.10.14 + command: ['bash', '-c'] + args: + - | + max_retry=20 + for ((i=1; i<=max_retry; i++)); do + curl -v 'http://{{ include "video-llama-lvm.fullname" . }}:{{ .Values.service.port }}/generate?video_url=https%3A%2F%2Fgithub.com%2FDAMO-NLP-SG%2FVideo-LLaMA%2Fraw%2Fmain%2Fexamples%2Fsilence_girl.mp4&start=0.0&duration=9&prompt=What%20is%20the%20person%20doing%3F&max_new_tokens=150' -sS --fail-with-body \ + -X POST \ + -H 'accept: */*' \ + -d '' && break; + curlcode=$? + if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi; + done; + if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi + restartPolicy: Never diff --git a/helm-charts/common/video-llama-lvm/values.yaml b/helm-charts/common/video-llama-lvm/values.yaml new file mode 100644 index 000000000..242005b95 --- /dev/null +++ b/helm-charts/common/video-llama-lvm/values.yaml @@ -0,0 +1,103 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# Default values for video-llama-lvm. + +replicaCount: 1 + +llmDownload: true + +image: + repository: opea/video-llama-lvm-server + # This sets the pull policy for images. + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "" +fullnameOverride: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + +service: + type: ClusterIP + port: 80 + containerPort: 9009 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 24 +readinessProbe: + tcpSocket: + port: http + initialDelaySeconds: 5 + periodSeconds: 5 +# Video-Llama Server may take upto 15 mins to startup completely. +startupProbe: + tcpSocket: + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 90 +# When LLM download is taking place, startup may take upto 60 mins. +startupProbeWithLlmDownload: + tcpSocket: + port: http + initialDelaySeconds: 5 + periodSeconds: 30 + failureThreshold: 120 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +global: + http_proxy: "" + https_proxy: "" + no_proxy: "" + HUGGINGFACEHUB_API_TOKEN: "insert-your-huggingface-token-here" + + # Choose where to save your downloaded models + # Set modelUseHostPath for local directory, this is good for one node test. Example: + # modelUseHostPath: /mnt/opea-models + # Set modelUsePVC for PersistentVolumeClaim(PVC), which is suitable for multinode deployment. Example: + # modelUsePVC: model-volume + # You can only set one of the following var, the behavior is not defined is both are set. + # By default, both var are set to empty, the model will be downloaded and saved to a tmp volume. + modelUseHostPath: "" + modelUsePVC: "" diff --git a/helm-charts/videoqna/.helmignore b/helm-charts/videoqna/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/helm-charts/videoqna/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm-charts/videoqna/Chart.yaml b/helm-charts/videoqna/Chart.yaml new file mode 100644 index 000000000..fb033383d --- /dev/null +++ b/helm-charts/videoqna/Chart.yaml @@ -0,0 +1,35 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v2 +name: videoqna +description: A Helm chart to deploy VideoQnA Application Suite +type: application +dependencies: + - name: data-prep + version: 1.0.0 + repository: "file://../common/data-prep" + - name: embedding-usvc + version: 1.0.0 + repository: "file://../common/embedding-usvc" + - name: reranking-usvc + version: 1.0.0 + repository: "file://../common/reranking-usvc" + - name: retriever-usvc + version: 1.0.0 + repository: "file://../common/retriever-usvc" + - name: vdms-vector-db + version: 1.0.0 + repository: "file://../common/vdms-vector-db" + - name: video-llama-lvm + version: 1.0.0 + repository: "file://../common/video-llama-lvm" + - name: lvm-uservice + version: 1.0.0 + repository: "file://../common/lvm-uservice" + - name: ui + version: 1.0.0 + alias: videoqna-ui + repository: "file://../common/ui" +version: 1.0.0 +appVersion: "v1.0" diff --git a/helm-charts/videoqna/README.md b/helm-charts/videoqna/README.md new file mode 100644 index 000000000..e9f10a8d5 --- /dev/null +++ b/helm-charts/videoqna/README.md @@ -0,0 +1,79 @@ +# VideoQnA + +Helm chart for deploying VideoQnA service. VideoQnA depends on the following other microservices: + +- [data-prep](../common/data-prep/README.md) +- [embedding-usvc](../common/embedding-usvc/README.md) +- [retriever-usvc](../common/retriever-usvc/README.md) +- [reranking-usvc](../common/reranking-usvc/README.md) +- [vdms-vector-db](../common/vdms-vector-db/README.md) +- [video-llama-lvm](../common/video-llama-lvm/README.md) +- [lvm-uservice](../common/lvm-uservice/README.md) + +## Installing the Chart + +To install the chart, run the following: + +```bash +cd GenAIInfra/helm-charts/ +./update_dependency.sh +helm dependency update videoqna + +# Set following required values for videoqna chart and various subcharts +export HFTOKEN="insert-your-huggingface-token-here" +export MODELDIR="/mnt/opea-models" +export CACHEDIR="/home/$USER/.cache" +export LLM_DOWNLOAD=true +export INDEX_NAME="mega-videoqna" + +# Set the proxy variables. Assign empty string if no proxy setup required. +export https_proxy="your_http_proxy" +export http_proxy="your_https_proxy" + +helm install videoqna videoqna --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set global.cacheUseHostPath=${CACHEDIR} --set video-llama-lvm.llmDownload=${LLM_DOWNLOAD} --set data-prep.indexName=${INDEX_NAME} --set retriever-usvc.indexName=${INDEX_NAME} --set global.https_proxy=${https_proxy} --set global.http_proxy=${http_proxy} --wait +``` + +### IMPORTANT NOTE + +1. Make sure your `MODELDIR` and `CACHEDIR` exists on the node where your workload is scheduled. These directories are used to cache the downloaded model for next time use. Otherwise, set `global.modelUseHostPath` and `global.cacheUseHostPath` to 'null' if you don't want to cache the models. + +## Verify + +To verify the installation, run the command `kubectl get pod` to make sure all pods are running. + +Curl command and UI are the two options that can be leveraged to verify the result. + +### Verify the workload through curl command + +Run the command `kubectl port-forward svc/videoqna 8888:8888` to expose the service for access. + +Open another terminal and run the following command to verify the service if working: + +```bash +curl http://localhost:8888/v1/videoqna -H "Content-Type: application/json" -d '{ + "messages": "What is the man doing?", + "stream": "True" + }' +``` + +### Verify the workload through UI + +The UI has already been installed via the Helm chart. To access it, use the external IP of one your Kubernetes node along with the NGINX port. You can find the NGINX port using the following command: + +```bash +export port=$(kubectl get service videoqna-nginx --output='jsonpath={.spec.ports[0].nodePort}') +echo $port +``` + +Open a browser and head to `http://:${port}` to use VideoQnA via a web UI. + +## Values + +| Key | Type | Default | Description | +| --------------------------------------------- | ------- | ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| image.repository | string | `"opea/videoqna"` | | +| service.port | string | `"8888"` | | +| global.modelUseHostPath | string | `"/mnt/opea-models"` | A directory to where model dir for video-llama-lvm service is mounted. | +| global.cacheUseHostPath | string | `"/home/$USER/.cache"` | A directory to where cache dir for several services are mounted. | +| video-llama-lvm.llmDownload | boolean | `true` | This value when true, makes video-llama-lvm download a model. Change it to false for stopping video-llama-lvm from re-downloading model. | +| data-prep.indexName, retriever-usvc.indexName | string | `"mega-videoqna"` | This value when true, makes video-llama-lvm download a model. Change it to false for stopping video-llama-lvm from re-downloading model. | diff --git a/helm-charts/videoqna/ci-values.yaml b/helm-charts/videoqna/ci-values.yaml new file mode 120000 index 000000000..7d1010096 --- /dev/null +++ b/helm-charts/videoqna/ci-values.yaml @@ -0,0 +1 @@ +values.yaml \ No newline at end of file diff --git a/helm-charts/videoqna/templates/_helpers.tpl b/helm-charts/videoqna/templates/_helpers.tpl new file mode 100644 index 000000000..be4c36252 --- /dev/null +++ b/helm-charts/videoqna/templates/_helpers.tpl @@ -0,0 +1,69 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "videoqna.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "videoqna.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "videoqna.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "videoqna.labels" -}} +helm.sh/chart: {{ include "videoqna.chart" . }} +{{ include "videoqna.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "videoqna.selectorLabels" -}} +app.kubernetes.io/name: {{ include "videoqna.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "videoqna.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "videoqna.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Define the name for nginx Chart. +*/}} +{{- define "nginx.fullname" -}} +{{ .Release.Name | trunc 57 | trimSuffix "-" }}-nginx +{{- end }} diff --git a/helm-charts/videoqna/templates/configmap.yaml b/helm-charts/videoqna/templates/configmap.yaml new file mode 100644 index 000000000..d043f4fd9 --- /dev/null +++ b/helm-charts/videoqna/templates/configmap.yaml @@ -0,0 +1,37 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "videoqna.fullname" . }}-config + labels: + {{- include "videoqna.labels" . | nindent 4 }} +data: + {{- if .Values.embeddingServiceHost }} + EMBEDDING_SERVICE_HOST_IP: {{ .Values.embeddingServiceHost | quote }} + {{- else }} + EMBEDDING_SERVICE_HOST_IP: "{{ .Release.Name }}-embedding-usvc" + {{- end }} + {{- if .Values.retrieverServiceHost }} + RETRIEVER_SERVICE_HOST_IP: {{ .Values.retrieverServiceHost | quote }} + {{- else }} + RETRIEVER_SERVICE_HOST_IP: "{{ .Release.Name }}-retriever-usvc" + {{- end }} + {{- if .Values.rerankServiceHost }} + RERANK_SERVICE_HOST_IP: {{ .Values.rerankServiceHost | quote }} + {{- else }} + RERANK_SERVICE_HOST_IP: "{{ .Release.Name }}-reranking-usvc" + {{- end }} + {{- if .Values.lvmServiceHost }} + LVM_SERVICE_HOST_IP: {{ .Values.lvmServiceHost | quote }} + {{- else }} + LVM_SERVICE_HOST_IP: "{{ .Release.Name }}-lvm-uservice" + {{- end }} + http_proxy: {{ .Values.global.http_proxy | quote }} + https_proxy: {{ .Values.global.https_proxy | quote }} + {{- if and (not .Values.lvmServiceHost) (or .Values.global.http_proxy .Values.global.https_proxy) }} + no_proxy: "{{ .Release.Name }}-embedding-usvc,{{ .Release.Name }}-retriever-usvc,{{ .Release.Name }}-lvm-uservice,{{ .Release.Name }}-reranking-usvc,{{ .Values.global.no_proxy }}" + {{- else }} + no_proxy: {{ .Values.global.no_proxy | quote }} + {{- end }} diff --git a/helm-charts/videoqna/templates/deployment.yaml b/helm-charts/videoqna/templates/deployment.yaml new file mode 100644 index 000000000..3f2711ae7 --- /dev/null +++ b/helm-charts/videoqna/templates/deployment.yaml @@ -0,0 +1,88 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "videoqna.fullname" . }} + labels: + {{- include "videoqna.labels" . | nindent 4 }} + app: {{ include "videoqna.fullname" . }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "videoqna.selectorLabels" . | nindent 6 }} + app: {{ include "videoqna.fullname" . }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "videoqna.labels" . | nindent 8 }} + app: {{ include "videoqna.fullname" . }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Release.Name }} + envFrom: + - configMapRef: + name: {{ include "videoqna.fullname" . }}-config + {{- if .Values.global.extraEnvConfig }} + - configMapRef: + name: {{ .Values.global.extraEnvConfig }} + optional: true + {{- end }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.entryCommand }} + command: {{ .Values.entryCommand }} + {{- end }} + {{- if .Values.extraArgs }} + args: + {{- range .Values.extraArgs }} + - {{ . | quote }} + {{- end }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.containerPort }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm-charts/videoqna/templates/nginx-deployment.yaml b/helm-charts/videoqna/templates/nginx-deployment.yaml new file mode 100644 index 000000000..31667dcec --- /dev/null +++ b/helm-charts/videoqna/templates/nginx-deployment.yaml @@ -0,0 +1,123 @@ +apiVersion: v1 +data: + default.conf: |+ + # Copyright (C) 2024 Intel Corporation + # SPDX-License-Identifier: Apache-2.0 + + + server { + listen 80; + listen [::]:80; + + location /home { + alias /usr/share/nginx/html/index.html; + } + + location / { + proxy_pass http://{{ include "ui.fullname" (index .Subcharts "videoqna-ui") }}:{{ index .Values "videoqna-ui" "service" "port" }}; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_cache_bypass $http_upgrade; + } + + location /v1/videoqna { + proxy_pass http://{{ include "videoqna.fullname" . }}:{{ .Values.service.port }}; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /v1/health_check { + proxy_pass http://{{ include "videoqna.fullname" . }}:{{ .Values.service.port }}; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /v1/dataprep/get_file { + proxy_pass http://{{ include "data-prep.fullname" (index .Subcharts "data-prep") }}:{{ index .Values "data-prep" "service" "port" }}; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /v1/dataprep/get_videos { + proxy_pass http://{{ include "data-prep.fullname" (index .Subcharts "data-prep") }}:{{ index .Values "data-prep" "service" "port" }}; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + + } + +kind: ConfigMap +metadata: + name: {{ include "nginx.fullname" . }}-config +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "nginx.fullname" . }} + labels: + {{- include "videoqna.labels" . | nindent 4 }} + app: {{ include "nginx.fullname" . }} +spec: + selector: + matchLabels: + {{- include "videoqna.selectorLabels" . | nindent 6 }} + app: {{ include "nginx.fullname" . }} + template: + metadata: + labels: + {{- include "videoqna.selectorLabels" . | nindent 8 }} + app: {{ include "nginx.fullname" . }} + spec: + containers: + - image: nginx:1.27.1 + imagePullPolicy: {{ .Values.image.pullPolicy }} + name: nginx + volumeMounts: + - mountPath: /etc/nginx/conf.d + name: nginx-config-volume + securityContext: {} + volumes: + - configMap: + defaultMode: 420 + name: {{ include "nginx.fullname" . }}-config + name: nginx-config-volume + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nginx.fullname" . }} +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + {{- include "videoqna.selectorLabels" . | nindent 4 }} + app: {{ include "nginx.fullname" . }} + type: NodePort diff --git a/helm-charts/videoqna/templates/service.yaml b/helm-charts/videoqna/templates/service.yaml new file mode 100644 index 000000000..0f74e52b6 --- /dev/null +++ b/helm-charts/videoqna/templates/service.yaml @@ -0,0 +1,19 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "videoqna.fullname" . }} + labels: + {{- include "videoqna.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.containerPort }} + protocol: TCP + name: http + selector: + {{- include "videoqna.selectorLabels" . | nindent 4 }} + app: {{ include "videoqna.fullname" . }} diff --git a/helm-charts/videoqna/templates/tests/test-pod.yaml b/helm-charts/videoqna/templates/tests/test-pod.yaml new file mode 100644 index 000000000..c570d4e24 --- /dev/null +++ b/helm-charts/videoqna/templates/tests/test-pod.yaml @@ -0,0 +1,29 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "videoqna.fullname" . }}-testpod" + labels: + {{- include "videoqna.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: curl + image: python:3.10.14 + command: ['bash', '-c'] + args: + - | + max_retry=20; + for ((i=1; i<=max_retry; i++)); do + curl http://{{ include "videoqna.fullname" . }}:{{ .Values.service.port }}/v1/videoqna -sS --fail-with-body \ + -X POST \ + -d '{"messages": "What is the man doing?","stream": "True"}' \ + -H 'Content-Type: application/json' && break; + curlcode=$? + if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi; + done; + if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi + restartPolicy: Never diff --git a/helm-charts/videoqna/values.yaml b/helm-charts/videoqna/values.yaml new file mode 100644 index 000000000..d8e847c27 --- /dev/null +++ b/helm-charts/videoqna/values.yaml @@ -0,0 +1,150 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# Default values for videoqna. +replicaCount: 1 + +image: + repository: opea/videoqna + # This sets the pull policy for images. + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "" +fullnameOverride: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + +service: + type: ClusterIP + port: 8888 + containerPort: 8888 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +embeddingServiceHost: "" +retrieverServiceHost: "" +rerankServiceHost: "" +lvmServiceHost: "" + +entryCommand: ["/bin/sh"] +extraArgs: ["-c", "sleep 45 && python videoqna.py"] + +videoqna-ui: + image: + repository: opea/videoqna-ui + pullPolicy: IfNotPresent + tag: "latest" + + # Overriding videoqna-ui chart name to avoid repetitive videoqna in the fullname. + nameOverride: "ui" + + # Following template value will be resolved in videoqna-ui ConfigMap + BACKEND_SERVICE_ENDPOINT: http://{{ printf "%s-nginx" .Release.Name | trunc 63 | trimSuffix "-" }}/v1/videoqna + + containerPort: 5173 + service: + type: ClusterIP + port: 5173 + +data-prep: + image: + repository: opea/dataprep-multimodal-vdms + pullPolicy: IfNotPresent + tag: "latest" + + indexName: "mega-videoqna" + vdmsHost: "" + vdmsPort: "8001" + entryCommand: ["/bin/sh"] + extraArgs: ["-c", "sleep 15 && python ingest_videos.py"] + +embedding-usvc: + image: + repository: opea/embedding-multimodal-clip + pullPolicy: IfNotPresent + tag: "latest" + + securityContext: + readOnlyRootFilesystem: false + +retriever-usvc: + image: + repository: opea/retriever-vdms + pullPolicy: IfNotPresent + tag: "latest" + + indexName: "mega-videoqna" + vdmsHost: "" + vdmsPort: "8001" + useClip: "1" + entryCommand: ["/bin/sh"] + extraArgs: ["-c", "sleep 30 && python retriever_vdms.py"] + +reranking-usvc: + image: + repository: opea/reranking-videoqna + pullPolicy: IfNotPresent + tag: "latest" + + chunkDuration: "" + + # Following template value will be resolved in rearanking-usvc ConfigMap + fileServerEndpoint: http://{{ printf "%s-nginx" .Release.Name | trunc 63 | trimSuffix "-" }}/v1/dataprep/get_file + getVideoListEndpoint: http://{{ printf "%s-nginx" .Release.Name | trunc 63 | trimSuffix "-" }}/v1/dataprep/get_videos + +lvm-uservice: + image: + repository: opea/lvm-video-llama + # This sets the pull policy for images. + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + + service: + type: ClusterIP + port: 9000 + containerPort: 9000 + + +global: + http_proxy: "" + https_proxy: "" + no_proxy: "" + HUGGINGFACEHUB_API_TOKEN: "insert-your-huggingface-token-here" + + # Set modelUseHostPath or modelUsePVC to use model cache. + modelUseHostPath: "" + modelUsePVC: ""