From 3c4ad75ac33b5551e055011d265e580574ef6612 Mon Sep 17 00:00:00 2001 From: alexsin368 Date: Tue, 28 Apr 2026 14:42:43 -0700 Subject: [PATCH 1/5] simplify env setup Signed-off-by: alexsin368 --- sample_solutions/RAGChatbot/.env.example | 32 +++++ sample_solutions/RAGChatbot/README.md | 131 +++++------------- sample_solutions/RAGChatbot/api/.env.example | 40 ------ .../RAGChatbot/docker-compose.yml | 2 +- 4 files changed, 65 insertions(+), 140 deletions(-) create mode 100644 sample_solutions/RAGChatbot/.env.example delete mode 100644 sample_solutions/RAGChatbot/api/.env.example diff --git a/sample_solutions/RAGChatbot/.env.example b/sample_solutions/RAGChatbot/.env.example new file mode 100644 index 00000000..d14569bb --- /dev/null +++ b/sample_solutions/RAGChatbot/.env.example @@ -0,0 +1,32 @@ +# Inference API Configuration +# Note: model names must match one of the deployed models with Enterprise Inference. +# INFERENCE_API_ENDPOINT: URL to your inference service (without /v1 suffix). Uncomment ONE of the below. +# - For GenAI Gateway: +#INFERENCE_API_ENDPOINT=https://api.example.com +# - For APISIX Gateway (default), requires the model name in the URL path: +INFERENCE_API_ENDPOINT=https://api.example.com/Llama-3.1-8B-Instruct + +# INFERENCE_API_TOKEN: Authentication token/API key for the inference service +# - For GenAI Gateway: Your GenAI Gateway API key +# - For APISIX Gateway: Your APISIX authentication token +INFERENCE_API_TOKEN=your-pre-generated-token-here + +# Model Configuration +# IMPORTANT: Use the full model names as they appear in your inference service +# Check available models: curl https://api.example.com/v1/models -H "Authorization: Bearer your-token" +INFERENCE_MODEL_NAME=meta-llama/Llama-3.1-8B-Instruct +EMBEDDING_MODEL_NAME=BAAI/bge-base-en-v1.5 + +# Embedding Model API (APISIX Gateway only) +# Set only when using APISIX Gateway (default). Comment out for GenAI Gateway +EMBEDDING_API_ENDPOINT=https://api.example.com/bge-base-en-v1.5 + +# Local URL Endpoint (only needed for non-public domains) +# If using a local domain like api.example.com mapped to localhost: +# Set this to: api.example.com (domain without https://) +# If using a public domain, set any placeholder value like: not-needed +LOCAL_URL_ENDPOINT=not-needed + +# SSL Verification Settings +# Set to false only for dev with self-signed certs +VERIFY_SSL=true diff --git a/sample_solutions/RAGChatbot/README.md b/sample_solutions/RAGChatbot/README.md index 403d1292..4ab96e41 100644 --- a/sample_solutions/RAGChatbot/README.md +++ b/sample_solutions/RAGChatbot/README.md @@ -86,6 +86,11 @@ Before you begin, ensure you have the following installed: This application supports multiple inference deployment patterns: +| API Configuration | Validated | +|---|:---:| +| GenAI Gateway | ✅ | +| Keycloak/APISIX | ✅ | + - **GenAI Gateway**: Provide your GenAI Gateway URL and API key - To generate the GenAI Gateway API key, use the [generate-vault-secrets.sh](https://github.com/opea-project/Enterprise-Inference/blob/main/core/scripts/generate-vault-secrets.sh) script - The API key is the `litellm_master_key` value from the generated `vault.yml` file @@ -94,21 +99,21 @@ This application supports multiple inference deployment patterns: - To generate the APISIX authentication token, use the [generate-token.sh](https://github.com/opea-project/Enterprise-Inference/blob/main/core/scripts/generate-token.sh) script - The token is generated using Keycloak client credentials -### Local Development Configuration - -**For Local Testing Only (Optional)** +### Deploy Required Model(s) -If you're testing with a local inference endpoint using a custom domain (e.g., `api.example.com` mapped to localhost in your hosts file): +The following models have been validated on different hardware platforms. -1. Edit `api/.env` and set: - ```bash - LOCAL_URL_ENDPOINT=api.example.com - ``` - (Use the domain name from your INFERENCE_API_ENDPOINT without `https://`) +At least one of the following models must be deployed. -2. This allows Docker containers to resolve your local domain correctly. +| Model | Xeon | Gaudi | +|---|:---:|:---:| +| **meta-llama/Llama-3.1-8B-Instruct** | ❌ | ✅ Validated on Dell XE7740 | +| **Qwen/Qwen3-4B-Instruct-2507** | ✅ Validated on Dell XE7740 | ❌ | -**Note:** For public domains or cloud-hosted endpoints, leave the default value `not-needed`. +In addition, an embedding model must be deployed. +| Model | Xeon | Gaudi | +|---|:---:|:---:| +| **BAAI/bge-base-en-v1.5** | ❌ | ✅ Validated on Dell XE7740 | ### Verify Docker Installation @@ -135,89 +140,29 @@ cd Enterprise-Inference/sample_solutions/RAGChatbot ### Set up the Environment -This application requires **two `.env` files** for proper configuration: - -1. **Root `.env` file** (for Docker Compose variables) -2. **`api/.env` file** (for backend application configuration) - -#### Step 1: Create Root `.env` File - -```bash -# From the RAGChatbot directory -cat > .env << EOF -# Docker Compose Configuration -LOCAL_URL_ENDPOINT=not-needed -EOF -``` - -**Note:** If using a local domain (e.g., `api.example.com` mapped to localhost), replace `not-needed` with your domain name (without `https://`). - -#### Step 2: Create `api/.env` File - -Copy from the example file and edit with your actual credentials: +This application requires an `.env` file in the root directory for proper configuration. Create it using [.env.example](./.env.example) with the commands below: ```bash -cp api/.env.example api/.env -``` - -Then edit `api/.env` to set your `INFERENCE_API_ENDPOINT` and `INFERENCE_API_TOKEN`. - -Or manually create `api/.env` with: - -```bash -# Inference API Configuration -# INFERENCE_API_ENDPOINT: URL to your inference service (without /v1 suffix) -# -# **GenAI Gateway**: Provide your GenAI Gateway URL and API key -# - URL format: https://genai-gateway.example.com -# - To generate the GenAI Gateway API key, use the [generate-vault-secrets.sh] script -# - The API key is the litellm_master_key value from the generated vault.yml file -# -# **APISIX Gateway**: Provide your APISIX Gateway URL and authentication token -# - For APISIX, include the model name in the INFERENCE_API_ENDPOINT path -# - Example: https://apisix-gateway.example.com/Llama-3.1-8B-Instruct -# - Set EMBEDDING_API_ENDPOINT separately for the embedding model -# - Example: https://apisix-gateway.example.com/bge-base-en-v1.5 -# - To generate the APISIX authentication token, use the [generate-token.sh] script -# - The token is generated using Keycloak client credentials -# -# INFERENCE_API_TOKEN: Authentication token/API key for the inference service -INFERENCE_API_ENDPOINT=https://api.example.com -INFERENCE_API_TOKEN=your-pre-generated-token-here - -# Model Configuration -EMBEDDING_MODEL_NAME=BAAI/bge-base-en-v1.5 -INFERENCE_MODEL_NAME=meta-llama/Llama-3.1-8B-Instruct - -# APISIX Gateway Endpoints -# Uncomment and set these when using APISIX Gateway: -# IMPORTANT: Use exact APISIX route paths: -# Example routes: /bge-base-en-v1.5/* and /Llama-3.1-8B-Instruct/* -# INFERENCE_API_ENDPOINT=https://api.example.com/Llama-3.1-8B-Instruct -# EMBEDDING_API_ENDPOINT=https://api.example.com/bge-base-en-v1.5 - -# Local URL Endpoint (only needed for non-public domains) -# If using a local domain like api.example.com mapped to localhost: -# Set this to: api.example.com (domain without https://) -# If using a public domain, set any placeholder value like: not-needed -LOCAL_URL_ENDPOINT=not-needed - -# SSL Verification Settings -# Set to false only for dev with self-signed certs -VERIFY_SSL=true +cp .env.example .env ``` +Then modify it as needed, with special consideration to certain environment variables mentioned below. Read through the .env file for full instructions. **Important Configuration Notes:** -- **INFERENCE_API_ENDPOINT**: Your actual inference service URL (replace `https://your-actual-api-endpoint.com`) - - For APISIX/Keycloak deployments, the model name must be included in the endpoint URL (e.g., `https://apisix-gateway.example.com/Llama-3.1-8B-Instruct`) +- **INFERENCE_API_ENDPOINT**: Your actual inference service URL (replace `https://api.example.com`) + - For APISIX/Keycloak deployments, the model name must be included in the endpoint URL (e.g., `https://api.example.com/Llama-3.1-8B-Instruct`) - **INFERENCE_API_TOKEN**: Your actual pre-generated authentication token -- **EMBEDDING_MODEL_NAME** and **INFERENCE_MODEL_NAME**: Use the exact model names from your inference service - - To check available models: `curl https://your-api-endpoint.com/v1/models -H "Authorization: Bearer your-token"` - - **Important for APISIX/Keycloak**: You need a separate endpoint for the embedding model. Configure `EMBEDDING_ENDPOINT` with the embedding model in the URL path (e.g., `https://apisix-gateway.example.com/bge-base-en-v1.5`) -- **LOCAL_URL_ENDPOINT**: Only needed if using local domain mapping (see [Local Development Configuration](#local-development-configuration)) - -**Note**: The docker-compose.yml file automatically loads environment variables from both `.env` (root) and `./api/.env` (backend) files. +- **EMBEDDING_MODEL_NAME**, **INFERENCE_MODEL_NAME**: Use the exact model name from your inference service + - To check available models: `curl https://api.example.com/v1/models -H "Authorization: Bearer your-token"` +- **EMBEDDING_API_ENDPOINT** (APISIX only): Your actual embedding service URL +- **LOCAL_URL_ENDPOINT**: Only needed if using local domain mapping (i.e. `api.example.com` mapped to localhost) for Docker containers to resolve correctly. + - Use the domain name from INFERENCE_API_ENDPOINT without `https://` + - For public domains or cloud-hosted endpoints, leave the default value `not-needed` +- **VERIFY_SSL**: Controls SSL certificate verification (default: `true`) + - Set to `false` only for development environments with self-signed certificates + - Keep as `true` for production environments + +**Note**: The docker-compose.yaml file automatically loads environment variables from `.env` for the backend service. ### Running the Application @@ -296,15 +241,3 @@ docker compose down For comprehensive troubleshooting guidance, common issues, and solutions, refer to: [Troubleshooting Guide - TROUBLESHOOTING.md](./TROUBLESHOOTING.md) - ---- - -## Additional Info - -The following models have been validated with RAGChatbot: - -| Model | Hardware | -|-------|----------| -| **meta-llama/Llama-3.1-8B-Instruct** | Gaudi | -| **BAAI/bge-base-en-v1.5** (embeddings) | Gaudi | -| **Qwen/Qwen3-4B-Instruct** | Xeon | diff --git a/sample_solutions/RAGChatbot/api/.env.example b/sample_solutions/RAGChatbot/api/.env.example deleted file mode 100644 index 5c72e049..00000000 --- a/sample_solutions/RAGChatbot/api/.env.example +++ /dev/null @@ -1,40 +0,0 @@ -# Inference API Configuration -# INFERENCE_API_ENDPOINT: URL to your inference service (without /v1 suffix) -# -# **GenAI Gateway**: Provide your GenAI Gateway URL and API key -# - URL format: https://genai-gateway.example.com -# - To generate the GenAI Gateway API key, use the [generate-vault-secrets.sh](https://github.com/opea-project/Enterprise-Inference/blob/main/core/scripts/generate-vault-secrets.sh) script -# - The API key is the litellm_master_key value from the generated vault.yml file -# -# **APISIX Gateway**: Provide your APISIX Gateway URL and authentication token -# - For APISIX, include the model name in the INFERENCE_API_ENDPOINT path -# - Example: https://apisix-gateway.example.com/Llama-3.1-8B-Instruct -# - Set EMBEDDING_API_ENDPOINT separately for the embedding model -# - Example: https://apisix-gateway.example.com/bge-base-en-v1.5 -# - To generate the APISIX authentication token, use the [generate-token.sh](https://github.com/opea-project/Enterprise-Inference/blob/main/core/scripts/generate-token.sh) script -# - The token is generated using Keycloak client credentials -# -# INFERENCE_API_TOKEN: Authentication token/API key for the inference service -INFERENCE_API_ENDPOINT=https://api.example.com -INFERENCE_API_TOKEN=your-pre-generated-token-here - -# Model Configuration -EMBEDDING_MODEL_NAME=BAAI/bge-base-en-v1.5 -INFERENCE_MODEL_NAME=meta-llama/Llama-3.1-8B-Instruct - -# APISIX Gateway Endpoints -# Uncomment and set these when using APISIX Gateway: -# IMPORTANT: Use exact APISIX route paths: -# Example routes: /bge-base-en-v1.5/* and /Llama-3.1-8B-Instruct/* -# INFERENCE_API_ENDPOINT=https://api.example.com/Llama-3.1-8B-Instruct -# EMBEDDING_API_ENDPOINT=https://api.example.com/bge-base-en-v1.5 - -# Local URL Endpoint (only needed for non-public domains) -# If using a local domain like api.example.com mapped to localhost: -# Set this to: api.example.com (domain without https://) -# If using a public domain, set any placeholder value like: not-needed -LOCAL_URL_ENDPOINT=not-needed - -# SSL Verification Settings -# Set to false only for dev with self-signed certs -VERIFY_SSL=true diff --git a/sample_solutions/RAGChatbot/docker-compose.yml b/sample_solutions/RAGChatbot/docker-compose.yml index e0f1382d..22ff88f8 100644 --- a/sample_solutions/RAGChatbot/docker-compose.yml +++ b/sample_solutions/RAGChatbot/docker-compose.yml @@ -8,7 +8,7 @@ services: ports: - "5001:5001" env_file: - - ./api/.env + - .env volumes: - ./api:/app networks: From a56d2440e2d18a4ba51634ad97a244942b9f3189 Mon Sep 17 00:00:00 2001 From: alexsin368 Date: Tue, 28 Apr 2026 16:46:32 -0700 Subject: [PATCH 2/5] update table, notes Signed-off-by: alexsin368 --- sample_solutions/RAGChatbot/.env.example | 3 +- sample_solutions/RAGChatbot/README.md | 48 +++++++++++------------- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/sample_solutions/RAGChatbot/.env.example b/sample_solutions/RAGChatbot/.env.example index d14569bb..54a78338 100644 --- a/sample_solutions/RAGChatbot/.env.example +++ b/sample_solutions/RAGChatbot/.env.example @@ -12,8 +12,7 @@ INFERENCE_API_ENDPOINT=https://api.example.com/Llama-3.1-8B-Instruct INFERENCE_API_TOKEN=your-pre-generated-token-here # Model Configuration -# IMPORTANT: Use the full model names as they appear in your inference service -# Check available models: curl https://api.example.com/v1/models -H "Authorization: Bearer your-token" +# IMPORTANT: Use the full model name i.e. model card INFERENCE_MODEL_NAME=meta-llama/Llama-3.1-8B-Instruct EMBEDDING_MODEL_NAME=BAAI/bge-base-en-v1.5 diff --git a/sample_solutions/RAGChatbot/README.md b/sample_solutions/RAGChatbot/README.md index 4ab96e41..805c443c 100644 --- a/sample_solutions/RAGChatbot/README.md +++ b/sample_solutions/RAGChatbot/README.md @@ -78,7 +78,21 @@ Below is the architecture as it consists of a server that waits for documents to Before you begin, ensure you have the following installed: - **Docker and Docker Compose** -- **Enterprise inference endpoint access** (token-based authentication) +- **Enterprise Inference endpoint access** (token-based authentication, see below for models and configs) + +#### Deploy Required Models + +See the table below for supported models, hardware, and gateway configuration. + +| Model | Xeon w/APISIX/Keycloak | Xeon w/GenAI Gateway | Gaudi w/APISIX/Keycloak | Gaudi w/GenAI Gateway | +|---|:---:|:---:|:---:|:---:| +| **meta-llama/Llama-3.1-8B-Instruct** | ❌ | ❌ | ✅ Validated on Dell XE7740 | ✅ Validated on Dell XE7740 | +| **Qwen/Qwen3-4B-Instruct-2507** | ✅ Validated on Dell XE7740 | ✅ Validated on Dell XE7740 | ❌ | ❌ | + +In addition, an embedding model must be deployed. +| Model | Xeon w/APISIX/Keycloak | Xeon w/GenAI Gateway | Gaudi w/APISIX/Keycloak | Gaudi w/GenAI Gateway | +|---|:---:|:---:|:---:|:---:| +| **BAAI/bge-base-en-v1.5** | ✅ Validated on Dell XE7740 | ❌ | ✅ Validated on Dell XE7740 | ✅ Validated on Dell XE7740 | ### Required API Configuration @@ -86,35 +100,17 @@ Before you begin, ensure you have the following installed: This application supports multiple inference deployment patterns: -| API Configuration | Validated | -|---|:---:| -| GenAI Gateway | ✅ | -| Keycloak/APISIX | ✅ | - -- **GenAI Gateway**: Provide your GenAI Gateway URL and API key +**GenAI Gateway**: Provide your GenAI Gateway URL and API key + - URL format: https://api.example.com - To generate the GenAI Gateway API key, use the [generate-vault-secrets.sh](https://github.com/opea-project/Enterprise-Inference/blob/main/core/scripts/generate-vault-secrets.sh) script - - The API key is the `litellm_master_key` value from the generated `vault.yml` file - -- **APISIX Gateway**: Provide your APISIX Gateway URL and authentication token + - The API key is the litellm_master_key value from the generated vault.yml file + +**APISIX Gateway**: Provide your APISIX Gateway URL and authentication token + - URL format: https://api.example.com/Llama-3.1-8B-Instruct + - Note: APISIX requires the model name in the URL path - To generate the APISIX authentication token, use the [generate-token.sh](https://github.com/opea-project/Enterprise-Inference/blob/main/core/scripts/generate-token.sh) script - The token is generated using Keycloak client credentials -### Deploy Required Model(s) - -The following models have been validated on different hardware platforms. - -At least one of the following models must be deployed. - -| Model | Xeon | Gaudi | -|---|:---:|:---:| -| **meta-llama/Llama-3.1-8B-Instruct** | ❌ | ✅ Validated on Dell XE7740 | -| **Qwen/Qwen3-4B-Instruct-2507** | ✅ Validated on Dell XE7740 | ❌ | - -In addition, an embedding model must be deployed. -| Model | Xeon | Gaudi | -|---|:---:|:---:| -| **BAAI/bge-base-en-v1.5** | ❌ | ✅ Validated on Dell XE7740 | - ### Verify Docker Installation ```bash From 2f0a3157a4134966d5c5e3d7ecdbd2159534750d Mon Sep 17 00:00:00 2001 From: alexsin368 Date: Tue, 28 Apr 2026 16:49:17 -0700 Subject: [PATCH 3/5] update table Signed-off-by: alexsin368 --- sample_solutions/RAGChatbot/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample_solutions/RAGChatbot/README.md b/sample_solutions/RAGChatbot/README.md index 805c443c..7ba472e7 100644 --- a/sample_solutions/RAGChatbot/README.md +++ b/sample_solutions/RAGChatbot/README.md @@ -92,7 +92,7 @@ See the table below for supported models, hardware, and gateway configuration. In addition, an embedding model must be deployed. | Model | Xeon w/APISIX/Keycloak | Xeon w/GenAI Gateway | Gaudi w/APISIX/Keycloak | Gaudi w/GenAI Gateway | |---|:---:|:---:|:---:|:---:| -| **BAAI/bge-base-en-v1.5** | ✅ Validated on Dell XE7740 | ❌ | ✅ Validated on Dell XE7740 | ✅ Validated on Dell XE7740 | +| **BAAI/bge-base-en-v1.5** | ✅ Validated on Dell XE7740 | ✅ Validated on Dell XE7740 | ✅ Validated on Dell XE7740 | ✅ Validated on Dell XE7740 | ### Required API Configuration From 157e297a93ec8cd719f55cd838e8fc7e4a56499d Mon Sep 17 00:00:00 2001 From: alexsin368 Date: Tue, 28 Apr 2026 16:56:29 -0700 Subject: [PATCH 4/5] change VECTOR_STORE_PATH to writable /tmp folder Signed-off-by: alexsin368 --- sample_solutions/RAGChatbot/api/services/vector_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample_solutions/RAGChatbot/api/services/vector_service.py b/sample_solutions/RAGChatbot/api/services/vector_service.py index 0884a9c0..22a6ac8d 100644 --- a/sample_solutions/RAGChatbot/api/services/vector_service.py +++ b/sample_solutions/RAGChatbot/api/services/vector_service.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) # Constants -VECTOR_STORE_PATH = "./dmv_index" +VECTOR_STORE_PATH = "/tmp/dmv_index" class CustomEmbeddings(Embeddings): From 94bd9ef47a03e5eec3c59bafdae4d84d97c1d66b Mon Sep 17 00:00:00 2001 From: alexsin368 Date: Tue, 28 Apr 2026 16:59:29 -0700 Subject: [PATCH 5/5] update note Signed-off-by: alexsin368 --- sample_solutions/RAGChatbot/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample_solutions/RAGChatbot/README.md b/sample_solutions/RAGChatbot/README.md index 7ba472e7..ab5985fb 100644 --- a/sample_solutions/RAGChatbot/README.md +++ b/sample_solutions/RAGChatbot/README.md @@ -150,7 +150,7 @@ Then modify it as needed, with special consideration to certain environment vari - **INFERENCE_API_TOKEN**: Your actual pre-generated authentication token - **EMBEDDING_MODEL_NAME**, **INFERENCE_MODEL_NAME**: Use the exact model name from your inference service - To check available models: `curl https://api.example.com/v1/models -H "Authorization: Bearer your-token"` -- **EMBEDDING_API_ENDPOINT** (APISIX only): Your actual embedding service URL +- **EMBEDDING_API_ENDPOINT** (APISIX only): Your actual embedding service URL. **Comment out** for GenAI Gateway. - **LOCAL_URL_ENDPOINT**: Only needed if using local domain mapping (i.e. `api.example.com` mapped to localhost) for Docker containers to resolve correctly. - Use the domain name from INFERENCE_API_ENDPOINT without `https://` - For public domains or cloud-hosted endpoints, leave the default value `not-needed`