release 1.4.0 (#17)

Author: euphoria0-0

docker-compose/.env.example

@@ -2,53 +2,96 @@
 COMPOSE_PROJECT_NAME=my_project
 VERSION_TAG=latest
 IMAGE_DIR=cryptolabinc/
+PULL_POLICY=always
 
-# es2 endpoint
-ES2E_HOST_PORT=50050  # Set your endpoint host port here
-ES2E_IMAGE_NAME=${IMAGE_DIR}es2e
-ES2E_TAG=${VERSION_TAG}
+# docker setup
+DOCKER_VOLUME_DIRECTORY=. # Set the docker volume mounting directory if needed
+
+# Log Level
+ENVECTOR_LOG_LEVEL=INFO
+
+# endpoint
+ENVECTOR_ENDPOINT_HOST_PORT=50050  # Set your endpoint host port here
+ENVECTOR_HTTP_HEALTH_HOST_PORT=18080  # Exposes /health and /health/ready on the host
+ENVECTOR_ADMIN_API_ENABLED=true
+ENVECTOR_ENDPOINT_IMAGE_NAME=${IMAGE_DIR}envector-endpoint
+ENVECTOR_ENDPOINT_TAG=${VERSION_TAG}
+# auth
+ENVECTOR_AUTH_OIDC_ISSUER=
+ENVECTOR_AUTH_ALLOWED_AUDIENCES=
+ENVECTOR_AUTH_ROLE_CLAIM=envector_roles
+ENVECTOR_AUTH_PRINCIPAL_CLAIM=principal_id
+ENVECTOR_AUTH_JWKS_URL=
+ENVECTOR_AUTH_CAPABILITY_POLICY_JSON=
 
-# es2 backend
-ES2B_IMAGE_NAME=${IMAGE_DIR}es2b
-ES2B_TAG=${VERSION_TAG}
+# backend
+ENVECTOR_BACKEND_IMAGE_NAME=${IMAGE_DIR}envector-backend
+ENVECTOR_BACKEND_TAG=${VERSION_TAG}
 
-# es2 scheduler
-ES2O_IMAGE_NAME=${IMAGE_DIR}es2o
-ES2O_TAG=${VERSION_TAG}
+# orchestrator
+ENVECTOR_ORCHESTRATOR_IMAGE_NAME=${IMAGE_DIR}envector-orchestrator
+ENVECTOR_ORCHESTRATOR_TAG=${VERSION_TAG}
 
-# es2 compute
-ES2C_IMAGE_NAME=${IMAGE_DIR}es2c
-ES2C_TAG=${VERSION_TAG}
+# shaper
+ENVECTOR_SHAPER_IMAGE_NAME=${IMAGE_DIR}envector-shaper
+ENVECTOR_SHAPER_TAG=${VERSION_TAG}
 
-# ES2C GPU ID
-ES2C_GPU0_ID=0
-ES2C_GPU1_ID=1
-ES2C_GPU2_ID=2
-ES2C_GPU3_ID=3
+# compute
+ENVECTOR_COMPUTE_IMAGE_NAME=${IMAGE_DIR}envector-compute
+ENVECTOR_COMPUTE_TAG=${VERSION_TAG}
+# Optional: set only if you want to override the auto-advertised host (defaults to POD_IP/hostname)
+# ENVECTOR_COMPUTE_ADVERTISE_HOST=envector-compute:${ENVECTOR_COMPUTE_PORT:-25123}
+
+# ENVECTOR_COMPUTE GPU ID
+ENVECTOR_COMPUTE_GPU0_ID=0
+ENVECTOR_COMPUTE_GPU1_ID=1
+ENVECTOR_COMPUTE_GPU2_ID=2
+ENVECTOR_COMPUTE_GPU3_ID=3
 
 # License Key
-ES2_LICENSE_TOKEN=
+# ENVECTOR_LICENSE_TOKEN=
 
 # DB
-ES2_DB_SERVICE_PORT=5432
-ES2_DB_SERVICE_USER=postgres
-ES2_DB_SERVICE_PASSWORD=abcdefghi
-ES2_DB_SERVICE_NAME=mydatabase
-ES2_DB_SERVICE_SCHEMA=public
-ES2_DB_SERVICE_URL="postgresql://postgres:abcdefghi@metadatadb:5432/mydatabase"
+ENVECTOR_DB_URL="postgresql://postgres:abcdefghi@metadatadb:5432/mydatabase"
+# Optional: set individual DB parameters instead of URL
+ENVECTOR_DB_HOST=metadatadb
+ENVECTOR_DB_PORT=5432
+ENVECTOR_DB_USER=postgres
+ENVECTOR_DB_PASSWORD=abcdefghi
+ENVECTOR_DB_NAME=mydatabase
+ENVECTOR_DB_SCHEMA=public
+ENVECTOR_DB_SSL=disable
+# Optional: external DB endpoint (host:port), e.g. AWS RDS.
+# ENVECTOR_DB_ADDRESS=mydb.abcdefghijkl.us-east-1.rds.amazonaws.com:5432
 
 # Storage
-ES2_STORAGE_PROVIDER=s3
-ES2_STORAGE_SERVICE_PORT=59000
-ES2_STORAGE_SERVICE_USER=minioadmin
-ES2_STORAGE_SERVICE_PASSWORD=abcdefghi
-ES2_STORAGE_SECURE=false # Set to true if using HTTPS
-ES2_STORAGE_BUCKET_NAME=es2
-ES2_STORAGE_SERVICE_CREATE_BUCKET=true
-ES2_STORAGE_SERVICE_HAS_HEALTHCHECK_ENDPOINT=true # Set to true if storage has health check endpoint
+ENVECTOR_STORAGE_PROVIDER=s3
+ENVECTOR_STORAGE_PORT=59000
+ENVECTOR_STORAGE_CONSOLE_PORT=59001
+ENVECTOR_STORAGE_USER=minioadmin
+ENVECTOR_STORAGE_PASSWORD=abcdefghi
+ENVECTOR_STORAGE_SECURE=false # Set to true if using HTTPS
+ENVECTOR_STORAGE_BUCKET_NAME=envector
+ENVECTOR_STORAGE_CREATE_BUCKET=true
+ENVECTOR_STORAGE_HAS_HEALTHCHECK_ENDPOINT=true # Set to true if storage has health check endpoint
+ENVECTOR_STORAGE_REGION=us-east-1
 
-# docker setup
-DOCKER_VOLUME_DIRECTORY=. # Set the docker volume mounting directory if needed
+# Advanced runtime options
+# Backend Development settings
+# ENVECTOR_MAX_NUM_VECTORS_PER_SHARD=4096
+# CTXTMAP_BATCH_SIZE=1024
+# CTXTMAP_INSERT_BATCH_SIZE=1024
+# CTXTMAP_UPDATE_BATCH_SIZE=1024
+# ENVECTOR_CLEANUP_GRACE_PERIOD=5m
+# VCT_CAPACITY=4096
 
-# Log Level
-ES2_LOG_LEVEL=INFO
+# Shaper Development settings
+# ENVECTOR_SHAPER_MAX_WORKERS=16
+# ENVECTOR_SHAPER_CLEANUP_PERIOD=1h
+# ENVECTOR_SHAPER_JOB_TTL=24h
+# ENVECTOR_SHAPER_SPLIT_UPLOAD_CONCURRENCY=1
+# ENVECTOR_SHAPER_MERGE_DOWNLOAD_CONCURRENCY=1
+
+# Compute Development settings
+# NUM_SEARCH_WORKERS=30
+# ENVECTOR_THREAD_POOL_SCALE=4

docker-compose/README.md

@@ -5,19 +5,19 @@
 The default tags come from `.env.example`:
 
 ```text
-es2e:     cryptolabinc/es2e:${VERSION_TAG}
-es2b:     cryptolabinc/es2b:${VERSION_TAG}
-es2o:     cryptolabinc/es2o:${VERSION_TAG}
-es2c:     cryptolabinc/es2c:${VERSION_TAG}
-postgres: postgres:14.9
-minio:    minio/minio:RELEASE.2023-03-20T20-16-18Z
+envector-endpoint:      cryptolabinc/envector-endpoint:${VERSION_TAG}
+envector-backend:       cryptolabinc/envector-backend:${VERSION_TAG}
+envector-orchestrator:  cryptolabinc/envector-orchestrator:${VERSION_TAG}
+envector-compute:       cryptolabinc/envector-compute:${VERSION_TAG}
+postgres:               postgres:14.9
+minio:                  minio/minio:RELEASE.2023-03-20T20-16-18Z
 ```
 
 ## 🧩 Compose File Layout
 
 ```text
 docker-compose.envector.yml    # Core application services
-docker-compose.gpu.yml         # GPU override for es2c
+docker-compose.gpu.yml         # GPU override for compute
 docker-compose.infra.yml       # Postgres + MinIO (adds readiness deps to core)
 ```
 
@@ -40,7 +40,7 @@ Optional: you can login manually ahead of time if you prefer.
 ## ✅ Step 2. Prepare Environment
 
 ```bash
-cd envector-deployment/docker-compose
+cd deployment/docker-compose
 # If .env is missing, ./start_envector.sh will auto-create it from .env.example
 cp .env.example .env  # optional
 ```
@@ -51,20 +51,20 @@ Edit `.env` as needed. `COMPOSE_PROJECT_NAME` customises the network/container p
 
 ## 🔐 License Token
 
-- Docker-mounted path: the container reads the token at `/es2/license/token.jwt` (source file on host: `docker-compose/token.jwt`).
+- Docker-mounted path: the container reads the token at `/envector/license/token.jwt` (source file on host: `docker-compose/token.jwt`).
 - If `token.jwt` is missing, `./start_envector.sh` will prompt for a path and copy the file to `docker-compose/token.jwt` automatically.
-- The `es2c` service reads the token from the Docker-mounted path `/es2/license/token.jwt`; the compose file mounts it for you:
+- The `envector-compute` service reads the token from the Docker-mounted path `/envector/license/token.jwt`; the compose file mounts it for you:
 
 ```yaml
 environment:
-  ES2_LICENSE_TOKEN: "${ES2_LICENSE_TOKEN:-/es2/license/token.jwt}"
+  ENVECTOR_LICENSE_TOKEN: "${ENVECTOR_LICENSE_TOKEN:-/envector/license/token.jwt}"
 # License file mount. Place your license token.jwt file in the same directory as this docker-compose file.
 volumes:
-  - ./token.jwt:/es2/license/token.jwt
+  - ./token.jwt:/envector/license/token.jwt
 ```
 
-- You normally don’t need to set `ES2_LICENSE_TOKEN` in `.env`; it matches the Docker-mounted path above.
-- If you change the token filename or path, update both `ES2_LICENSE_TOKEN` and the `volumes` mapping in `docker-compose/docker-compose.envector.yml` accordingly.
+- You normally don’t need to set `ENVECTOR_LICENSE_TOKEN` in `.env`; it matches the Docker-mounted path above.
+- If you change the token filename or path, update both `ENVECTOR_LICENSE_TOKEN` and the `volumes` mapping in `docker-compose/docker-compose.envector.yml` accordingly.
 
 ---
 
@@ -83,18 +83,20 @@ Recommended (helper script in this directory):
 ./start_envector.sh --gpu
 
 # Scale workers
-./start_envector.sh --num-es2c 4          # CPU-only: scale es2c=4
-./start_envector.sh --gpu --num-es2c 2    # GPU: gpu0 + gpu1
+./start_envector.sh --num-compute 4          # CPU-only: scale compute=4
+./start_envector.sh --gpu --num-compute 2    # GPU: gpu0 + gpu1
 
 # Project/env/log options
-./start_envector.sh -p my-es2 --env-file ./.env --log-file ./docker-logs.log
+./start_envector.sh -p my-envector --env-file ./.env --log-file ./docker-logs.log
 
 # Inline env overrides (higher precedence than .env)
-./start_envector.sh ES2E_HOST_PORT=50055 VERSION_TAG=dev
+./start_envector.sh ENVECTOR_ENDPOINT_HOST_PORT=50055 VERSION_TAG=dev
+./start_envector.sh ENVECTOR_HTTP_HEALTH_HOST_PORT=18081
+./start_envector.sh ENVECTOR_ADMIN_API_ENABLED=true
 
 # Stop the stack (use -p if you set a project)
 ./start_envector.sh --down    # also tears down GPU services automatically
-# e.g., ./start_envector.sh -p my-es2 --down
+# e.g., ./start_envector.sh -p my-envector --down
 # Remove volumes as well when stopping
 ./start_envector.sh --down --down-volumes
 ```
@@ -126,17 +128,41 @@ docker compose \
   up -d
 ```
 
+HTTP health endpoints:
+
+```bash
+curl http://localhost:${ENVECTOR_HTTP_HEALTH_HOST_PORT:-18080}/health
+curl http://localhost:${ENVECTOR_HTTP_HEALTH_HOST_PORT:-18080}/health/ready
+```
+
+Admin API:
+
+```bash
+curl http://localhost:${ENVECTOR_HTTP_HEALTH_HOST_PORT:-18080}/admin/services
+curl http://localhost:${ENVECTOR_HTTP_HEALTH_HOST_PORT:-18080}/admin/indexes
+curl http://localhost:${ENVECTOR_HTTP_HEALTH_HOST_PORT:-18080}/admin/keys
+curl "http://localhost:${ENVECTOR_HTTP_HEALTH_HOST_PORT:-18080}/admin/indexes/sample-index/operations/request-123?operation_type=INSERT"
+curl http://localhost:${ENVECTOR_HTTP_HEALTH_HOST_PORT:-18080}/admin/keys/key-1
+```
+
+Enable it with:
+
+```bash
+ENVECTOR_ADMIN_API_ENABLED=true
+```
+
+Swagger:
+
+```bash
+open http://localhost:${ENVECTOR_HTTP_HEALTH_HOST_PORT:-18080}/swagger/
+```
+
 To inspect the final configuration before starting:
 
 ```bash
 docker compose -f docker-compose.envector.yml -f docker-compose.infra.yml --env-file .env config
 ```
 
-Notes on scaling
-- CPU-only: `--scale es2c=N` (manual) or `./start_envector.sh --num-es2c N` (script).
-- GPU: `./start_envector.sh --gpu --num-es2c N` enables N GPU workers (base gpu0 plus additional).
-  Extend beyond 4 GPUs by editing `docker-compose.gpu.yml`.
-
 ---
 
 ## ✅ Step 4. Collect Logs