docker-compose.yml

x-common-env: &common-env
  # common
  COMMON_BLOCKED_DATASETS: ${COMMON_BLOCKED_DATASETS}
  COMMON_HF_ENDPOINT: ${COMMON_HF_ENDPOINT}
  COMMON_HF_TOKEN: ${COMMON_HF_TOKEN}
  # log
  LOG_LEVEL: ${LOG_LEVEL}
  # huggingface_hub
  HF_ENDPOINT: ${COMMON_HF_ENDPOINT} # see https://github.com/huggingface/datasets/pull/5196#issuecomment-1322191411
  # cache
  CACHE_MONGO_URL: ${CACHE_MONGO_URL} # use mongo container by default (with custom port)
  CACHE_MONGO_DATABASE: ${CACHE_MONGO_DATABASE}
  # queue
  QUEUE_MONGO_URL: ${QUEUE_MONGO_URL} # use mongo container by default (with custom port)
  QUEUE_MONGO_DATABASE: ${QUEUE_MONGO_DATABASE}
  # assets
  ASSETS_BASE_URL: ${ASSETS_BASE_URL} # <- must be set, no default value. If local, set to work with the reverse-proxy, like http://localhost:${PORT_REVERSE_PROXY-8000}/assets
  ASSETS_STORAGE_ROOT: ${ASSETS_STORAGE_ROOT}
  ASSETS_STORAGE_PROTOCOL: ${ASSETS_STORAGE_PROTOCOL}
  # cached assets
  CACHED_ASSETS_BASE_URL: ${CACHED_ASSETS_BASE_URL} # <- must be set, no default value. If local, set to work with the reverse-proxy, like http://localhost:${PORT_REVERSE_PROXY-8000}/cached-assets
  CACHED_ASSETS_STORAGE_ROOT: ${CACHED_ASSETS_STORAGE_ROOT}
  CACHED_ASSETS_STORAGE_PROTOCOL: ${CACHED_ASSETS_STORAGE_PROTOCOL}
  # S3
  S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
  S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
  S3_REGION_NAME: ${S3_REGION_NAME}
  # cloudfront (for signed URLs)
  CLOUDFRONT_EXPIRATION_SECONDS: ${CLOUDFRONT_EXPIRATION_SECONDS}
  CLOUDFRONT_KEY_PAIR_ID: ${CLOUDFRONT_KEY_PAIR_ID}
  CLOUDFRONT_PRIVATE_KEY: ${CLOUDFRONT_PRIVATE_KEY}
  # prometheus
  PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR}
  # datasets
  NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR}
  # libviewer
  LIBVIEWER_ENABLE_FOR_DATASETS: ${LIBVIEWER_ENABLE_FOR_DATASETS}

x-service-env: &service-env
  <<: *common-env
  # common for uvicorn services
  API_HF_AUTH_PATH: ${API_HF_AUTH_PATH}
  API_HF_JWT_PUBLIC_KEY_URL: ${API_HF_JWT_PUBLIC_KEY_URL}
  API_HF_JWT_ADDITIONAL_PUBLIC_KEYS: ${API_HF_JWT_ADDITIONAL_PUBLIC_KEYS}
  API_HF_JWT_ALGORITHM: ${API_HF_JWT_ALGORITHM}
  API_HF_TIMEOUT_SECONDS: ${API_HF_TIMEOUT_SECONDS}
  API_MAX_AGE_LONG: ${API_MAX_AGE_LONG}
  API_MAX_AGE_SHORT: ${API_MAX_AGE_SHORT}

services:
  reverse-proxy:
    image: docker.io/nginx:1.25.3
    volumes:
      - ./services/reverse-proxy/nginx-templates/:/etc/nginx/templates:ro
      - storage:${STORAGE_DIRECTORY}:ro
    ports:
      - "${PORT_REVERSE_PROXY-8000}:80"
    environment:
      ASSETS_STORAGE_ROOT: ${REVERSE_PROXY_ASSETS_STORAGE_ROOT-false} # <- if not set, returns 404 on /assets
      CACHED_ASSETS_STORAGE_ROOT: ${REVERSE_PROXY_CACHED_ASSETS_STORAGE_ROOT-false} # <- if not set, returns 404 on /cached-assets
      OPENAPI_FILE: ${OPENAPI_FILE-docs/source/openapi.json}
      HOST: localhost
      PORT: 80
      URL_ADMIN: http://admin:${ADMIN_UVICORN_PORT}
      URL_API: http://api:${API_UVICORN_PORT}
      URL_ROWS: http://rows:${ROWS_UVICORN_PORT}
      URL_SEARCH: http://search:${SEARCH_UVICORN_PORT}
      URL_SSE_API: http://sse-api:${SSE_API_UVICORN_PORT}
      URL_WEBHOOK: http://webhook:${WEBHOOK_UVICORN_PORT}
    depends_on:
      admin:
        condition: service_started
      api:
        condition: service_started
      rows:
        condition: service_started
      search:
        condition: service_started
  admin:
    build:
      target: admin
    volumes:
      - storage:${STORAGE_DIRECTORY}:rw
      - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY}:ro
    environment:
      <<: *service-env
      # service
      ADMIN_HF_ORGANIZATION: ${ADMIN_HF_ORGANIZATION}
      ADMIN_CACHE_REPORTS_NUM_RESULTS: ${ADMIN_CACHE_REPORTS_NUM_RESULTS}
      ADMIN_CACHE_REPORTS_WITH_CONTENT_NUM_RESULTS: ${ADMIN_CACHE_REPORTS_WITH_CONTENT_NUM_RESULTS}
      ADMIN_HF_TIMEOUT_SECONDS: ${ADMIN_HF_TIMEOUT_SECONDS}
      ADMIN_HF_WHOAMI_PATH: ${ADMIN_HF_WHOAMI_PATH}
      ADMIN_MAX_AGE: ${ADMIN_MAX_AGE}
      # uvicorn
      ADMIN_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
      ADMIN_UVICORN_NUM_WORKERS: ${ADMIN_UVICORN_NUM_WORKERS}
      ADMIN_UVICORN_PORT: ${ADMIN_UVICORN_PORT}
      PARQUET_METADATA_STORAGE_DIRECTORY: ${PARQUET_METADATA_STORAGE_DIRECTORY}
    depends_on:
      mongodb:
        condition: service_healthy
    restart: always
    ports:
      - ${ADMIN_UVICORN_PORT}:${ADMIN_UVICORN_PORT}
  api:
    build:
      target: api
    volumes:
      - storage:${STORAGE_DIRECTORY}:rw
    environment:
      <<: *service-env
      API_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
      API_UVICORN_NUM_WORKERS: ${API_UVICORN_NUM_WORKERS}
      API_UVICORN_PORT: ${API_UVICORN_PORT}
    ports:
      - ${API_UVICORN_PORT}:${API_UVICORN_PORT}
    depends_on:
      mongodb:
        condition: service_healthy
    restart: unless-stopped
  rows:
    build:
      target: rows
    volumes:
      - storage:${STORAGE_DIRECTORY}
      - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY}
    environment:
      <<: *service-env
      PARQUET_METADATA_STORAGE_DIRECTORY: ${PARQUET_METADATA_STORAGE_DIRECTORY}
      ROWS_INDEX_MAX_ARROW_DATA_IN_MEMORY: ${ROWS_INDEX_MAX_ARROW_DATA_IN_MEMORY}
      # uvicorn
      API_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
      API_UVICORN_NUM_WORKERS: ${ROWS_UVICORN_NUM_WORKERS}
      API_UVICORN_PORT: ${ROWS_UVICORN_PORT}
    ports:
      # for debug
      - ${ROWS_UVICORN_PORT}:${ROWS_UVICORN_PORT}
    depends_on:
      mongodb:
        condition: service_healthy
    restart: unless-stopped
  search:
    build:
      target: search
    volumes:
      - storage:${STORAGE_DIRECTORY}
      - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY}:rw
      - duckdb-index:${DUCKDB_INDEX_CACHE_DIRECTORY}:rw
    environment:
      <<: *service-env
      DUCKDB_INDEX_CACHE_DIRECTORY: ${DUCKDB_INDEX_CACHE_DIRECTORY}
      DUCKDB_INDEX_EXTENSIONS_DIRECTORY: ${DUCKDB_INDEX_EXTENSIONS_DIRECTORY}
      PARQUET_METADATA_STORAGE_DIRECTORY: ${PARQUET_METADATA_STORAGE_DIRECTORY}
      # uvicorn
      API_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
      API_UVICORN_NUM_WORKERS: ${SEARCH_UVICORN_NUM_WORKERS}
      API_UVICORN_PORT: ${SEARCH_UVICORN_PORT}
      HF_HUB_ENABLE_HF_TRANSFER: 1
    ports:
      # for debug
      - ${SEARCH_UVICORN_PORT}:${SEARCH_UVICORN_PORT}
    depends_on:
      mongodb:
        condition: service_healthy
    restart: unless-stopped
  sse-api:
    build:
      target: sse-api
    environment:
      <<: *service-env
      # uvicorn
      API_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
      API_UVICORN_NUM_WORKERS: ${SSE_API_UVICORN_NUM_WORKERS}
      API_UVICORN_PORT: ${SSE_API_UVICORN_PORT}
    ports:
      # for debug
      - ${SSE_API_UVICORN_PORT}:${SSE_API_UVICORN_PORT}
    depends_on:
      mongodb:
        condition: service_healthy
    restart: unless-stopped
  worker:
    build:
      target: worker
    volumes:
      - storage:${STORAGE_DIRECTORY}:rw
      - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY}:rw
      - descriptive-statistics:${DESCRIPTIVE_STATISTICS_CACHE_DIRECTORY}:rw
    environment:
      <<: *common-env
      COMMITTER_HF_TOKEN: ${COMMITTER_HF_TOKEN}
      CONFIG_NAMES_MAX_NUMBER: ${CONFIG_NAMES_MAX_NUMBER}
      DESCRIPTIVE_STATISTICS_CACHE_DIRECTORY: ${DESCRIPTIVE_STATISTICS_CACHE_DIRECTORY}
      DESCRIPTIVE_STATISTICS_MAX_SPLIT_SIZE_BYTES: ${DESCRIPTIVE_STATISTICS_MAX_SPLIT_SIZE_BYTES}
      FIRST_ROWS_MAX_BYTES: ${FIRST_ROWS_MAX_BYTES}
      FIRST_ROWS_MIN_CELL_BYTES: ${FIRST_ROWS_MIN_CELL_BYTES}
      FIRST_ROWS_MIN_NUMBER: ${FIRST_ROWS_MIN_NUMBER}
      FIRST_ROWS_COLUMNS_MAX_NUMBER: ${FIRST_ROWS_COLUMNS_MAX_NUMBER}
      HF_HUB_ENABLE_HF_TRANSFER: 1
      OPT_IN_OUT_URLS_SCAN_COLUMNS_MAX_NUMBER: ${OPT_IN_OUT_URLS_SCAN_COLUMNS_MAX_NUMBER}
      OPT_IN_OUT_URLS_SCAN_MAX_CONCURRENT_REQUESTS_NUMBER: ${OPT_IN_OUT_URLS_SCAN_MAX_CONCURRENT_REQUESTS_NUMBER}
      OPT_IN_OUT_URLS_SCAN_MAX_REQUESTS_PER_SECOND: ${OPT_IN_OUT_URLS_SCAN_MAX_REQUESTS_PER_SECOND}
      OPT_IN_OUT_URLS_SCAN_ROWS_MAX_NUMBER: ${OPT_IN_OUT_URLS_SCAN_ROWS_MAX_NUMBER}
      OPT_IN_OUT_URLS_SCAN_SPAWNING_TOKEN: ${OPT_IN_OUT_URLS_SCAN_SPAWNING_TOKEN}
      OPT_IN_OUT_URLS_SCAN_URLS_NUMBER_PER_BATCH: ${OPT_IN_OUT_URLS_SCAN_URLS_NUMBER_PER_BATCH}
      OPT_IN_OUT_URLS_SCAN_SPAWNING_URL: ${OPT_IN_OUT_URLS_SCAN_SPAWNING_URL}
      PARQUET_AND_INFO_COMMIT_MESSAGE: ${PARQUET_AND_INFO_COMMIT_MESSAGE}
      PARQUET_AND_INFO_MAX_DATASET_SIZE_BYTES: ${PARQUET_AND_INFO_MAX_DATASET_SIZE_BYTES}
      PARQUET_AND_INFO_SOURCE_REVISION: ${PARQUET_AND_INFO_SOURCE_REVISION}
      PARQUET_AND_INFO_TARGET_REVISION: ${PARQUET_AND_INFO_TARGET_REVISION}
      PARQUET_AND_INFO_URL_TEMPLATE: ${PARQUET_AND_INFO_URL_TEMPLATE}
      PARQUET_METADATA_STORAGE_DIRECTORY: ${PARQUET_METADATA_STORAGE_DIRECTORY}
      ROWS_INDEX_MAX_ARROW_DATA_IN_MEMORY: ${ROWS_INDEX_MAX_ARROW_DATA_IN_MEMORY}
      # uvicorn
      WORKER_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
      WORKER_UVICORN_NUM_WORKERS: ${WORKER_UVICORN_NUM_WORKERS}
      WORKER_UVICORN_PORT: ${WORKER_UVICORN_PORT}
      # datasets
      DATASETS_BASED_HF_DATASETS_CACHE: ${HF_DATASETS_CACHE}
      HF_MODULES_CACHE: ${HF_MODULES_CACHE}
      # worker
      WORKER_CONTENT_MAX_BYTES: ${WORKER_CONTENT_MAX_BYTES}
      WORKER_KILL_LONG_JOB_INTERVAL_SECONDS: ${WORKER_KILL_LONG_JOB_INTERVAL_SECONDS}
      WORKER_KILL_ZOMBIES_INTERVAL_SECONDS: ${WORKER_KILL_ZOMBIES_INTERVAL_SECONDS}
      WORKER_MAX_JOB_DURATION_SECONDS: ${WORKER_MAX_JOB_DURATION_SECONDS}
      WORKER_MAX_MISSING_HEARTBEATS: ${WORKER_MAX_MISSING_HEARTBEATS}
      WORKER_MAX_LOAD_PCT: ${WORKER_MAX_LOAD_PCT}
      WORKER_MAX_MEMORY_PCT: ${WORKER_MAX_MEMORY_PCT}
      WORKER_SLEEP_SECONDS: ${WORKER_SLEEP_SECONDS}
    ports:
      - ${WORKER_UVICORN_PORT}:${WORKER_UVICORN_PORT}
    depends_on:
      mongodb:
        condition: service_healthy
    restart: always
  webhook:
    build:
      target: webhook
    volumes:
      - storage:${STORAGE_DIRECTORY}:rw
    environment:
      <<: *service-env
      # uvicorn
      API_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
      API_UVICORN_NUM_WORKERS: ${WEBHOOK_UVICORN_NUM_WORKERS}
      API_UVICORN_PORT: ${WEBHOOK_UVICORN_PORT}
    ports:
      # for debug
      - ${WEBHOOK_UVICORN_PORT}:${WEBHOOK_UVICORN_PORT}
    depends_on:
      mongodb:
        condition: service_healthy
    restart: unless-stopped
  mongodb:
    image: "mongo:6.0.9"
    ports:
      - ${MONGO_PORT}:${MONGO_PORT}
    command: mongod --port ${MONGO_PORT} --replSet ${MONGO_REPLICASET} --bind_ip_all
    volumes:
      - mongo:/data/db:rw
    healthcheck:
      test: test $$(mongosh --port ${MONGO_PORT} --quiet --eval "try{rs.initiate({_id:'${MONGO_REPLICASET}',version:1,members:[{_id:0,host:'mongodb:${MONGO_PORT}'}]})} catch(e) {rs.status().ok}") -eq 1
      interval: 2s
      timeout: 20s
      retries: 10
      start_period: 20s
      start_interval: 2s
volumes:
  storage:
  mongo:
  parquet-metadata:
  duckdb-index:
  descriptive-statistics: