models.json

{
  "docker": {
    "image": "vllm/vllm-openai-cpu:v0.24.0",
    "port": "8000:8000",
    "environment": {
      "VLLM_CPU_SGL_KERNEL": "1",
      "VLLM_CPU_KVCACHE_SPACE": "40",
      "VLLM_RPC_TIMEOUT": "100000",
      "VLLM_ALLOW_LONG_MAX_MODEL_LEN": "1",
      "VLLM_ENGINE_ITERATION_TIMEOUT_S": "120",
      "VLLM_CPU_NUM_OF_RESERVED_CPU": "0"
    },
    "volumes": [
      "/root/.cache/huggingface:/root/.cache/huggingface",
      "/opt/vllm/examples:/workspace/examples"
    ]
  },
  "global_defaults": {
    "block_size": 128,
    "dtype": "bfloat16",
    "distributed_executor_backend": "mp",
    "trust_remote_code": true,
    "enable_chunked_prefill": true,
    "enforce_eager": true,
    "max_num_batched_tokens": 2048,
    "max_num_seqs": 256,
    "disable_log_requests": true,
    "enable_auto_tool_choice": true
  },
  "models": {
    "llama-8B": {
      "display_name": "Llama 3.1 8B Instruct",
      "model_path": "meta-llama/Llama-3.1-8B-Instruct",
      "vllm_args": {
        "max_model_len": 32768,
        "tool_call_parser": "llama3_json",
        "chat_template": "examples/tool_chat_template_llama3.1_json.jinja"
      }
    },
    "qwen-14B": {
      "display_name": "Qwen 3 14B",
      "model_path": "Qwen/Qwen3-14B",
      "vllm_args": {
        "max_model_len": 16384,
        "tool_call_parser": "hermes"
      }
    },
    "mistral-7B": {
      "display_name": "Mistral 7B Instruct v0.3",
      "model_path": "mistralai/Mistral-7B-Instruct-v0.3",
      "vllm_args": {
        "max_model_len": 32768,
        "tool_call_parser": "mistral",
        "chat_template": "examples/tool_chat_template_mistral_parallel.jinja"
      }
    }
  }
}