Skip to content

Commit 6e52db5

Browse files
author
Xuan-Son Nguyen
authored
server: add --cors-* options (ggml-org#25655)
* server: add --cors-* options * add special "localhost" value * add tests * fix test * add link to PR
1 parent 236ab57 commit 6e52db5

6 files changed

Lines changed: 177 additions & 21 deletions

File tree

‎common/arg.cpp‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -697,7 +697,7 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
697697
}
698698
};
699699

700-
// parse the first time to get -hf option (used for remote preset)
700+
// parse all CLI args now, so that -hf is available below for remote preset resolution
701701
parse_cli_args();
702702

703703
postprocess_cpu_params(params.cpuparams, nullptr);
@@ -748,6 +748,11 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
748748
params.kv_overrides.back().key[0] = 0;
749749
}
750750

751+
if (!params.server_tools.empty() && !params.cors_origins_explicit) {
752+
LOG_WRN("server tools are enabled, using localhost as default CORS origin (change via --cors-origins)\n");
753+
params.cors_origins = "localhost";
754+
}
755+
751756
// pad tensor_buft_overrides for llama_params_fit:
752757
const size_t ntbo = llama_max_tensor_buft_overrides();
753758
while (params.tensor_buft_overrides.size() < ntbo) {
@@ -3047,6 +3052,42 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
30473052
params.public_path = value;
30483053
}
30493054
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_STATIC_PATH"));
3055+
add_opt(common_arg(
3056+
{"--cors-origins"}, "ORIGINS",
3057+
string_format(
3058+
"comma-separated list of allowed origins for CORS (default: %s)\n"
3059+
"if set to special value 'localhost', reflect the Origin header only if it is localhost",
3060+
params.cors_origins.c_str()),
3061+
[](common_params & params, const std::string & value) {
3062+
params.cors_origins = value;
3063+
params.cors_origins_explicit = true;
3064+
}
3065+
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_ORIGINS"));
3066+
add_opt(common_arg(
3067+
{"--cors-methods"}, "METHODS",
3068+
string_format("comma-separated list of allowed methods for CORS (default: %s)", params.cors_methods.c_str()),
3069+
[](common_params & params, const std::string & value) {
3070+
params.cors_methods = value;
3071+
}
3072+
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_METHODS"));
3073+
add_opt(common_arg(
3074+
{"--cors-headers"}, "HEADERS",
3075+
string_format("comma-separated list of allowed headers for CORS (default: %s)", params.cors_headers.c_str()),
3076+
[](common_params & params, const std::string & value) {
3077+
params.cors_headers = value;
3078+
}
3079+
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_HEADERS"));
3080+
add_opt(common_arg(
3081+
{"--cors-credentials"},
3082+
{"--no-cors-credentials"},
3083+
string_format(
3084+
"whether to allow credentials for CORS (default: %s)\n"
3085+
"note: if this is enabled and --cors-origins is set to * (default), the Origin header will be echoed back, and credentials will always be allowed",
3086+
params.cors_credentials ? "enabled" : "disabled"),
3087+
[](common_params & params, bool value) {
3088+
params.cors_credentials = value;
3089+
}
3090+
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_CREDENTIALS"));
30503091
add_opt(common_arg(
30513092
{"--api-prefix"}, "PREFIX",
30523093
string_format("prefix path the server serves from, without the trailing slash (default: %s)", params.api_prefix.c_str()),
@@ -3080,15 +3121,17 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
30803121
{"--tools"}, "TOOL1,TOOL2,...",
30813122
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
30823123
"specify \"all\" to enable all tools\n"
3083-
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime",
3124+
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime\n"
3125+
"note: for security reasons, this will limit --cors-origins to localhost by default",
30843126
[](common_params & params, const std::string & value) {
30853127
params.server_tools = parse_csv_row(value);
30863128
}
30873129
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS"));
30883130
add_opt(common_arg(
30893131
{"-ag", "--agent"},
30903132
{"-no-ag", "--no-agent"},
3091-
"whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)",
3133+
"whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)\n"
3134+
"note: for security reasons, this will limit --cors-origins to localhost by default",
30923135
[](common_params & params, bool value) {
30933136
if (value) {
30943137
params.server_tools = {"all"};
@@ -3097,6 +3140,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
30973140
params.server_tools.clear();
30983141
params.ui_mcp_proxy = false;
30993142
}
3143+
// note: do not modify cors_origins here, as the options are not evaluated in order (user may explicitly set --cors-origins before --agent)
31003144
}
31013145
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_AGENT"));
31023146
add_opt(common_arg(

‎common/common.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -631,6 +631,14 @@ struct common_params {
631631
std::string api_prefix = ""; // NOLINT
632632
std::string chat_template = ""; // NOLINT
633633
bool use_jinja = true; // NOLINT
634+
635+
// server CORS params
636+
std::string cors_origins = "*";
637+
std::string cors_methods = "GET, POST, DELETE, OPTIONS";
638+
std::string cors_headers = "*";
639+
bool cors_credentials = true;
640+
bool cors_origins_explicit = false; // for --agent option
641+
634642
bool enable_chat_template = true;
635643
bool force_pure_content_parser = false;
636644
common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;

‎tools/server/server-http.cpp‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,16 @@ static void log_server_request(const httplib::Request & req, const httplib::Resp
4747
SRV_DBG("response: %s\n", res.body.c_str());
4848
}
4949

50+
// returns true if the Origin header value's host is localhost / 127.0.0.1 / ::1 (any port)
51+
static bool origin_is_localhost(const std::string & origin) {
52+
try {
53+
const std::string host = common_http_parse_url(origin).host;
54+
return host == "localhost" || host == "127.0.0.1" || host == "::1";
55+
} catch (const std::exception &) {
56+
return false;
57+
}
58+
}
59+
5060
// For Google Cloud Platform deployment compatibility
5161
struct gcp_params {
5262
bool enabled;
@@ -266,13 +276,26 @@ bool server_http_context::init(const common_params & params) {
266276
};
267277

268278
// register server middlewares
269-
srv->set_pre_routing_handler([middleware_validate_api_key, middleware_server_state](const httplib::Request & req, httplib::Response & res) {
270-
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
279+
srv->set_pre_routing_handler([&params, middleware_validate_api_key, middleware_server_state](const httplib::Request & req, httplib::Response & res) {
280+
if (params.cors_credentials && params.cors_origins == "*") {
281+
// special case: echo back the Origin header to allow any origin to access the server with credentials
282+
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
283+
} else if (params.cors_origins == "localhost") {
284+
// special case: only reflect the Origin header if it is a localhost origin
285+
std::string origin = req.get_header_value("Origin");
286+
if (origin_is_localhost(origin)) {
287+
res.set_header("Access-Control-Allow-Origin", origin);
288+
} else {
289+
SRV_WRN("(CORS) skip non-localhost origin: %s\n", origin.c_str());
290+
}
291+
} else {
292+
res.set_header("Access-Control-Allow-Origin", params.cors_origins);
293+
}
271294
// If this is OPTIONS request, skip validation because browsers don't include Authorization header
272295
if (req.method == "OPTIONS") {
273-
res.set_header("Access-Control-Allow-Credentials", "true");
274-
res.set_header("Access-Control-Allow-Methods", "GET, POST");
275-
res.set_header("Access-Control-Allow-Headers", "*");
296+
res.set_header("Access-Control-Allow-Credentials", params.cors_credentials ? "true" : "false");
297+
res.set_header("Access-Control-Allow-Methods", params.cors_methods);
298+
res.set_header("Access-Control-Allow-Headers", params.cors_headers);
276299
res.set_content("", "text/html"); // blank response, no data
277300
return httplib::Server::HandlerResponse::Handled; // skip further processing
278301
}

‎tools/server/server.cpp‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -303,14 +303,24 @@ int llama_server(common_params & params, int argc, char ** argv) {
303303
return res;
304304
};
305305

306-
// CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)
307-
if (params.ui_mcp_proxy) {
306+
if (params.cors_origins == "*" && params.api_keys.empty()) {
308307
SRV_WRN("%s", "-----------------\n");
309-
SRV_WRN("%s", "CORS proxy is enabled, do not expose server to untrusted environments\n");
310-
SRV_WRN("%s", "This feature is EXPERIMENTAL and may be removed or changed in future versions\n");
308+
SRV_WRN("%s", "CORS is set to allow all origins ('*') and no API key is set\n");
309+
SRV_WRN("%s", "this can be a security risk (cross-origin attacks)\n");
310+
SRV_WRN("%s", "more info: https://github.com/ggml-org/llama.cpp/pull/25655\n");
311311
SRV_WRN("%s", "-----------------\n");
312+
}
313+
314+
// CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)
315+
std::vector<std::string> warn_names;
316+
if (is_router_server) {
317+
warn_names.push_back("router mode");
318+
}
319+
320+
if (params.ui_mcp_proxy) {
312321
ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));
313322
ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));
323+
warn_names.push_back("MCP proxy (experimental)");
314324
} else {
315325
ctx_http.get ("/cors-proxy", ex_wrapper(res_403));
316326
ctx_http.post("/cors-proxy", ex_wrapper(res_403));
@@ -324,17 +334,24 @@ int llama_server(common_params & params, int argc, char ** argv) {
324334
SRV_ERR("tools setup failed: %s\n", e.what());
325335
return 1;
326336
}
327-
SRV_WRN("%s", "-----------------\n");
328-
SRV_WRN("%s", "Built-in tools are enabled, do not expose server to untrusted environments\n");
329-
SRV_WRN("%s", "This feature is EXPERIMENTAL and may be changed in the future\n");
330-
SRV_WRN("%s", "-----------------\n");
331337
ctx_http.get ("/tools", ex_wrapper(tools.handle_get));
332338
ctx_http.post("/tools", ex_wrapper(tools.handle_post));
339+
warn_names.push_back("built-in tools (experimental)");
333340
} else {
334341
ctx_http.get ("/tools", ex_wrapper(res_403));
335342
ctx_http.post("/tools", ex_wrapper(res_403));
336343
}
337344

345+
if (warn_names.size() > 0) {
346+
SRV_WRN("%s", "-----------------\n");
347+
SRV_WRN("%s", "the following feature(s) are enabled:\n");
348+
for (const auto & name : warn_names) {
349+
SRV_WRN(" %s\n", name.c_str());
350+
}
351+
SRV_WRN("%s", "do not expose the server to untrusted environments\n");
352+
SRV_WRN("%s", "-----------------\n");
353+
}
354+
338355
//
339356
// Handle downloading model
340357
//
@@ -452,9 +469,6 @@ int llama_server(common_params & params, int argc, char ** argv) {
452469
SRV_INF("listening on %s\n", ctx_http.listening_address.c_str());
453470

454471
if (is_router_server) {
455-
SRV_WRN("%s", "NOTE: router mode is experimental\n");
456-
SRV_WRN("%s", " it is not recommended to use this mode in untrusted environments\n");
457-
458472
if (!params.models_preset_hf.empty()) {
459473
SRV_WRN( "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str());
460474
SRV_WRN("%s", " please only use presets that you can trust! Unknown presets may be unsafe\n");

‎tools/server/tests/unit/test_security.py‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ def test_openai_library_correct_api_key():
9191
("localhost", "Access-Control-Allow-Origin", "localhost"),
9292
("web.mydomain.fr", "Access-Control-Allow-Origin", "web.mydomain.fr"),
9393
("origin", "Access-Control-Allow-Credentials", "true"),
94-
("web.mydomain.fr", "Access-Control-Allow-Methods", "GET, POST"),
94+
("web.mydomain.fr", "Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS"),
9595
("web.mydomain.fr", "Access-Control-Allow-Headers", "*"),
9696
])
9797
def test_cors_options(origin: str, cors_header: str, cors_header_value: str):
@@ -107,6 +107,70 @@ def test_cors_options(origin: str, cors_header: str, cors_header_value: str):
107107
assert res.headers[cors_header] == cors_header_value
108108

109109

110+
@pytest.mark.parametrize("origin", [
111+
"http://localhost",
112+
"http://localhost:8080",
113+
"http://127.0.0.1",
114+
"http://127.0.0.1:3000",
115+
"http://[::1]",
116+
"http://[::1]:3000",
117+
])
118+
def test_cors_origins_localhost_reflects(origin: str):
119+
global server
120+
server = ServerPreset.router()
121+
server.cors_origins = "localhost"
122+
server.start()
123+
res = server.make_request("OPTIONS", "/completions", headers={
124+
"Origin": origin,
125+
"Access-Control-Request-Method": "POST",
126+
"Access-Control-Request-Headers": "Authorization",
127+
})
128+
assert res.status_code == 200
129+
assert res.headers["Access-Control-Allow-Origin"] == origin
130+
131+
132+
@pytest.mark.parametrize("origin", [
133+
"http://web.mydomain.fr",
134+
"http://evil.com",
135+
"http://notlocalhost",
136+
"http://localhost.evil.com",
137+
])
138+
def test_cors_origins_localhost_rejects(origin: str):
139+
global server
140+
server = ServerPreset.router()
141+
server.cors_origins = "localhost"
142+
server.start()
143+
res = server.make_request("OPTIONS", "/completions", headers={
144+
"Origin": origin,
145+
"Access-Control-Request-Method": "POST",
146+
"Access-Control-Request-Headers": "Authorization",
147+
})
148+
assert res.status_code == 200
149+
assert "Access-Control-Allow-Origin" not in res.headers
150+
151+
152+
def test_cors_origins_defaults_to_localhost_with_tools_enabled():
153+
global server
154+
server = ServerPreset.router()
155+
server.server_tools = "all"
156+
server.start()
157+
res = server.make_request("OPTIONS", "/completions", headers={
158+
"Origin": "http://localhost:8080",
159+
"Access-Control-Request-Method": "POST",
160+
"Access-Control-Request-Headers": "Authorization",
161+
})
162+
assert res.status_code == 200
163+
assert res.headers["Access-Control-Allow-Origin"] == "http://localhost:8080"
164+
165+
res = server.make_request("OPTIONS", "/completions", headers={
166+
"Origin": "http://evil.com",
167+
"Access-Control-Request-Method": "POST",
168+
"Access-Control-Request-Headers": "Authorization",
169+
})
170+
assert res.status_code == 200
171+
assert "Access-Control-Allow-Origin" not in res.headers
172+
173+
110174
def test_cors_proxy_only_forwards_explicit_proxy_headers():
111175
class CaptureHeadersHandler(BaseHTTPRequestHandler):
112176
def do_GET(self):

‎tools/server/tests/utils.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ class ServerProcess:
114114
backend_sampling: bool = False
115115
gcp_compat: bool = False
116116
server_tools: str | None = None
117+
cors_origins: str | None = None
117118

118119
# session variables
119120
process: subprocess.Popen | None = None
@@ -170,6 +171,8 @@ def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None:
170171
server_args.extend(["--models-max", self.models_max])
171172
if self.models_preset:
172173
server_args.extend(["--models-preset", self.models_preset])
174+
if self.cors_origins:
175+
server_args.extend(["--cors-origins", self.cors_origins])
173176
if self.n_batch:
174177
server_args.extend(["--batch-size", self.n_batch])
175178
if self.n_ubatch:
@@ -359,7 +362,7 @@ def make_request(
359362
if parse_body:
360363
try:
361364
result.body = response.json()
362-
except JSONDecodeError:
365+
except (JSONDecodeError, requests.exceptions.JSONDecodeError):
363366
result.body = response.text
364367
else:
365368
result.body = None

0 commit comments

Comments
 (0)