Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 29 additions & 16 deletions python/openai/openai_frontend/engine/utils/triton.py
Original file line number Diff line number Diff line change
Expand Up @@ -617,6 +617,34 @@ def _get_guided_json_from_tool(
return None


def _validate_lora_path_trtllm(repo_path: str, lora_path: str, lora_name: str):
if os.path.isabs(lora_path):
raise ValueError(
f"LoRA path '{lora_path}' for '{lora_name}' must be a relative path inside its model repository"
)

# NOTE: Error messages should never contain the real/absolute paths.
realpath_repo = os.path.realpath(repo_path)
realpath_lora = os.path.realpath(os.path.join(realpath_repo, lora_path))
# Always check if the LoRA path is inside the model repository before checking its existence.
if os.path.commonpath([realpath_repo, realpath_lora]) != realpath_repo:
raise ValueError(
f"LoRA path '{lora_path}' for '{lora_name}' must be inside its model repository"
)
if not os.path.exists(realpath_lora):
raise ServerError(
f"LoRA directory '{lora_path}' not found for '{lora_name}' in its model repository"
)

# Check if the files exist
for lora_file in ["model.lora_weights.npy", "model.lora_config.npy"]:
lora_file_path = os.path.join(realpath_lora, lora_file)
if not os.path.exists(lora_file_path):
raise ServerError(
f"LoRA file '{lora_file}' not found for '{lora_name}' at path: {lora_file_path}"
)


def _parse_lora_configs(
model_repository: str | list[str], model_name: str, model_version: int, backend: str
) -> None | List[tuple[str, str]]:
Expand Down Expand Up @@ -683,25 +711,10 @@ def _parse_lora_configs(
with open(lora_config_path, "r") as f:
lora_config = json.load(f)
for lora_name, lora_path in lora_config.items():
print(f"backend: {backend}")
if backend == "vllm":
lora_configs.append(TritonLoraConfig(name=lora_name))
else:
lora_weights_path = os.path.join(
lora_path, "model.lora_weights.npy"
)
lora_config_path = os.path.join(
lora_path, "model.lora_config.npy"
)
if not os.path.exists(lora_weights_path):
raise ServerError(
f"LoRA weights file not found for '{lora_name}' at path: {lora_weights_path}"
)
if not os.path.exists(lora_config_path):
raise ServerError(
f"LoRA config file not found for '{lora_name}' at path: {lora_config_path}"
)

_validate_lora_path_trtllm(repo_path, lora_path, lora_name)
lora_configs.append(
TritonLoraConfig(
name=lora_name, path=lora_path, task_id=lora_task_id
Expand Down
75 changes: 68 additions & 7 deletions python/openai/tests/test_lora.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
Expand Down Expand Up @@ -35,10 +35,22 @@
from openai_frontend.engine.utils.triton import (
_parse_lora_configs as parse_lora_configs,
)
from openai_frontend.engine.utils.triton import (
_validate_lora_path_trtllm as validate_lora_path_trtllm,
)

from .utils import OpenAIServer


def is_vllm_installed():
try:
import vllm as _

return True
except ImportError:
return False


@pytest.mark.parametrize(
"model_repository,model_name,expect_error",
[
Expand Down Expand Up @@ -74,13 +86,62 @@ def test_parse_lora_configs(model_repository: str, model_name: str, expect_error
)


def is_vllm_installed():
@pytest.mark.skipif(
is_vllm_installed(),
reason="VLLM backend does not validate LoRA paths",
)
@pytest.mark.parametrize(
"lora_path,expect_error,error_message",
[
# Valid relative path inside repo (requires .npy files to exist at runtime).
("tensorrt_llm_bls/1/luotuo-lora-7b-0.1-weights", False, None),
("tensorrt_llm_bls/1/Japanese-Alpaca-LoRA-7b-v0-weights", False, None),
# Absolute path not allowed.
(
os.path.join(
os.path.abspath(os.curdir),
"tests/tensorrtllm_models",
"tensorrt_llm_bls/1/luotuo-lora-7b-0.1-weights",
),
True,
f"must be a relative path inside its model repository",
),
("/etc/passwd", True, "must be a relative path inside its model repository"),
# Path outside repo (traversal).
("tensorrt_llm_bls/1//../1/luotuo-lora-7b-0.1-weights", False, None),
("../outside/lora", True, "must be inside its model repository"),
("subdir/../../etc/passwd", True, "must be inside its model repository"),
# LoRA directory not found.
("tensorrt_llm_bls/10", True, "LoRA directory 'tensorrt_llm_bls/10' not found"),
(
"tensorrt_llm_bls/1/non_exist",
True,
"LoRA directory 'tensorrt_llm_bls/1/non_exist' not found",
),
# LoRA file not found.
("tensorrt_llm_bls/1", True, "LoRA file 'model.lora_weights.npy' not found"),
],
)
def test_validate_lora_path_trtllm(
lora_path: str,
expect_error: bool,
error_message: str,
):
lora_name = ""
repo_path = "tests/tensorrtllm_models"
try:
import vllm as _

return True
except ImportError:
return False
validate_lora_path_trtllm(repo_path, lora_path, lora_name)
except Exception as e:
if not expect_error:
raise pytest.fail(
f"repo_path='{repo_path}' raised exception unexpectedly: {e}"
)
assert error_message in str(e)
else:
if expect_error:
raise pytest.fail(
f"lora_path='{repo_path}' did not raise exception as expected."
)


class LoRATest(unittest.TestCase):
Expand Down
Loading