configuring genai model deployment with model readonly filesystem (#4139)

### 🛠 Summary

CVS-186376

Allows starting generative models on readonly filesystem with option to
change runtime parameters and accepts models without graph.pbtxt.
It can be done with `--model_path` and `--task` along with other task
specific paramters.
There is also removed default --task value as text_generation. Task
should be explicit if graph.pbtxt should be created.
The logic is:
- if --source_model is provided, OVMS will assume to path of pulling the
model from HF if missing on the local target path. It requires RW
filesystem. Should be used always with --task.
- if --source_model is not provided and deployment is with --model_name
and --model_path, task is optional. With empty --task, graph from local
model folder will be used. With set task, all runtime params will be
from CLI and graph will be in memory without saving to model folder.
 
### 🧪 Checklist

- [ ] Unit tests added.
- [ ] The documentation updated.
- [ ] Change follows security best practices.
``

---------

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: dtrawins <20400806+dtrawins@users.noreply.github.com>

Author: 3 people

src/BUILD

@@ -649,6 +649,7 @@ ovms_cc_library(
         "//src/dags:pipelinedefinition",
         "//src/filesystem:libovmsfilesystem",
         "//src/filesystem:libovmsfilesystemfactory",
+        "//src/graph_export:graph_export",
         "//src/metrics:libovms_metric_provider",
         "//src/metrics:libovmsmetrics",
         "@com_github_tencent_rapidjson//:rapidjson",
@@ -938,6 +939,7 @@ ovms_cc_library(
             "//src/kfserving_api:kfserving_api_cpp",
             "capimodule",
             "//src/pull_module:hf_pull_model_module",
+            "//src/graph_export:graph_export",
             "//src/servables_config_manager_module:servablesconfigmanagermodule",
             "predict_request_validation_utils", # to be removed when capi has its own lib and added there @atobisze
             "kfs_backend_impl",

src/capi_frontend/server_settings.hpp

@@ -90,6 +90,7 @@ enum OvmsServerMode : int {
     HF_PULL_AND_START_MODE,
     LIST_MODELS_MODE,
     MODIFY_CONFIG_MODE,
+    IN_MEMORY_GRAPH_MODE,
     UNKNOWN_MODE
 };
 
@@ -171,7 +172,7 @@ struct HFSettingsImpl {
     std::string downloadPath = "";
     bool overwriteModels = false;
     ModelDownlaodType downloadType = GIT_CLONE_DOWNLOAD;
-    GraphExportType task = TEXT_GENERATION_GRAPH;
+    GraphExportType task = UNKNOWN_GRAPH;
     std::variant<TextGenGraphSettingsImpl, RerankGraphSettingsImpl, EmbeddingsGraphSettingsImpl, TextToSpeechGraphSettingsImpl, SpeechToTextGraphSettingsImpl, ImageGenerationGraphSettingsImpl> graphSettings;
 };
 

src/cli_parser.cpp

@@ -344,8 +344,8 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
 
         result = std::make_unique<cxxopts::ParseResult>(options->parse(argc, argv));
 
-        // HF pull mode or pull and start mode
-        if (isHFPullOrPullAndStart(this->result)) {
+        // HF pull mode or pull and start mode or starting from local folder with graph created in memory
+        if (isHFPullOrPullAndStart(this->result) || isInMemoryGraphMode(this->result)) {
             std::vector<std::string> unmatchedOptions;
             GraphExportType task;
             if (result->count("task")) {
@@ -692,13 +692,27 @@ void CLIParser::prepareModel(ModelsSettingsImpl& modelsSettings, HFSettingsImpl&
 }
 
 bool CLIParser::isHFPullOrPullAndStart(const std::unique_ptr<cxxopts::ParseResult>& result) {
+    // Keep `--task` in the broad mutually exclusive task/pull CLI category so
+    // parse-time checks that rely on this helper continue to reject combining
+    // task-based flows with config-management modes. More specific mode
+    // differentiation is handled by isInMemoryGraphMode().
     return (result->count("pull") || result->count("task"));
 }
 
+bool CLIParser::isInMemoryGraphMode(const std::unique_ptr<cxxopts::ParseResult>& result) {
+    return (result->count("task") && !result->count("source_model") && !result->count("pull"));
+}
+
 void CLIParser::prepareGraph(ServerSettingsImpl& serverSettings, HFSettingsImpl& hfSettings, const std::string& modelName) {
+    // Always propagate source_model so validation can detect misuse
+    if (result->count("source_model")) {
+        hfSettings.sourceModel = result->operator[]("source_model").as<std::string>();
+    }
     // Ovms Pull models mode || pull and start models mode
-    if (isHFPullOrPullAndStart(this->result)) {
-        if (result->count("pull")) {
+    if (isHFPullOrPullAndStart(this->result) || isInMemoryGraphMode(this->result)) {
+        if (isInMemoryGraphMode(this->result)) {
+            serverSettings.serverMode = IN_MEMORY_GRAPH_MODE;
+        } else if (result->count("pull")) {
             serverSettings.serverMode = HF_PULL_MODE;
         } else {
             serverSettings.serverMode = HF_PULL_AND_START_MODE;
@@ -711,8 +725,11 @@ void CLIParser::prepareGraph(ServerSettingsImpl& serverSettings, HFSettingsImpl&
             hfSettings.overwriteModels = result->operator[]("overwrite_models").as<bool>();
         }
         if (result->count("source_model")) {
+            // Already set above, but keep the original flow for downloadType logic
             hfSettings.sourceModel = result->operator[]("source_model").as<std::string>();
-        } else if (result->count("model_name")) {
+        } else if (result->count("model_name") && !result->count("model_path")) {
+            // Only use model_name as source_model when model_path is not set
+            // (when model_path is set, user wants to use local model without HF pull)
             hfSettings.sourceModel = result->operator[]("model_name").as<std::string>();
         }
         if ((result->count("weight-format") || result->count("extra_quantization_params")) && isOptimumCliDownload(hfSettings.sourceModel, hfSettings.ggufFilename)) {
@@ -732,6 +749,11 @@ void CLIParser::prepareGraph(ServerSettingsImpl& serverSettings, HFSettingsImpl&
         if (result->count("vocoder"))
             hfSettings.exportSettings.vocoder = result->operator[]("vocoder").as<std::string>();
         hfSettings.downloadPath = result->operator[]("model_repository_path").as<std::string>();
+        // When --task is used with --model_path but without --pull/--source_model,
+        // use model_path as the model location (no HF download needed)
+        if (!result->count("pull") && !result->count("source_model") && result->count("model_path")) {
+            hfSettings.exportSettings.modelPath = result->operator[]("model_path").as<std::string>();
+        }
         if (result->count("task")) {
             hfSettings.task = stringToEnum(result->operator[]("task").as<std::string>());
             switch (hfSettings.task) {
@@ -798,7 +820,8 @@ void CLIParser::prepareGraph(ServerSettingsImpl& serverSettings, HFSettingsImpl&
         if (!serverSettings.cacheDir.empty()) {
             hfSettings.exportSettings.pluginConfig.cacheDir = serverSettings.cacheDir;
         }
-    // No pull nor pull and start mode
+
+    // No pull nor pull and start mode and no start with local model_path
     } else {
         if (result->count("weight-format")) {
             throw std::logic_error("--weight-format parameter unsupported for Openvino huggingface organization models.");
@@ -840,11 +863,14 @@ void CLIParser::prepareGraphStart(HFSettingsImpl& hfSettings, ModelsSettingsImpl
     // Model settings
     if (result->count("model_name")) {
         modelsSettings.modelName = result->operator[]("model_name").as<std::string>();
-    } else {
+    } else if (!hfSettings.sourceModel.empty()) {
         modelsSettings.modelName = hfSettings.sourceModel;
     }
 
-    modelsSettings.modelPath = FileSystem::joinPath({hfSettings.downloadPath, hfSettings.sourceModel});
+    // Only override modelPath if it wasn't already set via --model_path
+    if (!result->count("model_path")) {
+        modelsSettings.modelPath = FileSystem::joinPath({hfSettings.downloadPath, hfSettings.sourceModel});
+    }
 }
 
 void CLIParser::prepare(ServerSettingsImpl* serverSettings, ModelsSettingsImpl* modelsSettings) {

src/cli_parser.hpp

@@ -51,6 +51,7 @@ class CLIParser {
     void prepareGraphStart(HFSettingsImpl& hfSettings, ModelsSettingsImpl& modelsSettings);
     void prepareConfigExport(ModelsSettingsImpl& modelsSettings);
     bool isHFPullOrPullAndStart(const std::unique_ptr<cxxopts::ParseResult>& result);
+    bool isInMemoryGraphMode(const std::unique_ptr<cxxopts::ParseResult>& result);
 };
 
 }  // namespace ovms

src/config.cpp

@@ -148,18 +148,22 @@ bool Config::validateUserSettingsInConfigAddRemoveModel(const ModelsSettingsImpl
 }
 
 bool Config::validate() {
-    if (this->serverSettings.serverMode == HF_PULL_MODE || this->serverSettings.serverMode == HF_PULL_AND_START_MODE) {
-        if (!serverSettings.hfSettings.sourceModel.size()) {
-            std::cerr << "source_model parameter is required for pull mode";
-            return false;
-        }
-        if (!serverSettings.hfSettings.downloadPath.size()) {
-            std::cerr << "model_repository_path parameter is required for pull mode";
-            return false;
-        }
-        if (this->serverSettings.hfSettings.task == UNKNOWN_GRAPH) {
-            std::cerr << "Error: --task parameter not set." << std::endl;
-            return false;
+    if (!this->serverSettings.hfSettings.sourceModel.empty() && this->serverSettings.hfSettings.task == UNKNOWN_GRAPH) {
+        std::cerr << "--source_model should be used combined with --task" << std::endl;
+        return false;
+    }
+    if (this->serverSettings.serverMode == HF_PULL_MODE || this->serverSettings.serverMode == HF_PULL_AND_START_MODE || this->serverSettings.serverMode == IN_MEMORY_GRAPH_MODE) {
+        // When --task is used with --model_path (no HF pulling), sourceModel and downloadPath are not required
+        bool taskWithModelPath = this->serverSettings.serverMode == IN_MEMORY_GRAPH_MODE && !this->modelsSettings.modelPath.empty();
+        if (!taskWithModelPath) {
+            if (!serverSettings.hfSettings.sourceModel.size()) {
+                std::cerr << "source_model parameter is required for pull mode";
+                return false;
+            }
+            if (!serverSettings.hfSettings.downloadPath.size()) {
+                std::cerr << "model_repository_path parameter is required for pull mode";
+                return false;
+            }
         }
         if (this->serverSettings.hfSettings.task == TEXT_GENERATION_GRAPH) {
             if (!std::holds_alternative<TextGenGraphSettingsImpl>(this->serverSettings.hfSettings.graphSettings)) {

src/graph_export/graph_export.cpp

@@ -52,6 +52,20 @@
 #endif
 namespace ovms {
 
+static std::string inMemoryGraphContent;
+
+bool GraphExport::hasInMemoryGraphContent() {
+    return !inMemoryGraphContent.empty();
+}
+
+const std::string& GraphExport::getInMemoryGraphContent() {
+    return inMemoryGraphContent;
+}
+
+void GraphExport::clearInMemoryGraphContent() {
+    inMemoryGraphContent.clear();
+}
+
 static const std::string OVMS_VERSION_GRAPH_LINE = std::string("# File created with: ") + PROJECT_NAME + std::string(" ") + PROJECT_VERSION + std::string("\n");
 
 static std::string constructModelsPath(const std::string& modelPath, const std::optional<std::string>& ggufFilenameOpt) {
@@ -91,22 +105,26 @@ std::string GraphExport::getDraftModelDirectoryPath(const std::string& directory
     }                                                                             \
     auto pluginConfigOpt = std::get<std::optional<std::string>>(pluginConfigOrStatus)
 
-static Status createPbtxtFile(const std::string& directoryPath, const std::string& pbtxtContent) {
+static Status createPbtxtFile(const std::string& directoryPath, const std::string& pbtxtContent, bool writeToFile) {
 #if (MEDIAPIPE_DISABLE == 0)
     ::mediapipe::CalculatorGraphConfig config;
-    SPDLOG_TRACE("Generated pbtxt: {}", pbtxtContent);
+    SPDLOG_TRACE("Created graph config file:\n{}", pbtxtContent);
     bool success = ::google::protobuf::TextFormat::ParseFromString(pbtxtContent, &config);
     if (!success) {
         SPDLOG_ERROR("Created graph config file couldn't be parsed - check used task parameters values.");
         return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID;
     }
 #endif
+    if (!writeToFile) {
+        inMemoryGraphContent = pbtxtContent;
+        return StatusCode::OK;
+    }
     // clang-format on
     std::string fullPath = FileSystem::joinPath({directoryPath, "graph.pbtxt"});
     return FileSystem::createFileOverwrite(fullPath, pbtxtContent);
 }
 
-static Status createTextGenerationGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) {
+static Status createTextGenerationGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) {
     if (!std::holds_alternative<TextGenGraphSettingsImpl>(hfSettings.graphSettings)) {
         SPDLOG_ERROR("Graph options not initialized for text generation.");
         return StatusCode::INTERNAL_ERROR;
@@ -198,10 +216,10 @@ static Status createTextGenerationGraphTemplate(const std::string& directoryPath
         }
     }
     })";
-    return createPbtxtFile(directoryPath, oss.str());
+    return createPbtxtFile(directoryPath, oss.str(), writeToFile);
 }
 
-static Status createRerankGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) {
+static Status createRerankGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) {
     if (!std::holds_alternative<RerankGraphSettingsImpl>(hfSettings.graphSettings)) {
         SPDLOG_ERROR("Graph options not initialized for reranking.");
         return StatusCode::INTERNAL_ERROR;
@@ -242,10 +260,10 @@ node {
         }
     }
 })";
-    return createPbtxtFile(directoryPath, oss.str());
+    return createPbtxtFile(directoryPath, oss.str(), writeToFile);
 }
 
-static Status createEmbeddingsGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) {
+static Status createEmbeddingsGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) {
     if (!std::holds_alternative<EmbeddingsGraphSettingsImpl>(hfSettings.graphSettings)) {
         SPDLOG_ERROR("Graph options not initialized for embeddings.");
         return StatusCode::INTERNAL_ERROR;
@@ -289,10 +307,10 @@ node {
     oss << R"(}
     }
 })";
-    return createPbtxtFile(directoryPath, oss.str());
+    return createPbtxtFile(directoryPath, oss.str(), writeToFile);
 }
 
-static Status createTextToSpeechGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) {
+static Status createTextToSpeechGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) {
     if (!std::holds_alternative<TextToSpeechGraphSettingsImpl>(hfSettings.graphSettings)) {
         SPDLOG_ERROR("Graph options not initialized for speech generation.");
         return StatusCode::INTERNAL_ERROR;
@@ -339,11 +357,15 @@ node {
     }
 #endif
     // clang-format on
+    if (!writeToFile) {
+        inMemoryGraphContent = oss.str();
+        return StatusCode::OK;
+    }
     std::string fullPath = FileSystem::joinPath({directoryPath, "graph.pbtxt"});
     return FileSystem::createFileOverwrite(fullPath, oss.str());
 }
 
-static Status createSpeechToTextGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) {
+static Status createSpeechToTextGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) {
     if (!std::holds_alternative<SpeechToTextGraphSettingsImpl>(hfSettings.graphSettings)) {
         SPDLOG_ERROR("Graph options not initialized for speech to text.");
         return StatusCode::INTERNAL_ERROR;
@@ -405,11 +427,15 @@ node {
     }
 #endif
     // clang-format on
+    if (!writeToFile) {
+        inMemoryGraphContent = oss.str();
+        return StatusCode::OK;
+    }
     std::string fullPath = FileSystem::joinPath({directoryPath, "graph.pbtxt"});
     return FileSystem::createFileOverwrite(fullPath, oss.str());
 }
 
-static Status createImageGenerationGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings) {
+static Status createImageGenerationGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) {
     if (!std::holds_alternative<ImageGenerationGraphSettingsImpl>(hfSettings.graphSettings)) {
         SPDLOG_ERROR("Graph options not initialized for image generation.");
         return StatusCode::INTERNAL_ERROR;
@@ -489,13 +515,13 @@ node: {
 }
 )";
     // clang-format on
-    return createPbtxtFile(directoryPath, oss.str());
+    return createPbtxtFile(directoryPath, oss.str(), writeToFile);
 }
 
 GraphExport::GraphExport() {
 }
 
-Status GraphExport::createServableConfig(const std::string& directoryPath, const HFSettingsImpl& hfSettings) {
+Status GraphExport::createServableConfig(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) {
     if (directoryPath.empty()) {
         SPDLOG_ERROR("Directory path empty: {}", directoryPath);
         return StatusCode::PATH_INVALID;
@@ -518,17 +544,17 @@ Status GraphExport::createServableConfig(const std::string& directoryPath, const
         }
     }
     if (hfSettings.task == TEXT_GENERATION_GRAPH) {
-        return createTextGenerationGraphTemplate(directoryPath, hfSettings);
+        return createTextGenerationGraphTemplate(directoryPath, hfSettings, writeToFile);
     } else if (hfSettings.task == EMBEDDINGS_GRAPH) {
-        return createEmbeddingsGraphTemplate(directoryPath, hfSettings);
+        return createEmbeddingsGraphTemplate(directoryPath, hfSettings, writeToFile);
     } else if (hfSettings.task == RERANK_GRAPH) {
-        return createRerankGraphTemplate(directoryPath, hfSettings);
+        return createRerankGraphTemplate(directoryPath, hfSettings, writeToFile);
     } else if (hfSettings.task == IMAGE_GENERATION_GRAPH) {
-        return createImageGenerationGraphTemplate(directoryPath, hfSettings);
+        return createImageGenerationGraphTemplate(directoryPath, hfSettings, writeToFile);
     } else if (hfSettings.task == TEXT_TO_SPEECH_GRAPH) {
-        return createTextToSpeechGraphTemplate(directoryPath, hfSettings);
+        return createTextToSpeechGraphTemplate(directoryPath, hfSettings, writeToFile);
     } else if (hfSettings.task == SPEECH_TO_TEXT_GRAPH) {
-        return createSpeechToTextGraphTemplate(directoryPath, hfSettings);
+        return createSpeechToTextGraphTemplate(directoryPath, hfSettings, writeToFile);
     } else if (hfSettings.task == UNKNOWN_GRAPH) {
         SPDLOG_ERROR("Graph options not initialized.");
         return StatusCode::INTERNAL_ERROR;

src/graph_export/graph_export.hpp

@@ -27,9 +27,13 @@ class Status;
 class GraphExport {
 public:
     GraphExport();
-    Status createServableConfig(const std::string& directoryPath, const HFSettingsImpl& graphSettings);
+    Status createServableConfig(const std::string& directoryPath, const HFSettingsImpl& graphSettings, bool writeToFile = true);
     static std::variant<std::optional<std::string>, Status> createPluginString(const ExportSettings& exportSettings);
     static std::string getDraftModelDirectoryName(std::string draftModel);
     static std::string getDraftModelDirectoryPath(const std::string& directoryPath, const std::string& draftModel);
+
+    static bool hasInMemoryGraphContent();
+    static const std::string& getInMemoryGraphContent();
+    static void clearInMemoryGraphContent();
 };
 }  // namespace ovms

src/mediapipe_internal/BUILD

@@ -79,6 +79,7 @@ ovms_cc_library(
         "//src:libovms_servable_name_checker",
         "//src/metrics:libovms_metric_provider",
         "//src/filesystem:libovmsfilesystem",
+        "//src/graph_export:graph_export",
         "//src:libovms_version",
         "//src:libovms_execution_context",
         "//src:libovmstimer",

src/mediapipe_internal/mediapipegraphconfig.cpp

@@ -25,6 +25,7 @@
 #include <spdlog/spdlog.h>
 
 #include "src/filesystem/filesystem.hpp"
+#include "src/graph_export/graph_export.hpp"
 #include "../status.hpp"
 
 namespace ovms {
@@ -129,6 +130,10 @@ Status MediapipeGraphConfig::parseNode(const rapidjson::Value& v) {
 }
 
 void MediapipeGraphConfig::logGraphConfigContent() const {
+    if (GraphExport::hasInMemoryGraphContent()) {
+        SPDLOG_DEBUG("Content of in-memory graph config:\n{}", GraphExport::getInMemoryGraphContent());
+        return;
+    }
     std::ifstream fileStream(this->graphPath);
     if (!fileStream.is_open()) {
         SPDLOG_ERROR("Failed to open file: {}", this->graphPath);