From 2377a254f91d1ce0e46007ad7b7b818cc793c132 Mon Sep 17 00:00:00 2001 From: Alejandro Gonzalez Canton Date: Wed, 25 Jun 2025 16:38:01 +0200 Subject: [PATCH 1/4] If metadata request fail, check if file exists --- .gitignore | 1 + CMakeLists.txt | 2 ++ include/huggingface_hub.h | 5 +++-- src/huggingface_hub.cpp | 42 ++++++++++++++++++++++++++++++++------- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 633674d..f6fbf10 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ # CMake build/ +.cache/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 85a9652..1bb14ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,8 @@ cmake_minimum_required(VERSION 3.10) project(HFHub CXX) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + # Find CURL package find_package(CURL REQUIRED) diff --git a/include/huggingface_hub.h b/include/huggingface_hub.h index ed57fe5..3789574 100644 --- a/include/huggingface_hub.h +++ b/include/huggingface_hub.h @@ -32,6 +32,7 @@ #ifndef HUGGINGFACE_HUB_H #define HUGGINGFACE_HUB_H +#include #include #include #include @@ -72,9 +73,9 @@ struct DownloadResult { * @param repo The repository name or ID. * @param file The file name within the repository. * @return A variant containing either the FileMetadata structure or an error - * message string. + * code. */ -std::variant +std::variant get_model_metadata_from_hf(const std::string &repo, const std::string &file); /** diff --git a/src/huggingface_hub.cpp b/src/huggingface_hub.cpp index 6f9b4f9..9838d88 100644 --- a/src/huggingface_hub.cpp +++ b/src/huggingface_hub.cpp @@ -21,7 +21,6 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include #include #include #include @@ -30,7 +29,6 @@ #include #include -#include #include #include #include @@ -173,11 +171,11 @@ FileMetadata extract_metadata(const std::string &json) { return metadata; } -std::variant +std::variant get_model_metadata_from_hf(const std::string &repo, const std::string &file) { CURL *curl = curl_easy_init(); if (!curl) { - return "Failed to initialize CURL"; + return CURLE_FAILED_INIT; } std::string response, headers; @@ -204,7 +202,7 @@ get_model_metadata_from_hf(const std::string &repo, const std::string &file) { curl_easy_cleanup(curl); if (res != CURLE_OK) { - return "CURL request failed: " + std::string(curl_easy_strerror(res)); + return res; } return extract_metadata(response); @@ -332,8 +330,38 @@ struct DownloadResult hf_hub_download(const std::string &repo_id, // 1. Check that model exists on Hugging Face auto metadata_result = get_model_metadata_from_hf(repo_id, filename); - if (std::holds_alternative(metadata_result)) { - log_error(std::get(metadata_result)); + if (std::holds_alternative(metadata_result)) { + CURLcode err = std::get(metadata_result); + + std::string refs_main_path = "models/" + repo_id; + size_t pos = 0; + while ((pos = refs_main_path.find("/", pos)) != std::string::npos) { + refs_main_path.replace(pos, 1, "--"); + pos += 2; + } + + std::filesystem::path cache_model_dir = + expand_user_home("~/.cache/huggingface/hub/" + refs_main_path + "/"); + std::filesystem::path refs_file_path = cache_model_dir / "refs/main"; + + if (std::filesystem::exists(refs_file_path)) { + std::ifstream refs_file(refs_file_path); + std::string commit; + refs_file >> commit; + refs_file.close(); + + std::filesystem::path snapshot_file_path = + cache_model_dir / "snapshots" / commit / filename; + if (std::filesystem::exists(snapshot_file_path)) { + log_info("Snapshot file exists. Skipping download..."); + result.success = true; + result.path = snapshot_file_path; + return result; + } + } + + log_error("CURL metadata request failed: " + + std::string(curl_easy_strerror(err))); result.success = false; return result; } From aa063712436fcff0f861eaf79fa63e956e257dc9 Mon Sep 17 00:00:00 2001 From: Alejandro Gonzalez Canton Date: Fri, 27 Jun 2025 17:44:53 +0200 Subject: [PATCH 2/4] Properly commit retrieval --- include/huggingface_hub.h | 1 - src/huggingface_hub.cpp | 172 ++++++++++++++++++++++++++------------ 2 files changed, 120 insertions(+), 53 deletions(-) diff --git a/include/huggingface_hub.h b/include/huggingface_hub.h index 3789574..8fca6a0 100644 --- a/include/huggingface_hub.h +++ b/include/huggingface_hub.h @@ -45,7 +45,6 @@ namespace huggingface_hub { * This structure contains the SHA-256 hash, commit ID, and size of a file. */ struct FileMetadata { - std::string commit; /**< Commit ID of the file */ std::string type; /**< Type of the file (e.g., "model", "dataset") */ std::string oid; /**< Object ID of the file */ uint64_t size; /**< Size of the file in bytes */ diff --git a/src/huggingface_hub.cpp b/src/huggingface_hub.cpp index 9838d88..18dd83b 100644 --- a/src/huggingface_hub.cpp +++ b/src/huggingface_hub.cpp @@ -112,7 +112,7 @@ std::string create_cache_system(const std::string &cache_dir, size_t write_string_data(void *ptr, size_t size, size_t nmemb, void *stream) { if (!stream) { - log_error("Error: stream is null!"); + log_error("Stream is null!"); return 0; } std::string *out = static_cast(stream); @@ -122,12 +122,12 @@ size_t write_string_data(void *ptr, size_t size, size_t nmemb, void *stream) { size_t write_file_data(void *ptr, size_t size, size_t nmemb, void *stream) { if (!stream) { - log_error("Error: stream is null!"); + log_error("Stream is null!"); return 0; } std::ofstream *out = static_cast(stream); if (!out->is_open()) { - log_error("Error: output file stream is not open!"); + log_error("Output file stream is not open!"); return 0; } out->write(static_cast(ptr), size * nmemb); @@ -161,16 +161,74 @@ FileMetadata extract_metadata(const std::string &json) { R"(\"lfs\"\s*:\s*\{[^}]*\"oid\"\s*:\s*\"([a-f0-9]{64})\")"))) metadata.sha256 = match[1]; - // Extract "commit" ID - if (std::regex_search( - json, match, - std::regex( - R"(\"lastCommit\"\s*:\s*\{[^}]*\"id\"\s*:\s*\"([a-f0-9]{40})\")"))) - metadata.commit = match[1]; - return metadata; } +std::string get_file_path(const std::string &cache_dir, + const std::string &repo_id, const std::string &file) { + std::string model_folder = "models/" + repo_id; + + size_t pos = 0; + while ((pos = model_folder.find("/", pos)) != std::string::npos) { + model_folder.replace(pos, 1, "--"); + pos += 2; + } + + std::filesystem::path expanded_cache_dir = expand_user_home(cache_dir); + std::filesystem::path refs_file_path = + expanded_cache_dir / model_folder / "refs" / "main"; + + if (!std::filesystem::exists(refs_file_path)) { + log_info("refs file does not exist"); + return ""; + } + std::ifstream refs_file(refs_file_path); + std::string commit; + refs_file >> commit; + refs_file.close(); + std::filesystem::path snapshot_file_path = + expanded_cache_dir / model_folder / "snapshots" / commit / file; + if (std::filesystem::exists(snapshot_file_path)) { + return snapshot_file_path.string(); + } else { + return ""; // File does not exist + } +} + +std::variant get_model_commit(const std::string &repo) { + CURL *curl = curl_easy_init(); + if (!curl) { + return CURLE_FAILED_INIT; + } + + std::string url = + "https://huggingface.co/api/models/" + repo + "/revision/main"; + std::string response; + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_string_data); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(curl, CURLOPT_HEADER, 0L); + + CURLcode res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + if (res != CURLE_OK) { + return res; + } + + std::smatch match; + std::regex pattern("\"sha\"\\s*:\\s*\"([a-fA-F0-9]{40})\""); + + if (std::regex_search(response, match, pattern) && match.size() > 1) { + return match[1]; + } else { + return std::string(); // Return empty string if not found + } +} + std::variant get_model_metadata_from_hf(const std::string &repo, const std::string &file) { CURL *curl = curl_easy_init(); @@ -205,6 +263,10 @@ get_model_metadata_from_hf(const std::string &repo, const std::string &file) { return res; } + if (response.empty() || response == "[]") { + return CURLE_REMOTE_FILE_NOT_FOUND; + } + return extract_metadata(response); } @@ -287,7 +349,6 @@ CURLcode perform_download(std::string url, std::ios::binary | std::ios::app); if (!file.is_open()) { - log_error("Error: failed to open file stream!"); return CURLE_FAILED_INIT; } @@ -328,51 +389,65 @@ struct DownloadResult hf_hub_download(const std::string &repo_id, struct DownloadResult result; result.success = true; - // 1. Check that model exists on Hugging Face - auto metadata_result = get_model_metadata_from_hf(repo_id, filename); - if (std::holds_alternative(metadata_result)) { - CURLcode err = std::get(metadata_result); - - std::string refs_main_path = "models/" + repo_id; - size_t pos = 0; - while ((pos = refs_main_path.find("/", pos)) != std::string::npos) { - refs_main_path.replace(pos, 1, "--"); - pos += 2; - } + log_info("Downloading " + filename + " from " + repo_id); - std::filesystem::path cache_model_dir = - expand_user_home("~/.cache/huggingface/hub/" + refs_main_path + "/"); - std::filesystem::path refs_file_path = cache_model_dir / "refs/main"; + // Check repo (accessibility and version) + auto commit_result = get_model_commit(repo_id); - if (std::filesystem::exists(refs_file_path)) { - std::ifstream refs_file(refs_file_path); - std::string commit; - refs_file >> commit; - refs_file.close(); + if (std::holds_alternative(commit_result)) { + CURLcode err = std::get(commit_result); - std::filesystem::path snapshot_file_path = - cache_model_dir / "snapshots" / commit / filename; - if (std::filesystem::exists(snapshot_file_path)) { - log_info("Snapshot file exists. Skipping download..."); + if (err == CURLE_COULDNT_RESOLVE_HOST || + err == CURLE_COULDNT_CONNECT) { // OFFLINE MODE + std::string file_path = get_file_path(cache_dir, repo_id, filename); + if (!file_path.empty()) { + log_info("No connection. Using cached file."); + result.path = file_path; result.success = true; - result.path = snapshot_file_path; + return result; + } else { + log_error("Could not resolve host or connect to Hugging Face. " + "Please check your internet connection."); + result.success = false; return result; } + + } else if (err == CURLE_HTTP_RETURNED_ERROR) { // REPOSITORY NOT FOUND + log_error("Repository not found: " + repo_id); + result.success = false; + return result; + } else { + log_error("Error getting model: " + std::string(curl_easy_strerror(err))); + result.success = false; + return result; } + } - log_error("CURL metadata request failed: " + + std::string latest_commit = std::get(commit_result); + if (latest_commit.empty()) { + log_error("Failed to retrieve the latest commit for repository: " + + repo_id); + result.success = false; + return result; + } + + // Check file accessibility + auto metadata_result = get_model_metadata_from_hf(repo_id, filename); + + if (std::holds_alternative(metadata_result)) { + CURLcode err = std::get(metadata_result); + log_error("Error getting metadata: " + std::string(curl_easy_strerror(err))); result.success = false; return result; } - // 2. Create Cache Dir Struct + // Create Cache Dir Struct std::string cache_model_dir = create_cache_system(cache_dir, repo_id); log_debug("Cache directory: " + cache_model_dir); - log_info("Downloading " + filename + " from " + repo_id); struct FileMetadata metadata = std::get(metadata_result); - log_debug("Commit: " + metadata.commit); + log_debug("Commit: " + latest_commit); log_debug("Blob ID: " + metadata.oid); log_debug("Size: " + std::to_string(metadata.size) + " bytes"); log_debug("SHA256: " + metadata.sha256); @@ -391,29 +466,22 @@ struct DownloadResult hf_hub_download(const std::string &repo_id, } std::filesystem::path snapshot_file_path(cache_model_dir + "snapshots/" + - metadata.commit + "/" + filename); + latest_commit + "/" + filename); std::filesystem::path refs_file_path(cache_model_dir + "refs/main"); result.path = snapshot_file_path; + std::ofstream refs_file(refs_file_path); + refs_file << latest_commit << std::endl; + refs_file.close(); + if (std::filesystem::exists(snapshot_file_path) && std::filesystem::exists(blob_file_path) && !force_download) { log_info("Snapshot file exists. Skipping download..."); return result; } - if (std::filesystem::exists(refs_file_path)) { - std::ifstream refs_file(refs_file_path); - std::string commit; - refs_file >> commit; - refs_file.close(); - } else { - std::ofstream refs_file(refs_file_path); - refs_file << metadata.commit; - refs_file.close(); - } - - // 3. Download the file + // 4. Download the file std::string url = "https://huggingface.co/" + repo_id + "/resolve/main/" + filename; std::filesystem::create_directories(snapshot_file_path.parent_path()); From 226dfa14fbf65d6591e2f14275ca57544bd02db4 Mon Sep 17 00:00:00 2001 From: Alejandro Gonzalez Canton Date: Fri, 27 Jun 2025 18:01:52 +0200 Subject: [PATCH 3/4] Allow the usage of outdated models if no connection --- src/huggingface_hub.cpp | 45 ++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/src/huggingface_hub.cpp b/src/huggingface_hub.cpp index 18dd83b..91d6cb9 100644 --- a/src/huggingface_hub.cpp +++ b/src/huggingface_hub.cpp @@ -85,9 +85,8 @@ std::filesystem::path expand_user_home(const std::string &path) { return std::filesystem::path(path); } -std::string create_cache_system(const std::string &cache_dir, - const std::string &repo_id) { - std::string model_folder = std::string("models/" + repo_id); +std::string get_model_repo_path(const std::string &repo_id) { + std::string model_folder = "models/" + repo_id; size_t pos = 0; while ((pos = model_folder.find("/", pos)) != std::string::npos) { @@ -95,6 +94,13 @@ std::string create_cache_system(const std::string &cache_dir, pos += 2; } + return model_folder; +} + +std::string create_cache_system(const std::string &cache_dir, + const std::string &repo_id) { + std::string model_folder = get_model_repo_path(repo_id); + std::string expanded_cache_dir = expand_user_home(cache_dir); std::string model_cache_path = expanded_cache_dir + "/" + model_folder + "/"; @@ -166,13 +172,7 @@ FileMetadata extract_metadata(const std::string &json) { std::string get_file_path(const std::string &cache_dir, const std::string &repo_id, const std::string &file) { - std::string model_folder = "models/" + repo_id; - - size_t pos = 0; - while ((pos = model_folder.find("/", pos)) != std::string::npos) { - model_folder.replace(pos, 1, "--"); - pos += 2; - } + std::string model_folder = get_model_repo_path(repo_id); std::filesystem::path expanded_cache_dir = expand_user_home(cache_dir); std::filesystem::path refs_file_path = @@ -406,12 +406,33 @@ struct DownloadResult hf_hub_download(const std::string &repo_id, result.success = true; return result; } else { + std::string model_path = get_model_repo_path(repo_id); + bool file_found = false; + for (const auto &version : std::filesystem::directory_iterator( + expand_user_home(cache_dir + "/" + model_path + "/snapshots"))) { + for (const auto &file : + std::filesystem::directory_iterator(version.path())) { + if (file.path().filename() == filename) { + result.path = file.path(); + file_found = true; + break; + } + } + if (file_found) { + break; + } + } + if (file_found) { + log_info("No connection. Using outdated cached file: " + result.path); + result.success = true; + return result; + } + log_error("Could not resolve host or connect to Hugging Face. " "Please check your internet connection."); result.success = false; return result; } - } else if (err == CURLE_HTTP_RETURNED_ERROR) { // REPOSITORY NOT FOUND log_error("Repository not found: " + repo_id); result.success = false; @@ -477,7 +498,7 @@ struct DownloadResult hf_hub_download(const std::string &repo_id, if (std::filesystem::exists(snapshot_file_path) && std::filesystem::exists(blob_file_path) && !force_download) { - log_info("Snapshot file exists. Skipping download..."); + log_info("Snapshot file exists. Using cached file."); return result; } From 88a97722d887cc085529eadad23f9d2013480146 Mon Sep 17 00:00:00 2001 From: Alejandro Gonzalez Canton Date: Fri, 27 Jun 2025 18:27:44 +0200 Subject: [PATCH 4/4] Refactor for offline models --- src/huggingface_hub.cpp | 83 ++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/src/huggingface_hub.cpp b/src/huggingface_hub.cpp index 91d6cb9..77c9187 100644 --- a/src/huggingface_hub.cpp +++ b/src/huggingface_hub.cpp @@ -97,6 +97,21 @@ std::string get_model_repo_path(const std::string &repo_id) { return model_folder; } +std::string find_outdated_file(const std::string &snapshot_dir, + const std::string &filename) { + for (const auto &version : + std::filesystem::directory_iterator(snapshot_dir)) { + for (const auto &file : + std::filesystem::directory_iterator(version.path())) { + if (file.path().filename() == filename) { + return file.path(); + break; + } + } + } + return ""; +} + std::string create_cache_system(const std::string &cache_dir, const std::string &repo_id) { std::string model_folder = get_model_repo_path(repo_id); @@ -179,7 +194,7 @@ std::string get_file_path(const std::string &cache_dir, expanded_cache_dir / model_folder / "refs" / "main"; if (!std::filesystem::exists(refs_file_path)) { - log_info("refs file does not exist"); + log_debug("refs file does not exist"); return ""; } std::ifstream refs_file(refs_file_path); @@ -397,51 +412,35 @@ struct DownloadResult hf_hub_download(const std::string &repo_id, if (std::holds_alternative(commit_result)) { CURLcode err = std::get(commit_result); - if (err == CURLE_COULDNT_RESOLVE_HOST || - err == CURLE_COULDNT_CONNECT) { // OFFLINE MODE - std::string file_path = get_file_path(cache_dir, repo_id, filename); - if (!file_path.empty()) { - log_info("No connection. Using cached file."); - result.path = file_path; - result.success = true; - return result; - } else { - std::string model_path = get_model_repo_path(repo_id); - bool file_found = false; - for (const auto &version : std::filesystem::directory_iterator( - expand_user_home(cache_dir + "/" + model_path + "/snapshots"))) { - for (const auto &file : - std::filesystem::directory_iterator(version.path())) { - if (file.path().filename() == filename) { - result.path = file.path(); - file_found = true; - break; - } - } - if (file_found) { - break; - } - } - if (file_found) { - log_info("No connection. Using outdated cached file: " + result.path); - result.success = true; - return result; - } - - log_error("Could not resolve host or connect to Hugging Face. " - "Please check your internet connection."); - result.success = false; - return result; - } - } else if (err == CURLE_HTTP_RETURNED_ERROR) { // REPOSITORY NOT FOUND - log_error("Repository not found: " + repo_id); - result.success = false; + std::string file_path = get_file_path(cache_dir, repo_id, filename); + if (!file_path.empty()) { + log_info("Using cached file."); + result.path = file_path; + result.success = true; return result; - } else { - log_error("Error getting model: " + std::string(curl_easy_strerror(err))); + } + + std::string model_path = get_model_repo_path(repo_id); + std::string snapshot_path = + expand_user_home(cache_dir + "/" + model_path + "/snapshots"); + if (!std::filesystem::exists(snapshot_path)) { + log_info(snapshot_path); + log_error("Repo not found (locally nor online): " + repo_id); result.success = false; return result; } + + std::string outdated_file = find_outdated_file(snapshot_path, filename); + if (!outdated_file.empty()) { + log_info("Using outdated cached file " + outdated_file); + result.path = outdated_file; + result.success = true; + return result; + } + + log_error("Error getting model: " + std::string(curl_easy_strerror(err))); + result.success = false; + return result; } std::string latest_commit = std::get(commit_result);