|
| 1 | +use colored::Colorize; |
| 2 | +use log::{debug, info}; |
| 3 | +use std::path::PathBuf; |
| 4 | +use std::sync::Arc; |
| 5 | + |
| 6 | +use hf_hub::api::tokio::{ApiBuilder, Progress}; |
| 7 | +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; |
| 8 | + |
| 9 | +use crate::downloader::downloader::{DownloadError, Downloader}; |
| 10 | + |
| 11 | +#[derive(Clone)] |
| 12 | +struct FileProgressBar { |
| 13 | + pb: ProgressBar, |
| 14 | +} |
| 15 | + |
| 16 | +impl Progress for FileProgressBar { |
| 17 | + async fn init(&mut self, size: usize, _filename: &str) { |
| 18 | + self.pb.set_length(size as u64); |
| 19 | + self.pb.reset(); |
| 20 | + self.pb.tick(); // Force render with correct size |
| 21 | + } |
| 22 | + |
| 23 | + async fn update(&mut self, size: usize) { |
| 24 | + self.pb.inc(size as u64); |
| 25 | + } |
| 26 | + |
| 27 | + async fn finish(&mut self) {} |
| 28 | +} |
| 29 | + |
| 30 | +pub struct HuggingFaceDownloader; |
| 31 | + |
| 32 | +impl HuggingFaceDownloader { |
| 33 | + pub fn new() -> Self { |
| 34 | + Self |
| 35 | + } |
| 36 | +} |
| 37 | + |
| 38 | +impl Default for HuggingFaceDownloader { |
| 39 | + fn default() -> Self { |
| 40 | + Self::new() |
| 41 | + } |
| 42 | +} |
| 43 | + |
| 44 | +impl Downloader for HuggingFaceDownloader { |
| 45 | + async fn download_model(&self, name: &str, cache_dir: &PathBuf) -> Result<(), DownloadError> { |
| 46 | + let start_time = std::time::Instant::now(); |
| 47 | + |
| 48 | + info!("Downloading model {} from Hugging Face...", name); |
| 49 | + |
| 50 | + // Build API without default progress bars (we have our own implementation) |
| 51 | + let api = if cache_dir.as_os_str().is_empty() { |
| 52 | + // Use default HF cache |
| 53 | + ApiBuilder::new().build().map_err(|e| { |
| 54 | + DownloadError::ApiError(format!("Failed to initialize Hugging Face API: {}", e)) |
| 55 | + })? |
| 56 | + } else { |
| 57 | + // Use custom cache directory |
| 58 | + ApiBuilder::new() |
| 59 | + .with_cache_dir(cache_dir.clone()) |
| 60 | + .build() |
| 61 | + .map_err(|e| { |
| 62 | + DownloadError::ApiError(format!( |
| 63 | + "Failed to initialize Hugging Face API with custom cache: {}", |
| 64 | + e |
| 65 | + )) |
| 66 | + })? |
| 67 | + }; |
| 68 | + |
| 69 | + // Download the entire model repository using snapshot download |
| 70 | + let repo = api.model(name.to_string()); |
| 71 | + |
| 72 | + // Get model info to list all files |
| 73 | + let model_info = repo.info().await.map_err(|e| { |
| 74 | + let err_str = e.to_string(); |
| 75 | + if err_str.contains("404") || err_str.contains("not found") { |
| 76 | + DownloadError::ModelNotFound(format!("Model '{}' not found", name)) |
| 77 | + } else if err_str.contains("401") || err_str.contains("403") { |
| 78 | + DownloadError::AuthError(format!("Authentication failed: {}", e)) |
| 79 | + } else if err_str.contains("network") || err_str.contains("connection") { |
| 80 | + DownloadError::NetworkError(format!("Network error: {}", e)) |
| 81 | + } else { |
| 82 | + DownloadError::ApiError(format!("Failed to fetch model info: {}", e)) |
| 83 | + } |
| 84 | + })?; |
| 85 | + |
| 86 | + debug!("Model info for {}: {:?}", name, model_info); |
| 87 | + |
| 88 | + // Create multi-progress for parallel downloads |
| 89 | + let multi_progress = Arc::new(MultiProgress::new()); |
| 90 | + |
| 91 | + // Progress bar style with block characters (chart-like, not #) |
| 92 | + let style = ProgressStyle::default_bar() |
| 93 | + .template("{msg:<30} [{elapsed_precise}] {bar:60.white} {bytes}/{total_bytes}") |
| 94 | + .unwrap() |
| 95 | + .progress_chars("▇▆▅▄▃▂▁ "); |
| 96 | + |
| 97 | + // Download all files in parallel |
| 98 | + let mut tasks = Vec::new(); |
| 99 | + |
| 100 | + for sibling in model_info.siblings { |
| 101 | + let api_clone = api.clone(); |
| 102 | + let model_name = name.to_string(); |
| 103 | + let filename = sibling.rfilename.clone(); |
| 104 | + |
| 105 | + let pb = multi_progress.add(ProgressBar::hidden()); |
| 106 | + pb.set_style(style.clone()); |
| 107 | + pb.set_message(filename.clone()); |
| 108 | + |
| 109 | + let task = tokio::spawn(async move { |
| 110 | + debug!("Downloading: {}", filename); |
| 111 | + |
| 112 | + let repo = api_clone.model(model_name); |
| 113 | + let progress = FileProgressBar { pb: pb.clone() }; |
| 114 | + |
| 115 | + let result = repo.download_with_progress(&filename, progress).await; |
| 116 | + |
| 117 | + match &result { |
| 118 | + Ok(_) => { |
| 119 | + pb.finish(); |
| 120 | + } |
| 121 | + Err(_) => { |
| 122 | + pb.abandon(); |
| 123 | + } |
| 124 | + } |
| 125 | + |
| 126 | + result.map_err(|e| { |
| 127 | + DownloadError::NetworkError(format!("Failed to download {}: {}", filename, e)) |
| 128 | + }) |
| 129 | + }); |
| 130 | + |
| 131 | + tasks.push(task); |
| 132 | + } |
| 133 | + |
| 134 | + // Wait for all downloads to complete |
| 135 | + for task in tasks { |
| 136 | + task.await |
| 137 | + .map_err(|e| DownloadError::ApiError(format!("Task join error: {}", e)))??; |
| 138 | + } |
| 139 | + |
| 140 | + let elapsed_time = start_time.elapsed(); |
| 141 | + |
| 142 | + println!( |
| 143 | + "\n{} {} {} {} {:.2?}", |
| 144 | + "✓".green().bold(), |
| 145 | + "Successfully downloaded model".bright_white(), |
| 146 | + name.cyan().bold(), |
| 147 | + "in".bright_white(), |
| 148 | + elapsed_time |
| 149 | + ); |
| 150 | + |
| 151 | + Ok(()) |
| 152 | + } |
| 153 | +} |
| 154 | + |
| 155 | +#[cfg(test)] |
| 156 | +mod tests { |
| 157 | + use super::*; |
| 158 | + |
| 159 | + #[tokio::test] |
| 160 | + async fn test_download_model_invalid() { |
| 161 | + let downloader = HuggingFaceDownloader::new(); |
| 162 | + let result = downloader |
| 163 | + .download_model("invalid-model-that-does-not-exist-12345", &PathBuf::new()) |
| 164 | + .await; |
| 165 | + assert!(result.is_err()); |
| 166 | + } |
| 167 | + |
| 168 | + #[tokio::test] |
| 169 | + async fn test_download_real_tiny_model() { |
| 170 | + let downloader = HuggingFaceDownloader::new(); |
| 171 | + // Use HF's official tiny test model (only a few KB) |
| 172 | + let result = downloader |
| 173 | + .download_model("InftyAI/tiny-random-gpt2", &PathBuf::new()) |
| 174 | + .await; |
| 175 | + assert!( |
| 176 | + result.is_ok(), |
| 177 | + "Failed to download tiny model: {:?}", |
| 178 | + result |
| 179 | + ); |
| 180 | + |
| 181 | + // Cleanup: remove the downloaded files from the default HF cache (~/.cache/huggingface/hub) |
| 182 | + if let Some(home_dir) = dirs::home_dir() { |
| 183 | + let cache_dir = home_dir |
| 184 | + .join(".cache") |
| 185 | + .join("huggingface") |
| 186 | + .join("hub") |
| 187 | + .join("models--InftyAI--tiny-random-gpt2"); |
| 188 | + |
| 189 | + if cache_dir.exists() { |
| 190 | + let _ = std::fs::remove_dir_all(&cache_dir); |
| 191 | + } |
| 192 | + } |
| 193 | + } |
| 194 | + |
| 195 | + #[tokio::test] |
| 196 | + async fn test_download_with_custom_cache() { |
| 197 | + use std::env; |
| 198 | + use std::fs; |
| 199 | + |
| 200 | + let downloader = HuggingFaceDownloader::new(); |
| 201 | + let temp_dir = env::temp_dir().join("puma_test_cache"); |
| 202 | + |
| 203 | + print!("Using temporary cache directory: {:?}\n", temp_dir); |
| 204 | + |
| 205 | + // Create the directory first |
| 206 | + fs::create_dir_all(&temp_dir).unwrap(); |
| 207 | + |
| 208 | + let result = downloader |
| 209 | + .download_model("InftyAI/tiny-random-gpt2", &temp_dir) |
| 210 | + .await; |
| 211 | + |
| 212 | + assert!( |
| 213 | + result.is_ok(), |
| 214 | + "Failed to download with custom cache: {:?}", |
| 215 | + result |
| 216 | + ); |
| 217 | + |
| 218 | + // Cleanup |
| 219 | + let _ = std::fs::remove_dir_all(&temp_dir); |
| 220 | + } |
| 221 | +} |
0 commit comments