|
5 | 5 | """ |
6 | 6 |
|
7 | 7 | import os |
| 8 | +import posixpath |
8 | 9 | import re |
9 | 10 | import threading |
10 | 11 | import time |
@@ -331,7 +332,214 @@ def parse_huggingface_url(url: str) -> Optional[Dict[str, str]]: |
331 | 332 |
|
332 | 333 | def get_huggingface_download_url(repo: str, filename: str, branch: str = "main") -> str: |
333 | 334 | """Generate a direct download URL for a HuggingFace file.""" |
334 | | - return f"https://huggingface.co/{repo}/resolve/{branch}/{quote(filename)}" |
| 335 | + return ( |
| 336 | + f"https://huggingface.co/{repo}/resolve/" |
| 337 | + f"{quote(branch, safe='')}/{quote(filename)}" |
| 338 | + ) |
| 339 | + |
| 340 | + |
| 341 | +def _normalize_huggingface_path(value: Any) -> str: |
| 342 | + return str(value or "").replace("\\", "/").strip("/") |
| 343 | + |
| 344 | + |
| 345 | +def _get_huggingface_variant_label(filename: str) -> str: |
| 346 | + """Return a concise precision/quantization label inferred from a filename.""" |
| 347 | + stem = os.path.splitext(get_filename_from_path(filename))[0].lower() |
| 348 | + patterns = ( |
| 349 | + ("nvfp4_mixed", "NVFP4 mixed"), |
| 350 | + ("fp8_scaled", "FP8 scaled"), |
| 351 | + ("int8_convrot", "INT8 convrot"), |
| 352 | + ("nvfp4", "NVFP4"), |
| 353 | + ("fp8", "FP8"), |
| 354 | + ("bf16", "BF16"), |
| 355 | + ("fp16", "FP16"), |
| 356 | + ("int8", "INT8"), |
| 357 | + ("int4", "INT4"), |
| 358 | + ) |
| 359 | + for token, label in patterns: |
| 360 | + if re.search(rf"(?:^|[_.-]){re.escape(token)}(?:$|[_.-])", stem): |
| 361 | + return label |
| 362 | + return "" |
| 363 | + |
| 364 | + |
| 365 | +def _normalize_huggingface_details_file( |
| 366 | + repo_id: str, |
| 367 | + branch: str, |
| 368 | + file_info: Dict[str, Any], |
| 369 | + target_path: str, |
| 370 | +) -> Optional[Dict[str, Any]]: |
| 371 | + file_path = _normalize_huggingface_path(file_info.get("path")) |
| 372 | + if not file_path: |
| 373 | + return None |
| 374 | + |
| 375 | + download_url = get_huggingface_download_url(repo_id, file_path, branch) |
| 376 | + if not looks_like_model_file(download_url): |
| 377 | + return None |
| 378 | + |
| 379 | + filename = get_filename_from_path(file_path) |
| 380 | + sha256 = _extract_huggingface_file_sha256(file_info) |
| 381 | + extension = os.path.splitext(filename)[1].lstrip(".") |
| 382 | + metadata = { |
| 383 | + "format": extension.upper() if extension else "", |
| 384 | + "fp": _get_huggingface_variant_label(filename), |
| 385 | + } |
| 386 | + metadata = {key: value for key, value in metadata.items() if value} |
| 387 | + page_url = ( |
| 388 | + f"https://huggingface.co/{repo_id}/blob/" |
| 389 | + f"{quote(branch, safe='')}/{quote(file_path, safe='/')}" |
| 390 | + ) |
| 391 | + |
| 392 | + return { |
| 393 | + "name": filename, |
| 394 | + "filename": filename, |
| 395 | + "path": file_path, |
| 396 | + "size": extract_file_size(file_info), |
| 397 | + "sha256": sha256, |
| 398 | + "hash": sha256, |
| 399 | + "hashes": {"SHA256": sha256} if sha256 else {}, |
| 400 | + "download_url": download_url, |
| 401 | + "url": page_url, |
| 402 | + "primary": bool(target_path and file_path == target_path), |
| 403 | + "metadata": metadata, |
| 404 | + } |
| 405 | + |
| 406 | + |
| 407 | +def get_huggingface_model_details( |
| 408 | + repo_id: str, |
| 409 | + file_path: str = "", |
| 410 | + branch: str = "main", |
| 411 | + token: Optional[str] = None, |
| 412 | +) -> Optional[Dict[str, Any]]: |
| 413 | + """Fetch model-file variants from the folder containing a matched HF file.""" |
| 414 | + normalized_repo_id = str(repo_id or "").strip().strip("/") |
| 415 | + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", normalized_repo_id): |
| 416 | + return None |
| 417 | + |
| 418 | + normalized_branch = str(branch or "main").strip() or "main" |
| 419 | + if len(normalized_branch) > 200 or any( |
| 420 | + character in normalized_branch for character in "\r\n?#" |
| 421 | + ): |
| 422 | + return None |
| 423 | + |
| 424 | + target_path = _normalize_huggingface_path(file_path) |
| 425 | + target_directory = posixpath.dirname(target_path) |
| 426 | + headers = {"Authorization": f"Bearer {token.strip()}"} if token and token.strip() else {} |
| 427 | + repo_tree = _get_repo_tree( |
| 428 | + normalized_repo_id, |
| 429 | + headers=headers, |
| 430 | + branch=normalized_branch, |
| 431 | + ) |
| 432 | + if not repo_tree: |
| 433 | + return None |
| 434 | + |
| 435 | + normalized_files = [] |
| 436 | + for file_info in repo_tree: |
| 437 | + candidate_path = _normalize_huggingface_path(file_info.get("path")) |
| 438 | + if not candidate_path: |
| 439 | + continue |
| 440 | + if target_path and posixpath.dirname(candidate_path) != target_directory: |
| 441 | + continue |
| 442 | + |
| 443 | + normalized_file = _normalize_huggingface_details_file( |
| 444 | + normalized_repo_id, |
| 445 | + normalized_branch, |
| 446 | + file_info, |
| 447 | + target_path, |
| 448 | + ) |
| 449 | + if normalized_file: |
| 450 | + normalized_files.append(normalized_file) |
| 451 | + |
| 452 | + if not normalized_files: |
| 453 | + return None |
| 454 | + |
| 455 | + normalized_files.sort( |
| 456 | + key=lambda item: ( |
| 457 | + not item.get("primary", False), |
| 458 | + str(item.get("name") or "").lower(), |
| 459 | + ) |
| 460 | + ) |
| 461 | + |
| 462 | + model_data: Dict[str, Any] = {} |
| 463 | + try: |
| 464 | + response_data = execute_provider_json_request( |
| 465 | + "HuggingFace model details", |
| 466 | + f"{HF_API_URL}/models/{normalized_repo_id}", |
| 467 | + headers=headers, |
| 468 | + timeout=15, |
| 469 | + ) |
| 470 | + if isinstance(response_data, dict): |
| 471 | + model_data = response_data |
| 472 | + except Exception as e: |
| 473 | + log.debug( |
| 474 | + f"Error getting HuggingFace model metadata for {normalized_repo_id}: {e}" |
| 475 | + ) |
| 476 | + |
| 477 | + card_data = ( |
| 478 | + model_data.get("cardData") |
| 479 | + if isinstance(model_data.get("cardData"), dict) |
| 480 | + else {} |
| 481 | + ) |
| 482 | + tags = model_data.get("tags") if isinstance(model_data.get("tags"), list) else [] |
| 483 | + repo_url = f"https://huggingface.co/{normalized_repo_id}" |
| 484 | + tree_url = ( |
| 485 | + f"{repo_url}/tree/{quote(normalized_branch, safe='')}" |
| 486 | + + ( |
| 487 | + f"/{quote(target_directory, safe='/')}" |
| 488 | + if target_directory |
| 489 | + else "" |
| 490 | + ) |
| 491 | + ) |
| 492 | + version = { |
| 493 | + "id": normalized_branch, |
| 494 | + "name": normalized_branch, |
| 495 | + "published_at": model_data.get("createdAt") or "", |
| 496 | + "updated_at": model_data.get("lastModified") or "", |
| 497 | + "description": "", |
| 498 | + "trained_words": [], |
| 499 | + "stats": { |
| 500 | + "downloads": model_data.get("downloads", 0), |
| 501 | + "thumbsUpCount": model_data.get("likes", 0), |
| 502 | + }, |
| 503 | + "files": normalized_files, |
| 504 | + "images": [], |
| 505 | + "url": tree_url, |
| 506 | + } |
| 507 | + owner = normalized_repo_id.split("/", 1)[0] |
| 508 | + description = ( |
| 509 | + model_data.get("description") |
| 510 | + or card_data.get("description") |
| 511 | + or ( |
| 512 | + f"Select a model file from `{target_directory}` in this Hugging Face " |
| 513 | + "repository." |
| 514 | + if target_directory |
| 515 | + else "Select a model file from this Hugging Face repository." |
| 516 | + ) |
| 517 | + ) |
| 518 | + |
| 519 | + return { |
| 520 | + "source": "huggingface", |
| 521 | + "details_source": "huggingface", |
| 522 | + "model_id": normalized_repo_id, |
| 523 | + "version_id": normalized_branch, |
| 524 | + "name": model_data.get("modelId") or normalized_repo_id, |
| 525 | + "type": model_data.get("pipeline_tag") |
| 526 | + or model_data.get("library_name") |
| 527 | + or "Model repository", |
| 528 | + "description": description, |
| 529 | + "creator": {"username": model_data.get("author") or owner}, |
| 530 | + "stats": { |
| 531 | + "downloads": model_data.get("downloads", 0), |
| 532 | + "thumbsUpCount": model_data.get("likes", 0), |
| 533 | + }, |
| 534 | + "tags": tags, |
| 535 | + "url": repo_url, |
| 536 | + "version_url": tree_url, |
| 537 | + "folder": target_directory, |
| 538 | + "branch": normalized_branch, |
| 539 | + "versions": [version], |
| 540 | + "selected_version": version, |
| 541 | + "images": [], |
| 542 | + } |
335 | 543 |
|
336 | 544 |
|
337 | 545 |
|
@@ -381,6 +589,7 @@ def _build_huggingface_result( |
381 | 589 | sha256=sha256, |
382 | 590 | repo_id=repo_id, |
383 | 591 | path=file_path, |
| 592 | + branch="main", |
384 | 593 | ) |
385 | 594 |
|
386 | 595 |
|
@@ -1077,6 +1286,7 @@ def quote_url_path(val): |
1077 | 1286 | repo_id=repo_id, |
1078 | 1287 | repo=repo_id, |
1079 | 1288 | path=file_path, |
| 1289 | + branch=branch, |
1080 | 1290 | page_url=page_url, |
1081 | 1291 | version_url=page_url, |
1082 | 1292 | custom_url=True, |
|
0 commit comments