|
429 | 429 | - filename: llama-cpp/models/Qwen3-4B-DFlash-GGUF-Test/Qwen3-4B-DFlash-q8_0.gguf |
430 | 430 | sha256: 5ecf02bb269fc42277f43961794387fea11ecc367ea2d99e86b2b71cc249aff6 |
431 | 431 | uri: https://huggingface.co/lym00/Qwen3-4B-DFlash-GGUF-Test/resolve/main/Qwen3-4B-DFlash-q8_0.gguf |
| 432 | +- name: "qwen3.5-4b-dflash" |
| 433 | + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" |
| 434 | + urls: |
| 435 | + - https://huggingface.co/Qwen/Qwen3.5-4B |
| 436 | + - https://huggingface.co/unsloth/Qwen3.5-4B-GGUF |
| 437 | + - https://huggingface.co/z-lab/Qwen3.5-4B-DFlash |
| 438 | + - https://huggingface.co/AtomicChat/Qwen3.5-4B-DFlash-GGUF |
| 439 | + description: | |
| 440 | + Qwen3.5-4B paired with its DFlash block-diffusion drafter for speculative decoding on the llama.cpp backend. |
| 441 | + |
| 442 | + DFlash produces a whole block of draft tokens in a single forward pass and injects the target model's hidden states into the drafter's attention, which keeps the drafter tiny while making drafting GPU-friendly. The Q4_K_M file carries the full Qwen3.5-4B target; the ~0.6 GB Q8_0 drafter (`draft-dflash`) accelerates generation without changing the target's outputs. The drafter is not a standalone chat model: it only runs paired with the target, which is why both are bundled here. |
| 443 | + |
| 444 | + Flash attention is required for DFlash and is enabled in this config. A GPU is recommended. |
| 445 | + |
| 446 | + License: Apache 2.0 (Qwen3.5-4B target) / MIT (z-lab DFlash drafter). |
| 447 | + license: "apache-2.0" |
| 448 | + tags: |
| 449 | + - llm |
| 450 | + - gguf |
| 451 | + - gpu |
| 452 | + - qwen |
| 453 | + - dflash |
| 454 | + - speculative |
| 455 | + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg |
| 456 | + overrides: |
| 457 | + backend: llama-cpp |
| 458 | + flash_attention: "on" |
| 459 | + draft_model: llama-cpp/models/Qwen3.5-4B-DFlash-GGUF/Qwen3.5-4B-DFlash.Q8_0.gguf |
| 460 | + function: |
| 461 | + automatic_tool_parsing_fallback: true |
| 462 | + grammar: |
| 463 | + disable: true |
| 464 | + known_usecases: |
| 465 | + - chat |
| 466 | + options: |
| 467 | + - use_jinja:true |
| 468 | + - spec_type:draft-dflash |
| 469 | + - spec_n_max:15 |
| 470 | + parameters: |
| 471 | + model: llama-cpp/models/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q4_K_M.gguf |
| 472 | + template: |
| 473 | + use_tokenizer_template: true |
| 474 | + files: |
| 475 | + - filename: llama-cpp/models/Qwen3.5-4B-GGUF/Qwen3.5-4B-Q4_K_M.gguf |
| 476 | + sha256: 00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4 |
| 477 | + uri: https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/Qwen3.5-4B-Q4_K_M.gguf |
| 478 | + - filename: llama-cpp/models/Qwen3.5-4B-DFlash-GGUF/Qwen3.5-4B-DFlash.Q8_0.gguf |
| 479 | + sha256: f0689eadab0d46468bc2629bb2706f4aba7f794b63e1c9f6eb291269d572a52b |
| 480 | + uri: https://huggingface.co/AtomicChat/Qwen3.5-4B-DFlash-GGUF/resolve/main/Qwen3.5-4B-DFlash.Q8_0.gguf |
432 | 481 | - name: "qwen3.5-9b-dflash" |
433 | 482 | url: "github:mudler/LocalAI/gallery/virtual.yaml@master" |
434 | 483 | urls: |
|
0 commit comments