Skip to content

Commit 87a97b3

Browse files
kashifkashif
authored
Merge branch 'ggml-org:master' into dflash-rebase
2 parents dc01f83 + dd69db2 commit 87a97b3

63 files changed

Lines changed: 4151 additions & 2059 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/release.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,13 @@ jobs:
4646

4747
steps:
4848
- id: check
49+
env:
50+
COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
4951
run: |
5052
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
5153
echo "should_release=true" >> $GITHUB_OUTPUT
5254
elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/master" ]]; then
53-
if echo "${{ github.event.head_commit.message }}" | grep -q '\[no release\]'; then
55+
if echo "$COMMIT_MESSAGE" | grep -q '\[no release\]'; then
5456
echo "should_release=false" >> $GITHUB_OUTPUT
5557
else
5658
echo "should_release=true" >> $GITHUB_OUTPUT
@@ -542,6 +544,7 @@ jobs:
542544
steps:
543545
- name: Set OpenVINO version output
544546
id: openvino_version
547+
shell: bash
545548
run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT
546549

547550
- name: Clone

‎app/llama.cpp‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,21 @@ int llama_fit_params(int argc, char ** argv);
2020
int llama_quantize(int argc, char ** argv);
2121
int llama_perplexity(int argc, char ** argv);
2222

23-
// hands the update over to the install script, which downloads and swaps the binary
23+
// Self-update is only supported for binaries built with llama-install.sh
2424
static int llama_update(int argc, char ** argv) {
2525
(void) argc;
2626
(void) argv;
2727

28+
#ifdef LLAMA_INSTALL_BUILD
2829
#if defined(_WIN32)
2930
return system("powershell -NoProfile -ExecutionPolicy Bypass -Command \"irm https://llama.app/install.ps1 | iex\"");
3031
#else
3132
return system("curl -fsSL https://llama.app/install.sh | sh");
3233
#endif
34+
#else
35+
printf("Updates are available only when installed from https://llama.app\n");
36+
return 1;
37+
#endif
3338
}
3439

3540
static const char * progname;
@@ -46,21 +51,29 @@ struct command {
4651
int (*func)(int, char **);
4752
};
4853

54+
#ifdef LLAMA_INSTALL_BUILD
55+
#define UPDATE_HIDDEN false
56+
#else
57+
#define UPDATE_HIDDEN true
58+
#endif
59+
4960
static const command cmds[] = {
50-
{"serve", "HTTP API server", {"server"}, false, llama_server },
51-
{"cli", "Command-line interactive interface", {"client"}, false, llama_cli },
52-
{"update", "Update llama to the latest release", {}, false, llama_update },
53-
{"completion", "Text completion", {"complete"}, true, llama_completion },
54-
{"bench", "Benchmark prompt processing and text generation", {}, true, llama_bench },
55-
{"batched-bench", "Benchmark batched decoding performance", {}, true, llama_batched_bench},
56-
{"fit-params", "Compute parameters to fit a model in device memory", {}, true, llama_fit_params },
57-
{"quantize", "Quantize a model", {}, true, llama_quantize },
58-
{"perplexity", "Compute model perplexity and KL divergence", {}, true, llama_perplexity },
59-
{"version", "Show version", {}, false, version },
60-
{"licenses", "Show third-party licenses", {"credits"}, false, licenses },
61-
{"help", "Show available commands", {}, false, help },
61+
{"serve", "HTTP API server", {"server"}, false, llama_server },
62+
{"cli", "Command-line interactive interface", {"client"}, false, llama_cli },
63+
{"update", "Update llama to the latest release", {}, UPDATE_HIDDEN, llama_update },
64+
{"completion", "Text completion", {"complete"}, true, llama_completion },
65+
{"bench", "Benchmark prompt processing and text generation", {}, true, llama_bench },
66+
{"batched-bench", "Benchmark batched decoding performance", {}, true, llama_batched_bench},
67+
{"fit-params", "Compute parameters to fit a model in device memory", {}, true, llama_fit_params },
68+
{"quantize", "Quantize a model", {}, true, llama_quantize },
69+
{"perplexity", "Compute model perplexity and KL divergence", {}, true, llama_perplexity },
70+
{"version", "Show version", {}, false, version },
71+
{"licenses", "Show third-party licenses", {"credits"}, false, licenses },
72+
{"help", "Show available commands", {}, false, help },
6273
};
6374

75+
#undef UPDATE_HIDDEN
76+
6477
static int version(int argc, char ** argv) {
6578
printf("%s\n", llama_build_info());
6679
return 0;

‎common/download.cpp‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -997,3 +997,87 @@ std::vector<common_cached_model_info> common_list_cached_models() {
997997

998998
return result;
999999
}
1000+
1001+
bool common_download_remove(const std::string & hf_repo_with_tag) {
1002+
namespace fs = std::filesystem;
1003+
1004+
auto [repo_id, tag] = common_download_split_repo_tag(hf_repo_with_tag);
1005+
1006+
if (tag.empty()) {
1007+
return hf_cache::remove_cached_repo(repo_id);
1008+
}
1009+
1010+
std::string tag_upper = tag;
1011+
for (char & c : tag_upper) {
1012+
c = (char) std::toupper((unsigned char) c);
1013+
}
1014+
1015+
auto files = hf_cache::get_cached_files(repo_id);
1016+
if (files.empty()) {
1017+
return false;
1018+
}
1019+
1020+
// collect snapshot entries whose tag matches
1021+
std::vector<fs::path> to_remove;
1022+
for (const auto & f : files) {
1023+
auto split = get_gguf_split_info(f.path);
1024+
if (split.tag == tag_upper) {
1025+
to_remove.emplace_back(f.local_path);
1026+
}
1027+
}
1028+
1029+
if (to_remove.empty()) {
1030+
return false;
1031+
}
1032+
1033+
// resolve blob paths from symlinks before deleting snapshot entries
1034+
std::vector<fs::path> blobs_to_check;
1035+
for (const auto & p : to_remove) {
1036+
std::error_code ec;
1037+
if (fs::is_symlink(p, ec)) {
1038+
auto target = fs::read_symlink(p, ec);
1039+
if (!ec) {
1040+
blobs_to_check.push_back((p.parent_path() / target).lexically_normal());
1041+
}
1042+
}
1043+
}
1044+
1045+
// remove snapshot entries
1046+
for (const auto & p : to_remove) {
1047+
std::error_code ec;
1048+
fs::remove(p, ec);
1049+
if (ec) {
1050+
LOG_WRN("%s: failed to remove %s: %s\n", __func__, p.string().c_str(), ec.message().c_str());
1051+
}
1052+
}
1053+
1054+
if (blobs_to_check.empty()) {
1055+
return true;
1056+
}
1057+
1058+
// collect blobs still referenced by remaining snapshot entries
1059+
std::unordered_set<std::string> still_referenced;
1060+
for (const auto & f : hf_cache::get_cached_files(repo_id)) {
1061+
fs::path p(f.local_path);
1062+
std::error_code ec;
1063+
if (fs::is_symlink(p, ec)) {
1064+
auto target = fs::read_symlink(p, ec);
1065+
if (!ec) {
1066+
still_referenced.insert((p.parent_path() / target).lexically_normal().string());
1067+
}
1068+
}
1069+
}
1070+
1071+
// remove orphaned blobs
1072+
for (const auto & blob : blobs_to_check) {
1073+
if (still_referenced.find(blob.string()) == still_referenced.end()) {
1074+
std::error_code ec;
1075+
fs::remove(blob, ec);
1076+
if (ec) {
1077+
LOG_WRN("%s: failed to remove blob %s: %s\n", __func__, blob.string().c_str(), ec.message().c_str());
1078+
}
1079+
}
1080+
}
1081+
1082+
return true;
1083+
}

‎common/download.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,10 @@ int common_download_file_single(const std::string & url,
115115
// resolve and download model from Docker registry
116116
// return local path to downloaded model file
117117
std::string common_docker_resolve_model(const std::string & docker);
118+
119+
// Remove a cached model from disk
120+
// input format: "user/model" or "user/model:tag"
121+
// - if tag is omitted, removes the entire repo cache directory
122+
// - if tag is present, removes only files matching that tag (and orphaned blobs)
123+
// returns true if anything was removed
124+
bool common_download_remove(const std::string & hf_repo_with_tag);

‎common/hf-cache.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -495,4 +495,19 @@ std::string finalize_file(const hf_file & file) {
495495
return file.final_path;
496496
}
497497

498+
bool remove_cached_repo(const std::string & repo_id) {
499+
if (!is_valid_repo_id(repo_id)) {
500+
LOG_WRN("%s: invalid repository: %s\n", __func__, repo_id.c_str());
501+
return false;
502+
}
503+
fs::path repo_path = get_repo_path(repo_id);
504+
std::error_code ec;
505+
auto removed = fs::remove_all(repo_path, ec);
506+
if (ec) {
507+
LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());
508+
return false;
509+
}
510+
return removed > 0;
511+
}
512+
498513
} // namespace hf_cache

‎common/hf-cache.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,7 @@ hf_files get_cached_files(const std::string & repo_id = {});
2929
// Create snapshot path (link or move/copy) and return it
3030
std::string finalize_file(const hf_file & file);
3131

32+
// Remove the entire cached directory for a repo, returns true if removed
33+
bool remove_cached_repo(const std::string & repo_id);
34+
3235
} // namespace hf_cache

‎docs/backend/SYCL.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,64 @@ You could update your test result in it directly.
161161

162162
Please refer to [Docker with SYCL](../docker.md#docker-with-sycl) for details.
163163

164+
## Quick Development WOW
165+
166+
This chapter is for quick development & try with SYCL backend on Intel GPU.
167+
168+
You need to install following sofeware before development:
169+
- Intel GPU driver
170+
- oneAPI package
171+
- other development tools.
172+
173+
Please refer to [Linux](#linux) or [Windows](#windows-1) for above installation and resolve the trouble in usage. There are the detailed guide.
174+
175+
- Linux
176+
177+
```
178+
## build from source code
179+
./examples/sycl/build.sh
180+
181+
## run CONV_2D_DW unit test cases
182+
./build/bin/test-backend-ops -b SYCL0 -o CONV_2D_DW
183+
184+
## run all unit test cases
185+
./build/bin/test-backend-ops -b SYCL0
186+
187+
## run with LLM on the first GPU
188+
./examples/sycl/test.sh -mg 0 -m xxxx.gguf
189+
190+
## run service with LLM on the first GPU
191+
export ONEAPI_DEVICE_SELECTOR="level_zero:0"
192+
./examples/sycl/start-svr.sh -m xxxx.gguf
193+
194+
## update the docs/ops.md for new/update OPs
195+
./examples/sycl/update-ops-doc.sh
196+
```
197+
198+
- Windows
199+
200+
```
201+
## build from source code
202+
examples\sycl\win-build-sycl.bat
203+
204+
## run CONV_2D_DW unit test cases
205+
build\bin\test-backend-ops.exe -b SYCL0 -o CONV_2D_DW
206+
207+
## run all unit test cases
208+
build\bin\test-backend-ops.exe -b SYCL0
209+
210+
## run LLM on the first GPU
211+
examples\sycl\win-test.bat -mg 0 -m xxxx.gguf
212+
213+
## run service with LLM on the first GPU
214+
set ONEAPI_DEVICE_SELECTOR="level_zero:0"
215+
examples\sycl\win-start-svr.bat -m xxxx.gguf
216+
217+
## update the docs/ops.md for new/update OPs
218+
examples\sycl\win-update-ops-doc.bat
219+
```
220+
221+
164222
## Linux
165223

166224
### I. Setup Environment
@@ -712,6 +770,7 @@ use 1 SYCL GPUs: [0] with Max compute units:512
712770
| Name | Value | Function |
713771
|-------------------|------------------|---------------------------------------------------------------------------------------------------------------------------|
714772
| GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function by macro: GGML_SYCL_DEBUG |
773+
| GGML_SYCL_DEV2DEV_MEMCPY | 0 (default) or 1 | Choose the SYCL or L0 API in dev2dev memory copy.<br>Value: <br>* 0: SYCL API (default)<br>* 1: L0 API -- L0 API is found to lead to abnormal crash in some case. This debug flag is used to check the issue.|
715774
| GGML_SYCL_ENABLE_FLASH_ATTN | 1 (default) or 0| Enable Flash-Attention. It can reduce memory usage. The performance impact depends on the LLM.|
716775
| GGML_SYCL_DISABLE_OPT | 0 (default) or 1 | Disable optimize features for Intel GPUs. (Recommended to 1 for Intel devices older than Gen 10) |
717776
| GGML_SYCL_DISABLE_GRAPH | 0 or 1 (default) | Disable running computations through SYCL Graphs feature. Disabled by default because SYCL Graph is still on development, no better performance. |
@@ -731,6 +790,7 @@ Pass these via `CXXFLAGS` or add a one-off `#define` to enable a flag on the spo
731790
| DEBUG_SYCL_POOL | Enable device memory pool logging on teardown. Useful for profiling allocations. |
732791
| DEBUG_SYCL_MALLOC | Enable verbose per-call logging of device pool alloc/free operations. |
733792

793+
734794
## Design Rule
735795

736796
- Open to all contributors.

‎docs/ops.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,11 @@ Legend:
2727
| COL2IM_1D | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
2828
| CONCAT | ❌ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
2929
| CONT | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
30-
| CONV_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ |
31-
| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
32-
| CONV_3D | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
30+
| CONV_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
31+
| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
32+
| CONV_3D | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
3333
| CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
34-
| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
34+
| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
3535
| COS | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
3636
| COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
3737
| CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |

0 commit comments

Comments
 (0)