Skip to content

Commit 0e4b8d2

Browse files
giladgdgiladgd
authored
feat: manual binding loading (#153)
* feat: manual binding loading - load the bindings using the `getLlama` method instead of it loading up by itself on import * feat: log settings - configure the log level or even set a custom logger for llama.cpp logs * fix: bugs
1 parent 187627e commit 0e4b8d2

78 files changed

Lines changed: 2411 additions & 855 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build.yml‎

Lines changed: 85 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ jobs:
99
name: Build
1010
runs-on: ubuntu-latest
1111
steps:
12-
- uses: actions/checkout@v3
13-
- uses: actions/setup-node@v3
12+
- uses: actions/checkout@v4
13+
- uses: actions/setup-node@v4
1414
with:
1515
node-version: "20"
1616
- name: Install modules
@@ -20,27 +20,21 @@ jobs:
2020
- name: Download latest llama.cpp release
2121
env:
2222
CI: true
23-
run: node ./dist/cli/cli.js download --release latest --skipBuild --noBundle --updateBinariesReleaseMetadataAndSaveGitBundle
23+
run: node ./dist/cli/cli.js download --release latest --skipBuild --noBundle --noUsageExample --updateBinariesReleaseMetadataAndSaveGitBundle
2424
- name: Upload build artifact
25-
uses: actions/upload-artifact@v3
25+
uses: actions/upload-artifact@v4
2626
with:
2727
name: "build"
2828
path: "dist"
29-
- name: Upload binariesGithubRelease.json artifact
30-
uses: actions/upload-artifact@v3
31-
with:
32-
name: "binariesGithubRelease"
33-
path: "llama/binariesGithubRelease.json"
3429
- name: Upload llama.cpp artifact
35-
uses: actions/upload-artifact@v3
30+
uses: actions/upload-artifact@v4
3631
with:
3732
name: "llama.cpp"
38-
path: "llama/llama.cpp"
39-
- name: Upload gitRelease.bundle artifact
40-
uses: actions/upload-artifact@v3
41-
with:
42-
name: "gitReleaseBundle"
43-
path: "llama/gitRelease.bundle"
33+
path: |
34+
llama/binariesGithubRelease.json
35+
llama/llama.cpp.info.json
36+
llama/llama.cpp
37+
llama/gitRelease.bundle
4438
4539
build-binaries:
4640
name: Build binaries - ${{ matrix.config.name }}
@@ -72,29 +66,29 @@ jobs:
7266
artifact: "mac"
7367

7468
steps:
75-
- uses: actions/checkout@v3
76-
- uses: actions/setup-node@v3
69+
- uses: actions/checkout@v4
70+
- uses: actions/setup-node@v4
7771
with:
7872
node-version: "20"
7973

8074
- name: Download build artifact
81-
uses: actions/download-artifact@v3
75+
uses: actions/download-artifact@v4
8276
with:
8377
name: build
8478
path: dist
8579

8680
- name: Download llama.cpp artifact
87-
uses: actions/download-artifact@v3
81+
uses: actions/download-artifact@v4
8882
with:
8983
name: llama.cpp
90-
path: llama/llama.cpp
84+
path: llama
9185

92-
- name: Install dependencies on windows
86+
- name: Install dependencies on Windows
9387
if: startsWith(matrix.config.os, 'windows')
9488
run: |
9589
choco install ninja cmake
9690
97-
- name: Install dependencies on ubuntu
91+
- name: Install dependencies on Ubuntu
9892
if: startsWith(matrix.config.name, 'Ubuntu GCC')
9993
run: |
10094
sudo apt-get update
@@ -106,7 +100,23 @@ jobs:
106100
which arm-linux-gnueabihf-gcc
107101
which arm-linux-gnueabihf-g++
108102
109-
- name: Install dependencies on macos
103+
- name: Install Cuda on Windows
104+
if: startsWith(matrix.config.os, 'windows')
105+
uses: Jimver/cuda-toolkit@v0.2.11
106+
with:
107+
cuda: '12.2.0'
108+
method: 'network'
109+
sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
110+
use-local-cache: false
111+
112+
- name: Install Cuda on Ubuntu
113+
if: startsWith(matrix.config.name, 'Ubuntu GCC')
114+
uses: Jimver/cuda-toolkit@v0.2.11
115+
with:
116+
cuda: '12.2.0'
117+
method: 'network'
118+
119+
- name: Install dependencies on macOS
110120
if: startsWith(matrix.config.os, 'macos')
111121
run: |
112122
brew install cmake ninja
@@ -148,50 +158,46 @@ jobs:
148158
return {versions, latestVersion};
149159
}
150160
151-
function getArches() {
152-
switch (process.env.ARTIFACT_NAME) {
153-
case "win":
154-
return ["x64" /*, "arm64" */ ]; // disabled arm64 for now as compilation doesn't work
155-
case "linux":
156-
return ["x64", "arm64", "armv7l"];
157-
case "mac":
158-
return ["x64", "arm64"];
159-
}
160-
161-
return ["x64"];
162-
}
163-
164161
const {versions: latestNodeVersions} = await getLatestNodeVersions(Date.now() - 1000 * 60 * 60 * 24 * 14);
165162
166163
const nodeVersion = latestNodeVersions.get(18);
167164
const windowsOnArmNodeVersion = latestNodeVersions.get(20);
168-
const arches = getArches();
169165
170166
if (nodeVersion == null || windowsOnArmNodeVersion == null) {
171167
throw new Error("Could not find node versions");
172168
}
173169
174-
console.log("Building for node version", nodeVersion, "and archs", arches);
175-
176170
await $`mkdir -p llamaBins`;
177171
178-
for (const arch of arches) {
179-
const additionalFlags = [];
180-
let buildNodeVersion = nodeVersion;
181-
182-
if (process.env.ARTIFACT_NAME === "win" && arch === "arm64") {
183-
buildNodeVersion = windowsOnArmNodeVersion;
184-
} else if (process.env.ARTIFACT_NAME === "mac" && arch === "arm64") {
185-
additionalFlags.push("--metal");
186-
} else if (process.env.ARTIFACT_NAME === "mac" && arch === "x64") {
187-
additionalFlags.push("--no-metal");
188-
}
189-
190-
console.log(`Building ${arch} for node ${buildNodeVersion}`);
172+
async function buildBinary(arch, flags = [], nodeTarget = nodeVersion) {
173+
console.log(`Building ${arch} for node ${nodeTarget} with flags`, flags);
191174
192-
const binName = `${process.env.ARTIFACT_NAME}-${arch}`;
193-
await $`node ./dist/cli/cli.js build --arch ${arch} --nodeTarget ${buildNodeVersion} ${additionalFlags}`;
194-
await $`mv ./llama/build/Release ${"./llamaBins/" + binName}`;
175+
await $`node ./dist/cli/cli.js build --noUsageExample --arch ${arch} --nodeTarget ${nodeVersion} ${flags}`;
176+
}
177+
178+
// build binaries
179+
if (process.env.ARTIFACT_NAME === "win") {
180+
await buildBinary("x64");
181+
await buildBinary("x64", ["--cuda"]);
182+
// await buildBinary("arm64", [], windowsOnArmNodeVersion); // disabled arm64 for now as compilation doesn't work
183+
} else if (process.env.ARTIFACT_NAME === "linux") {
184+
await buildBinary("x64");
185+
await buildBinary("x64", ["--cuda"]);
186+
await buildBinary("arm64");
187+
await buildBinary("armv7l");
188+
} else if (process.env.ARTIFACT_NAME === "mac") {
189+
await buildBinary("x64", ["--metal"]);
190+
await buildBinary("arm64", ["--no-metal"]);
191+
}
192+
193+
// move binaries to llamaBins
194+
const localBuildsDirectoryPath = path.join(process.cwd(), "llama", "localBuilds");
195+
const llamaBinsDirectoryPath = path.join(process.cwd(), "llamaBins");
196+
for (const folderName of await fs.readdir(localBuildsDirectoryPath)) {
197+
await fs.move(
198+
path.join(localBuildsDirectoryPath, folderName, "Release"),
199+
path.join(llamaBinsDirectoryPath, folderName)
200+
);
195201
}
196202
197203
await $`echo "Built binaries:"`;
@@ -200,7 +206,7 @@ jobs:
200206
EOF
201207
202208
- name: Publish artifact
203-
uses: actions/upload-artifact@v3
209+
uses: actions/upload-artifact@v4
204210
with:
205211
name: "bins-${{ matrix.config.artifact }}"
206212
path: "llamaBins/*"
@@ -211,22 +217,22 @@ jobs:
211217
needs:
212218
- build
213219
steps:
214-
- uses: actions/checkout@v3
215-
- uses: actions/setup-node@v3
220+
- uses: actions/checkout@v4
221+
- uses: actions/setup-node@v4
216222
with:
217223
node-version: "20"
218224

219225
- name: Download build artifact
220-
uses: actions/download-artifact@v3
226+
uses: actions/download-artifact@v4
221227
with:
222228
name: build
223229
path: dist
224230

225231
- name: Download llama.cpp artifact
226-
uses: actions/download-artifact@v3
232+
uses: actions/download-artifact@v4
227233
with:
228234
name: llama.cpp
229-
path: llama/llama.cpp
235+
path: llama
230236

231237
- name: Install dependencies on ubuntu
232238
run: |
@@ -248,22 +254,22 @@ jobs:
248254
needs:
249255
- build
250256
steps:
251-
- uses: actions/checkout@v3
252-
- uses: actions/setup-node@v3
257+
- uses: actions/checkout@v4
258+
- uses: actions/setup-node@v4
253259
with:
254260
node-version: "20"
255261

256262
- name: Download build artifact
257-
uses: actions/download-artifact@v3
263+
uses: actions/download-artifact@v4
258264
with:
259265
name: build
260266
path: dist
261267

262268
- name: Download llama.cpp artifact
263-
uses: actions/download-artifact@v3
269+
uses: actions/download-artifact@v4
264270
with:
265271
name: llama.cpp
266-
path: llama/llama.cpp
272+
path: llama
267273

268274
- name: Install dependencies on macOS
269275
run: |
@@ -278,7 +284,7 @@ jobs:
278284

279285
- name: Cache models
280286
id: cache-test-models
281-
uses: actions/cache@v3
287+
uses: actions/cache@v4
282288
with:
283289
path: "test/.models/**.gguf"
284290
key: cache-test-models-${{ runner.os }}-${{ github.workflow }}
@@ -307,15 +313,15 @@ jobs:
307313
- build
308314
- build-binaries
309315
steps:
310-
- uses: actions/checkout@v3
316+
- uses: actions/checkout@v4
311317
with:
312318
lfs: true
313-
- uses: actions/setup-node@v3
319+
- uses: actions/setup-node@v4
314320
with:
315321
node-version: "20"
316322
- name: Install modules
317323
run: npm ci
318-
- uses: actions/download-artifact@v3
324+
- uses: actions/download-artifact@v4
319325
with:
320326
path: artifacts
321327
- name: Move artifacts
@@ -324,13 +330,16 @@ jobs:
324330
mv artifacts/bins-*/* llamaBins/
325331
mv artifacts/build dist/
326332
327-
cp -r artifacts/llama.cpp/grammars llama/grammars
333+
cp -r artifacts/llama.cpp/llama.cpp/grammars llama/grammars
328334
329335
rm -f ./llama/binariesGithubRelease.json
330-
mv artifacts/binariesGithubRelease/binariesGithubRelease.json ./llama/binariesGithubRelease.json
336+
mv artifacts/llama.cpp/binariesGithubRelease.json ./llama/binariesGithubRelease.json
337+
338+
rm -f ./llama/llama.cpp.info.json
339+
mv artifacts/llama.cpp/llama.cpp.info.json ./llama/llama.cpp.info.json
331340
332341
rm -f ./llama/gitRelease.bundle
333-
mv artifacts/gitReleaseBundle/gitRelease.bundle ./llama/gitRelease.bundle
342+
mv artifacts/llama.cpp/gitRelease.bundle ./llama/gitRelease.bundle
334343
335344
echo "Built binaries:"
336345
ls llamaBins
@@ -356,12 +365,12 @@ jobs:
356365
npm run docs:build
357366
- name: Upload docs to GitHub Pages
358367
if: steps.set-npm-url.outputs.npm-url != '' && github.ref == 'refs/heads/master'
359-
uses: actions/upload-pages-artifact@v2
368+
uses: actions/upload-artifact@v4
360369
with:
361370
name: pages-docs
362371
path: docs-site
363372
- name: Deploy docs to GitHub Pages
364373
if: steps.set-npm-url.outputs.npm-url != '' && github.ref == 'refs/heads/master'
365-
uses: actions/deploy-pages@v2
374+
uses: actions/deploy-pages@v4
366375
with:
367376
artifact_name: pages-docs

‎.github/workflows/test.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ jobs:
55
name: Test
66
runs-on: ubuntu-latest
77
steps:
8-
- uses: actions/checkout@v3
9-
- uses: actions/setup-node@v3
8+
- uses: actions/checkout@v4
9+
- uses: actions/setup-node@v4
1010
with:
1111
node-version: "18"
1212
- name: Install modules
@@ -20,10 +20,10 @@ jobs:
2020
name: Test docs compilation
2121
runs-on: ubuntu-latest
2222
steps:
23-
- uses: actions/checkout@v3
23+
- uses: actions/checkout@v4
2424
with:
2525
lfs: true
26-
- uses: actions/setup-node@v3
26+
- uses: actions/setup-node@v4
2727
with:
2828
node-version: "20"
2929
- name: Install modules

‎.gitignore‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,16 @@ node_modules
1515

1616
/llama/compile_commands.json
1717
/llama/llama.cpp
18-
/llama/llama.cpp.tag.json
18+
/llama/llama.cpp.lock
19+
/llama/llama.cpp.info.json
20+
/llama/lastBuild.json
1921
/llama/gitRelease.bundle
2022
/llama/.temp
21-
/llama/build
23+
/llama/localBuilds
2224
/llama/Release
2325
/llama/Debug
2426
/llama/xpack/cache
2527
/llama/xpack/store
2628
/llama/xpack/xpacks
29+
/llama/xpack/cmakeInstall.lock
2730
/llamaBins

0 commit comments

Comments
 (0)