Skip to content

Commit 4f3c27b

Browse files
arthwarthw
committed
add help script for windows
1 parent 422343c commit 4f3c27b

3 files changed

Lines changed: 371 additions & 6 deletions

File tree

examples/sycl/start_svr.sh renamed to examples/sycl/start-svr.sh

File renamed without changes.

‎examples/sycl/win-start-svr.bat‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
:: MIT license
2+
:: Copyright (C) 2024 Intel Corporation
3+
:: SPDX-License-Identifier: MIT
4+
5+
@echo off
6+
setlocal EnableExtensions EnableDelayedExpansion
7+
8+
set "BIN_FILE=.\build\bin\llama-server.exe"
9+
set "SEED=0"
10+
set "GPUS_SETTING="
11+
12+
set "MODEL_FILE=..\models\Qwen3.5-4B-Q4_0.gguf"
13+
set "NGL=99"
14+
set "CONTEXT=4096"
15+
set "GGML_SYCL_DEVICE=-1"
16+
set "SPLIT_MODE=layer"
17+
set "LOG_VERBOSE=3"
18+
19+
if "%~1"=="" goto after_args
20+
21+
:parse_args
22+
if "%~1"=="" goto after_args
23+
24+
if /I "%~1"=="-c" (
25+
if "%~2"=="" goto missing_value
26+
set "CONTEXT=%~2"
27+
shift
28+
shift
29+
goto parse_args
30+
)
31+
if /I "%~1"=="--context" (
32+
if "%~2"=="" goto missing_value
33+
set "CONTEXT=%~2"
34+
shift
35+
shift
36+
goto parse_args
37+
)
38+
39+
if /I "%~1"=="-m" (
40+
if "%~2"=="" goto missing_value
41+
set "MODEL_FILE=%~2"
42+
shift
43+
shift
44+
goto parse_args
45+
)
46+
if /I "%~1"=="--model" (
47+
if "%~2"=="" goto missing_value
48+
set "MODEL_FILE=%~2"
49+
shift
50+
shift
51+
goto parse_args
52+
)
53+
54+
if /I "%~1"=="-mg" (
55+
if "%~2"=="" goto missing_value
56+
set "GGML_SYCL_DEVICE=%~2"
57+
set "SPLIT_MODE=none"
58+
shift
59+
shift
60+
goto parse_args
61+
)
62+
if /I "%~1"=="--main-gpu" (
63+
if "%~2"=="" goto missing_value
64+
set "GGML_SYCL_DEVICE=%~2"
65+
set "SPLIT_MODE=none"
66+
shift
67+
shift
68+
goto parse_args
69+
)
70+
71+
if /I "%~1"=="-sm" (
72+
if "%~2"=="" goto missing_value
73+
set "SPLIT_MODE=%~2"
74+
shift
75+
shift
76+
goto parse_args
77+
)
78+
if /I "%~1"=="--split-mode" (
79+
if "%~2"=="" goto missing_value
80+
set "SPLIT_MODE=%~2"
81+
shift
82+
shift
83+
goto parse_args
84+
)
85+
86+
if /I "%~1"=="-ngl" (
87+
if "%~2"=="" goto missing_value
88+
set "NGL=%~2"
89+
shift
90+
shift
91+
goto parse_args
92+
)
93+
if /I "%~1"=="--n-gpu-layers" (
94+
if "%~2"=="" goto missing_value
95+
set "NGL=%~2"
96+
shift
97+
shift
98+
goto parse_args
99+
)
100+
101+
if /I "%~1"=="-lv" (
102+
if "%~2"=="" goto missing_value
103+
set "LOG_VERBOSE=%~2"
104+
shift
105+
shift
106+
goto parse_args
107+
)
108+
if /I "%~1"=="--log-verbosity" (
109+
if "%~2"=="" goto missing_value
110+
set "LOG_VERBOSE=%~2"
111+
shift
112+
shift
113+
goto parse_args
114+
)
115+
116+
if /I "%~1"=="-h" goto help
117+
if /I "%~1"=="--help" goto help
118+
119+
echo Invalid option: %~1
120+
exit /b 1
121+
122+
:missing_value
123+
echo Missing value for option: %~1
124+
exit /b 1
125+
126+
:help
127+
echo Usage: %~n0 [OPTIONS]
128+
echo.
129+
echo This script processes files with specified options.
130+
echo.
131+
echo Options:
132+
echo -h, --help Display this help message and exit.
133+
echo -c, --context ^<value^> Set context length. Bigger need more memory.
134+
echo -m, --model ^<value^> Full model file path.
135+
echo -mg,--main-gpu ^<value^> Set main GPU ID (0 - n) for single GPU mode.
136+
echo -sm,--split-mode ^<value^> How to split the model across multiple GPUs, one of:
137+
echo - none: use one GPU only
138+
echo - layer (default): split layers and KV across GPUs
139+
echo - row: split rows across GPUs
140+
echo -ngl,--n-gpu-layers ^<value^> Max. number of layers to store in VRAM (default: -1)
141+
echo -lv,--log-verbosity ^<value^> Set the verbosity threshold. Messages with a higher verbosity will be
142+
echo ignored. Values:
143+
echo - 0: generic output
144+
echo - 1: error
145+
echo - 2: warning
146+
echo - 3: info
147+
echo - 4: debug
148+
exit /b 0
149+
150+
:after_args
151+
152+
REM In Windows CMD, source is not available; call oneAPI setvars if present.
153+
if exist "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" (
154+
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" >nul
155+
) else (
156+
echo Warning: oneAPI setvars.bat not found. Continuing without environment setup.
157+
)
158+
159+
REM Support malloc device memory more than 4GB.
160+
set "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1"
161+
echo UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=%UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS%
162+
163+
if not "%GGML_SYCL_DEVICE%"=="-1" (
164+
echo Use %GGML_SYCL_DEVICE% as main GPU
165+
REM Use single GPU only.
166+
set "GPUS_SETTING=-mg %GGML_SYCL_DEVICE% -sm %SPLIT_MODE%"
167+
set "ONEAPI_DEVICE_SELECTOR=level_zero:%GGML_SYCL_DEVICE%"
168+
echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%
169+
) else (
170+
echo Use all Intel GPUs, including iGPU ^& dGPU
171+
)
172+
173+
echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap --host 0.0.0.0 --port 8000
174+
set "ZES_ENABLE_SYSMAN=1"
175+
%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap --host 0.0.0.0 --port 8000
176+
177+
endlocal

‎examples/sycl/win-test.bat‎

Lines changed: 194 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,198 @@
22
:: Copyright (C) 2024 Intel Corporation
33
:: SPDX-License-Identifier: MIT
44

5-
set INPUT2="Building a website can be done in 10 simple steps:\nStep 1:"
6-
@call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force
75

8-
:: support malloc device memory more than 4GB.
9-
set UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1
10-
set LOAD_MODE="--mmap"
11-
.\build\bin\llama-completion.exe -m models\llama-2-7b.Q4_0.gguf -no-cnv -p %INPUT2% -n 400 -e -ngl 99 -s 0 %LOAD_MODE%
6+
@echo off
7+
setlocal EnableExtensions EnableDelayedExpansion
8+
9+
REM MIT license
10+
REM Copyright (C) 2024 Intel Corporation
11+
REM SPDX-License-Identifier: MIT
12+
13+
set "BIN_FILE=.\build\bin\llama-completion.exe"
14+
set "SEED=0"
15+
set "GPUS_SETTING="
16+
17+
set "INPUT_PROMPT=Building a website can be done in 10 simple steps:^nStep 1:"
18+
set "MODEL_FILE=..\models\llama-2-7b.Q4_0.gguf"
19+
set "NGL=99"
20+
set "CONTEXT=4096"
21+
set "GGML_SYCL_DEVICE=-1"
22+
set "SPLIT_MODE=layer"
23+
set "LOG_VERBOSE=3"
24+
25+
if "%~1"=="" goto after_args
26+
27+
:parse_args
28+
if "%~1"=="" goto after_args
29+
30+
if /I "%~1"=="-c" (
31+
if "%~2"=="" goto missing_value
32+
set "CONTEXT=%~2"
33+
shift
34+
shift
35+
goto parse_args
36+
)
37+
if /I "%~1"=="--context" (
38+
if "%~2"=="" goto missing_value
39+
set "CONTEXT=%~2"
40+
shift
41+
shift
42+
goto parse_args
43+
)
44+
45+
if /I "%~1"=="-p" (
46+
if "%~2"=="" goto missing_value
47+
set "INPUT_PROMPT=%~2"
48+
shift
49+
shift
50+
goto parse_args
51+
)
52+
if /I "%~1"=="--promote" (
53+
if "%~2"=="" goto missing_value
54+
set "INPUT_PROMPT=%~2"
55+
shift
56+
shift
57+
goto parse_args
58+
)
59+
60+
if /I "%~1"=="-m" (
61+
if "%~2"=="" goto missing_value
62+
set "MODEL_FILE=%~2"
63+
shift
64+
shift
65+
goto parse_args
66+
)
67+
if /I "%~1"=="--model" (
68+
if "%~2"=="" goto missing_value
69+
set "MODEL_FILE=%~2"
70+
shift
71+
shift
72+
goto parse_args
73+
)
74+
75+
if /I "%~1"=="-mg" (
76+
if "%~2"=="" goto missing_value
77+
set "GGML_SYCL_DEVICE=%~2"
78+
set "SPLIT_MODE=none"
79+
shift
80+
shift
81+
goto parse_args
82+
)
83+
if /I "%~1"=="--main-gpu" (
84+
if "%~2"=="" goto missing_value
85+
set "GGML_SYCL_DEVICE=%~2"
86+
set "SPLIT_MODE=none"
87+
shift
88+
shift
89+
goto parse_args
90+
)
91+
92+
if /I "%~1"=="-sm" (
93+
if "%~2"=="" goto missing_value
94+
set "SPLIT_MODE=%~2"
95+
shift
96+
shift
97+
goto parse_args
98+
)
99+
if /I "%~1"=="--split-mode" (
100+
if "%~2"=="" goto missing_value
101+
set "SPLIT_MODE=%~2"
102+
shift
103+
shift
104+
goto parse_args
105+
)
106+
107+
if /I "%~1"=="-ngl" (
108+
if "%~2"=="" goto missing_value
109+
set "NGL=%~2"
110+
shift
111+
shift
112+
goto parse_args
113+
)
114+
if /I "%~1"=="--n-gpu-layers" (
115+
if "%~2"=="" goto missing_value
116+
set "NGL=%~2"
117+
shift
118+
shift
119+
goto parse_args
120+
)
121+
122+
if /I "%~1"=="-lv" (
123+
if "%~2"=="" goto missing_value
124+
set "LOG_VERBOSE=%~2"
125+
shift
126+
shift
127+
goto parse_args
128+
)
129+
if /I "%~1"=="--log-verbosity" (
130+
if "%~2"=="" goto missing_value
131+
set "LOG_VERBOSE=%~2"
132+
shift
133+
shift
134+
goto parse_args
135+
)
136+
137+
if /I "%~1"=="-h" goto help
138+
if /I "%~1"=="--help" goto help
139+
140+
echo Invalid option: %~1
141+
exit /b 1
142+
143+
:missing_value
144+
echo Missing value for option: %~1
145+
exit /b 1
146+
147+
:help
148+
echo Usage: %~n0 [OPTIONS]
149+
echo.
150+
echo This script processes files with specified options.
151+
echo.
152+
echo Options:
153+
echo -h, --help Display this help message and exit.
154+
echo -c, --context ^<value^> Set context length. Bigger need more memory.
155+
echo -p, --promote ^<value^> Prompt to start generation with.
156+
echo -m, --model ^<value^> Full model file path.
157+
echo -mg,--main-gpu ^<value^> Set main GPU ID (0 - n) for single GPU mode.
158+
echo -sm,--split-mode ^<value^> How to split the model across multiple GPUs, one of:
159+
echo - none: use one GPU only
160+
echo - layer (default): split layers and KV across GPUs
161+
echo - row: split rows across GPUs
162+
echo -ngl,--n-gpu-layers ^<value^> Max. number of layers to store in VRAM (default: -1)
163+
echo -lv,--log-verbosity ^<value^> Set the verbosity threshold. Messages with a higher verbosity will be
164+
echo ignored. Values:
165+
echo - 0: generic output
166+
echo - 1: error
167+
echo - 2: warning
168+
echo - 3: info
169+
echo - 4: debug
170+
exit /b 0
171+
172+
:after_args
173+
174+
REM In Windows CMD, source is not available; call oneAPI setvars if present.
175+
if exist "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" (
176+
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" >nul
177+
) else (
178+
echo Warning: oneAPI setvars.bat not found. Continuing without environment setup.
179+
)
180+
181+
REM Support malloc device memory more than 4GB.
182+
set "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1"
183+
echo UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=%UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS%
184+
185+
if not "%GGML_SYCL_DEVICE%"=="-1" (
186+
echo Use %GGML_SYCL_DEVICE% as main GPU
187+
REM Use single GPU only.
188+
set "GPUS_SETTING=-mg %GGML_SYCL_DEVICE% -sm %SPLIT_MODE%"
189+
set "ONEAPI_DEVICE_SELECTOR=level_zero:%GGML_SYCL_DEVICE%"
190+
echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%
191+
) else (
192+
echo Use all Intel GPUs, including iGPU ^& dGPU
193+
)
194+
195+
echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m %MODEL_FILE% -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap
196+
set "ZES_ENABLE_SYSMAN=1"
197+
%BIN_FILE% -m "%MODEL_FILE%" -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap
198+
199+
endlocal

0 commit comments

Comments
 (0)