|
4 | 4 | # |
5 | 5 | # 1. Augment the kv_cache_types[] allow-list so `LoadModel` accepts the |
6 | 6 | # fork-specific `turbo2` / `turbo3` / `turbo4` cache types. |
7 | | -# 2. Replace `get_media_marker()` (added upstream in ggml-org/llama.cpp#21962, |
8 | | -# server-side random per-instance marker) with the legacy "<__media__>" |
9 | | -# literal. The fork branched before that PR, so server-common.cpp has no |
10 | | -# get_media_marker symbol. The fork's mtmd_default_marker() still returns |
11 | | -# "<__media__>", and Go-side tooling falls back to that sentinel when the |
12 | | -# backend does not expose media_marker, so substituting the literal keeps |
13 | | -# behavior identical on the turboquant path. |
14 | | -# 3. Revert the `common_params_speculative` field references to the |
15 | | -# pre-refactor flat layout. Upstream ggml-org/llama.cpp#22397 split the |
16 | | -# struct into nested `draft` / `ngram_simple` / `ngram_mod` / etc. members; |
17 | | -# the turboquant fork branched before that PR and still exposes the flat |
18 | | -# `n_max`, `mparams_dft`, `ngram_size_n`, ... fields. The substitutions |
19 | | -# below map the new nested paths back to the legacy flat names so the |
20 | | -# shared grpc-server.cpp keeps compiling against the fork's common.h. |
21 | | -# Drop this block once the fork rebases past #22397. |
| 7 | +# 2. Define LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP at the top of the file |
| 8 | +# so the grpc-server option parser skips the two references to |
| 9 | +# common_params::checkpoint_min_step (the default and the option handler). |
| 10 | +# That field does not exist in the fork yet; drop this once it does. |
| 11 | +# |
| 12 | +# The fork used to lag upstream on the whole common_params_speculative refactor |
| 13 | +# (ggml-org/llama.cpp#22397/#22838/#22964), the model_tgt rename (#22838) and |
| 14 | +# get_media_marker (#21962), which required a much larger compat shim here |
| 15 | +# (flat-field sed renames + a coarse LOCALAI_LEGACY_LLAMA_CPP_SPEC define). The |
| 16 | +# fork has since rebased past all of those, so the only remaining gap is |
| 17 | +# checkpoint_min_step. If a future bump reintroduces a divergence, add a narrow |
| 18 | +# guard in grpc-server.cpp keyed on a fork-specific macro and inject it here |
| 19 | +# rather than resurrecting the coarse one. |
22 | 20 | # |
23 | 21 | # We patch the *copy* sitting in turboquant-<flavor>-build/, never the original |
24 | 22 | # under backend/cpp/llama-cpp/, so the stock llama-cpp build keeps compiling |
|
72 | 70 | echo "==> KV allow-list patch OK" |
73 | 71 | fi |
74 | 72 |
|
75 | | -if grep -q 'get_media_marker()' "$SRC"; then |
76 | | - echo "==> patching $SRC to replace get_media_marker() with legacy \"<__media__>\" literal" |
77 | | - # Only one call site today (ModelMetadata), but replace all occurrences to |
78 | | - # stay robust if upstream adds more. Use a temp file to avoid relying on |
79 | | - # sed -i portability (the builder image uses GNU sed, but keeping this |
80 | | - # consistent with the awk block above). |
81 | | - sed 's/get_media_marker()/"<__media__>"/g' "$SRC" > "$SRC.tmp" |
82 | | - mv "$SRC.tmp" "$SRC" |
83 | | - echo "==> get_media_marker() substitution OK" |
84 | | -else |
85 | | - echo "==> $SRC has no get_media_marker() call, skipping media-marker patch" |
86 | | -fi |
87 | | - |
88 | | -if grep -q 'params\.speculative\.draft\.\|params\.speculative\.ngram_simple\.' "$SRC"; then |
89 | | - echo "==> patching $SRC to revert common_params_speculative refs to pre-#22397 flat layout" |
90 | | - # Each substitution is the exact post-refactor path → legacy flat field. |
91 | | - # Order doesn't matter because the source paths are disjoint, but we keep |
92 | | - # the most-specific (mparams.path) first for readability. |
93 | | - sed -E \ |
94 | | - -e 's/params\.speculative\.draft\.mparams\.path/params.speculative.mparams_dft.path/g' \ |
95 | | - -e 's/params\.speculative\.draft\.n_max/params.speculative.n_max/g' \ |
96 | | - -e 's/params\.speculative\.draft\.n_min/params.speculative.n_min/g' \ |
97 | | - -e 's/params\.speculative\.draft\.p_min/params.speculative.p_min/g' \ |
98 | | - -e 's/params\.speculative\.draft\.p_split/params.speculative.p_split/g' \ |
99 | | - -e 's/params\.speculative\.draft\.n_gpu_layers/params.speculative.n_gpu_layers/g' \ |
100 | | - -e 's/params\.speculative\.draft\.n_ctx/params.speculative.n_ctx/g' \ |
101 | | - -e 's/params\.speculative\.ngram_simple\.size_n/params.speculative.ngram_size_n/g' \ |
102 | | - -e 's/params\.speculative\.ngram_simple\.size_m/params.speculative.ngram_size_m/g' \ |
103 | | - -e 's/params\.speculative\.ngram_simple\.min_hits/params.speculative.ngram_min_hits/g' \ |
104 | | - "$SRC" > "$SRC.tmp" |
105 | | - mv "$SRC.tmp" "$SRC" |
106 | | - echo "==> speculative field rename OK" |
107 | | -else |
108 | | - echo "==> $SRC has no post-#22397 speculative field refs, skipping spec rename patch" |
109 | | -fi |
110 | | - |
111 | | -# 4. Revert the `ctx_server.impl->model_tgt` rename introduced by upstream |
112 | | -# ggml-org/llama.cpp#22838 (parallel drafting). The turboquant fork still |
113 | | -# exposes the field as `model` on `server_context_impl`. The two call sites |
114 | | -# are in the Rerank and ModelMetadata RPC handlers. |
115 | | -if grep -q 'ctx_server\.impl->model_tgt' "$SRC"; then |
116 | | - echo "==> patching $SRC to revert ctx_server.impl->model_tgt -> ctx_server.impl->model" |
117 | | - sed -E 's/ctx_server\.impl->model_tgt/ctx_server.impl->model/g' "$SRC" > "$SRC.tmp" |
118 | | - mv "$SRC.tmp" "$SRC" |
119 | | - echo "==> model_tgt rename OK" |
120 | | -else |
121 | | - echo "==> $SRC has no ctx_server.impl->model_tgt refs, skipping model_tgt rename patch" |
122 | | -fi |
123 | | - |
124 | | -# 5. Define LOCALAI_LEGACY_LLAMA_CPP_SPEC at the top of the file so the |
125 | | -# grpc-server option parser skips the new option-handler blocks (ngram_mod, |
126 | | -# ngram_map_k, ngram_map_k4v, ngram_cache, draft.cache_type_*, draft.cpuparams*, |
127 | | -# draft.tensor_buft_overrides) introduced for the post-#22838 layout, the |
128 | | -# draft.tensor_buft_overrides sentinel termination, and the |
129 | | -# common_params::checkpoint_min_step default/option (added with the |
130 | | -# 35c9b1f3 bump). Those blocks reference struct fields that simply do not |
131 | | -# exist in the fork. |
132 | | -if grep -q '^#define LOCALAI_LEGACY_LLAMA_CPP_SPEC' "$SRC"; then |
133 | | - echo "==> $SRC already defines LOCALAI_LEGACY_LLAMA_CPP_SPEC, skipping" |
| 73 | +# 2. Define LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP at the top of the file so |
| 74 | +# the grpc-server option parser skips the two references to |
| 75 | +# common_params::checkpoint_min_step (the default assignment and the option |
| 76 | +# handler). That field does not exist in the fork yet. Drop this block once |
| 77 | +# the fork rebases past the bump that added checkpoint_min_step. |
| 78 | +if grep -q '^#define LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP' "$SRC"; then |
| 79 | + echo "==> $SRC already defines LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP, skipping" |
134 | 80 | else |
135 | | - echo "==> patching $SRC to define LOCALAI_LEGACY_LLAMA_CPP_SPEC at the top" |
136 | | - # Insert the define before the very first `#include` so it precedes all the |
137 | | - # speculative-decoding code paths. |
| 81 | + echo "==> patching $SRC to define LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP at the top" |
| 82 | + # Insert the define before the very first `#include` so it precedes the |
| 83 | + # checkpoint_min_step references. |
138 | 84 | awk ' |
139 | 85 | !done && /^#include/ { |
140 | | - print "#define LOCALAI_LEGACY_LLAMA_CPP_SPEC 1" |
| 86 | + print "#define LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP 1" |
141 | 87 | print "// ^ injected by backend/cpp/turboquant/patch-grpc-server.sh" |
142 | 88 | print "" |
143 | 89 | done = 1 |
144 | 90 | } |
145 | 91 | { print } |
146 | 92 | END { |
147 | 93 | if (!done) { |
148 | | - print "patch-grpc-server.sh: no #include anchor found to insert LOCALAI_LEGACY_LLAMA_CPP_SPEC" > "/dev/stderr" |
| 94 | + print "patch-grpc-server.sh: no #include anchor found to insert LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP" > "/dev/stderr" |
149 | 95 | exit 1 |
150 | 96 | } |
151 | 97 | } |
152 | 98 | ' "$SRC" > "$SRC.tmp" |
153 | 99 | mv "$SRC.tmp" "$SRC" |
154 | | - echo "==> LOCALAI_LEGACY_LLAMA_CPP_SPEC define OK" |
| 100 | + echo "==> LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP define OK" |
155 | 101 | fi |
156 | 102 |
|
157 | 103 | echo "==> all patches applied" |
0 commit comments