ci(bump-deps): register ds4 + move version pin into the Makefile

.agents/adding-backends.md

@@ -34,7 +34,55 @@ The build matrix is data-only YAML at `.github/backend-matrix.yml` (not inside `
 
 **Without an entry here no image is ever built or pushed, and the gallery entry in `backend/index.yaml` will point at a tag that does not exist.** The `dockerfile:` field must point at `./backend/Dockerfile.<lang>` matching the language bucket from step 1 (e.g. `Dockerfile.python`, `Dockerfile.golang`, `Dockerfile.rust`). The `tag-suffix` must match the `uri:` in the corresponding `backend/index.yaml` image entry exactly.
 
-If you add a new language bucket, `scripts/changed-backends.js` also needs a branch in `inferBackendPath` so PR change-detection routes file edits correctly.
+**`scripts/changed-backends.js` registration — REQUIRED for any new dockerfile suffix.** This is the single most common omission, because it has no effect on the PR that adds the backend (when no prior path filter could catch it anyway) — it only breaks the *next* PR that touches your backend's directory, which then gets zero CI jobs and looks broken for unrelated reasons. Edit `scripts/changed-backends.js:inferBackendPath` and add a branch BEFORE the more-generic suffixes:
+
+```js
+if (item.dockerfile.endsWith("<your-dockerfile-suffix>")) {
+    return `backend/cpp/<your-backend>/`;   // or backend/python|go|rust/...
+}
+```
+
+The `endsWith()` test is against the matrix entry's `dockerfile:` value (e.g. `./backend/Dockerfile.ds4` → `endsWith("ds4")`). Specificity order matters here just like it does for importers: more-specific suffixes go BEFORE more-generic ones (e.g. `ds4` before `llama-cpp` even though both end with letters, because some upstream might one day call itself `super-ds4-llama-cpp`). Verify locally before pushing:
+
+```bash
+# Confirm your dockerfile suffix is unique enough
+node -e "
+const yaml = require('js-yaml'); const fs = require('fs');
+const m = yaml.load(fs.readFileSync('.github/backend-matrix.yml','utf8'));
+for (const e of m.include.filter(e => e.backend === '<your-backend>')) {
+  console.log(e.dockerfile, '->', e.dockerfile.endsWith('<suffix>'));
+}"
+```
+
+A quick way to find the right insertion point: `grep -n 'item.dockerfile.endsWith' scripts/changed-backends.js`.
+
+**`bump_deps.yaml` registration — REQUIRED for any backend pinning an upstream commit.** If your backend's Makefile has a `*_VERSION?=<sha>` pin to a third-party repo, the daily auto-bump bot at `.github/workflows/bump_deps.yaml` won't notice it unless you register the backend in its matrix. The bot runs `.github/bump_deps.sh` which `grep`s for `^$VAR?=` in the Makefile you list — so the pin MUST live in the Makefile (not in a separate shell script). The bump for ds4 (#9761) had to walk this back because the original landed the pin in `prepare.sh`, which the bot can't see. Pattern (for `antirez/ds4`):
+
+```yaml
+# .github/workflows/bump_deps.yaml
+matrix:
+  include:
+    - repository: "antirez/ds4"
+      variable: "DS4_VERSION"
+      branch: "main"
+      file: "backend/cpp/ds4/Makefile"
+```
+
+And the corresponding Makefile shape (mirror `backend/cpp/llama-cpp/Makefile`):
+
+```makefile
+DS4_VERSION?=ae302c2fa18cc6d9aefc021d0f27ae03c9ad2fc0
+DS4_REPO?=https://github.com/antirez/ds4
+...
+ds4:
+	mkdir -p ds4
+	cd ds4 && git init -q && \
+	git remote add origin $(DS4_REPO) && \
+	git fetch --depth 1 origin $(DS4_VERSION) && \
+	git checkout FETCH_HEAD
+```
+
+If you have a `prepare.sh` doing the clone, delete it — the recipe belongs in the Makefile target so `make purge && make` works as a clean-and-rebuild and so the bump bot finds the pin.
 
 **Placement in file:**
 - CPU builds: Add after other CPU builds (e.g., after `cpu-chatterbox`)

.agents/ds4-backend.md

@@ -6,8 +6,12 @@ LocalAI wraps the engine's C API (`ds4/ds4.h`) with a fresh C++ gRPC server at
 
 ## Pin
 
-`backend/cpp/ds4/prepare.sh` clones `antirez/ds4` at `DS4_VERSION`. Bump that
-commit to follow upstream.
+`backend/cpp/ds4/Makefile` pins `DS4_VERSION?=<sha>` at the top. The `ds4`
+target in the Makefile clones `antirez/ds4` at that commit (mirroring the
+llama-cpp / ik-llama-cpp / turboquant pattern). The bump-deps bot
+(`.github/workflows/bump_deps.yaml`) finds this pin via grep and opens a
+daily PR to update it. To bump manually: edit the `DS4_VERSION?=` line,
+then `make purge && make` (or rely on CI's clean build).
 
 ## Wire shape
 

.github/workflows/bump_deps.yaml

@@ -22,6 +22,10 @@ jobs:
             variable: "TURBOQUANT_VERSION"
             branch: "feature/turboquant-kv-cache"
             file: "backend/cpp/turboquant/Makefile"
+          - repository: "antirez/ds4"
+            variable: "DS4_VERSION"
+            branch: "main"
+            file: "backend/cpp/ds4/Makefile"
           - repository: "ggml-org/whisper.cpp"
             variable: "WHISPER_CPP_VERSION"
             branch: "master"

backend/cpp/ds4/Makefile

@@ -1,7 +1,14 @@
 # ds4 backend Makefile.
-CURDIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-DS4_DIR := $(CURDIR)ds4
-BUILD_DIR := $(CURDIR)build
+#
+# Upstream pin lives below as DS4_VERSION?= so the bump-deps bot
+# (.github/bump_deps.sh) can find and update it - matches the
+# llama-cpp / ik-llama-cpp / turboquant convention.
+
+DS4_VERSION?=ae302c2fa18cc6d9aefc021d0f27ae03c9ad2fc0
+DS4_REPO?=https://github.com/antirez/ds4
+
+CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+BUILD_DIR := build
 
 BUILD_TYPE ?=
 NATIVE ?= false
@@ -27,37 +34,45 @@ ifneq ($(NATIVE),true)
     CMAKE_ARGS += -DDS4_NATIVE=OFF
 endif
 
-.PHONY: prepare grpc-server package clean purge test all
+.PHONY: grpc-server package clean purge test all
 all: grpc-server
 
-prepare:
-	bash $(CURDIR)prepare.sh
+# Clone the upstream ds4 source at the pinned commit. Directory acts as the
+# target so make only re-clones when missing. After a DS4_VERSION bump,
+# run 'make purge && make' to refetch (or rely on CI's clean build).
+ds4:
+	mkdir -p ds4
+	cd ds4 && \
+	git init -q && \
+	git remote add origin $(DS4_REPO) && \
+	git fetch --depth 1 origin $(DS4_VERSION) && \
+	git checkout FETCH_HEAD
 
 # Build ds4's engine object files via its own Makefile, which already encodes
 # the right per-platform compile flags (Objective-C/Metal on Darwin, nvcc on Linux+CUDA).
-$(DS4_DIR)/ds4.o: prepare
+ds4/ds4.o: ds4
 ifeq ($(BUILD_TYPE),cublas)
-	+$(MAKE) -C $(DS4_DIR) ds4.o ds4_cuda.o
+	+$(MAKE) -C ds4 ds4.o ds4_cuda.o
 else ifeq ($(UNAME_S),Darwin)
-	+$(MAKE) -C $(DS4_DIR) ds4.o ds4_metal.o
+	+$(MAKE) -C ds4 ds4.o ds4_metal.o
 else
-	+$(MAKE) -C $(DS4_DIR) ds4_cpu.o
+	+$(MAKE) -C ds4 ds4_cpu.o
 endif
 
-grpc-server: $(DS4_DIR)/ds4.o
+grpc-server: ds4/ds4.o
 	mkdir -p $(BUILD_DIR)
-	cd $(BUILD_DIR) && cmake $(CMAKE_ARGS) -DDS4_DIR=$(DS4_DIR) $(CURDIR) && cmake --build . --config Release -j $(JOBS)
-	cp $(BUILD_DIR)/grpc-server $(CURDIR)grpc-server
+	cd $(BUILD_DIR) && cmake $(CMAKE_ARGS) $(CURRENT_MAKEFILE_DIR) && cmake --build . --config Release -j $(JOBS)
+	cp $(BUILD_DIR)/grpc-server grpc-server
 
 package: grpc-server
-	bash $(CURDIR)package.sh
+	bash package.sh
 
 test:
 	@echo "ds4 backend: e2e coverage at tests/e2e-backends/ (BACKEND_BINARY mode)"
 
 clean:
-	rm -rf $(BUILD_DIR) $(CURDIR)grpc-server $(CURDIR)package
-	if [ -d $(DS4_DIR) ]; then $(MAKE) -C $(DS4_DIR) clean; fi
+	rm -rf $(BUILD_DIR) grpc-server package
+	if [ -d ds4 ]; then $(MAKE) -C ds4 clean; fi
 
 purge: clean
-	rm -rf $(DS4_DIR)
+	rm -rf ds4

scripts/changed-backends.js

@@ -32,6 +32,9 @@ function inferBackendPath(item) {
     // via a thin wrapper Makefile. Changes to either dir should retrigger it.
     return `backend/cpp/turboquant/`;
   }
+  if (item.dockerfile.endsWith("ds4")) {
+    return `backend/cpp/ds4/`;
+  }
   if (item.dockerfile.endsWith("llama-cpp")) {
     return `backend/cpp/llama-cpp/`;
   }