Skip to content

Commit 56cc4f6

Browse files
localai-botmudler
localai-bot
and
mudler
authored
feat(backend): locate-anything-cpp (open-vocabulary object detection via ggml) (#10264)
* feat(backend): add locate-anything-cpp backend (open-vocab detection via la_capi) A Go/purego backend wrapping locate-anything.cpp's la_capi C ABI, implementing the gRPC Detect RPC: image + open-vocabulary text prompt -> labeled boxes. Mirrors backend/go/rfdetr-cpp; static-links ggml into a per-CPU-variant .so. Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci(backend): register locate-anything-cpp in build matrix Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(gallery): locate-anything gallery entry + model importer Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * test(backend): locate-anything-cpp Load+Detect wire test Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(gallery): add locate-anything-3b model to the gallery index Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci(backend): register locate-anything.cpp in bump_deps auto-bump Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: mudler <mudler@localai.io> * ci(test): e2e smoke for locate-anything-cpp in test-extra (loads the 3B + image, runs Detect) Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: mudler <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Signed-off-by: mudler <mudler@localai.io> Co-authored-by: mudler <mudler@localai.io>
1 parent a53f34e commit 56cc4f6

18 files changed

Lines changed: 1351 additions & 0 deletions

File tree

‎.github/backend-matrix.yml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -703,6 +703,19 @@ include:
703703
dockerfile: "./backend/Dockerfile.golang"
704704
context: "./"
705705
ubuntu-version: '2404'
706+
- build-type: 'cublas'
707+
cuda-major-version: "12"
708+
cuda-minor-version: "8"
709+
platforms: 'linux/amd64'
710+
tag-latest: 'auto'
711+
tag-suffix: '-gpu-nvidia-cuda-12-locate-anything-cpp'
712+
runs-on: 'ubuntu-latest'
713+
base-image: "ubuntu:24.04"
714+
skip-drivers: 'false'
715+
backend: "locate-anything-cpp"
716+
dockerfile: "./backend/Dockerfile.golang"
717+
context: "./"
718+
ubuntu-version: '2404'
706719
- build-type: 'cublas'
707720
cuda-major-version: "12"
708721
cuda-minor-version: "8"
@@ -1543,6 +1556,19 @@ include:
15431556
dockerfile: "./backend/Dockerfile.golang"
15441557
context: "./"
15451558
ubuntu-version: '2404'
1559+
- build-type: 'cublas'
1560+
cuda-major-version: "13"
1561+
cuda-minor-version: "0"
1562+
platforms: 'linux/amd64'
1563+
tag-latest: 'auto'
1564+
tag-suffix: '-gpu-nvidia-cuda-13-locate-anything-cpp'
1565+
runs-on: 'ubuntu-latest'
1566+
base-image: "ubuntu:24.04"
1567+
skip-drivers: 'false'
1568+
backend: "locate-anything-cpp"
1569+
dockerfile: "./backend/Dockerfile.golang"
1570+
context: "./"
1571+
ubuntu-version: '2404'
15461572
- build-type: 'cublas'
15471573
cuda-major-version: "13"
15481574
cuda-minor-version: "0"
@@ -1569,6 +1595,19 @@ include:
15691595
backend: "rfdetr-cpp"
15701596
dockerfile: "./backend/Dockerfile.golang"
15711597
context: "./"
1598+
- build-type: 'cublas'
1599+
cuda-major-version: "13"
1600+
cuda-minor-version: "0"
1601+
platforms: 'linux/arm64'
1602+
skip-drivers: 'false'
1603+
tag-latest: 'auto'
1604+
tag-suffix: '-nvidia-l4t-cuda-13-arm64-locate-anything-cpp'
1605+
base-image: "ubuntu:24.04"
1606+
ubuntu-version: '2404'
1607+
runs-on: 'ubuntu-24.04-arm'
1608+
backend: "locate-anything-cpp"
1609+
dockerfile: "./backend/Dockerfile.golang"
1610+
context: "./"
15721611
- build-type: 'cublas'
15731612
cuda-major-version: "13"
15741613
cuda-minor-version: "0"
@@ -2806,6 +2845,74 @@ include:
28062845
dockerfile: "./backend/Dockerfile.golang"
28072846
context: "./"
28082847
ubuntu-version: '2404'
2848+
# locate-anything-cpp
2849+
- build-type: ''
2850+
cuda-major-version: ""
2851+
cuda-minor-version: ""
2852+
platforms: 'linux/amd64'
2853+
tag-latest: 'auto'
2854+
tag-suffix: '-cpu-locate-anything-cpp'
2855+
runs-on: 'ubuntu-latest'
2856+
base-image: "ubuntu:24.04"
2857+
skip-drivers: 'false'
2858+
backend: "locate-anything-cpp"
2859+
dockerfile: "./backend/Dockerfile.golang"
2860+
context: "./"
2861+
ubuntu-version: '2404'
2862+
- build-type: 'sycl_f32'
2863+
cuda-major-version: ""
2864+
cuda-minor-version: ""
2865+
platforms: 'linux/amd64'
2866+
tag-latest: 'auto'
2867+
tag-suffix: '-gpu-intel-sycl-f32-locate-anything-cpp'
2868+
runs-on: 'ubuntu-latest'
2869+
base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
2870+
skip-drivers: 'false'
2871+
backend: "locate-anything-cpp"
2872+
dockerfile: "./backend/Dockerfile.golang"
2873+
context: "./"
2874+
ubuntu-version: '2404'
2875+
- build-type: 'sycl_f16'
2876+
cuda-major-version: ""
2877+
cuda-minor-version: ""
2878+
platforms: 'linux/amd64'
2879+
tag-latest: 'auto'
2880+
tag-suffix: '-gpu-intel-sycl-f16-locate-anything-cpp'
2881+
runs-on: 'ubuntu-latest'
2882+
base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
2883+
skip-drivers: 'false'
2884+
backend: "locate-anything-cpp"
2885+
dockerfile: "./backend/Dockerfile.golang"
2886+
context: "./"
2887+
ubuntu-version: '2404'
2888+
- build-type: 'vulkan'
2889+
cuda-major-version: ""
2890+
cuda-minor-version: ""
2891+
platforms: 'linux/amd64'
2892+
platform-tag: 'amd64'
2893+
tag-latest: 'auto'
2894+
tag-suffix: '-gpu-vulkan-locate-anything-cpp'
2895+
runs-on: 'ubuntu-latest'
2896+
base-image: "ubuntu:24.04"
2897+
skip-drivers: 'false'
2898+
backend: "locate-anything-cpp"
2899+
dockerfile: "./backend/Dockerfile.golang"
2900+
context: "./"
2901+
ubuntu-version: '2404'
2902+
- build-type: 'vulkan'
2903+
cuda-major-version: ""
2904+
cuda-minor-version: ""
2905+
platforms: 'linux/arm64'
2906+
platform-tag: 'arm64'
2907+
tag-latest: 'auto'
2908+
tag-suffix: '-gpu-vulkan-locate-anything-cpp'
2909+
runs-on: 'ubuntu-24.04-arm'
2910+
base-image: "ubuntu:24.04"
2911+
skip-drivers: 'false'
2912+
backend: "locate-anything-cpp"
2913+
dockerfile: "./backend/Dockerfile.golang"
2914+
context: "./"
2915+
ubuntu-version: '2404'
28092916
- build-type: 'sycl_f32'
28102917
cuda-major-version: ""
28112918
cuda-minor-version: ""
@@ -2899,6 +3006,19 @@ include:
28993006
dockerfile: "./backend/Dockerfile.golang"
29003007
context: "./"
29013008
ubuntu-version: '2204'
3009+
- build-type: 'cublas'
3010+
cuda-major-version: "12"
3011+
cuda-minor-version: "0"
3012+
platforms: 'linux/arm64'
3013+
skip-drivers: 'false'
3014+
tag-latest: 'auto'
3015+
tag-suffix: '-nvidia-l4t-arm64-locate-anything-cpp'
3016+
base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
3017+
runs-on: 'ubuntu-24.04-arm'
3018+
backend: "locate-anything-cpp"
3019+
dockerfile: "./backend/Dockerfile.golang"
3020+
context: "./"
3021+
ubuntu-version: '2204'
29023022
# whisper
29033023
- build-type: ''
29043024
cuda-major-version: ""

‎.github/workflows/bump_deps.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ jobs:
6262
variable: "RFDETR_VERSION"
6363
branch: "main"
6464
file: "backend/go/rfdetr-cpp/Makefile"
65+
- repository: "mudler/locate-anything.cpp"
66+
variable: "LOCATEANYTHING_VERSION"
67+
branch: "master"
68+
file: "backend/go/locate-anything-cpp/Makefile"
6569
- repository: "predict-woo/qwen3-tts.cpp"
6670
variable: "QWEN3TTS_CPP_VERSION"
6771
branch: "main"

‎.github/workflows/test-extra.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ jobs:
3838
acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }}
3939
qwen3-tts-cpp: ${{ steps.detect.outputs.qwen3-tts-cpp }}
4040
rfdetr-cpp: ${{ steps.detect.outputs.rfdetr-cpp }}
41+
locate-anything-cpp: ${{ steps.detect.outputs.locate-anything-cpp }}
4142
vibevoice-cpp: ${{ steps.detect.outputs.vibevoice-cpp }}
4243
localvqe: ${{ steps.detect.outputs.localvqe }}
4344
voxtral: ${{ steps.detect.outputs.voxtral }}
@@ -901,6 +902,45 @@ jobs:
901902
- name: Test rfdetr-cpp
902903
run: |
903904
make --jobs=5 --output-sync=target -C backend/go/rfdetr-cpp test
905+
# Per-backend e2e for locate-anything-cpp: builds the .so + Go binary and
906+
# runs `make -C backend/go/locate-anything-cpp test`. test.sh fetches the
907+
# locate-anything-q8_0 GGUF (~6.3 GB, NVIDIA LocateAnything-3B) from the
908+
# published mudler/locate-anything.cpp-gguf HF repo + a COCO image, then the
909+
# Go wire test loads the model and runs an open-vocabulary Detect, asserting
910+
# at least one labeled box. Heavier than the other Go backends (it is a 3B),
911+
# so it is gated to changes under backend/go/locate-anything-cpp/.
912+
tests-locate-anything-cpp:
913+
needs: detect-changes
914+
if: needs.detect-changes.outputs.locate-anything-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
915+
runs-on: ubuntu-latest
916+
steps:
917+
- name: Clone
918+
uses: actions/checkout@v6
919+
with:
920+
submodules: true
921+
- name: Dependencies
922+
run: |
923+
sudo apt-get update
924+
sudo apt-get install -y build-essential cmake curl libopenblas-dev
925+
- name: Setup Go
926+
uses: actions/setup-go@v5
927+
- name: Display Go version
928+
run: go version
929+
- name: Proto Dependencies
930+
run: |
931+
# Install protoc
932+
curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \
933+
unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
934+
rm protoc.zip
935+
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
936+
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
937+
PATH="$PATH:$HOME/go/bin" make protogen-go
938+
- name: Build locate-anything-cpp
939+
run: |
940+
make --jobs=5 --output-sync=target -C backend/go/locate-anything-cpp
941+
- name: Test locate-anything-cpp
942+
run: |
943+
make --jobs=5 --output-sync=target -C backend/go/locate-anything-cpp test
904944
# Per-backend smoke for vibevoice-cpp: builds the .so + Go binary and
905945
# runs `make -C backend/go/vibevoice-cpp test`. test.sh auto-downloads
906946
# the published mudler/vibevoice.cpp-models bundle (TTS Q8_0 + ASR Q4_K

‎Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,7 @@ prepare-test-extra: protogen-python
566566
$(MAKE) -C backend/python/speaker-recognition
567567
$(MAKE) -C backend/rust/kokoros kokoros-grpc
568568
$(MAKE) -C backend/go/rfdetr-cpp
569+
$(MAKE) -C backend/go/locate-anything-cpp
569570

570571
test-extra: prepare-test-extra
571572
$(MAKE) -C backend/python/transformers test
@@ -593,6 +594,7 @@ test-extra: prepare-test-extra
593594
$(MAKE) -C backend/python/speaker-recognition test
594595
$(MAKE) -C backend/rust/kokoros test
595596
$(MAKE) -C backend/go/rfdetr-cpp test
597+
$(MAKE) -C backend/go/locate-anything-cpp test
596598

597599
##
598600
## End-to-end gRPC tests that exercise a built backend container image.

‎backend/go/locate-anything-cpp/.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
sources/
2+
build*/
3+
package/
4+
liblocateanythingcpp*.so
5+
locate-anything-cpp
6+
test-models/
7+
test-data/

‎backend/go/locate-anything-cpp/CMakeLists.txt‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
cmake_minimum_required(VERSION 3.18)
2+
project(liblocateanythingcpp LANGUAGES C CXX)
3+
4+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
5+
set(CMAKE_CXX_STANDARD 17)
6+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
7+
8+
# Static-link ggml + locate_anything so the resulting .so has no runtime
9+
# dependency on extra ggml/locate_anything shared libraries — only on
10+
# libc/libstdc++/libgomp, which the LocalAI package step bundles into the
11+
# docker image.
12+
set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries" FORCE)
13+
14+
# locate-anything.cpp build switches: skip CLI/tests, keep static lib.
15+
set(LA_BUILD_CLI OFF CACHE BOOL "Disable locate-anything CLI" FORCE)
16+
set(LA_BUILD_TESTS OFF CACHE BOOL "Disable locate-anything tests" FORCE)
17+
set(LA_SHARED OFF CACHE BOOL "Build locate_anything as static lib" FORCE)
18+
19+
# Unlike rt-detr.cpp, locate-anything.cpp ships no in-tree ggml patches, so
20+
# there is no apply_ggml_patches.sh hook to shim here.
21+
add_subdirectory(./sources/locate-anything.cpp)
22+
23+
# locate-anything.cpp's top-level CMakeLists points its own target's include
24+
# dirs at ${CMAKE_SOURCE_DIR}/{include,src,third_party,...}. CMAKE_SOURCE_DIR
25+
# is the *top-level* source dir of the whole CMake tree, so when we pull it in
26+
# via add_subdirectory it resolves to OUR directory, not theirs, and the
27+
# locate_anything target fails to find its own headers (la_capi.h, stb_image.h,
28+
# la_gguf_keys.h). Re-add the correct, subdir-relative include paths to the
29+
# already-defined target so it compiles regardless of where it's nested.
30+
set(LA_SRC ${CMAKE_CURRENT_SOURCE_DIR}/sources/locate-anything.cpp)
31+
target_include_directories(locate_anything PRIVATE
32+
${LA_SRC}/include
33+
${LA_SRC}/src
34+
${LA_SRC}/third_party
35+
${LA_SRC}/third_party/stb)
36+
37+
# locate-anything.cpp's C-API symbols already live inside liblocate_anything
38+
# (src/la_capi.cpp is compiled into the lib). We re-export them via a MODULE
39+
# library that links locate_anything so the symbols are visible at dlopen time.
40+
add_library(locateanythingcpp MODULE
41+
sources/locate-anything.cpp/src/la_capi.cpp)
42+
43+
target_include_directories(locateanythingcpp PRIVATE
44+
sources/locate-anything.cpp/include
45+
sources/locate-anything.cpp/src
46+
sources/locate-anything.cpp/third_party
47+
sources/locate-anything.cpp/third_party/stb
48+
)
49+
50+
target_link_libraries(locateanythingcpp PRIVATE locate_anything ggml)
51+
52+
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
53+
target_link_libraries(locateanythingcpp PRIVATE stdc++fs)
54+
endif()
55+
56+
set_property(TARGET locateanythingcpp PROPERTY CXX_STANDARD 17)
57+
set_target_properties(locateanythingcpp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

0 commit comments

Comments
 (0)