Skip to content

Commit 706cf5d

Browse files
mudlermudler
authored
feat(sam.cpp): add sam.cpp detection backend (#9288)
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 13a6ed7 commit 706cf5d

21 files changed

Lines changed: 1134 additions & 17 deletions

File tree

‎.github/workflows/backend.yml‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -574,6 +574,19 @@ jobs:
574574
dockerfile: "./backend/Dockerfile.golang"
575575
context: "./"
576576
ubuntu-version: '2404'
577+
- build-type: 'cublas'
578+
cuda-major-version: "12"
579+
cuda-minor-version: "8"
580+
platforms: 'linux/amd64'
581+
tag-latest: 'auto'
582+
tag-suffix: '-gpu-nvidia-cuda-12-sam3-cpp'
583+
runs-on: 'ubuntu-latest'
584+
base-image: "ubuntu:24.04"
585+
skip-drivers: 'false'
586+
backend: "sam3-cpp"
587+
dockerfile: "./backend/Dockerfile.golang"
588+
context: "./"
589+
ubuntu-version: '2404'
577590
- build-type: 'cublas'
578591
cuda-major-version: "12"
579592
cuda-minor-version: "8"
@@ -1147,6 +1160,32 @@ jobs:
11471160
backend: "stablediffusion-ggml"
11481161
dockerfile: "./backend/Dockerfile.golang"
11491162
context: "./"
1163+
- build-type: 'cublas'
1164+
cuda-major-version: "13"
1165+
cuda-minor-version: "0"
1166+
platforms: 'linux/amd64'
1167+
tag-latest: 'auto'
1168+
tag-suffix: '-gpu-nvidia-cuda-13-sam3-cpp'
1169+
runs-on: 'ubuntu-latest'
1170+
base-image: "ubuntu:24.04"
1171+
skip-drivers: 'false'
1172+
backend: "sam3-cpp"
1173+
dockerfile: "./backend/Dockerfile.golang"
1174+
context: "./"
1175+
ubuntu-version: '2404'
1176+
- build-type: 'cublas'
1177+
cuda-major-version: "13"
1178+
cuda-minor-version: "0"
1179+
platforms: 'linux/arm64'
1180+
skip-drivers: 'false'
1181+
tag-latest: 'auto'
1182+
tag-suffix: '-nvidia-l4t-cuda-13-arm64-sam3-cpp'
1183+
base-image: "ubuntu:24.04"
1184+
ubuntu-version: '2404'
1185+
runs-on: 'ubuntu-24.04-arm'
1186+
backend: "sam3-cpp"
1187+
dockerfile: "./backend/Dockerfile.golang"
1188+
context: "./"
11501189
- build-type: 'cublas'
11511190
cuda-major-version: "13"
11521191
cuda-minor-version: "0"
@@ -1907,6 +1946,59 @@ jobs:
19071946
dockerfile: "./backend/Dockerfile.golang"
19081947
context: "./"
19091948
ubuntu-version: '2404'
1949+
# sam3-cpp
1950+
- build-type: ''
1951+
cuda-major-version: ""
1952+
cuda-minor-version: ""
1953+
platforms: 'linux/amd64'
1954+
tag-latest: 'auto'
1955+
tag-suffix: '-cpu-sam3-cpp'
1956+
runs-on: 'ubuntu-latest'
1957+
base-image: "ubuntu:24.04"
1958+
skip-drivers: 'false'
1959+
backend: "sam3-cpp"
1960+
dockerfile: "./backend/Dockerfile.golang"
1961+
context: "./"
1962+
ubuntu-version: '2404'
1963+
- build-type: 'sycl_f32'
1964+
cuda-major-version: ""
1965+
cuda-minor-version: ""
1966+
platforms: 'linux/amd64'
1967+
tag-latest: 'auto'
1968+
tag-suffix: '-gpu-intel-sycl-f32-sam3-cpp'
1969+
runs-on: 'ubuntu-latest'
1970+
base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
1971+
skip-drivers: 'false'
1972+
backend: "sam3-cpp"
1973+
dockerfile: "./backend/Dockerfile.golang"
1974+
context: "./"
1975+
ubuntu-version: '2404'
1976+
- build-type: 'sycl_f16'
1977+
cuda-major-version: ""
1978+
cuda-minor-version: ""
1979+
platforms: 'linux/amd64'
1980+
tag-latest: 'auto'
1981+
tag-suffix: '-gpu-intel-sycl-f16-sam3-cpp'
1982+
runs-on: 'ubuntu-latest'
1983+
base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
1984+
skip-drivers: 'false'
1985+
backend: "sam3-cpp"
1986+
dockerfile: "./backend/Dockerfile.golang"
1987+
context: "./"
1988+
ubuntu-version: '2404'
1989+
- build-type: 'vulkan'
1990+
cuda-major-version: ""
1991+
cuda-minor-version: ""
1992+
platforms: 'linux/amd64,linux/arm64'
1993+
tag-latest: 'auto'
1994+
tag-suffix: '-gpu-vulkan-sam3-cpp'
1995+
runs-on: 'ubuntu-latest'
1996+
base-image: "ubuntu:24.04"
1997+
skip-drivers: 'false'
1998+
backend: "sam3-cpp"
1999+
dockerfile: "./backend/Dockerfile.golang"
2000+
context: "./"
2001+
ubuntu-version: '2404'
19102002
- build-type: 'sycl_f32'
19112003
cuda-major-version: ""
19122004
cuda-minor-version: ""
@@ -1959,6 +2051,19 @@ jobs:
19592051
dockerfile: "./backend/Dockerfile.golang"
19602052
context: "./"
19612053
ubuntu-version: '2204'
2054+
- build-type: 'cublas'
2055+
cuda-major-version: "12"
2056+
cuda-minor-version: "0"
2057+
platforms: 'linux/arm64'
2058+
skip-drivers: 'false'
2059+
tag-latest: 'auto'
2060+
tag-suffix: '-nvidia-l4t-arm64-sam3-cpp'
2061+
base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
2062+
runs-on: 'ubuntu-24.04-arm'
2063+
backend: "sam3-cpp"
2064+
dockerfile: "./backend/Dockerfile.golang"
2065+
context: "./"
2066+
ubuntu-version: '2204'
19622067
# whisper
19632068
- build-type: ''
19642069
cuda-major-version: ""

‎.github/workflows/bump_deps.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ jobs:
3434
variable: "ACESTEP_CPP_VERSION"
3535
branch: "master"
3636
file: "backend/go/acestep-cpp/Makefile"
37+
- repository: "PABannier/sam3.cpp"
38+
variable: "SAM3_VERSION"
39+
branch: "main"
40+
file: "backend/go/sam3-cpp/Makefile"
3741
runs-on: ubuntu-latest
3842
steps:
3943
- uses: actions/checkout@v6

‎Makefile‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Disable parallel execution for backend builds
2-
.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros
2+
.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp
33

44
GOCMD=go
55
GOTEST=$(GOCMD) test
@@ -593,6 +593,9 @@ BACKEND_LLAMA_CPP_QUANTIZATION = llama-cpp-quantization|python|.|false|true
593593
# Rust backends
594594
BACKEND_KOKOROS = kokoros|rust|.|false|true
595595

596+
# C++ backends (Go wrapper with purego)
597+
BACKEND_SAM3_CPP = sam3-cpp|golang|.|false|true
598+
596599
# Helper function to build docker image for a backend
597600
# Usage: $(call docker-build-backend,BACKEND_NAME,DOCKERFILE_TYPE,BUILD_CONTEXT,PROGRESS_FLAG,NEEDS_BACKEND_ARG)
598601
define docker-build-backend
@@ -652,12 +655,13 @@ $(eval $(call generate-docker-build-target,$(BACKEND_MLX_DISTRIBUTED)))
652655
$(eval $(call generate-docker-build-target,$(BACKEND_TRL)))
653656
$(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP_QUANTIZATION)))
654657
$(eval $(call generate-docker-build-target,$(BACKEND_KOKOROS)))
658+
$(eval $(call generate-docker-build-target,$(BACKEND_SAM3_CPP)))
655659

656660
# Pattern rule for docker-save targets
657661
docker-save-%: backend-images
658662
docker save local-ai-backend:$* -o backend-images/$*.tar
659663

660-
docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros
664+
docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros docker-build-sam3-cpp
661665

662666
########################################################
663667
### Mock Backend for E2E Tests

‎backend/backend.proto‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,10 @@ message Message {
444444

445445
message DetectOptions {
446446
string src = 1;
447+
string prompt = 2; // Text prompt (for SAM 3 PCS mode)
448+
repeated float points = 3; // Point coordinates as [x1, y1, label1, x2, y2, label2, ...] (label: 1=pos, 0=neg)
449+
repeated float boxes = 4; // Box coordinates as [x1, y1, x2, y2, ...]
450+
float threshold = 5; // Detection confidence threshold
447451
}
448452

449453
message Detection {
@@ -453,6 +457,7 @@ message Detection {
453457
float height = 4;
454458
float confidence = 5;
455459
string class_name = 6;
460+
bytes mask = 7; // PNG-encoded binary segmentation mask
456461
}
457462

458463
message DetectResponse {

‎backend/go/sam3-cpp/.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
sources/
2+
build*/
3+
package/
4+
libgosam3*.so
5+
sam3-cpp
6+
test-models/
7+
test-data/

‎backend/go/sam3-cpp/CMakeLists.txt‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
cmake_minimum_required(VERSION 3.14)
2+
project(gosam3 LANGUAGES C CXX)
3+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
4+
5+
# Build ggml as static libraries to avoid runtime .so dependencies
6+
set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries" FORCE)
7+
8+
set(SAM3_BUILD_EXAMPLES OFF CACHE BOOL "Disable sam3.cpp examples" FORCE)
9+
set(SAM3_BUILD_TESTS OFF CACHE BOOL "Disable sam3.cpp tests" FORCE)
10+
11+
add_subdirectory(./sources/sam3.cpp)
12+
13+
add_library(gosam3 MODULE gosam3.cpp)
14+
target_link_libraries(gosam3 PRIVATE sam3 ggml)
15+
16+
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
17+
target_link_libraries(gosam3 PRIVATE stdc++fs)
18+
endif()
19+
20+
target_include_directories(gosam3 PUBLIC
21+
sources/sam3.cpp
22+
sources/sam3.cpp/ggml/include
23+
)
24+
25+
set_property(TARGET gosam3 PROPERTY CXX_STANDARD 14)
26+
set_target_properties(gosam3 PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

‎backend/go/sam3-cpp/Makefile‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
CMAKE_ARGS?=
2+
BUILD_TYPE?=
3+
NATIVE?=false
4+
5+
GOCMD?=go
6+
GO_TAGS?=
7+
JOBS?=$(shell nproc --ignore=1)
8+
9+
# sam3.cpp
10+
SAM3_REPO?=https://github.com/PABannier/sam3.cpp
11+
SAM3_VERSION?=8cc6e62bc740d7972746fcd47465ddf1c2b1e3c3
12+
13+
ifeq ($(NATIVE),false)
14+
CMAKE_ARGS+=-DGGML_NATIVE=OFF
15+
endif
16+
17+
# If build type is cublas, then we set -DGGML_CUDA=ON to CMAKE_ARGS automatically
18+
ifeq ($(BUILD_TYPE),cublas)
19+
CMAKE_ARGS+=-DGGML_CUDA=ON
20+
else ifeq ($(BUILD_TYPE),openblas)
21+
CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
22+
else ifeq ($(BUILD_TYPE),clblas)
23+
CMAKE_ARGS+=-DGGML_CLBLAST=ON
24+
else ifeq ($(BUILD_TYPE),hipblas)
25+
ROCM_HOME ?= /opt/rocm
26+
ROCM_PATH ?= /opt/rocm
27+
export CXX=$(ROCM_HOME)/llvm/bin/clang++
28+
export CC=$(ROCM_HOME)/llvm/bin/clang
29+
AMDGPU_TARGETS?=gfx803,gfx900,gfx906,gfx908,gfx90a,gfx942,gfx1010,gfx1030,gfx1032,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
30+
CMAKE_ARGS+=-DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
31+
else ifeq ($(BUILD_TYPE),vulkan)
32+
CMAKE_ARGS+=-DGGML_VULKAN=ON
33+
else ifeq ($(OS),Darwin)
34+
ifneq ($(BUILD_TYPE),metal)
35+
CMAKE_ARGS+=-DGGML_METAL=OFF
36+
else
37+
CMAKE_ARGS+=-DGGML_METAL=ON
38+
CMAKE_ARGS+=-DGGML_METAL_EMBED_LIBRARY=ON
39+
endif
40+
endif
41+
42+
ifeq ($(BUILD_TYPE),sycl_f16)
43+
CMAKE_ARGS+=-DGGML_SYCL=ON \
44+
-DCMAKE_C_COMPILER=icx \
45+
-DCMAKE_CXX_COMPILER=icpx \
46+
-DGGML_SYCL_F16=ON
47+
endif
48+
49+
ifeq ($(BUILD_TYPE),sycl_f32)
50+
CMAKE_ARGS+=-DGGML_SYCL=ON \
51+
-DCMAKE_C_COMPILER=icx \
52+
-DCMAKE_CXX_COMPILER=icpx
53+
endif
54+
55+
sources/sam3.cpp:
56+
git clone --recursive $(SAM3_REPO) sources/sam3.cpp && \
57+
cd sources/sam3.cpp && \
58+
git checkout $(SAM3_VERSION) && \
59+
git submodule update --init --recursive --depth 1 --single-branch
60+
61+
# Detect OS
62+
UNAME_S := $(shell uname -s)
63+
64+
# Only build CPU variants on Linux
65+
ifeq ($(UNAME_S),Linux)
66+
VARIANT_TARGETS = libgosam3-avx.so libgosam3-avx2.so libgosam3-avx512.so libgosam3-fallback.so
67+
else
68+
# On non-Linux (e.g., Darwin), build only fallback variant
69+
VARIANT_TARGETS = libgosam3-fallback.so
70+
endif
71+
72+
sam3-cpp: main.go gosam3.go $(VARIANT_TARGETS)
73+
CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o sam3-cpp ./
74+
75+
package: sam3-cpp
76+
bash package.sh
77+
78+
build: package
79+
80+
clean: purge
81+
rm -rf libgosam3*.so sam3-cpp package sources
82+
83+
purge:
84+
rm -rf build*
85+
86+
# Build all variants (Linux only)
87+
ifeq ($(UNAME_S),Linux)
88+
libgosam3-avx.so: sources/sam3.cpp
89+
$(MAKE) purge
90+
$(info ${GREEN}I sam3-cpp build info:avx${RESET})
91+
SO_TARGET=libgosam3-avx.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgosam3-custom
92+
rm -rfv build*
93+
94+
libgosam3-avx2.so: sources/sam3.cpp
95+
$(MAKE) purge
96+
$(info ${GREEN}I sam3-cpp build info:avx2${RESET})
97+
SO_TARGET=libgosam3-avx2.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgosam3-custom
98+
rm -rfv build*
99+
100+
libgosam3-avx512.so: sources/sam3.cpp
101+
$(MAKE) purge
102+
$(info ${GREEN}I sam3-cpp build info:avx512${RESET})
103+
SO_TARGET=libgosam3-avx512.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgosam3-custom
104+
rm -rfv build*
105+
endif
106+
107+
# Build fallback variant (all platforms)
108+
libgosam3-fallback.so: sources/sam3.cpp
109+
$(MAKE) purge
110+
$(info ${GREEN}I sam3-cpp build info:fallback${RESET})
111+
SO_TARGET=libgosam3-fallback.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgosam3-custom
112+
rm -rfv build*
113+
114+
libgosam3-custom: CMakeLists.txt gosam3.cpp gosam3.h
115+
mkdir -p build-$(SO_TARGET) && \
116+
cd build-$(SO_TARGET) && \
117+
cmake .. $(CMAKE_ARGS) && \
118+
cmake --build . --config Release -j$(JOBS) && \
119+
cd .. && \
120+
mv build-$(SO_TARGET)/libgosam3.so ./$(SO_TARGET)
121+
122+
all: sam3-cpp package

0 commit comments

Comments
 (0)