Skip to content

Commit cd4bd28

Browse files
mudlermudler
committed
feat(backends): add sglang
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 8487058 commit cd4bd28

21 files changed

Lines changed: 895 additions & 2 deletions

‎.github/workflows/backend.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,19 @@ jobs:
6666
dockerfile: "./backend/Dockerfile.python"
6767
context: "./"
6868
ubuntu-version: '2404'
69+
- build-type: ''
70+
cuda-major-version: ""
71+
cuda-minor-version: ""
72+
platforms: 'linux/amd64'
73+
tag-latest: 'auto'
74+
tag-suffix: '-cpu-sglang'
75+
runs-on: 'ubuntu-latest'
76+
base-image: "ubuntu:24.04"
77+
skip-drivers: 'true'
78+
backend: "sglang"
79+
dockerfile: "./backend/Dockerfile.python"
80+
context: "./"
81+
ubuntu-version: '2404'
6982
- build-type: ''
7083
cuda-major-version: ""
7184
cuda-minor-version: ""
@@ -392,6 +405,19 @@ jobs:
392405
dockerfile: "./backend/Dockerfile.python"
393406
context: "./"
394407
ubuntu-version: '2404'
408+
- build-type: 'cublas'
409+
cuda-major-version: "12"
410+
cuda-minor-version: "8"
411+
platforms: 'linux/amd64'
412+
tag-latest: 'auto'
413+
tag-suffix: '-gpu-nvidia-cuda-12-sglang'
414+
runs-on: 'arc-runner-set'
415+
base-image: "ubuntu:24.04"
416+
skip-drivers: 'false'
417+
backend: "sglang"
418+
dockerfile: "./backend/Dockerfile.python"
419+
context: "./"
420+
ubuntu-version: '2404'
395421
- build-type: 'cublas'
396422
cuda-major-version: "12"
397423
cuda-minor-version: "8"
@@ -1408,6 +1434,19 @@ jobs:
14081434
dockerfile: "./backend/Dockerfile.python"
14091435
context: "./"
14101436
ubuntu-version: '2404'
1437+
- build-type: 'hipblas'
1438+
cuda-major-version: ""
1439+
cuda-minor-version: ""
1440+
platforms: 'linux/amd64'
1441+
tag-latest: 'auto'
1442+
tag-suffix: '-gpu-rocm-hipblas-sglang'
1443+
runs-on: 'arc-runner-set'
1444+
base-image: "rocm/dev-ubuntu-24.04:7.2.1"
1445+
skip-drivers: 'false'
1446+
backend: "sglang"
1447+
dockerfile: "./backend/Dockerfile.python"
1448+
context: "./"
1449+
ubuntu-version: '2404'
14111450
- build-type: 'hipblas'
14121451
cuda-major-version: ""
14131452
cuda-minor-version: ""
@@ -1670,6 +1709,19 @@ jobs:
16701709
dockerfile: "./backend/Dockerfile.python"
16711710
context: "./"
16721711
ubuntu-version: '2404'
1712+
- build-type: 'intel'
1713+
cuda-major-version: ""
1714+
cuda-minor-version: ""
1715+
platforms: 'linux/amd64'
1716+
tag-latest: 'auto'
1717+
tag-suffix: '-gpu-intel-sglang'
1718+
runs-on: 'arc-runner-set'
1719+
base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
1720+
skip-drivers: 'false'
1721+
backend: "sglang"
1722+
dockerfile: "./backend/Dockerfile.python"
1723+
context: "./"
1724+
ubuntu-version: '2404'
16731725
- build-type: 'intel'
16741726
cuda-major-version: ""
16751727
cuda-minor-version: ""

‎.github/workflows/test-extra.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ jobs:
3333
ik-llama-cpp: ${{ steps.detect.outputs.ik-llama-cpp }}
3434
turboquant: ${{ steps.detect.outputs.turboquant }}
3535
vllm: ${{ steps.detect.outputs.vllm }}
36+
sglang: ${{ steps.detect.outputs.sglang }}
3637
acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }}
3738
qwen3-tts-cpp: ${{ steps.detect.outputs.qwen3-tts-cpp }}
3839
voxtral: ${{ steps.detect.outputs.voxtral }}
@@ -589,6 +590,32 @@ jobs:
589590
# - name: Build vllm (cpu) backend image and run gRPC e2e tests
590591
# run: |
591592
# make test-extra-backend-vllm
593+
tests-sglang-grpc:
594+
needs: detect-changes
595+
if: needs.detect-changes.outputs.sglang == 'true' || needs.detect-changes.outputs.run-all == 'true'
596+
runs-on: ubuntu-latest
597+
timeout-minutes: 90
598+
steps:
599+
- name: Clone
600+
uses: actions/checkout@v6
601+
with:
602+
submodules: true
603+
- name: Dependencies
604+
run: |
605+
sudo apt-get update
606+
sudo apt-get install -y --no-install-recommends \
607+
make build-essential curl unzip ca-certificates git tar
608+
- name: Setup Go
609+
uses: actions/setup-go@v5
610+
with:
611+
go-version: '1.25.4'
612+
- name: Free disk space
613+
run: |
614+
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL || true
615+
df -h
616+
- name: Build sglang (cpu) backend image and run gRPC e2e tests
617+
run: |
618+
make test-extra-backend-sglang
592619
tests-acestep-cpp:
593620
needs: detect-changes
594621
if: needs.detect-changes.outputs.acestep-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'

‎Makefile‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Disable parallel execution for backend builds
2-
.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp
2+
.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp
33

44
GOCMD=go
55
GOTEST=$(GOCMD) test
@@ -419,6 +419,7 @@ prepare-test-extra: protogen-python
419419
$(MAKE) -C backend/python/chatterbox
420420
$(MAKE) -C backend/python/vllm
421421
$(MAKE) -C backend/python/vllm-omni
422+
$(MAKE) -C backend/python/sglang
422423
$(MAKE) -C backend/python/vibevoice
423424
$(MAKE) -C backend/python/moonshine
424425
$(MAKE) -C backend/python/pocket-tts
@@ -550,6 +551,16 @@ test-extra-backend-vllm: docker-build-vllm
550551
BACKEND_TEST_OPTIONS=tool_parser:hermes \
551552
$(MAKE) test-extra-backend
552553

554+
## sglang mirrors the vllm setup: HuggingFace model id, same tiny Qwen,
555+
## tool-call extraction via sglang's native qwen parser. CPU builds use
556+
## sglang's upstream pyproject_cpu.toml recipe (see backend/python/sglang/install.sh).
557+
test-extra-backend-sglang: docker-build-sglang
558+
BACKEND_IMAGE=local-ai-backend:sglang \
559+
BACKEND_TEST_MODEL_NAME=Qwen/Qwen2.5-0.5B-Instruct \
560+
BACKEND_TEST_CAPS=health,load,predict,stream,tools \
561+
BACKEND_TEST_OPTIONS=tool_parser:qwen \
562+
$(MAKE) test-extra-backend
563+
553564
## mlx is Apple-Silicon-first — the MLX backend auto-detects the right tool
554565
## parser from the chat template, so no tool_parser: option is needed (it
555566
## would be ignored at runtime). Run this on macOS / arm64 with Metal; the
@@ -689,6 +700,7 @@ BACKEND_NEUTTS = neutts|python|.|false|true
689700
BACKEND_KOKORO = kokoro|python|.|false|true
690701
BACKEND_VLLM = vllm|python|.|false|true
691702
BACKEND_VLLM_OMNI = vllm-omni|python|.|false|true
703+
BACKEND_SGLANG = sglang|python|.|false|true
692704
BACKEND_DIFFUSERS = diffusers|python|.|--progress=plain|true
693705
BACKEND_CHATTERBOX = chatterbox|python|.|false|true
694706
BACKEND_VIBEVOICE = vibevoice|python|.|--progress=plain|true
@@ -758,6 +770,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_NEUTTS)))
758770
$(eval $(call generate-docker-build-target,$(BACKEND_KOKORO)))
759771
$(eval $(call generate-docker-build-target,$(BACKEND_VLLM)))
760772
$(eval $(call generate-docker-build-target,$(BACKEND_VLLM_OMNI)))
773+
$(eval $(call generate-docker-build-target,$(BACKEND_SGLANG)))
761774
$(eval $(call generate-docker-build-target,$(BACKEND_DIFFUSERS)))
762775
$(eval $(call generate-docker-build-target,$(BACKEND_CHATTERBOX)))
763776
$(eval $(call generate-docker-build-target,$(BACKEND_VIBEVOICE)))
@@ -785,7 +798,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SAM3_CPP)))
785798
docker-save-%: backend-images
786799
docker save local-ai-backend:$* -o backend-images/$*.tar
787800

788-
docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros docker-build-sam3-cpp docker-build-qwen3-tts-cpp
801+
docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros docker-build-sam3-cpp docker-build-qwen3-tts-cpp
789802

790803
########################################################
791804
### Mock Backend for E2E Tests

‎backend/index.yaml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,28 @@
227227
intel: "intel-vllm"
228228
nvidia-cuda-12: "cuda12-vllm"
229229
cpu: "cpu-vllm"
230+
- &sglang
231+
name: "sglang"
232+
license: apache-2.0
233+
urls:
234+
- https://github.com/sgl-project/sglang
235+
tags:
236+
- text-to-text
237+
- multimodal
238+
icon: https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png
239+
description: |
240+
SGLang is a fast serving framework for large language models and vision language models.
241+
It co-designs the backend runtime (RadixAttention, continuous batching, structured
242+
decoding) and the frontend language to make interaction with models faster and more
243+
controllable. Features include fast backend runtime, flexible frontend language,
244+
extensive model support, and an active community.
245+
alias: "sglang"
246+
capabilities:
247+
nvidia: "cuda12-sglang"
248+
amd: "rocm-sglang"
249+
intel: "intel-sglang"
250+
nvidia-cuda-12: "cuda12-sglang"
251+
cpu: "cpu-sglang"
230252
- &vllm-omni
231253
name: "vllm-omni"
232254
license: apache-2.0
@@ -1738,6 +1760,54 @@
17381760
uri: "quay.io/go-skynet/local-ai-backends:master-cpu-vllm"
17391761
mirrors:
17401762
- localai/localai-backends:master-cpu-vllm
1763+
# sglang
1764+
- !!merge <<: *sglang
1765+
name: "sglang-development"
1766+
capabilities:
1767+
nvidia: "cuda12-sglang-development"
1768+
amd: "rocm-sglang-development"
1769+
intel: "intel-sglang-development"
1770+
cpu: "cpu-sglang-development"
1771+
- !!merge <<: *sglang
1772+
name: "cuda12-sglang"
1773+
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-sglang"
1774+
mirrors:
1775+
- localai/localai-backends:latest-gpu-nvidia-cuda-12-sglang
1776+
- !!merge <<: *sglang
1777+
name: "rocm-sglang"
1778+
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-sglang"
1779+
mirrors:
1780+
- localai/localai-backends:latest-gpu-rocm-hipblas-sglang
1781+
- !!merge <<: *sglang
1782+
name: "intel-sglang"
1783+
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sglang"
1784+
mirrors:
1785+
- localai/localai-backends:latest-gpu-intel-sglang
1786+
- !!merge <<: *sglang
1787+
name: "cpu-sglang"
1788+
uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-sglang"
1789+
mirrors:
1790+
- localai/localai-backends:latest-cpu-sglang
1791+
- !!merge <<: *sglang
1792+
name: "cuda12-sglang-development"
1793+
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-sglang"
1794+
mirrors:
1795+
- localai/localai-backends:master-gpu-nvidia-cuda-12-sglang
1796+
- !!merge <<: *sglang
1797+
name: "rocm-sglang-development"
1798+
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-sglang"
1799+
mirrors:
1800+
- localai/localai-backends:master-gpu-rocm-hipblas-sglang
1801+
- !!merge <<: *sglang
1802+
name: "intel-sglang-development"
1803+
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sglang"
1804+
mirrors:
1805+
- localai/localai-backends:master-gpu-intel-sglang
1806+
- !!merge <<: *sglang
1807+
name: "cpu-sglang-development"
1808+
uri: "quay.io/go-skynet/local-ai-backends:master-cpu-sglang"
1809+
mirrors:
1810+
- localai/localai-backends:master-cpu-sglang
17411811
# vllm-omni
17421812
- !!merge <<: *vllm-omni
17431813
name: "vllm-omni-development"

‎backend/python/sglang/Makefile‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
.PHONY: sglang
2+
sglang:
3+
bash install.sh
4+
5+
.PHONY: run
6+
run: sglang
7+
@echo "Running sglang..."
8+
bash run.sh
9+
@echo "sglang run."
10+
11+
.PHONY: protogen-clean
12+
protogen-clean:
13+
$(RM) backend_pb2_grpc.py backend_pb2.py
14+
15+
.PHONY: clean
16+
clean: protogen-clean
17+
rm -rf venv __pycache__

0 commit comments

Comments
 (0)