Skip to content

Commit cf2bbc1

Browse files
csg-pr-botganisback
csg-pr-bot
and
ganisback
authored
Feat/llama cpp upgrade and support rocm image (#1216)
Co-authored-by: ganisback <370036720@qq.com>
1 parent 0d59803 commit cf2bbc1

9 files changed

Lines changed: 364 additions & 28 deletions

File tree

‎builder/deploy/deployer.go‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -731,14 +731,17 @@ func (d *deployer) UpdateDeploy(ctx context.Context, dur *types.DeployUpdateReq,
731731
deploy.EngineArgs = *dur.EngineArgs
732732
}
733733

734-
if dur.Entrypoint != nil {
735-
if deploy.RuntimeFramework == string(types.LlamaCpp) {
736-
newVarStr, err := buildVariables(dur)
734+
if isGGUFRuntime(deploy.RuntimeFramework) {
735+
if dur.Entrypoint != nil {
736+
newVarStr, err := buildVariables(deploy.Variables, dur)
737737
if err != nil {
738738
return fmt.Errorf("build variables for llama cpp error: %w", err)
739739
}
740740
dur.Variables = &newVarStr
741741
}
742+
if !hasGGUFEntryPoint(dur.Variables, deploy.Variables) {
743+
return fmt.Errorf("entrypoint is required for llama.cpp or ktransformers")
744+
}
742745
}
743746

744747
if dur.Variables != nil {
@@ -761,8 +764,13 @@ func (d *deployer) UpdateDeploy(ctx context.Context, dur *types.DeployUpdateReq,
761764
return nil
762765
}
763766

764-
func buildVariables(dur *types.DeployUpdateReq) (string, error) {
765-
varStr := ""
767+
func isGGUFRuntime(runtimeFramework string) bool {
768+
name := strings.ToLower(runtimeFramework)
769+
return strings.Contains(name, string(types.LlamaCpp)) || strings.Contains(name, string(types.Ktransformers))
770+
}
771+
772+
func buildVariables(existingVariables string, dur *types.DeployUpdateReq) (string, error) {
773+
varStr := existingVariables
766774
if dur.Variables != nil {
767775
varStr = *dur.Variables
768776
}
@@ -779,6 +787,18 @@ func buildVariables(dur *types.DeployUpdateReq) (string, error) {
779787
return varStr, nil
780788
}
781789

790+
func hasGGUFEntryPoint(variables *string, existingVariables string) bool {
791+
varStr := existingVariables
792+
if variables != nil {
793+
varStr = *variables
794+
}
795+
varMap, err := hubcom.JsonStrToMap(varStr)
796+
if err != nil {
797+
return false
798+
}
799+
return strings.TrimSpace(varMap[types.GGUFEntryPoint]) != ""
800+
}
801+
782802
func (d *deployer) StartDeploy(ctx context.Context, deploy *database.Deploy) error {
783803
deploy.Status = common.Pending
784804
// update deploy table

‎component/model.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1216,7 +1216,7 @@ func (c *modelComponentImpl) Wakeup(ctx context.Context, namespace, name string,
12161216

12171217
func (c *modelComponentImpl) buildVariables(req types.ModelRunReq, frame *database.RuntimeFramework) (string, error) {
12181218
engineName := strings.ToLower(frame.FrameName)
1219-
if engineName == string(types.LlamaCpp) || engineName == string(types.Ktransformers) {
1219+
if strings.Contains(engineName, string(types.LlamaCpp)) || strings.Contains(engineName, string(types.Ktransformers)) {
12201220
//check entrypoint for llama.cpp
12211221
if len(req.Entrypoint) < 1 {
12221222
err := fmt.Errorf("entrypoint is required for llama.cpp or ktransformers")

‎configs/inference/amd-llama.cpp.json‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
{
2+
"engine_name": "amd-llama.cpp",
3+
"enabled": 1,
4+
"model_format": "gguf",
5+
"container_port": 8000,
6+
"description": "llama.cpp OpenAI-compatible GGUF text generation service for ROCm",
7+
"engine_images": [
8+
{
9+
"compute_type": "gpu",
10+
"image": "opencsghq/llama.cpp-rocm:rocm7.2.2-b9787",
11+
"driver_version": "7.2.2",
12+
"engine_version": "b9787"
13+
}
14+
],
15+
"engine_args": [
16+
{ "name": "ctx-size", "value": "8192", "format": "-c %s" },
17+
{ "name": "parallel", "value": "1", "format": "-np %s" },
18+
{ "name": "batch-size", "value": "2048", "format": "-b %s" },
19+
{ "name": "predict", "value": "-1", "format": "-n %s" },
20+
{ "name": "threads", "value": "-1", "format": "-t %s" },
21+
{ "name": "gpu-layers", "value": "-1", "format": "-ngl %s" },
22+
{ "name": "custom-options", "value": "", "format": "%s" }
23+
],
24+
"supported_archs": [
25+
"llama",
26+
"llama4",
27+
"deci",
28+
"falcon",
29+
"baichuan",
30+
"grok",
31+
"gpt2",
32+
"gptj",
33+
"gptneox",
34+
"mpt",
35+
"starcoder",
36+
"refact",
37+
"bert",
38+
"modern-bert",
39+
"nomic-bert",
40+
"nomic-bert-moe",
41+
"neo-bert",
42+
"jina-bert-v2",
43+
"jina-bert-v3",
44+
"eurobert",
45+
"bloom",
46+
"stablelm",
47+
"qwen",
48+
"qwen2",
49+
"qwen2moe",
50+
"qwen2vl",
51+
"qwen3",
52+
"qwen3moe",
53+
"qwen3next",
54+
"qwen3vl",
55+
"qwen3vlmoe",
56+
"qwen35",
57+
"qwen35moe",
58+
"phi2",
59+
"phi3",
60+
"phimoe",
61+
"plamo",
62+
"plamo2",
63+
"plamo3",
64+
"codeshell",
65+
"orion",
66+
"internlm2",
67+
"minicpm",
68+
"minicpm3",
69+
"gemma",
70+
"gemma2",
71+
"gemma3",
72+
"gemma3n",
73+
"gemma4",
74+
"gemma4-assistant",
75+
"gemma-embedding",
76+
"starcoder2",
77+
"rwkv6",
78+
"rwkv6qwen2",
79+
"rwkv7",
80+
"arwkv7",
81+
"mamba",
82+
"mamba2",
83+
"jamba",
84+
"xverse",
85+
"command-r",
86+
"cohere2",
87+
"cohere2moe",
88+
"dbrx",
89+
"olmo",
90+
"olmo2",
91+
"olmoe",
92+
"openelm",
93+
"arctic",
94+
"deepseek",
95+
"deepseek2",
96+
"deepseek2-ocr",
97+
"deepseek32",
98+
"chatglm",
99+
"glm4",
100+
"glm4moe",
101+
"glm-dsa",
102+
"bitnet",
103+
"t5",
104+
"t5encoder",
105+
"jais",
106+
"jais2",
107+
"nemotron",
108+
"nemotron_h",
109+
"nemotron_h_moe",
110+
"exaone",
111+
"exaone4",
112+
"exaone-moe",
113+
"granite",
114+
"granitemoe",
115+
"granitehybrid",
116+
"chameleon",
117+
"wavtokenizer-dec",
118+
"plm",
119+
"bailingmoe",
120+
"bailingmoe2",
121+
"dots1",
122+
"arcee",
123+
"afmoe",
124+
"ernie4_5",
125+
"ernie4_5-moe",
126+
"falcon-h1",
127+
"hunyuan-moe",
128+
"hunyuan-dense",
129+
"hunyuan_vl",
130+
"smollm3",
131+
"gpt-oss",
132+
"lfm2",
133+
"lfm2moe",
134+
"dream",
135+
"smallthinker",
136+
"llada",
137+
"llada-moe",
138+
"seed_oss",
139+
"grovemoe",
140+
"apertus",
141+
"minimax-m2",
142+
"cogvlm",
143+
"rnd1",
144+
"pangu-embedded",
145+
"mistral3",
146+
"eagle3",
147+
"mistral4",
148+
"paddleocr",
149+
"mimo2",
150+
"step35",
151+
"llama-embed",
152+
"maincoder",
153+
"kimi-linear",
154+
"talkie",
155+
"mellum"
156+
]
157+
}

‎configs/inference/llama.cpp.json‎

Lines changed: 74 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,14 @@
66
"engine_images": [
77
{
88
"compute_type": "gpu",
9-
"image": "opencsghq/llama.cpp:b5215",
9+
"image": "opencsghq/llama.cpp:b9787",
1010
"driver_version": "12.4",
11-
"engine_version": "b5215"
12-
},
13-
{
14-
"compute_type": "gpu",
15-
"image": "opencsghq/llama.cpp:b5215-cu121",
16-
"driver_version": "12.1",
17-
"engine_version": "b5215"
11+
"engine_version": "b9787"
1812
},
1913
{
2014
"compute_type": "cpu",
21-
"image": "opencsghq/llama.cpp:b5215-cpu",
22-
"engine_version": "b5215"
15+
"image": "opencsghq/llama.cpp:b9787-cpu",
16+
"engine_version": "b9787"
2317
}
2418
],
2519
"engine_args": [
@@ -36,18 +30,22 @@
3630
"llama4",
3731
"deci",
3832
"falcon",
33+
"baichuan",
3934
"grok",
4035
"gpt2",
4136
"gptj",
4237
"gptneox",
4338
"mpt",
44-
"baichuan",
4539
"starcoder",
4640
"refact",
4741
"bert",
42+
"modern-bert",
4843
"nomic-bert",
4944
"nomic-bert-moe",
45+
"neo-bert",
5046
"jina-bert-v2",
47+
"jina-bert-v3",
48+
"eurobert",
5149
"bloom",
5250
"stablelm",
5351
"qwen",
@@ -56,10 +54,17 @@
5654
"qwen2vl",
5755
"qwen3",
5856
"qwen3moe",
57+
"qwen3next",
58+
"qwen3vl",
59+
"qwen3vlmoe",
60+
"qwen35",
61+
"qwen35moe",
5962
"phi2",
6063
"phi3",
6164
"phimoe",
6265
"plamo",
66+
"plamo2",
67+
"plamo3",
6368
"codeshell",
6469
"orion",
6570
"internlm2",
@@ -68,11 +73,22 @@
6873
"gemma",
6974
"gemma2",
7075
"gemma3",
76+
"gemma3n",
77+
"gemma4",
78+
"gemma4-assistant",
79+
"gemma-embedding",
7180
"starcoder2",
81+
"rwkv6",
82+
"rwkv6qwen2",
83+
"rwkv7",
84+
"arwkv7",
7285
"mamba",
86+
"mamba2",
87+
"jamba",
7388
"xverse",
7489
"command-r",
7590
"cohere2",
91+
"cohere2moe",
7692
"dbrx",
7793
"olmo",
7894
"olmo2",
@@ -81,23 +97,65 @@
8197
"arctic",
8298
"deepseek",
8399
"deepseek2",
100+
"deepseek2-ocr",
101+
"deepseek32",
84102
"chatglm",
85103
"glm4",
104+
"glm4moe",
105+
"glm-dsa",
86106
"bitnet",
87107
"t5",
88108
"t5encoder",
89109
"jais",
110+
"jais2",
90111
"nemotron",
112+
"nemotron_h",
113+
"nemotron_h_moe",
91114
"exaone",
92-
"rwkv6",
93-
"rwkv6qwen2",
94-
"rwkv7",
95-
"arwkv7",
115+
"exaone4",
116+
"exaone-moe",
96117
"granite",
97118
"granitemoe",
119+
"granitehybrid",
98120
"chameleon",
99121
"wavtokenizer-dec",
100122
"plm",
101-
"bailingmoe"
123+
"bailingmoe",
124+
"bailingmoe2",
125+
"dots1",
126+
"arcee",
127+
"afmoe",
128+
"ernie4_5",
129+
"ernie4_5-moe",
130+
"falcon-h1",
131+
"hunyuan-moe",
132+
"hunyuan-dense",
133+
"hunyuan_vl",
134+
"smollm3",
135+
"gpt-oss",
136+
"lfm2",
137+
"lfm2moe",
138+
"dream",
139+
"smallthinker",
140+
"llada",
141+
"llada-moe",
142+
"seed_oss",
143+
"grovemoe",
144+
"apertus",
145+
"minimax-m2",
146+
"cogvlm",
147+
"rnd1",
148+
"pangu-embedded",
149+
"mistral3",
150+
"eagle3",
151+
"mistral4",
152+
"paddleocr",
153+
"mimo2",
154+
"step35",
155+
"llama-embed",
156+
"maincoder",
157+
"kimi-linear",
158+
"talkie",
159+
"mellum"
102160
]
103161
}

0 commit comments

Comments
 (0)