Skip to content

Commit 2c4abd0

Browse files
brosenpodbrosenpod
committed
feat: CE-1579: Add support for deploying serverless with models
1 parent 7e5f9ee commit 2c4abd0

5 files changed

Lines changed: 220 additions & 2 deletions

File tree

‎cmd/serverless/create.go‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ examples:
2424
# create from a template
2525
runpodctl serverless create --template-id <id> --gpu-id "NVIDIA GeForce RTX 4090"
2626
27+
# create from a template and attach a model
28+
runpodctl serverless create --template-id <id> --gpu-id ADA_24 --model-reference https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct:main
29+
2730
# create from a hub repo
2831
runpodctl hub search vllm # find the hub id
2932
runpodctl serverless create --hub-id <id> --gpu-id "NVIDIA GeForce RTX 4090"
@@ -46,6 +49,7 @@ var (
4649
createDataCenterIDs string
4750
createNetworkVolumeID string
4851
createEnvVars []string
52+
createModelReferences []string
4953
)
5054

5155
func init() {
@@ -60,6 +64,7 @@ func init() {
6064
createCmd.Flags().StringVar(&createDataCenterIDs, "data-center-ids", "", "comma-separated list of data center ids")
6165
createCmd.Flags().StringVar(&createNetworkVolumeID, "network-volume-id", "", "network volume id to attach")
6266
createCmd.Flags().StringSliceVar(&createEnvVars, "env", nil, "env vars in KEY=VALUE format; overrides hub defaults (repeatable)")
67+
createCmd.Flags().StringArrayVar(&createModelReferences, "model-reference", nil, "model reference to attach to the endpoint (repeatable)")
6368
}
6469

6570
func runCreate(cmd *cobra.Command, args []string) error {
@@ -69,6 +74,13 @@ func runCreate(cmd *cobra.Command, args []string) error {
6974
if createTemplateID != "" && createHubID != "" {
7075
return fmt.Errorf("--template-id and --hub-id are mutually exclusive; use one or the other")
7176
}
77+
if createHubID != "" && len(createModelReferences) > 0 {
78+
return fmt.Errorf("--model-reference is only supported with --template-id")
79+
}
80+
computeType := strings.ToUpper(strings.TrimSpace(createComputeType))
81+
if len(createModelReferences) > 0 && computeType != "" && computeType != "GPU" {
82+
return fmt.Errorf("--model-reference is only supported with --compute-type GPU")
83+
}
7284

7385
client, err := api.NewClient()
7486
if err != nil {
@@ -79,7 +91,7 @@ func runCreate(cmd *cobra.Command, args []string) error {
7991
req := &api.EndpointCreateRequest{
8092
Name: createName,
8193
TemplateID: createTemplateID,
82-
ComputeType: strings.ToUpper(strings.TrimSpace(createComputeType)),
94+
ComputeType: computeType,
8395
GpuCount: createGpuCount,
8496
WorkersMin: createWorkersMin,
8597
WorkersMax: createWorkersMax,
@@ -211,6 +223,18 @@ func runCreate(cmd *cobra.Command, args []string) error {
211223
req.DataCenterIDs = strings.Split(createDataCenterIDs, ",")
212224
}
213225

226+
if len(createModelReferences) > 0 {
227+
gqlReq := buildTemplateEndpointGQLInput(req, gpuTypeID, createDataCenterIDs, createModelReferences)
228+
endpoint, err := client.CreateEndpointGQL(gqlReq)
229+
if err != nil {
230+
output.Error(err)
231+
return fmt.Errorf("failed to create endpoint: %w", err)
232+
}
233+
234+
format := output.ParseFormat(cmd.Flag("output").Value.String())
235+
return output.Print(endpoint, &output.Config{Format: format})
236+
}
237+
214238
endpoint, err := client.CreateEndpoint(req)
215239
if err != nil {
216240
output.Error(err)
@@ -221,6 +245,21 @@ func runCreate(cmd *cobra.Command, args []string) error {
221245
return output.Print(endpoint, &output.Config{Format: format})
222246
}
223247

248+
func buildTemplateEndpointGQLInput(req *api.EndpointCreateRequest, gpuTypeID, locations string, modelReferences []string) *api.EndpointCreateGQLInput {
249+
// saveEndpoint derives GPU by default when instanceIds is omitted; computeType is not part of EndpointInput.
250+
return &api.EndpointCreateGQLInput{
251+
Name: req.Name,
252+
TemplateID: req.TemplateID,
253+
GpuIDs: gpuTypeID,
254+
GpuCount: req.GpuCount,
255+
WorkersMin: req.WorkersMin,
256+
WorkersMax: req.WorkersMax,
257+
Locations: locations,
258+
NetworkVolumeID: req.NetworkVolumeID,
259+
ModelReferences: modelReferences,
260+
}
261+
}
262+
224263
func randomString(n int) string {
225264
const letters = "abcdefghijklmnopqrstuvwxyz0123456789"
226265
b := make([]byte, n)

‎cmd/serverless/serverless_test.go‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,11 @@ package serverless
22

33
import (
44
"bytes"
5+
"strings"
56
"testing"
67

8+
"github.com/runpod/runpodctl/internal/api"
9+
710
"github.com/spf13/cobra"
811
)
912

@@ -65,6 +68,92 @@ func TestCreateCmd_Flags(t *testing.T) {
6568
if flags.Lookup("workers-max") == nil {
6669
t.Error("expected --workers-max flag")
6770
}
71+
if flags.Lookup("model-reference") == nil {
72+
t.Error("expected --model-reference flag")
73+
}
74+
}
75+
76+
func TestCreateCmd_RejectsHubWithModelReference(t *testing.T) {
77+
oldTemplateID := createTemplateID
78+
oldHubID := createHubID
79+
oldModelReferences := createModelReferences
80+
t.Cleanup(func() {
81+
createTemplateID = oldTemplateID
82+
createHubID = oldHubID
83+
createModelReferences = oldModelReferences
84+
})
85+
86+
createTemplateID = ""
87+
createHubID = "hub-123"
88+
createModelReferences = []string{"https://local/user/model:hash"}
89+
90+
err := runCreate(&cobra.Command{}, nil)
91+
if err == nil {
92+
t.Fatal("expected error")
93+
}
94+
if !strings.Contains(err.Error(), "--model-reference is only supported with --template-id") {
95+
t.Fatalf("unexpected error: %v", err)
96+
}
97+
}
98+
99+
func TestCreateCmd_RejectsCPUWithModelReference(t *testing.T) {
100+
oldTemplateID := createTemplateID
101+
oldHubID := createHubID
102+
oldComputeType := createComputeType
103+
oldModelReferences := createModelReferences
104+
t.Cleanup(func() {
105+
createTemplateID = oldTemplateID
106+
createHubID = oldHubID
107+
createComputeType = oldComputeType
108+
createModelReferences = oldModelReferences
109+
})
110+
111+
createTemplateID = "tpl-123"
112+
createHubID = ""
113+
createComputeType = "CPU"
114+
createModelReferences = []string{"https://local/user/model:hash"}
115+
116+
err := runCreate(&cobra.Command{}, nil)
117+
if err == nil {
118+
t.Fatal("expected error")
119+
}
120+
if !strings.Contains(err.Error(), "--model-reference is only supported with --compute-type GPU") {
121+
t.Fatalf("unexpected error: %v", err)
122+
}
123+
}
124+
125+
func TestBuildTemplateEndpointGQLInput(t *testing.T) {
126+
modelReferences := []string{"https://local/user/model:hash"}
127+
gqlReq := buildTemplateEndpointGQLInput(&api.EndpointCreateRequest{
128+
Name: "test-endpoint",
129+
TemplateID: "tpl-123",
130+
GpuCount: 2,
131+
WorkersMin: 1,
132+
WorkersMax: 3,
133+
NetworkVolumeID: "vol-123",
134+
}, "ADA_24", "US-KS-2", modelReferences)
135+
136+
if gqlReq.Name != "test-endpoint" {
137+
t.Errorf("expected name test-endpoint, got %s", gqlReq.Name)
138+
}
139+
if gqlReq.TemplateID != "tpl-123" {
140+
t.Errorf("expected template id tpl-123, got %s", gqlReq.TemplateID)
141+
}
142+
if gqlReq.GpuIDs != "ADA_24" {
143+
t.Errorf("expected gpu ids ADA_24, got %s", gqlReq.GpuIDs)
144+
}
145+
if gqlReq.GpuCount != 2 || gqlReq.WorkersMin != 1 || gqlReq.WorkersMax != 3 {
146+
t.Fatalf("unexpected worker/gpu settings: %#v", gqlReq)
147+
}
148+
if gqlReq.Locations != "US-KS-2" {
149+
t.Errorf("expected locations US-KS-2, got %s", gqlReq.Locations)
150+
}
151+
if gqlReq.NetworkVolumeID != "vol-123" {
152+
t.Errorf("expected network volume id vol-123, got %s", gqlReq.NetworkVolumeID)
153+
}
154+
if len(gqlReq.ModelReferences) != 1 || gqlReq.ModelReferences[0] != modelReferences[0] {
155+
t.Fatalf("unexpected model references: %#v", gqlReq.ModelReferences)
156+
}
68157
}
69158

70159
func TestUpdateCmd_Flags(t *testing.T) {

‎docs/runpodctl_serverless_create.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ examples:
1313
# create from a template
1414
runpodctl serverless create --template-id <id> --gpu-id "NVIDIA GeForce RTX 4090"
1515

16+
# create from a template and attach a model
17+
runpodctl serverless create --template-id <id> --gpu-id ADA_24 --model-reference https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct:main
18+
1619
# create from a hub repo
1720
runpodctl hub search vllm # find the hub id
1821
runpodctl serverless create --hub-id <id> --gpu-id "NVIDIA GeForce RTX 4090"
@@ -34,6 +37,7 @@ runpodctl serverless create [flags]
3437
--gpu-id string gpu id (from 'runpodctl gpu list')
3538
-h, --help help for create
3639
--hub-id string hub listing id; accepts both SERVERLESS and POD types (alternative to --template-id)
40+
--model-reference stringArray model reference to attach to the endpoint (repeatable)
3741
--name string endpoint name
3842
--network-volume-id string network volume id to attach
3943
--template-id string template id (required if no --hub-id)

‎internal/api/endpoints.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ type Endpoint struct {
2020
WorkersMin int `json:"workersMin,omitempty"`
2121
WorkersMax int `json:"workersMax,omitempty"`
2222
GpuCount int `json:"gpuCount,omitempty"`
23+
ModelReferences []string `json:"modelReferences,omitempty"`
2324
Template map[string]interface{} `json:"template,omitempty"`
2425
Workers []interface{} `json:"workers,omitempty"`
2526
}
@@ -146,7 +147,7 @@ func (c *Client) DeleteEndpoint(endpointID string) error {
146147
// EndpointCreateGQLInput is the input for creating an endpoint via GraphQL
147148
// Used when hubReleaseId is needed (REST API doesn't support it)
148149
type EndpointCreateGQLInput struct {
149-
Name string `json:"name"`
150+
Name string `json:"name,omitempty"`
150151
HubReleaseID string `json:"hubReleaseId,omitempty"`
151152
TemplateID string `json:"templateId,omitempty"`
152153
Template *EndpointTemplateInput `json:"template,omitempty"`
@@ -156,6 +157,7 @@ type EndpointCreateGQLInput struct {
156157
WorkersMax int `json:"workersMax,omitempty"`
157158
Locations string `json:"locations,omitempty"`
158159
NetworkVolumeID string `json:"networkVolumeId,omitempty"`
160+
ModelReferences []string `json:"modelReferences,omitempty"`
159161
}
160162

161163
// EndpointTemplateInput is the inline template for endpoint creation via GraphQL
@@ -174,6 +176,7 @@ func (c *Client) CreateEndpointGQL(req *EndpointCreateGQLInput) (*Endpoint, erro
174176
saveEndpoint(input: $input) {
175177
id
176178
name
179+
templateId
177180
gpuIds
178181
networkVolumeId
179182
locations
@@ -183,6 +186,7 @@ func (c *Client) CreateEndpointGQL(req *EndpointCreateGQLInput) (*Endpoint, erro
183186
workersMin
184187
workersMax
185188
gpuCount
189+
modelReferences
186190
}
187191
}
188192
`

‎internal/api/endpoints_test.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@ import (
44
"encoding/json"
55
"net/http"
66
"net/http/httptest"
7+
"strings"
78
"testing"
9+
10+
"github.com/spf13/viper"
811
)
912

1013
func TestListEndpoints(t *testing.T) {
@@ -93,6 +96,85 @@ func TestCreateEndpoint(t *testing.T) {
9396
}
9497
}
9598

99+
func TestEndpointCreateGQLInputOmitsEmptyName(t *testing.T) {
100+
data, err := json.Marshal(EndpointCreateGQLInput{
101+
TemplateID: "tpl-123",
102+
})
103+
if err != nil {
104+
t.Fatalf("failed to marshal request: %v", err)
105+
}
106+
if strings.Contains(string(data), `"name"`) {
107+
t.Fatalf("expected empty name to be omitted, got %s", data)
108+
}
109+
}
110+
111+
func TestCreateEndpointGQLIncludesModelReferences(t *testing.T) {
112+
modelReference := "https://local/user/model:hash"
113+
oldAPIURL := viper.GetString("apiUrl")
114+
t.Cleanup(func() {
115+
viper.Set("apiUrl", oldAPIURL)
116+
})
117+
118+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
119+
if r.Method != http.MethodPost {
120+
t.Errorf("expected POST, got %s", r.Method)
121+
}
122+
123+
var body struct {
124+
Query string `json:"query"`
125+
Variables struct {
126+
Input EndpointCreateGQLInput `json:"input"`
127+
} `json:"variables"`
128+
}
129+
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
130+
t.Fatalf("failed to decode request: %v", err)
131+
}
132+
if !strings.Contains(body.Query, "modelReferences") {
133+
t.Error("expected query to select modelReferences")
134+
}
135+
if !strings.Contains(body.Query, "templateId") {
136+
t.Error("expected query to select templateId")
137+
}
138+
if len(body.Variables.Input.ModelReferences) != 1 || body.Variables.Input.ModelReferences[0] != modelReference {
139+
t.Fatalf("unexpected model references: %#v", body.Variables.Input.ModelReferences)
140+
}
141+
142+
json.NewEncoder(w).Encode(map[string]interface{}{
143+
"data": map[string]interface{}{
144+
"saveEndpoint": map[string]interface{}{
145+
"id": "new-ep-id",
146+
"name": body.Variables.Input.Name,
147+
"templateId": body.Variables.Input.TemplateID,
148+
"modelReferences": body.Variables.Input.ModelReferences,
149+
},
150+
},
151+
})
152+
}))
153+
defer server.Close()
154+
155+
viper.Set("apiUrl", server.URL)
156+
client := &Client{
157+
baseURL: "http://rest.example",
158+
apiKey: "test-key",
159+
httpClient: server.Client(),
160+
}
161+
162+
endpoint, err := client.CreateEndpointGQL(&EndpointCreateGQLInput{
163+
Name: "test-endpoint",
164+
TemplateID: "tpl-123",
165+
ModelReferences: []string{modelReference},
166+
})
167+
if err != nil {
168+
t.Fatalf("unexpected error: %v", err)
169+
}
170+
if endpoint.TemplateID != "tpl-123" {
171+
t.Errorf("expected template id tpl-123, got %s", endpoint.TemplateID)
172+
}
173+
if len(endpoint.ModelReferences) != 1 || endpoint.ModelReferences[0] != modelReference {
174+
t.Fatalf("unexpected response model references: %#v", endpoint.ModelReferences)
175+
}
176+
}
177+
96178
func TestUpdateEndpoint(t *testing.T) {
97179
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
98180
if r.Method != http.MethodPatch {

0 commit comments

Comments
 (0)