Skip to content

Commit 793753a

Browse files
radu-malliuclaude
andcommitted
feat: allow MPS shared GPU requests greater than one
Honor FailRequestsGreaterThanOne in the MPS sharing strategy the same way it is honored for time-slicing. Previously this flag was forcibly set to true and then ignored, making it impossible to request more than one shared GPU unit under MPS. Now the flag defaults to false, allowing multi-unit MPS requests unless explicitly restricted. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 202c835 commit 793753a

4 files changed

Lines changed: 17 additions & 27 deletions

File tree

api/config/v1/config.go

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,11 @@ func NewConfig(c *cli.Context, flags []cli.Flag) (*Config, error) {
6868
config.Flags.NvidiaDevRoot = config.Flags.NvidiaDriverRoot
6969
}
7070

71-
// We explicitly set sharing.mps.failRequestsGreaterThanOne = true
72-
// This can be relaxed in certain cases -- such as a single GPU -- but
73-
// requires additional logic around when it's OK to combine requests and
74-
// makes the semantics of a request unclear.
75-
if config.Sharing.MPS != nil {
76-
config.Sharing.MPS.FailRequestsGreaterThanOne = true
71+
// Default sharing.mps.failRequestsGreaterThanOne to true if not explicitly set in config.
72+
// Set it to false in the device plugin config to allow pods to request more than one MPS-shared GPU unit.
73+
if config.Sharing.MPS != nil && config.Sharing.MPS.FailRequestsGreaterThanOne == nil {
74+
t := true
75+
config.Sharing.MPS.FailRequestsGreaterThanOne = &t
7776
}
7877

7978
return config, nil

api/config/v1/replicas.go

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828
// ReplicatedResources defines generic options for replicating devices.
2929
type ReplicatedResources struct {
3030
RenameByDefault bool `json:"renameByDefault,omitempty" yaml:"renameByDefault,omitempty"`
31-
FailRequestsGreaterThanOne bool `json:"failRequestsGreaterThanOne,omitempty" yaml:"failRequestsGreaterThanOne,omitempty"`
31+
FailRequestsGreaterThanOne *bool `json:"failRequestsGreaterThanOne,omitempty" yaml:"failRequestsGreaterThanOne,omitempty"`
3232
Resources []ReplicatedResource `json:"resources,omitempty" yaml:"resources,omitempty"`
3333
}
3434

@@ -179,14 +179,11 @@ func (s *ReplicatedResources) UnmarshalJSON(b []byte) error {
179179
return err
180180
}
181181

182-
failRequestsGreaterThanOne, exists := ts["failRequestsGreaterThanOne"]
183-
if !exists {
184-
failRequestsGreaterThanOne = []byte(`false`)
185-
}
186-
187-
err = json.Unmarshal(failRequestsGreaterThanOne, &s.FailRequestsGreaterThanOne)
188-
if err != nil {
189-
return err
182+
if failRequestsGreaterThanOne, exists := ts["failRequestsGreaterThanOne"]; exists {
183+
err = json.Unmarshal(failRequestsGreaterThanOne, &s.FailRequestsGreaterThanOne)
184+
if err != nil {
185+
return err
186+
}
190187
}
191188

192189
resources, exists := ts["resources"]

internal/rm/rm.go

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -74,20 +74,14 @@ func (r *resourceManager) ValidateRequest(ids AnnotatedIDs) error {
7474
// error out if more than one resource is being allocated.
7575
includesReplicas := ids.AnyHasAnnotations()
7676
numRequestedDevices := len(ids)
77+
failRequestsGTOne := r.config.Sharing.ReplicatedResources().FailRequestsGreaterThanOne
7778
switch r.config.Sharing.SharingStrategy() {
7879
case spec.SharingStrategyTimeSlicing:
79-
if includesReplicas && numRequestedDevices > 1 && r.config.Sharing.ReplicatedResources().FailRequestsGreaterThanOne {
80+
if includesReplicas && numRequestedDevices > 1 && failRequestsGTOne != nil && *failRequestsGTOne {
8081
return fmt.Errorf("%w: maximum request size for shared resources is 1; found %d", errInvalidRequest, numRequestedDevices)
8182
}
8283
case spec.SharingStrategyMPS:
83-
// For MPS sharing, we explicitly ignore the FailRequestsGreaterThanOne
84-
// value in the sharing settings.
85-
// This setting was added to timeslicing after the initial release and
86-
// is set to `false` to maintain backward compatibility with existing
87-
// deployments. If we do extend MPS to allow multiple devices to be
88-
// requested, the MPS API will be extended separately from the
89-
// time-slicing API.
90-
if includesReplicas && numRequestedDevices > 1 {
84+
if includesReplicas && numRequestedDevices > 1 && failRequestsGTOne != nil && *failRequestsGTOne {
9185
return fmt.Errorf("%w: maximum request size for shared resources is 1; found %d", errInvalidRequest, numRequestedDevices)
9286
}
9387
}

internal/rm/rm_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"testing"
2121

2222
"github.com/stretchr/testify/require"
23+
"k8s.io/utils/ptr"
2324

2425
spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
2526
)
@@ -94,7 +95,7 @@ func TestValidateRequest(t *testing.T) {
9495
description: "timeslicing with two devices -- failRequestsGreaterThanOne",
9596
sharing: spec.Sharing{
9697
TimeSlicing: spec.ReplicatedResources{
97-
FailRequestsGreaterThanOne: true,
98+
FailRequestsGreaterThanOne: ptr.To(true),
9899
Resources: []spec.ReplicatedResource{
99100
{
100101
Name: "nvidia.com/gpu",
@@ -151,13 +152,12 @@ func TestValidateRequest(t *testing.T) {
151152
"device1::1": nil,
152153
},
153154
requestDevicesIDs: []string{"device0::1", "device1::0"},
154-
expectedError: errInvalidRequest,
155155
},
156156
{
157157
description: "MPS with two devices -- failRequestsGreaterThanOne",
158158
sharing: spec.Sharing{
159159
MPS: &spec.ReplicatedResources{
160-
FailRequestsGreaterThanOne: true,
160+
FailRequestsGreaterThanOne: ptr.To(true),
161161
Resources: []spec.ReplicatedResource{
162162
{
163163
Name: "nvidia.com/gpu",

0 commit comments

Comments
 (0)