Skip to content

Commit 14a952a

Browse files
committed
[wsl2] discover all .so files at nvidia driver store path
We currently define a hardcoded list of required files that must be discovered at the driver store. This commit extends this logic to also search for additional .so files that may be present in the nvidia driver store path. fixes #1864 Signed-off-by: Christopher Desiniotis <cdesiniotis@nvidia.com>
1 parent 59c0420 commit 14a952a

1 file changed

Lines changed: 101 additions & 8 deletions

File tree

pkg/nvcdi/driver-wsl.go

Lines changed: 101 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,28 @@ package nvcdi
1919
import (
2020
"fmt"
2121
"path/filepath"
22+
"slices"
2223

2324
"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
2425
"github.com/NVIDIA/nvidia-container-toolkit/internal/dxcore"
2526
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
2627
"github.com/NVIDIA/nvidia-container-toolkit/pkg/lookup"
2728
)
2829

30+
const (
31+
libcudaSo = "libcuda.so.1.1"
32+
)
33+
34+
var dxcoreLibraries = []string{
35+
"libdxcore.so", /* Core library for dxcore support */
36+
}
37+
2938
var requiredDriverStoreFiles = []string{
3039
"libcuda.so.1.1", /* Core library for cuda support */
3140
"libcuda_loader.so", /* Core library for cuda support on WSL */
3241
"libnvidia-ptxjitcompiler.so.1", /* Core library for PTX Jit support */
3342
"libnvidia-ml.so.1", /* Core library for nvml */
3443
"libnvidia-ml_loader.so", /* Core library for nvml on WSL */
35-
"libdxcore.so", /* Core library for dxcore support */
3644
"libnvdxgdmal.so.1", /* dxgdmal library for cuda */
3745
"nvcubins.bin", /* Binary containing GPU code for cuda */
3846
"nvidia-smi", /* nvidia-smi binary*/
@@ -56,40 +64,125 @@ func (l *wsllib) newWSLDriverDiscoverer() (discover.Discover, error) {
5664
if len(driverStorePaths) > 1 {
5765
l.logger.Warningf("Found multiple driver store paths: %v", driverStorePaths)
5866
}
59-
l.logger.Infof("Using WSL driver store paths: %v", driverStorePaths)
6067

61-
driverStorePaths = append(driverStorePaths, "/usr/lib/wsl/lib")
68+
nvDriverStorePath, err := l.getNVIDIADriverStorePath(driverStorePaths)
69+
if err != nil {
70+
return nil, fmt.Errorf("failed to find NVIDIA driver store path: %w", err)
71+
}
72+
73+
l.logger.Infof("Using WSL driver store path: %v", nvDriverStorePath)
6274

63-
driverStoreMounts := discover.NewMounts(
75+
dxcoreMounts := discover.NewMounts(
6476
l.logger,
6577
lookup.NewFileLocator(
6678
lookup.WithLogger(l.logger),
6779
lookup.WithSearchPaths(
68-
driverStorePaths...,
80+
"/usr/lib/wsl/lib",
81+
),
82+
lookup.WithCount(1),
83+
),
84+
l.driver.Root,
85+
dxcoreLibraries,
86+
)
87+
88+
requiredDriverStoreMounts := discover.NewMounts(
89+
l.logger,
90+
lookup.NewFileLocator(
91+
lookup.WithLogger(l.logger),
92+
lookup.WithSearchPaths(
93+
nvDriverStorePath,
6994
),
7095
lookup.WithCount(1),
7196
),
7297
l.driver.Root,
7398
requiredDriverStoreFiles,
7499
)
75100

101+
additionalDriverStoreMounts, err := l.getAdditionalMountsFromDriverStore(nvDriverStorePath)
102+
if err != nil {
103+
return nil, fmt.Errorf("failed to get additional mounts from driver store: %w", err)
104+
}
105+
76106
symlinkHook := nvidiaSMISimlinkHook{
77107
logger: l.logger,
78-
mountsFrom: driverStoreMounts,
108+
mountsFrom: requiredDriverStoreMounts,
79109
hookCreator: l.hookCreator,
80110
}
81111

82-
ldcacheHook, _ := discover.NewLDCacheUpdateHook(l.logger, driverStoreMounts, l.hookCreator)
112+
ldcacheHook, _ := discover.NewLDCacheUpdateHook(l.logger, requiredDriverStoreMounts, l.hookCreator)
83113

84114
d := discover.Merge(
85-
driverStoreMounts,
115+
dxcoreMounts,
116+
requiredDriverStoreMounts,
117+
additionalDriverStoreMounts,
86118
symlinkHook,
87119
ldcacheHook,
88120
)
89121

90122
return d, nil
91123
}
92124

125+
// getNVIDIADriverStorePath returns the driver store path associated with NVIDIA GPUs
126+
func (l *wsllib) getNVIDIADriverStorePath(driverStorePaths []string) (string, error) {
127+
fileLocator := lookup.NewFileLocator(
128+
lookup.WithLogger(l.logger),
129+
lookup.WithSearchPaths(
130+
driverStorePaths...,
131+
),
132+
lookup.WithCount(1),
133+
)
134+
matches, err := fileLocator.Locate(libcudaSo)
135+
if err != nil {
136+
return "", fmt.Errorf("failed to locate %s at WSL driver store paths: %w", libcudaSo, err)
137+
}
138+
if len(matches) == 0 {
139+
return "", fmt.Errorf("could not locate %s at WSL driver store paths", libcudaSo)
140+
}
141+
142+
return filepath.Dir(matches[0]), nil
143+
}
144+
145+
// getAdditionalMountsFromDriverStore discovers additional NVIDIA libraries (.so files) from the
146+
// driver store that are not in the required list of libraries.
147+
func (l *wsllib) getAdditionalMountsFromDriverStore(driverStore string) (discover.Discover, error) {
148+
additionalLibs, err := l.getAdditionalLibsFromDriverStore(driverStore, requiredDriverStoreFiles)
149+
if err != nil {
150+
return nil, fmt.Errorf("failed to lookup additional libraries in driver store: %w", err)
151+
}
152+
153+
mounts := discover.NewMounts(
154+
l.logger,
155+
lookup.NewFileLocator(
156+
lookup.WithLogger(l.logger),
157+
lookup.WithRoot(l.driver.Root),
158+
),
159+
l.driver.Root,
160+
additionalLibs,
161+
)
162+
163+
return mounts, nil
164+
}
165+
166+
func (l *wsllib) getAdditionalLibsFromDriverStore(driverStore string, excludeFiles []string) ([]string, error) {
167+
fileLocator := lookup.AsOptional(
168+
lookup.NewFileLocator(
169+
lookup.WithLogger(l.logger),
170+
lookup.WithSearchPaths(driverStore),
171+
lookup.WithFilter(func(s string) error {
172+
if slices.Contains(excludeFiles, filepath.Base(s)) {
173+
return fmt.Errorf("file is excluded")
174+
}
175+
return nil
176+
}),
177+
))
178+
179+
found, err := fileLocator.Locate("*.so*")
180+
if err != nil {
181+
return nil, fmt.Errorf("failed to find additional files at driver store: %w", err)
182+
}
183+
return found, nil
184+
}
185+
93186
type nvidiaSMISimlinkHook struct {
94187
discover.None
95188
logger logger.Interface

0 commit comments

Comments
 (0)