diff --git a/annotator/list/list.go b/annotator/list/list.go index d2293aa14..c9fc9fa24 100644 --- a/annotator/list/list.go +++ b/annotator/list/list.go @@ -31,38 +31,48 @@ import ( "github.com/google/osv-scalibr/annotator/osduplicate/cos" "github.com/google/osv-scalibr/annotator/osduplicate/dpkg" "github.com/google/osv-scalibr/annotator/osduplicate/rpm" + "github.com/google/osv-scalibr/plugin/config" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" ) // InitFn is the annotator initializer function. -type InitFn func(cfg *cpb.PluginConfig) (annotator.Annotator, error) +type InitFn func(cfg *config.PluginConfig) (annotator.Annotator, error) + +func protoCfg(f func(cfg *cpb.PluginConfig) (annotator.Annotator, error)) InitFn { + return func(cfg *config.PluginConfig) (annotator.Annotator, error) { + if cfg == nil || cfg.ProtoConfig == nil { + return f(&cpb.PluginConfig{}) + } + return f(cfg.ProtoConfig) + } +} // InitMap is a map of annotator names to their initers. type InitMap map[string][]InitFn // VEX generation related annotators. var VEX = InitMap{ - apk.Name: {apk.New}, - cachedir.Name: {cachedir.New}, - cos.Name: {cos.New}, - dpkg.Name: {dpkg.New}, - rpm.Name: {rpm.New}, - noexecutabledpkg.Name: {noexecutabledpkg.New}, + apk.Name: {protoCfg(apk.New)}, + cachedir.Name: {protoCfg(cachedir.New)}, + cos.Name: {protoCfg(cos.New)}, + dpkg.Name: {protoCfg(dpkg.New)}, + rpm.Name: {protoCfg(rpm.New)}, + noexecutabledpkg.Name: {protoCfg(noexecutabledpkg.New)}, } // Misc annotators. var Misc = InitMap{ - npmsource.Name: {npmsource.New}, - dpkgsource.Name: {dpkgsource.New}, - brewsource.Name: {brewsource.New}, + npmsource.Name: {protoCfg(npmsource.New)}, + dpkgsource.Name: {protoCfg(dpkgsource.New)}, + brewsource.Name: {protoCfg(brewsource.New)}, } // FFA (Full Filesystem Accountability) related annotators. -var FFA = InitMap{unknownbinariesanno.Name: {unknownbinariesanno.New}} +var FFA = InitMap{unknownbinariesanno.Name: {protoCfg(unknownbinariesanno.New)}} // Default detectors that are recommended to be enabled. -var Default = InitMap{cachedir.Name: {cachedir.New}} +var Default = InitMap{cachedir.Name: {protoCfg(cachedir.New)}} // All annotators. var All = concat( @@ -94,7 +104,7 @@ func vals(initMap InitMap) []InitFn { } // AnnotatorsFromName returns a list of annotators from a name. -func AnnotatorsFromName(name string, cfg *cpb.PluginConfig) ([]annotator.Annotator, error) { +func AnnotatorsFromName(name string, cfg *config.PluginConfig) ([]annotator.Annotator, error) { if initers, ok := annotatorNames[name]; ok { result := []annotator.Annotator{} for _, initer := range initers { diff --git a/annotator/list/list_test.go b/annotator/list/list_test.go index 9c36da5f1..ba83bb96b 100644 --- a/annotator/list/list_test.go +++ b/annotator/list/list_test.go @@ -19,7 +19,7 @@ import ( "testing" al "github.com/google/osv-scalibr/annotator/list" - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) var ( @@ -29,7 +29,7 @@ var ( func TestPluginNamesValid(t *testing.T) { for _, initers := range al.All { for _, initer := range initers { - p, err := initer(&cpb.PluginConfig{}) + p, err := initer(config.DefaultPluginConfig()) if err != nil { t.Fatalf("initer(): %v", err) } diff --git a/binary/cli/cli.go b/binary/cli/cli.go index 4a9e82c5e..f4cc304a5 100644 --- a/binary/cli/cli.go +++ b/binary/cli/cli.go @@ -45,6 +45,7 @@ import ( "google.golang.org/protobuf/encoding/prototext" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) // Array is a type to be passed to flag.Var that supports arrays passed as repeated flags, @@ -319,7 +320,7 @@ func validateGlob(arg string) error { func validateDependency(pluginNames []string, requireExtractors bool) error { f := &Flags{PluginsToRun: pluginNames} - plugins, _, err := f.pluginsToRun() + plugins, _, err := f.resolvePluginsAndConfig() if err != nil { return err } @@ -356,7 +357,7 @@ type extractorOverride struct { // GetScanConfig constructs a SCALIBR scan config from the provided CLI flags. func (f *Flags) GetScanConfig() (*scalibr.ScanConfig, error) { - plugins, pluginCFG, err := f.pluginsToRun() + plugins, pluginCFG, err := f.resolvePluginsAndConfig() if err != nil { return nil, err } @@ -526,13 +527,15 @@ func (f *Flags) WriteScanResults(result *scalibr.ScanResult) error { } // TODO(b/279413691): Allow commas in argument names. -func (f *Flags) pluginsToRun() ([]plugin.Plugin, *cpb.PluginConfig, error) { +func (f *Flags) resolvePluginsAndConfig() ([]plugin.Plugin, *config.PluginConfig, error) { result := make([]plugin.Plugin, 0, len(f.PluginsToRun)) pluginNames := multiStringToList(f.PluginsToRun) - pluginCFG, err := pluginCFGFromFlags(f.PluginCFG) + protoCFG, err := pluginProtoCFGFromFlags(f.PluginCFG) if err != nil { return nil, nil, err } + pluginCFG := config.DefaultPluginConfig() + pluginCFG.ProtoConfig = protoCFG extractorNames := addPluginPrefixToGroups("extractors/", multiStringToList(f.ExtractorsToRun)) detectorNames := addPluginPrefixToGroups("detectors/", multiStringToList(f.DetectorsToRun)) @@ -570,9 +573,9 @@ func addPluginPrefixToGroups(prefix string, pluginNames []string) []string { return result } -// pluginCFGFromFlags parses individually provided +// pluginProtoCFGFromFlags parses individually provided // plugin config strings into one proto. -func pluginCFGFromFlags(flags []string) (*cpb.PluginConfig, error) { +func pluginProtoCFGFromFlags(flags []string) (*cpb.PluginConfig, error) { var cfgString strings.Builder for _, flag := range flags { pluginCFG := &cpb.PluginConfig{} diff --git a/binary/cli/cli_test.go b/binary/cli/cli_test.go index c4f310265..813ba894c 100644 --- a/binary/cli/cli_test.go +++ b/binary/cli/cli_test.go @@ -28,6 +28,7 @@ import ( "github.com/google/osv-scalibr/binary/cli" "github.com/google/osv-scalibr/extractor/filesystem/language/golang/gobinary" "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/plugin/config" "google.golang.org/protobuf/testing/protocmp" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" @@ -580,25 +581,29 @@ func TestGetScanConfig_PluginConfig(t *testing.T) { for _, tc := range []struct { desc string cfgFlags []string - wantCFG *cpb.PluginConfig + wantCFG *config.PluginConfig wantMaxFileSizeBytes int64 wantVersionFromContent bool }{ { desc: "single_setting_in_one_flag", cfgFlags: []string{"max_file_size_bytes:1234"}, - wantCFG: &cpb.PluginConfig{ - MaxFileSizeBytes: 1234, + wantCFG: &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + MaxFileSizeBytes: 1234, + }, }, wantMaxFileSizeBytes: 1234, }, { desc: "multiple_settings_in_one_flag", cfgFlags: []string{"max_file_size_bytes:1234 plugin_specific:{go_binary:{version_from_content:true}}"}, - wantCFG: &cpb.PluginConfig{ - MaxFileSizeBytes: 1234, - PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_GoBinary{GoBinary: &cpb.GoBinaryConfig{VersionFromContent: true}}}, + wantCFG: &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + MaxFileSizeBytes: 1234, + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_GoBinary{GoBinary: &cpb.GoBinaryConfig{VersionFromContent: true}}}, + }, }, }, wantMaxFileSizeBytes: 1234, @@ -610,10 +615,12 @@ func TestGetScanConfig_PluginConfig(t *testing.T) { "max_file_size_bytes:1234", "plugin_specific:{go_binary:{version_from_content:true}}", }, - wantCFG: &cpb.PluginConfig{ - MaxFileSizeBytes: 1234, - PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_GoBinary{GoBinary: &cpb.GoBinaryConfig{VersionFromContent: true}}}, + wantCFG: &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + MaxFileSizeBytes: 1234, + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_GoBinary{GoBinary: &cpb.GoBinaryConfig{VersionFromContent: true}}}, + }, }, }, wantMaxFileSizeBytes: 1234, @@ -622,9 +629,11 @@ func TestGetScanConfig_PluginConfig(t *testing.T) { { desc: "plugin_specific_config_short_version", cfgFlags: []string{"go_binary:{version_from_content:true}"}, - wantCFG: &cpb.PluginConfig{ - PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_GoBinary{GoBinary: &cpb.GoBinaryConfig{VersionFromContent: true}}}, + wantCFG: &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_GoBinary{GoBinary: &cpb.GoBinaryConfig{VersionFromContent: true}}}, + }, }, }, wantVersionFromContent: true, @@ -641,7 +650,7 @@ func TestGetScanConfig_PluginConfig(t *testing.T) { t.Errorf("%v.GetScanConfig(): %v", flags, err) } - if diff := cmp.Diff(tc.wantCFG, scanConfig.RequiredPluginConfig, protocmp.Transform()); diff != "" { + if diff := cmp.Diff(tc.wantCFG.ProtoConfig, scanConfig.RequiredPluginConfig.ProtoConfig, protocmp.Transform()); diff != "" { t.Errorf("%v.GetScanConfig() ScanRoots got diff (-want +got):\n%s", flags, diff) } if len(scanConfig.Plugins) != 1 { diff --git a/clients/datasource/insights.go b/clients/datasource/insights.go index e2ecdf2c4..a4619b5c7 100644 --- a/clients/datasource/insights.go +++ b/clients/datasource/insights.go @@ -110,6 +110,16 @@ func NewCachedInsightsClient(addr string, userAgent string) (*CachedInsightsClie }, nil } +// NewCachedInsightsClientWithConn creates a CachedInsightsClient with a provided connection. +func NewCachedInsightsClientWithConn(conn grpc.ClientConnInterface) *CachedInsightsClient { + return &CachedInsightsClient{ + InsightsClient: pb.NewInsightsClient(conn), + packageCache: NewRequestCache[packageKey, *pb.Package](), + versionCache: NewRequestCache[versionKey, *pb.Version](), + requirementsCache: NewRequestCache[versionKey, *pb.Requirements](), + } +} + // GetPackage returns metadata about a package by querying deps.dev API. func (c *CachedInsightsClient) GetPackage(ctx context.Context, in *pb.GetPackageRequest, opts ...grpc.CallOption) (*pb.Package, error) { return c.packageCache.Get(makePackageKey(in.GetPackageKey()), func() (*pb.Package, error) { diff --git a/detector/list/list.go b/detector/list/list.go index f65cc67f6..0a3dc6251 100644 --- a/detector/list/list.go +++ b/detector/list/list.go @@ -39,70 +39,80 @@ import ( "github.com/google/osv-scalibr/detector/weakcredentials/etcshadow" "github.com/google/osv-scalibr/detector/weakcredentials/filebrowser" "github.com/google/osv-scalibr/detector/weakcredentials/winlocal" + "github.com/google/osv-scalibr/plugin/config" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" ) // InitFn is the detector initializer function. -type InitFn func(cfg *cpb.PluginConfig) (detector.Detector, error) +type InitFn func(cfg *config.PluginConfig) (detector.Detector, error) + +func protoCfg(f func(cfg *cpb.PluginConfig) (detector.Detector, error)) InitFn { + return func(cfg *config.PluginConfig) (detector.Detector, error) { + if cfg == nil || cfg.ProtoConfig == nil { + return f(&cpb.PluginConfig{}) + } + return f(cfg.ProtoConfig) + } +} // InitMap is a map of detector names to their initers. type InitMap map[string][]InitFn // CIS scanning related detectors. var CIS = InitMap{ - etcpasswdpermissions.Name: {etcpasswdpermissions.New}, + etcpasswdpermissions.Name: {protoCfg(etcpasswdpermissions.New)}, } // Govulncheck detectors. -var Govulncheck = InitMap{binary.Name: {binary.New}} +var Govulncheck = InitMap{binary.Name: {protoCfg(binary.New)}} // EndOfLife detectors. -var EndOfLife = InitMap{linuxdistro.Name: {linuxdistro.New}} +var EndOfLife = InitMap{linuxdistro.Name: {protoCfg(linuxdistro.New)}} // Untested CVE scanning related detectors - since they don't have proper testing they // might not work as expected in the future. // TODO(b/405223999): Add tests. var Untested = InitMap{ // CVE-2023-38408 OpenSSH detector. - cve202338408.Name: {cve202338408.New}, + cve202338408.Name: {protoCfg(cve202338408.New)}, // CVE-2022-33891 Spark UI detector. - cve202233891.Name: {cve202233891.New}, + cve202233891.Name: {protoCfg(cve202233891.New)}, // CVE-2020-16846 Salt detector. - cve202016846.Name: {cve202016846.New}, + cve202016846.Name: {protoCfg(cve202016846.New)}, // CVE-2023-6019 Ray Dashboard detector. - cve20236019.Name: {cve20236019.New}, + cve20236019.Name: {protoCfg(cve20236019.New)}, // CVE-2020-11978 Apache Airflow detector. - cve202011978.Name: {cve202011978.New}, + cve202011978.Name: {protoCfg(cve202011978.New)}, // CVE-2024-2912 BentoML detector. - cve20242912.Name: {cve20242912.New}, + cve20242912.Name: {protoCfg(cve20242912.New)}, } // Weakcredentials detectors for weak credentials. var Weakcredentials = InitMap{ - codeserver.Name: {codeserver.New}, - etcshadow.Name: {etcshadow.New}, - filebrowser.Name: {filebrowser.New}, - winlocal.Name: {winlocal.New}, + codeserver.Name: {protoCfg(codeserver.New)}, + etcshadow.Name: {protoCfg(etcshadow.New)}, + filebrowser.Name: {protoCfg(filebrowser.New)}, + winlocal.Name: {protoCfg(winlocal.New)}, } // Misc detectors for miscellaneous security issues. var Misc = InitMap{ - cronjobprivesc.Name: {cronjobprivesc.New}, - dockersocket.Name: {dockersocket.New}, - pammisconfig.Name: {pammisconfig.New}, + cronjobprivesc.Name: {protoCfg(cronjobprivesc.New)}, + dockersocket.Name: {protoCfg(dockersocket.New)}, + pammisconfig.Name: {protoCfg(pammisconfig.New)}, } // CVE for vulnerabilities that have a CVE associated var CVE = InitMap{ // CVE-2025-7775 detector - cve20257775.Name: {cve20257775.New}, + cve20257775.Name: {protoCfg(cve20257775.New)}, } // SupplyChain related vulnerability detectors. var SupplyChain = InitMap{ // Malicious NPM for CanisterWorm - canisterworm.Name: {canisterworm.New}, + canisterworm.Name: {protoCfg(canisterworm.New)}, } // Default detectors that are recommended to be enabled. @@ -148,7 +158,7 @@ func vals(initMap InitMap) []InitFn { } // DetectorsFromName returns a list of detectors from a name. -func DetectorsFromName(name string, cfg *cpb.PluginConfig) ([]detector.Detector, error) { +func DetectorsFromName(name string, cfg *config.PluginConfig) ([]detector.Detector, error) { if initers, ok := detectorNames[name]; ok { result := []detector.Detector{} for _, initer := range initers { diff --git a/detector/list/list_test.go b/detector/list/list_test.go index 29de05c2a..758f5eb3e 100644 --- a/detector/list/list_test.go +++ b/detector/list/list_test.go @@ -20,8 +20,8 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" dl "github.com/google/osv-scalibr/detector/list" + "github.com/google/osv-scalibr/plugin/config" ) var ( @@ -31,7 +31,7 @@ var ( func TestPluginNamesValid(t *testing.T) { for _, initers := range dl.All { for _, initer := range initers { - p, err := initer(&cpb.PluginConfig{}) + p, err := initer(config.DefaultPluginConfig()) if err != nil { t.Fatalf("initer(): %v", err) } @@ -85,7 +85,7 @@ func TestDetectorsFromName(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - got, err := dl.DetectorsFromName(tc.name, &cpb.PluginConfig{}) + got, err := dl.DetectorsFromName(tc.name, config.DefaultPluginConfig()) if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { t.Errorf("dl.DetectorsFromName(%v) error got diff (-want +got):\n%s", tc.name, diff) } diff --git a/enricher/enricherlist/list.go b/enricher/enricherlist/list.go index 32c5b1253..dd633a7f9 100644 --- a/enricher/enricherlist/list.go +++ b/enricher/enricherlist/list.go @@ -85,10 +85,20 @@ import ( "github.com/google/osv-scalibr/veles/secrets/urlcreds" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) // InitFn is the enricher initializer function. -type InitFn func(cfg *cpb.PluginConfig) (enricher.Enricher, error) +type InitFn func(cfg *config.PluginConfig) (enricher.Enricher, error) + +func protoCfg(f func(cfg *cpb.PluginConfig) (enricher.Enricher, error)) InitFn { + return func(cfg *config.PluginConfig) (enricher.Enricher, error) { + if cfg == nil || cfg.ProtoConfig == nil { + return f(&cpb.PluginConfig{}) + } + return f(cfg.ProtoConfig) + } +} // InitMap is a map of names to enricher initializer functions. type InitMap map[string][]InitFn @@ -97,7 +107,7 @@ var ( // LayerDetails enrichers. LayerDetails = InitMap{ - baseimage.Name: {baseimage.New}, + baseimage.Name: {protoCfg(baseimage.New)}, } // License enrichers. @@ -113,7 +123,7 @@ var ( // VEX related enrichers. VEX = InitMap{ - filter.Name: {filter.New}, + filter.Name: {protoCfg(filter.New)}, } // SecretsValidate lists secret validators. @@ -200,25 +210,25 @@ var ( // Reachability enrichers. Reachability = InitMap{ java.Name: {java.New}, - govcsource.Name: {govcsource.New}, - rust.Name: {rust.New}, + govcsource.Name: {protoCfg(govcsource.New)}, + rust.Name: {protoCfg(rust.New)}, } // TransitiveDependency enrichers. TransitiveDependency = InitMap{ - requirements.Name: {requirements.New}, - pomxml.Name: {pomxml.New}, + requirements.Name: {protoCfg(requirements.New)}, + pomxml.Name: {protoCfg(pomxml.New)}, } // PackageDeprecation enricher. PackageDeprecation = InitMap{ - packagedeprecation.Name: {packagedeprecation.New}, + packagedeprecation.Name: {protoCfg(packagedeprecation.New)}, } // FFA enrichers. FFA = InitMap{ - baseimage.Name: {baseimage.New}, - baseimageattr.Name: {baseimageattr.New}, + baseimage.Name: {protoCfg(baseimage.New)}, + baseimageattr.Name: {protoCfg(baseimageattr.New)}, } // Default enrichers. @@ -271,7 +281,7 @@ func vals(initMap InitMap) []InitFn { } // EnricherFromName returns a single enricher based on its exact name. -func EnricherFromName(name string, cfg *cpb.PluginConfig) (enricher.Enricher, error) { +func EnricherFromName(name string, cfg *config.PluginConfig) (enricher.Enricher, error) { initers, ok := enricherNames[name] if !ok { return nil, fmt.Errorf("unknown enricher %q", name) @@ -290,7 +300,7 @@ func EnricherFromName(name string, cfg *cpb.PluginConfig) (enricher.Enricher, er } // EnrichersFromName returns a list of enrichers from a name. -func EnrichersFromName(name string, cfg *cpb.PluginConfig) ([]enricher.Enricher, error) { +func EnrichersFromName(name string, cfg *config.PluginConfig) ([]enricher.Enricher, error) { if initers, ok := enricherNames[name]; ok { var result []enricher.Enricher for _, initer := range initers { @@ -312,14 +322,14 @@ type velesPlugin struct { func fromVeles[S veles.Secret](validator veles.Validator[S], name string, version int) velesPlugin { return velesPlugin{ - initFunc: convert.FromVelesValidator(validator, name, version), + initFunc: protoCfg(convert.FromVelesValidator(validator, name, version)), name: name, } } -func fromVelesWithCfg(initFunc InitFn, name string) velesPlugin { +func fromVelesWithCfg(initFunc func(cfg *cpb.PluginConfig) (enricher.Enricher, error), name string) velesPlugin { return velesPlugin{ - initFunc: initFunc, + initFunc: protoCfg(initFunc), name: name, } } diff --git a/enricher/enricherlist/list_test.go b/enricher/enricherlist/list_test.go index de2520850..c9ab08c51 100644 --- a/enricher/enricherlist/list_test.go +++ b/enricher/enricherlist/list_test.go @@ -20,9 +20,9 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" "github.com/google/osv-scalibr/enricher/baseimage" el "github.com/google/osv-scalibr/enricher/enricherlist" + "github.com/google/osv-scalibr/plugin/config" ) var ( @@ -32,7 +32,7 @@ var ( func TestPluginNamesValid(t *testing.T) { for _, initers := range el.All { for _, initer := range initers { - p, err := initer(&cpb.PluginConfig{}) + p, err := initer(config.DefaultPluginConfig()) if err != nil { t.Fatalf("initer(): %v", err) } @@ -64,7 +64,7 @@ func TestEnrichersFromName(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - got, err := el.EnrichersFromName(tc.name, &cpb.PluginConfig{}) + got, err := el.EnrichersFromName(tc.name, config.DefaultPluginConfig()) if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { t.Errorf("el.EnrichersFromName(%v) error got diff (-want +got):\n%s", tc.name, diff) } diff --git a/enricher/hcpidentity/hcpidentity.go b/enricher/hcpidentity/hcpidentity.go index a8fc1d0f1..e6260887d 100644 --- a/enricher/hcpidentity/hcpidentity.go +++ b/enricher/hcpidentity/hcpidentity.go @@ -29,6 +29,7 @@ import ( "github.com/google/osv-scalibr/veles/secrets/hcp" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) const ( @@ -48,9 +49,17 @@ type Enricher struct { } // New creates a new Enricher with default configuration. -func New(cfg *cpb.PluginConfig) (enricher.Enricher, error) { +func New(cfg *config.PluginConfig) (enricher.Enricher, error) { + if cfg == nil || cfg.ClientFactories == nil { + return nil, fmt.Errorf("client factories not configured for %s", Name) + } + httpClient := cfg.ClientFactories.HTTPClient() + if httpClient == nil { + return nil, fmt.Errorf("HTTP client is nil for %s", Name) + } + baseURL := defaultBaseURL - specific := plugin.FindConfig(cfg, func(c *cpb.PluginSpecificConfig) *cpb.HCPIdentityConfig { + specific := plugin.FindConfig(cfg.ProtoConfig, func(c *cpb.PluginSpecificConfig) *cpb.HCPIdentityConfig { return c.GetHcpIdentity() }) if specific.GetBaseUrl() != "" { @@ -59,7 +68,7 @@ func New(cfg *cpb.PluginConfig) (enricher.Enricher, error) { return &Enricher{ baseURL: baseURL, - httpClient: http.DefaultClient, + httpClient: httpClient, }, nil } diff --git a/enricher/hcpidentity/hcpidentity_test.go b/enricher/hcpidentity/hcpidentity_test.go index 30c4dc14f..6bfee3c44 100644 --- a/enricher/hcpidentity/hcpidentity_test.go +++ b/enricher/hcpidentity/hcpidentity_test.go @@ -25,6 +25,7 @@ import ( "github.com/google/osv-scalibr/veles/secrets/hcp" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) func TestEnrich_PopulatesServicePrincipal(t *testing.T) { @@ -51,10 +52,13 @@ func TestEnrich_PopulatesServicePrincipal(t *testing.T) { })) defer srv.Close() - cfg := &cpb.PluginConfig{ - PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_HcpIdentity{HcpIdentity: &cpb.HCPIdentityConfig{BaseUrl: srv.URL}}}, + cfg := &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_HcpIdentity{HcpIdentity: &cpb.HCPIdentityConfig{BaseUrl: srv.URL}}}, + }, }, + ClientFactories: config.NewDefaultClientFactories(), } e, err := New(cfg) if err != nil { @@ -85,10 +89,13 @@ func TestEnrich_SkipsOnNon200(t *testing.T) { })) defer srv.Close() - cfg := &cpb.PluginConfig{ - PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_HcpIdentity{HcpIdentity: &cpb.HCPIdentityConfig{BaseUrl: srv.URL}}}, + cfg := &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_HcpIdentity{HcpIdentity: &cpb.HCPIdentityConfig{BaseUrl: srv.URL}}}, + }, }, + ClientFactories: config.NewDefaultClientFactories(), } e, err := New(cfg) if err != nil { @@ -109,10 +116,13 @@ func TestEnrich_ConnectionError(t *testing.T) { base := srv.URL srv.Close() - cfg := &cpb.PluginConfig{ - PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_HcpIdentity{HcpIdentity: &cpb.HCPIdentityConfig{BaseUrl: base}}}, + cfg := &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_HcpIdentity{HcpIdentity: &cpb.HCPIdentityConfig{BaseUrl: base}}}, + }, }, + ClientFactories: config.NewDefaultClientFactories(), } e, err := New(cfg) if err != nil { @@ -129,7 +139,7 @@ func TestEnrich_ConnectionError(t *testing.T) { } func TestEnrich_SkipsNonHCPSecret(t *testing.T) { - e, err := New(&cpb.PluginConfig{}) + e, err := New(config.DefaultPluginConfig()) if err != nil { t.Fatalf("New: %v", err) } @@ -140,7 +150,7 @@ func TestEnrich_SkipsNonHCPSecret(t *testing.T) { } func TestEnrich_ContextCanceled(t *testing.T) { - e, err := New(&cpb.PluginConfig{}) + e, err := New(config.DefaultPluginConfig()) if err != nil { t.Fatalf("New: %v", err) } diff --git a/enricher/herokuexpiration/herokuexpiration.go b/enricher/herokuexpiration/herokuexpiration.go index f10b14395..7caa07018 100644 --- a/enricher/herokuexpiration/herokuexpiration.go +++ b/enricher/herokuexpiration/herokuexpiration.go @@ -30,6 +30,7 @@ import ( "github.com/google/osv-scalibr/veles/secrets/herokuplatformkey" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) const ( @@ -49,9 +50,17 @@ type Enricher struct { } // New creates a new Enricher with default configuration. -func New(cfg *cpb.PluginConfig) (enricher.Enricher, error) { +func New(cfg *config.PluginConfig) (enricher.Enricher, error) { + if cfg == nil || cfg.ClientFactories == nil { + return nil, fmt.Errorf("client factories not configured for %s", Name) + } + httpClient := cfg.ClientFactories.HTTPClient() + if httpClient == nil { + return nil, fmt.Errorf("HTTP client is nil for %s", Name) + } + baseURL := defaultBaseURL - specific := plugin.FindConfig(cfg, func(c *cpb.PluginSpecificConfig) *cpb.HerokuExpirationConfig { + specific := plugin.FindConfig(cfg.ProtoConfig, func(c *cpb.PluginSpecificConfig) *cpb.HerokuExpirationConfig { return c.GetHerokuExpiration() }) if specific.GetBaseUrl() != "" { @@ -59,7 +68,7 @@ func New(cfg *cpb.PluginConfig) (enricher.Enricher, error) { } return &Enricher{ baseURL: baseURL, - httpClient: http.DefaultClient, + httpClient: httpClient, }, nil } diff --git a/enricher/herokuexpiration/herokuexpiration_test.go b/enricher/herokuexpiration/herokuexpiration_test.go index 9ddf5b44b..89e698c96 100644 --- a/enricher/herokuexpiration/herokuexpiration_test.go +++ b/enricher/herokuexpiration/herokuexpiration_test.go @@ -26,6 +26,7 @@ import ( "github.com/google/osv-scalibr/veles/secrets/herokuplatformkey" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) const ( @@ -91,10 +92,13 @@ func TestEnrich_DefiniteExpireTime(t *testing.T) { })) defer srv.Close() - cfg := &cpb.PluginConfig{ - PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_HerokuExpiration{HerokuExpiration: &cpb.HerokuExpirationConfig{BaseUrl: srv.URL}}}, + cfg := &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_HerokuExpiration{HerokuExpiration: &cpb.HerokuExpirationConfig{BaseUrl: srv.URL}}}, + }, }, + ClientFactories: config.NewDefaultClientFactories(), } e, err := New(cfg) if err != nil { @@ -125,10 +129,13 @@ func TestEnrich_IndefiniteExpireTime(t *testing.T) { })) defer srv.Close() - cfg := &cpb.PluginConfig{ - PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_HerokuExpiration{HerokuExpiration: &cpb.HerokuExpirationConfig{BaseUrl: srv.URL}}}, + cfg := &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_HerokuExpiration{HerokuExpiration: &cpb.HerokuExpirationConfig{BaseUrl: srv.URL}}}, + }, }, + ClientFactories: config.NewDefaultClientFactories(), } e, err := New(cfg) if err != nil { @@ -153,10 +160,13 @@ func TestEnrich_SkipsOnNon200(t *testing.T) { })) defer srv.Close() - cfg := &cpb.PluginConfig{ - PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_HerokuExpiration{HerokuExpiration: &cpb.HerokuExpirationConfig{BaseUrl: srv.URL}}}, + cfg := &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_HerokuExpiration{HerokuExpiration: &cpb.HerokuExpirationConfig{BaseUrl: srv.URL}}}, + }, }, + ClientFactories: config.NewDefaultClientFactories(), } e, err := New(cfg) if err != nil { @@ -177,10 +187,13 @@ func TestEnrich_ConnectionError(t *testing.T) { base := srv.URL srv.Close() - cfg := &cpb.PluginConfig{ - PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_HerokuExpiration{HerokuExpiration: &cpb.HerokuExpirationConfig{BaseUrl: base}}}, + cfg := &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_HerokuExpiration{HerokuExpiration: &cpb.HerokuExpirationConfig{BaseUrl: base}}}, + }, }, + ClientFactories: config.NewDefaultClientFactories(), } e, err := New(cfg) if err != nil { @@ -197,7 +210,7 @@ func TestEnrich_ConnectionError(t *testing.T) { } func TestEnrich_SkipsNonHerokuSecret(t *testing.T) { - e, err := New(&cpb.PluginConfig{}) + e, err := New(config.DefaultPluginConfig()) if err != nil { t.Fatalf("New: %v", err) } @@ -208,7 +221,7 @@ func TestEnrich_SkipsNonHerokuSecret(t *testing.T) { } func TestEnrich_ContextCanceled(t *testing.T) { - e, err := New(&cpb.PluginConfig{}) + e, err := New(config.DefaultPluginConfig()) if err != nil { t.Fatalf("New: %v", err) } diff --git a/enricher/huggingfacemeta/huggingfacemeta.go b/enricher/huggingfacemeta/huggingfacemeta.go index 75915ccb9..c3a81ae73 100644 --- a/enricher/huggingfacemeta/huggingfacemeta.go +++ b/enricher/huggingfacemeta/huggingfacemeta.go @@ -29,6 +29,7 @@ import ( "github.com/google/osv-scalibr/veles/secrets/huggingfaceapikey" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) const ( @@ -48,9 +49,17 @@ type Enricher struct { } // New creates a new Enricher using the default Veles Validators. -func New(cfg *cpb.PluginConfig) (enricher.Enricher, error) { +func New(cfg *config.PluginConfig) (enricher.Enricher, error) { + if cfg == nil || cfg.ClientFactories == nil { + return nil, fmt.Errorf("client factories not configured for %s", Name) + } + httpClient := cfg.ClientFactories.HTTPClient() + if httpClient == nil { + return nil, fmt.Errorf("HTTP client is nil for %s", Name) + } + baseURL := defaultBaseURL - specific := plugin.FindConfig(cfg, func(c *cpb.PluginSpecificConfig) *cpb.HuggingfaceMetaConfig { + specific := plugin.FindConfig(cfg.ProtoConfig, func(c *cpb.PluginSpecificConfig) *cpb.HuggingfaceMetaConfig { return c.GetHuggingfaceMeta() }) if specific.GetBaseUrl() != "" { @@ -58,7 +67,7 @@ func New(cfg *cpb.PluginConfig) (enricher.Enricher, error) { } return &Enricher{ baseURL: baseURL, - httpClient: http.DefaultClient, + httpClient: httpClient, }, nil } diff --git a/enricher/huggingfacemeta/huggingfacemeta_test.go b/enricher/huggingfacemeta/huggingfacemeta_test.go index 5d5731540..6f12a2d17 100644 --- a/enricher/huggingfacemeta/huggingfacemeta_test.go +++ b/enricher/huggingfacemeta/huggingfacemeta_test.go @@ -29,6 +29,7 @@ import ( "github.com/google/osv-scalibr/veles/secrets/huggingfaceapikey" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) func TestEnricher(t *testing.T) { @@ -166,10 +167,13 @@ func TestEnricher(t *testing.T) { defer ts.Close() // Use enricher configured against the mock server - cfg := &cpb.PluginConfig{ - PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_HuggingfaceMeta{HuggingfaceMeta: &cpb.HuggingfaceMetaConfig{BaseUrl: ts.URL}}}, + cfg := &config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_HuggingfaceMeta{HuggingfaceMeta: &cpb.HuggingfaceMetaConfig{BaseUrl: ts.URL}}}, + }, }, + ClientFactories: config.NewDefaultClientFactories(), } enricher, err := huggingfacemeta.New(cfg) if err != nil { diff --git a/enricher/license/license.go b/enricher/license/license.go index bdfa9af41..915c477f0 100644 --- a/enricher/license/license.go +++ b/enricher/license/license.go @@ -25,13 +25,14 @@ import ( "github.com/google/osv-scalibr/enricher" "github.com/google/osv-scalibr/inventory" "github.com/google/osv-scalibr/plugin" - scalibrversion "github.com/google/osv-scalibr/version" "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" depsdevpb "deps.dev/api/v3" - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" + scalibrversion "github.com/google/osv-scalibr/version" + "google.golang.org/grpc" ) const ( @@ -47,11 +48,12 @@ var _ enricher.Enricher = &Enricher{} // Enricher adds license data to software packages by querying deps.dev type Enricher struct { Client Client + Config *config.PluginConfig } // New creates a new Enricher -func New(_ *cpb.PluginConfig) (enricher.Enricher, error) { - return &Enricher{}, nil +func New(cfg *config.PluginConfig) (enricher.Enricher, error) { + return &Enricher{Config: cfg}, nil } // Name of the Enricher. @@ -82,11 +84,18 @@ func (Enricher) RequiredPlugins() []string { // Enrich adds license data to all the packages using deps.dev func (e *Enricher) Enrich(ctx context.Context, _ *enricher.ScanInput, inv *inventory.Inventory) error { if e.Client == nil { - depsDevAPIClient, err := datasource.NewCachedInsightsClient(depsdev.DepsdevAPI, "osv-scalibr/"+scalibrversion.ScannerVersion) + if e.Config == nil || e.Config.ClientFactories == nil { + return fmt.Errorf("client factories not configured for %s", Name) + } + userAgent := "osv-scalibr/" + scalibrversion.ScannerVersion + conn, err := e.Config.ClientFactories.GRPCClientConn(depsdev.DepsdevAPI, grpc.WithUserAgent(userAgent)) if err != nil { - return fmt.Errorf("cannot connect with deps.dev %w", err) + return fmt.Errorf("failed to get gRPC connection for %s: %w", Name, err) + } + if conn == nil { + return fmt.Errorf("gRPC connection is nil for %s", Name) } - e.Client = depsDevAPIClient + e.Client = datasource.NewCachedInsightsClientWithConn(conn) } queries := make([]*depsdevpb.GetVersionRequest, len(inv.Packages)) diff --git a/enricher/license/license_test.go b/enricher/license/license_test.go index 9d905ae7f..c8e9e0a85 100644 --- a/enricher/license/license_test.go +++ b/enricher/license/license_test.go @@ -30,7 +30,7 @@ import ( "google.golang.org/protobuf/proto" depsdevpb "deps.dev/api/v3" - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) func TestEnrich(t *testing.T) { @@ -49,7 +49,7 @@ func TestEnrich(t *testing.T) { } cli := fakeclient.New(licenseMap) - e, err := license.New(&cpb.PluginConfig{}) + e, err := license.New(&config.PluginConfig{}) if err != nil { t.Fatalf("license.New(): %v", err) } diff --git a/enricher/reachability/java/java.go b/enricher/reachability/java/java.go index 9fd71f043..3c6734a3a 100644 --- a/enricher/reachability/java/java.go +++ b/enricher/reachability/java/java.go @@ -39,7 +39,7 @@ import ( "github.com/google/osv-scalibr/log" "github.com/google/osv-scalibr/plugin" - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) const ( @@ -90,9 +90,16 @@ func (Enricher) RequiredPlugins() []string { } // New creates a new Enricher with default configuration. -func New(cfg *cpb.PluginConfig) (enricher.Enricher, error) { +func New(cfg *config.PluginConfig) (enricher.Enricher, error) { + if cfg == nil || cfg.ClientFactories == nil { + return nil, fmt.Errorf("client factories not configured for %s", Name) + } + httpClient := cfg.ClientFactories.HTTPClient() + if httpClient == nil { + return nil, fmt.Errorf("HTTP client is nil for %s", Name) + } return &Enricher{ - Client: http.DefaultClient, + Client: httpClient, }, nil } @@ -100,7 +107,7 @@ func New(cfg *cpb.PluginConfig) (enricher.Enricher, error) { func (enr Enricher) Enrich(ctx context.Context, input *enricher.ScanInput, inv *inventory.Inventory) error { client := enr.Client if client == nil { - client = http.DefaultClient + return fmt.Errorf("client not configured for %s", Name) } jars := make(map[string]struct{}) for i := range inv.Packages { diff --git a/enricher/reachability/java/java_test.go b/enricher/reachability/java/java_test.go index 882b229b2..7d2ea1af6 100644 --- a/enricher/reachability/java/java_test.go +++ b/enricher/reachability/java/java_test.go @@ -32,7 +32,7 @@ import ( "github.com/google/osv-scalibr/inventory/vex" "github.com/google/osv-scalibr/purl" - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) const ( @@ -49,7 +49,7 @@ const ( func TestScan(t *testing.T) { jar := filepath.Join("testdata", reachableJar) - enr, err := java.New(&cpb.PluginConfig{}) + enr, err := java.New(config.DefaultPluginConfig()) if err != nil { t.Fatalf("Javareach enricher init failed: %s", err) } diff --git a/enricher/vulnmatch/osvdev/osvdev.go b/enricher/vulnmatch/osvdev/osvdev.go index 09a313846..cbc678105 100644 --- a/enricher/vulnmatch/osvdev/osvdev.go +++ b/enricher/vulnmatch/osvdev/osvdev.go @@ -18,6 +18,7 @@ package osvdev import ( "context" "errors" + "fmt" "maps" "slices" "time" @@ -33,6 +34,7 @@ import ( "osv.dev/bindings/go/osvdevexperimental" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" osvpb "github.com/ossf/osv-schema/bindings/go/osvschema" osvapipb "osv.dev/bindings/go/api" ) @@ -59,15 +61,26 @@ type Enricher struct { } // New creates a new Enricher with the given configuration. -func New(cfg *cpb.PluginConfig) (enricher.Enricher, error) { +func New(cfg *config.PluginConfig) (enricher.Enricher, error) { + if cfg == nil || cfg.ClientFactories == nil { + return nil, fmt.Errorf("client factories not configured for %s", Name) + } + httpClient := cfg.ClientFactories.HTTPClient() + if httpClient == nil { + return nil, fmt.Errorf("HTTP client is nil for %s", Name) + } + client := osvdev.DefaultClient() + client.HTTPClient = httpClient client.Config.UserAgent = "osv-scalibr/" + scalibrversion.ScannerVersion initialQueryTimeout := 5 * time.Minute - specific := plugin.FindConfig(cfg, func(c *cpb.PluginSpecificConfig) *cpb.OSVDevConfig { return c.GetOsvdev() }) - if specific != nil { - if specific.InitialQueryTimeoutSeconds > 0 { - initialQueryTimeout = time.Duration(specific.InitialQueryTimeoutSeconds) * time.Second + if cfg.ProtoConfig != nil { + specific := plugin.FindConfig(cfg.ProtoConfig, func(c *cpb.PluginSpecificConfig) *cpb.OSVDevConfig { return c.GetOsvdev() }) + if specific != nil { + if specific.InitialQueryTimeoutSeconds > 0 { + initialQueryTimeout = time.Duration(specific.InitialQueryTimeoutSeconds) * time.Second + } } } diff --git a/enricher/vulnmatch/osvlocal/matcher.go b/enricher/vulnmatch/osvlocal/matcher.go index cdb890449..99d67c791 100644 --- a/enricher/vulnmatch/osvlocal/matcher.go +++ b/enricher/vulnmatch/osvlocal/matcher.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "net/http" "os" "path" @@ -39,10 +40,11 @@ type localMatcher struct { // failedDBs keeps track of the errors when getting databases for each ecosystem failedDBs map[osvconstants.Ecosystem]error // userAgent sets the user agent requests for db zips are made with - userAgent string + userAgent string + httpClient *http.Client } -func newlocalMatcher(localDBPath string, userAgent string, downloadDB bool, zippedDBRemoteHost string) (*localMatcher, error) { +func newlocalMatcher(localDBPath string, userAgent string, downloadDB bool, zippedDBRemoteHost string, httpClient *http.Client) (*localMatcher, error) { dbBasePath, err := setupLocalDBDirectory(localDBPath) if err != nil { return nil, fmt.Errorf("could not create %s: %w", dbBasePath, err) @@ -56,6 +58,7 @@ func newlocalMatcher(localDBPath string, userAgent string, downloadDB bool, zipp downloadDB: downloadDB, userAgent: userAgent, failedDBs: make(map[osvconstants.Ecosystem]error), + httpClient: httpClient, }, nil } @@ -107,6 +110,7 @@ func (matcher *localMatcher) loadDBFromCache(ctx context.Context, eco osvconstan matcher.userAgent, !matcher.downloadDB, invs, + matcher.httpClient, ) if err != nil { diff --git a/enricher/vulnmatch/osvlocal/osvlocal.go b/enricher/vulnmatch/osvlocal/osvlocal.go index af9bf12c7..685d6ac33 100644 --- a/enricher/vulnmatch/osvlocal/osvlocal.go +++ b/enricher/vulnmatch/osvlocal/osvlocal.go @@ -17,7 +17,9 @@ package osvlocal import ( "context" + "fmt" "maps" + "net/http" "slices" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" @@ -26,6 +28,7 @@ import ( "github.com/google/osv-scalibr/inventory" "github.com/google/osv-scalibr/inventory/vex" "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/plugin/config" scalibrversion "github.com/google/osv-scalibr/version" ) @@ -41,23 +44,32 @@ var _ enricher.Enricher = &Enricher{} type Enricher struct { zippedDBRemoteHost string - userAgent string - localPath string - download bool + userAgent string + localPath string + download bool + httpClient *http.Client } // New makes a new osvlocal.Enricher with the given config. -func New(cfg *cpb.PluginConfig) (enricher.Enricher, error) { +func New(cfg *config.PluginConfig) (enricher.Enricher, error) { + if cfg == nil || cfg.ClientFactories == nil { + return nil, fmt.Errorf("client factories not configured for %s", Name) + } + httpClient := cfg.ClientFactories.HTTPClient() + if httpClient == nil { + return nil, fmt.Errorf("HTTP client is nil for %s", Name) + } + userAgent := "osv-scalibr/" + scalibrversion.ScannerVersion remoteHost := "https://osv-vulnerabilities.storage.googleapis.com" localPath := "" download := true - if cfg.UserAgent != "" { - userAgent = cfg.UserAgent + if cfg.ProtoConfig != nil && cfg.ProtoConfig.UserAgent != "" { + userAgent = cfg.ProtoConfig.UserAgent } - specific := plugin.FindConfig(cfg, func(c *cpb.PluginSpecificConfig) *cpb.OSVLocalConfig { return c.GetOsvlocal() }) + specific := plugin.FindConfig(cfg.ProtoConfig, func(c *cpb.PluginSpecificConfig) *cpb.OSVLocalConfig { return c.GetOsvlocal() }) if specific != nil { remoteHost = specific.RemoteHost localPath = specific.LocalPath @@ -66,15 +78,21 @@ func New(cfg *cpb.PluginConfig) (enricher.Enricher, error) { return &Enricher{ zippedDBRemoteHost: remoteHost, - - userAgent: userAgent, - localPath: localPath, - download: download, + userAgent: userAgent, + localPath: localPath, + download: download, + httpClient: httpClient, }, nil } func newForTesting(zippedDBRemoteHost string) enricher.Enricher { - return &Enricher{zippedDBRemoteHost, "", "", true} + return &Enricher{ + zippedDBRemoteHost: zippedDBRemoteHost, + userAgent: "", + localPath: "", + download: true, + httpClient: http.DefaultClient, + } } // Name of the Enricher. @@ -110,11 +128,15 @@ func (Enricher) RequiredPlugins() []string { // Enrich checks for vulnerabilities in the inventory packages using zip files exported by osv.dev func (e *Enricher) Enrich(ctx context.Context, _ *enricher.ScanInput, inv *inventory.Inventory) error { + if e.httpClient == nil { + return fmt.Errorf("client not configured for %s", Name) + } dbs, err := newlocalMatcher( e.localPath, e.userAgent, e.download, e.zippedDBRemoteHost, + e.httpClient, ) if err != nil { diff --git a/enricher/vulnmatch/osvlocal/osvlocal_test.go b/enricher/vulnmatch/osvlocal/osvlocal_test.go index 7745ebbda..a1e1690ce 100644 --- a/enricher/vulnmatch/osvlocal/osvlocal_test.go +++ b/enricher/vulnmatch/osvlocal/osvlocal_test.go @@ -30,6 +30,7 @@ import ( "github.com/google/osv-scalibr/inventory" "github.com/google/osv-scalibr/inventory/vex" "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/plugin/config" "github.com/google/osv-scalibr/purl" osvpb "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" @@ -42,7 +43,7 @@ func TestRequirements(t *testing.T) { var err error // we should be online by default - e, err = New(&cpb.PluginConfig{}) + e, err = New(config.DefaultPluginConfig()) if err != nil { t.Errorf("New() = %v, want nil", err) @@ -53,11 +54,16 @@ func TestRequirements(t *testing.T) { } // we should not be online - e, err = New(&cpb.PluginConfig{PluginSpecific: []*cpb.PluginSpecificConfig{ - {Config: &cpb.PluginSpecificConfig_Osvlocal{ - Osvlocal: &cpb.OSVLocalConfig{Download: false}, - }}, - }}) + e, err = New(&config.PluginConfig{ + ProtoConfig: &cpb.PluginConfig{ + PluginSpecific: []*cpb.PluginSpecificConfig{ + {Config: &cpb.PluginSpecificConfig_Osvlocal{ + Osvlocal: &cpb.OSVLocalConfig{Download: false}, + }}, + }, + }, + ClientFactories: config.NewDefaultClientFactories(), + }) if err != nil { t.Errorf("New() = %v, want nil", err) diff --git a/enricher/vulnmatch/osvlocal/zip.go b/enricher/vulnmatch/osvlocal/zip.go index 9a802f956..99aeed075 100644 --- a/enricher/vulnmatch/osvlocal/zip.go +++ b/enricher/vulnmatch/osvlocal/zip.go @@ -47,19 +47,20 @@ type zipDB struct { // the vulnerabilities that are loaded into this database Vulnerabilities []*osvpb.Vulnerability // User agent to query with - UserAgent string + UserAgent string + httpClient *http.Client } var errOfflineDatabaseNotFound = errors.New("no offline version of the OSV database is available") -func fetchRemoteArchiveCRC32CHash(ctx context.Context, url string) (uint32, error) { +func fetchRemoteArchiveCRC32CHash(ctx context.Context, url string, httpClient *http.Client) (uint32, error) { req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) if err != nil { return 0, err } - resp, err := http.DefaultClient.Do(req) + resp, err := httpClient.Do(req) if err != nil { return 0, err } @@ -102,7 +103,7 @@ func (db *zipDB) fetchZip(ctx context.Context) ([]byte, error) { } if err == nil { - remoteHash, err := fetchRemoteArchiveCRC32CHash(ctx, db.ArchiveURL) + remoteHash, err := fetchRemoteArchiveCRC32CHash(ctx, db.ArchiveURL, db.httpClient) if err != nil { return nil, err @@ -123,7 +124,7 @@ func (db *zipDB) fetchZip(ctx context.Context) ([]byte, error) { req.Header.Set("User-Agent", db.UserAgent) } - resp, err := http.DefaultClient.Do(req) + resp, err := db.httpClient.Do(req) if err != nil { return nil, fmt.Errorf("could not retrieve OSV database archive: %w", err) } @@ -232,13 +233,14 @@ func (db *zipDB) load(ctx context.Context, names []string) error { return nil } -func newZippedDB(ctx context.Context, dbBasePath, name, url, userAgent string, offline bool, invs []*extractor.Package) (*zipDB, error) { +func newZippedDB(ctx context.Context, dbBasePath, name, url, userAgent string, offline bool, invs []*extractor.Package, httpClient *http.Client) (*zipDB, error) { db := &zipDB{ Name: name, ArchiveURL: url, Offline: offline, StoredAt: path.Join(dbBasePath, name, "all.zip"), UserAgent: userAgent, + httpClient: httpClient, } names := make([]string, 0, len(invs)) diff --git a/enricher/vulnmatch/osvlocal/zip_test.go b/enricher/vulnmatch/osvlocal/zip_test.go index ecb8959cd..2d0918ef7 100644 --- a/enricher/vulnmatch/osvlocal/zip_test.go +++ b/enricher/vulnmatch/osvlocal/zip_test.go @@ -115,7 +115,7 @@ func TestNewZippedDB_Offline_WithoutCache(t *testing.T) { t.Errorf("a server request was made when running offline") }) - _, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, true, nil) + _, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, true, nil, http.DefaultClient) if !errors.Is(err, errOfflineDatabaseNotFound) { t.Errorf("expected \"%v\" error but got \"%v\"", errOfflineDatabaseNotFound, err) @@ -145,7 +145,7 @@ func TestNewZippedDB_Offline_WithCache(t *testing.T) { "GHSA-5.json": {Id: "GHSA-5"}, })) - db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, true, nil) + db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, true, nil, http.DefaultClient) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -161,7 +161,7 @@ func TestNewZippedDB_BadZip(t *testing.T) { _, _ = w.Write([]byte("this is not a zip")) }) - _, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil) + _, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil, http.DefaultClient) if err == nil { t.Errorf("expected an error but did not get one") @@ -171,7 +171,7 @@ func TestNewZippedDB_BadZip(t *testing.T) { func TestNewZippedDB_UnsupportedProtocol(t *testing.T) { testDir := createTestDir(t) - _, err := newZippedDB(t.Context(), testDir, "my-db", "file://hello-world", userAgent, false, nil) + _, err := newZippedDB(t.Context(), testDir, "my-db", "file://hello-world", userAgent, false, nil, http.DefaultClient) if err == nil { t.Errorf("expected an error but did not get one") @@ -199,7 +199,7 @@ func TestNewZippedDB_Online_WithoutCache(t *testing.T) { }) }) - db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil) + db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil, http.DefaultClient) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -229,7 +229,7 @@ func TestNewZippedDB_Online_WithoutCacheAndNoHashHeader(t *testing.T) { })) }) - db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil) + db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil, http.DefaultClient) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -265,7 +265,7 @@ func TestNewZippedDB_Online_WithSameCache(t *testing.T) { cacheWrite(t, determineStoredAtPath(testDir, "my-db"), cache) - db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil) + db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil, http.DefaultClient) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -301,7 +301,7 @@ func TestNewZippedDB_Online_WithDifferentCache(t *testing.T) { "GHSA-3.json": {Id: "GHSA-3"}, })) - db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil) + db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil, http.DefaultClient) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -329,7 +329,7 @@ func TestNewZippedDB_Online_WithCacheButNoHashHeader(t *testing.T) { "GHSA-3.json": {Id: "GHSA-3"}, })) - _, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil) + _, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil, http.DefaultClient) if err == nil { t.Errorf("expected an error but did not get one") @@ -355,7 +355,7 @@ func TestNewZippedDB_Online_WithBadCache(t *testing.T) { cacheWriteBad(t, determineStoredAtPath(testDir, "my-db"), "this is not json!") - db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil) + db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil, http.DefaultClient) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -379,7 +379,7 @@ func TestNewZippedDB_FileChecks(t *testing.T) { }) }) - db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil) + db, err := newZippedDB(t.Context(), testDir, "my-db", ts.URL, userAgent, false, nil, http.DefaultClient) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -437,6 +437,7 @@ func TestNewZippedDB_WithSpecificPackages(t *testing.T) { userAgent, false, []*extractor.Package{{Name: "pkg-1"}, {Name: "pkg-3"}}, + http.DefaultClient, ) if err != nil { diff --git a/extractor/filesystem/list/list.go b/extractor/filesystem/list/list.go index ecb838602..6f12888d6 100644 --- a/extractor/filesystem/list/list.go +++ b/extractor/filesystem/list/list.go @@ -183,10 +183,20 @@ import ( "github.com/google/osv-scalibr/veles/secrets/vapid" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) // InitFn is the extractor initializer function. -type InitFn func(cfg *cpb.PluginConfig) (filesystem.Extractor, error) +type InitFn func(cfg *config.PluginConfig) (filesystem.Extractor, error) + +func protoCfg(f func(cfg *cpb.PluginConfig) (filesystem.Extractor, error)) InitFn { + return func(cfg *config.PluginConfig) (filesystem.Extractor, error) { + if cfg == nil || cfg.ProtoConfig == nil { + return f(&cpb.PluginConfig{}) + } + return f(cfg.ProtoConfig) + } +} // InitMap is a map of extractor names to their initers. type InitMap map[string][]InitFn @@ -196,173 +206,173 @@ var ( // Language extractors. // CppSource extractors for C++. - CppSource = InitMap{conanlock.Name: {conanlock.New}} + CppSource = InitMap{conanlock.Name: {protoCfg(conanlock.New)}} // JavaSource extractors for Java. JavaSource = InitMap{ - gradlelockfile.Name: {gradlelockfile.New}, - gradleverificationmetadataxml.Name: {gradleverificationmetadataxml.New}, - pomxml.Name: {pomxml.New}, + gradlelockfile.Name: {protoCfg(gradlelockfile.New)}, + gradleverificationmetadataxml.Name: {protoCfg(gradleverificationmetadataxml.New)}, + pomxml.Name: {protoCfg(pomxml.New)}, } // JavaArtifact extractors for Java. JavaArtifact = InitMap{ - javaarchive.Name: {javaarchive.New}, + javaarchive.Name: {protoCfg(javaarchive.New)}, } // JavascriptSource extractors for Javascript. JavascriptSource = InitMap{ - packagejson.Name: {packagejson.New}, - packagelockjson.Name: {packagelockjson.New}, - denojson.Name: {denojson.New}, - denotssource.Name: {denotssource.New}, - pnpmlock.Name: {pnpmlock.New}, - yarnlock.Name: {yarnlock.New}, - bunlock.Name: {bunlock.New}, + packagejson.Name: {protoCfg(packagejson.New)}, + packagelockjson.Name: {protoCfg(packagelockjson.New)}, + denojson.Name: {protoCfg(denojson.New)}, + denotssource.Name: {protoCfg(denotssource.New)}, + pnpmlock.Name: {protoCfg(pnpmlock.New)}, + yarnlock.Name: {protoCfg(yarnlock.New)}, + bunlock.Name: {protoCfg(bunlock.New)}, } // JavascriptArtifact extractors for Javascript. JavascriptArtifact = InitMap{ - packagejson.Name: {packagejson.New}, - denojson.Name: {denojson.New}, + packagejson.Name: {protoCfg(packagejson.New)}, + denojson.Name: {protoCfg(denojson.New)}, } // PythonSource extractors for Python. PythonSource = InitMap{ // requirements extraction for environments with and without network access. - requirements.Name: {requirements.New}, - setup.Name: {setup.New}, - pipfilelock.Name: {pipfilelock.New}, - pdmlock.Name: {pdmlock.New}, - poetrylock.Name: {poetrylock.New}, - pylock.Name: {pylock.New}, - condameta.Name: {condameta.New}, - uvlock.Name: {uvlock.New}, + requirements.Name: {protoCfg(requirements.New)}, + setup.Name: {protoCfg(setup.New)}, + pipfilelock.Name: {protoCfg(pipfilelock.New)}, + pdmlock.Name: {protoCfg(pdmlock.New)}, + poetrylock.Name: {protoCfg(poetrylock.New)}, + pylock.Name: {protoCfg(pylock.New)}, + condameta.Name: {protoCfg(condameta.New)}, + uvlock.Name: {protoCfg(uvlock.New)}, } // PythonArtifact extractors for Python. PythonArtifact = InitMap{ - wheelegg.Name: {wheelegg.New}, + wheelegg.Name: {protoCfg(wheelegg.New)}, } // GoSource extractors for Go. GoSource = InitMap{ - gomod.Name: {gomod.New}, + gomod.Name: {protoCfg(gomod.New)}, } // GoArtifact extractors for Go. GoArtifact = InitMap{ - gobinary.Name: {gobinary.New}, + gobinary.Name: {protoCfg(gobinary.New)}, } // DartSource extractors for Dart. - DartSource = InitMap{pubspec.Name: {pubspec.New}} + DartSource = InitMap{pubspec.Name: {protoCfg(pubspec.New)}} // ErlangSource extractors for Erlang. - ErlangSource = InitMap{mixlock.Name: {mixlock.New}} + ErlangSource = InitMap{mixlock.Name: {protoCfg(mixlock.New)}} // GleamSource extractors for Gleam. - GleamSource = InitMap{gleamtoml.Name: {gleamtoml.New}} + GleamSource = InitMap{gleamtoml.Name: {protoCfg(gleamtoml.New)}} // NimSource extractors for Nim. - NimSource = InitMap{nimble.Name: {nimble.New}} + NimSource = InitMap{nimble.Name: {protoCfg(nimble.New)}} // LuaSource extractors for Lua. - LuaSource = InitMap{luarocks.Name: {luarocks.New}} + LuaSource = InitMap{luarocks.Name: {protoCfg(luarocks.New)}} // OcamlSource extractors for OCaml. - OcamlSource = InitMap{opam.Name: {opam.New}} + OcamlSource = InitMap{opam.Name: {protoCfg(opam.New)}} // ElixirSource extractors for Elixir. - ElixirSource = InitMap{elixir.Name: {elixir.New}} + ElixirSource = InitMap{elixir.Name: {protoCfg(elixir.New)}} // HaskellSource extractors for Haskell. HaskellSource = InitMap{ - stacklock.Name: {stacklock.New}, - cabal.Name: {cabal.New}, + stacklock.Name: {protoCfg(stacklock.New)}, + cabal.Name: {protoCfg(cabal.New)}, } // RSource extractors for R source extractors - RSource = InitMap{renvlock.Name: {renvlock.New}} + RSource = InitMap{renvlock.Name: {protoCfg(renvlock.New)}} // RubySource extractors for Ruby. RubySource = InitMap{ - gemspec.Name: {gemspec.New}, - gemfilelock.Name: {gemfilelock.New}, + gemspec.Name: {protoCfg(gemspec.New)}, + gemfilelock.Name: {protoCfg(gemfilelock.New)}, } // RustSource extractors for Rust. RustSource = InitMap{ - cargolock.Name: {cargolock.New}, - cargotoml.Name: {cargotoml.New}, + cargolock.Name: {protoCfg(cargolock.New)}, + cargotoml.Name: {protoCfg(cargotoml.New)}, } // CPANSource extractors for Perl. - CPANSource = InitMap{cpan.Name: {cpan.New}} + CPANSource = InitMap{cpan.Name: {protoCfg(cpan.New)}} // RustArtifact extractors for Rust. RustArtifact = InitMap{ - cargoauditable.Name: {cargoauditable.New}, + cargoauditable.Name: {protoCfg(cargoauditable.New)}, } // JuliaSource extractors for Julia. JuliaSource = InitMap{ - projecttoml.Name: {projecttoml.New}, - manifesttoml.Name: {manifesttoml.New}, + projecttoml.Name: {protoCfg(projecttoml.New)}, + manifesttoml.Name: {protoCfg(manifesttoml.New)}, } // JuliaArtifact extractors for Julia. JuliaArtifact = InitMap{ - manifesttoml.Name: {manifesttoml.New}, + manifesttoml.Name: {protoCfg(manifesttoml.New)}, } // SBOM extractors. SBOM = InitMap{ - cdx.Name: {cdx.New}, - spdx.Name: {spdx.New}, + cdx.Name: {protoCfg(cdx.New)}, + spdx.Name: {protoCfg(spdx.New)}, } // DotnetSource extractors for Dotnet (.NET). DotnetSource = InitMap{ - csproj.Name: {csproj.New}, - depsjson.Name: {depsjson.New}, - nugetcpm.Name: {nugetcpm.New}, - packagesconfig.Name: {packagesconfig.New}, - packageslockjson.Name: {packageslockjson.New}, - paketdependencies.Name: {paketdependencies.New}, - paketlock.Name: {paketlock.New}, + csproj.Name: {protoCfg(csproj.New)}, + depsjson.Name: {protoCfg(depsjson.New)}, + nugetcpm.Name: {protoCfg(nugetcpm.New)}, + packagesconfig.Name: {protoCfg(packagesconfig.New)}, + packageslockjson.Name: {protoCfg(packageslockjson.New)}, + paketdependencies.Name: {protoCfg(paketdependencies.New)}, + paketlock.Name: {protoCfg(paketlock.New)}, } // DotnetArtifact extractors for Dotnet (.NET). DotnetArtifact = InitMap{ - dotnetpe.Name: {dotnetpe.New}, + dotnetpe.Name: {protoCfg(dotnetpe.New)}, } // PHPSource extractors for PHP Source extractors. - PHPSource = InitMap{composerlock.Name: {composerlock.New}} + PHPSource = InitMap{composerlock.Name: {protoCfg(composerlock.New)}} // SwiftSource extractors for Swift. SwiftSource = InitMap{ - packageresolved.Name: {packageresolved.New}, - podfilelock.Name: {podfilelock.New}, + packageresolved.Name: {protoCfg(packageresolved.New)}, + podfilelock.Name: {protoCfg(podfilelock.New)}, } // Containers extractors. Containers = InitMap{ - containerd.Name: {containerd.New}, - k8simage.Name: {k8simage.New}, - podman.Name: {podman.New}, - dockerbaseimage.Name: {dockerbaseimage.New}, - dockercomposeimage.Name: {dockercomposeimage.New}, + containerd.Name: {protoCfg(containerd.New)}, + k8simage.Name: {protoCfg(k8simage.New)}, + podman.Name: {protoCfg(podman.New)}, + dockerbaseimage.Name: {protoCfg(dockerbaseimage.New)}, + dockercomposeimage.Name: {protoCfg(dockercomposeimage.New)}, } // OS extractors. OS = InitMap{ - dpkg.Name: {dpkg.New}, - apk.Name: {apk.New}, - rpm.Name: {rpm.New}, - cos.Name: {cos.New}, - snap.Name: {snap.New}, - nix.Name: {nix.New}, - module.Name: {module.New}, - vmlinuz.Name: {vmlinuz.New}, - pacman.Name: {pacman.New}, - portage.Name: {portage.New}, - flatpak.Name: {flatpak.New}, - spack.Name: {spack.New}, - homebrew.Name: {homebrew.New}, - macapps.Name: {macapps.New}, - macports.Name: {macports.New}, - winget.Name: {winget.New}, - chocolatey.Name: {chocolatey.New}, - chisel.Name: {chisel.New}, + dpkg.Name: {protoCfg(dpkg.New)}, + apk.Name: {protoCfg(apk.New)}, + rpm.Name: {protoCfg(rpm.New)}, + cos.Name: {protoCfg(cos.New)}, + snap.Name: {protoCfg(snap.New)}, + nix.Name: {protoCfg(nix.New)}, + module.Name: {protoCfg(module.New)}, + vmlinuz.Name: {protoCfg(vmlinuz.New)}, + pacman.Name: {protoCfg(pacman.New)}, + portage.Name: {protoCfg(portage.New)}, + flatpak.Name: {protoCfg(flatpak.New)}, + spack.Name: {protoCfg(spack.New)}, + homebrew.Name: {protoCfg(homebrew.New)}, + macapps.Name: {protoCfg(macapps.New)}, + macports.Name: {protoCfg(macports.New)}, + winget.Name: {protoCfg(winget.New)}, + chocolatey.Name: {protoCfg(chocolatey.New)}, + chisel.Name: {protoCfg(chisel.New)}, } // SecretExtractors for Extractor interface. SecretExtractors = InitMap{ - mysqlmylogin.Name: {mysqlmylogin.New}, - pgpass.Name: {pgpass.New}, - onepasswordconnecttoken.Name: {onepasswordconnecttoken.New}, - mariadb.Name: {mariadb.New}, - awsaccesskey.Name: {awsaccesskey.New}, - composerpackagist.Name: {composerpackagist.New}, - codecatalyst.Name: {codecatalyst.New}, - codecommit.Name: {codecommit.New}, - bitbucket.Name: {bitbucket.New}, - cloudflareapitoken.Name: {cloudflareapitoken.New}, - bitwardenoauth2access.Name: {bitwardenoauth2access.New}, + mysqlmylogin.Name: {protoCfg(mysqlmylogin.New)}, + pgpass.Name: {protoCfg(pgpass.New)}, + onepasswordconnecttoken.Name: {protoCfg(onepasswordconnecttoken.New)}, + mariadb.Name: {protoCfg(mariadb.New)}, + awsaccesskey.Name: {protoCfg(awsaccesskey.New)}, + composerpackagist.Name: {protoCfg(composerpackagist.New)}, + codecatalyst.Name: {protoCfg(codecatalyst.New)}, + codecommit.Name: {protoCfg(codecommit.New)}, + bitbucket.Name: {protoCfg(bitbucket.New)}, + cloudflareapitoken.Name: {protoCfg(cloudflareapitoken.New)}, + bitwardenoauth2access.Name: {protoCfg(bitwardenoauth2access.New)}, } // SecretDetectors for Detector interface. @@ -455,35 +465,35 @@ var ( // Misc artifact extractors. Misc = InitMap{ - vscodeextensions.Name: {vscodeextensions.New}, - wordpressplugins.Name: {wordpressplugins.New}, - chromeextensions.Name: {chromeextensions.New}, - netscaler.Name: {netscaler.New}, + vscodeextensions.Name: {protoCfg(vscodeextensions.New)}, + wordpressplugins.Name: {protoCfg(wordpressplugins.New)}, + chromeextensions.Name: {protoCfg(chromeextensions.New)}, + netscaler.Name: {protoCfg(netscaler.New)}, } // MiscSource extractors for miscellaneous purposes. MiscSource = InitMap{ - asdf.Name: {asdf.New}, - githubactions.Name: {githubactions.New}, - mise.Name: {mise.New}, - nvm.Name: {nvm.New}, - nodeversion.Name: {nodeversion.New}, + asdf.Name: {protoCfg(asdf.New)}, + githubactions.Name: {protoCfg(githubactions.New)}, + mise.Name: {protoCfg(mise.New)}, + nvm.Name: {protoCfg(nvm.New)}, + nodeversion.Name: {protoCfg(nodeversion.New)}, } // EmbeddedFS extractors. EmbeddedFS = InitMap{ - archive.Name: {archive.New}, - vdi.Name: {vdi.New}, - vmdk.Name: {vmdk.New}, - ova.Name: {ova.New}, - qcow2.Name: {qcow2.New}, + archive.Name: {protoCfg(archive.New)}, + vdi.Name: {protoCfg(vdi.New)}, + vmdk.Name: {protoCfg(vmdk.New)}, + ova.Name: {protoCfg(ova.New)}, + qcow2.Name: {protoCfg(qcow2.New)}, } // FFA extractor. FFA = InitMap{ - unknownbinariesextr.Name: {unknownbinariesextr.New}, - asdf.Name: {asdf.New}, - bazelmaven.Name: {bazelmaven.New}, + unknownbinariesextr.Name: {protoCfg(unknownbinariesextr.New)}, + asdf.Name: {protoCfg(asdf.New)}, + bazelmaven.Name: {protoCfg(bazelmaven.New)}, } // Collections of extractors. @@ -607,7 +617,7 @@ func vals(initMap InitMap) []InitFn { } // ExtractorsFromName returns a list of extractors from a name. -func ExtractorsFromName(name string, cfg *cpb.PluginConfig) ([]filesystem.Extractor, error) { +func ExtractorsFromName(name string, cfg *config.PluginConfig) ([]filesystem.Extractor, error) { if initers, ok := extractorNames[name]; ok { var result []filesystem.Extractor for _, initer := range initers { @@ -631,7 +641,7 @@ type velesPlugin struct { func initMapFromVelesPlugins(plugins []velesPlugin) InitMap { result := InitMap{} for _, p := range plugins { - result[p.name] = []InitFn{convert.FromVelesDetector(p.detector, p.name, p.version)} + result[p.name] = []InitFn{protoCfg(convert.FromVelesDetector(p.detector, p.name, p.version))} } return result } diff --git a/extractor/filesystem/list/list_test.go b/extractor/filesystem/list/list_test.go index ce4935f70..e76aebbeb 100644 --- a/extractor/filesystem/list/list_test.go +++ b/extractor/filesystem/list/list_test.go @@ -20,8 +20,8 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" el "github.com/google/osv-scalibr/extractor/filesystem/list" + "github.com/google/osv-scalibr/plugin/config" ) var ( @@ -31,7 +31,7 @@ var ( func TestPluginNamesValid(t *testing.T) { for _, initers := range el.All { for _, initer := range initers { - p, err := initer(&cpb.PluginConfig{}) + p, err := initer(config.DefaultPluginConfig()) if err != nil { t.Fatalf("initer(): %v", err) } @@ -64,7 +64,7 @@ func TestExtractorsFromName(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - got, err := el.ExtractorsFromName(tc.name, &cpb.PluginConfig{}) + got, err := el.ExtractorsFromName(tc.name, config.DefaultPluginConfig()) if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { t.Errorf("el.ExtractorsFromName(%v) error got diff (-want +got):\n%s", tc.name, diff) } diff --git a/extractor/standalone/list/list.go b/extractor/standalone/list/list.go index f8dc200f0..8763fcf21 100644 --- a/extractor/standalone/list/list.go +++ b/extractor/standalone/list/list.go @@ -20,6 +20,7 @@ import ( "maps" "slices" + cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" "github.com/google/osv-scalibr/extractor/standalone" "github.com/google/osv-scalibr/extractor/standalone/containers/containerd" "github.com/google/osv-scalibr/extractor/standalone/containers/docker" @@ -28,37 +29,36 @@ import ( "github.com/google/osv-scalibr/extractor/standalone/windows/ospackages" "github.com/google/osv-scalibr/extractor/standalone/windows/regosversion" "github.com/google/osv-scalibr/extractor/standalone/windows/regpatchlevel" - - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) // InitFn is the extractor initializer function. -type InitFn func(cfg *cpb.PluginConfig) (standalone.Extractor, error) +type InitFn func(cfg *config.PluginConfig) (standalone.Extractor, error) // InitMap is a map of extractor names to their initers. type InitMap map[string][]InitFn var ( // Windows standalone extractors. - Windows = InitMap{dismpatch.Name: {dismpatch.New}} + Windows = InitMap{dismpatch.Name: {protoCfg(dismpatch.New)}} // WindowsExperimental defines experimental extractors. Note that experimental does not mean // dangerous. WindowsExperimental = InitMap{ - ospackages.Name: {ospackages.New}, - regosversion.Name: {regosversion.New}, - regpatchlevel.Name: {regpatchlevel.New}, + ospackages.Name: {protoCfg(ospackages.New)}, + regosversion.Name: {protoCfg(regosversion.New)}, + regpatchlevel.Name: {protoCfg(regpatchlevel.New)}, } // OSExperimental defines experimental OS extractors. OSExperimental = InitMap{ - netports.Name: {netports.New}, + netports.Name: {protoCfg(netports.New)}, } // Containers standalone extractors. Containers = InitMap{ - containerd.Name: {containerd.New}, - docker.Name: {docker.New}, + containerd.Name: {protoCfg(containerd.New)}, + docker.Name: {protoCfg(docker.New)}, } // Default standalone extractors. @@ -91,8 +91,17 @@ func vals(initMap InitMap) []InitFn { return slices.Concat(slices.Collect(maps.Values(initMap))...) } +func protoCfg(f func(cfg *cpb.PluginConfig) (standalone.Extractor, error)) InitFn { + return func(cfg *config.PluginConfig) (standalone.Extractor, error) { + if cfg == nil || cfg.ProtoConfig == nil { + return f(&cpb.PluginConfig{}) + } + return f(cfg.ProtoConfig) + } +} + // ExtractorsFromName returns a list of extractors from a name. -func ExtractorsFromName(name string, cfg *cpb.PluginConfig) ([]standalone.Extractor, error) { +func ExtractorsFromName(name string, cfg *config.PluginConfig) ([]standalone.Extractor, error) { if initers, ok := extractorNames[name]; ok { result := []standalone.Extractor{} for _, initer := range initers { diff --git a/extractor/standalone/list/list_test.go b/extractor/standalone/list/list_test.go index 38716067e..728a98168 100644 --- a/extractor/standalone/list/list_test.go +++ b/extractor/standalone/list/list_test.go @@ -18,8 +18,8 @@ import ( "regexp" "testing" - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" el "github.com/google/osv-scalibr/extractor/standalone/list" + "github.com/google/osv-scalibr/plugin/config" ) var ( @@ -29,7 +29,7 @@ var ( func TestPluginNamesValid(t *testing.T) { for _, initers := range el.All { for _, initer := range initers { - p, err := initer(&cpb.PluginConfig{}) + p, err := initer(config.DefaultPluginConfig()) if err != nil { t.Fatalf("initer(): %v", err) } diff --git a/plugin/config/config.go b/plugin/config/config.go new file mode 100644 index 000000000..ee69bd2af --- /dev/null +++ b/plugin/config/config.go @@ -0,0 +1,112 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package config provides configuration structures and client factories for plugins. +package config + +import ( + "context" + "crypto/x509" + "net/http" + "sync" + + cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "golang.org/x/oauth2/google" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +// PluginConfig wraps the proto generated PluginConfig to allow passing arbitrary pointers/interfaces. +type PluginConfig struct { + ProtoConfig *cpb.PluginConfig + ClientFactories ClientFactories +} + +// ClientFactories defines methods to obtain shared clients and connections. +type ClientFactories interface { + HTTPClient() *http.Client + GRPCClientConn(url string, dialOpts ...grpc.DialOption) (grpc.ClientConnInterface, error) + GoogleHTTPClient(scope ...string) (*http.Client, error) +} + +// DefaultClientFactories provides a default implementation of ClientFactories. +type DefaultClientFactories struct { + mu sync.Mutex + httpClient *http.Client + googleHTTPClient *http.Client + grpcClientConns map[string]*grpc.ClientConn +} + +// NewDefaultClientFactories returns a new DefaultClientFactories. +func NewDefaultClientFactories() *DefaultClientFactories { + return &DefaultClientFactories{ + grpcClientConns: make(map[string]*grpc.ClientConn), + } +} + +// DefaultPluginConfig returns a PluginConfig with default client factories. +func DefaultPluginConfig() *PluginConfig { + return &PluginConfig{ + ProtoConfig: &cpb.PluginConfig{}, + ClientFactories: NewDefaultClientFactories(), + } +} + +// HTTPClient returns a shared HTTP client. +func (c *DefaultClientFactories) HTTPClient() *http.Client { + c.mu.Lock() + defer c.mu.Unlock() + if c.httpClient == nil { + c.httpClient = &http.Client{} + } + return c.httpClient +} + +// GRPCClientConn returns a shared gRPC connection. +func (c *DefaultClientFactories) GRPCClientConn(url string, dialOpts ...grpc.DialOption) (grpc.ClientConnInterface, error) { + c.mu.Lock() + defer c.mu.Unlock() + if conn, ok := c.grpcClientConns[url]; ok { + return conn, nil + } + + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + creds := credentials.NewClientTLSFromCert(certPool, "") + ourDialOpts := []grpc.DialOption{grpc.WithTransportCredentials(creds)} + ourDialOpts = append(ourDialOpts, dialOpts...) + conn, err := grpc.NewClient(url, ourDialOpts...) + if err != nil { + return nil, err + } + + c.grpcClientConns[url] = conn + return conn, nil +} + +// GoogleHTTPClient returns a shared HTTP client authenticated for Google services. +func (c *DefaultClientFactories) GoogleHTTPClient(scope ...string) (*http.Client, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.googleHTTPClient == nil { + client, err := google.DefaultClient(context.Background(), scope...) + if err != nil { + return nil, err + } + c.googleHTTPClient = client + } + return c.googleHTTPClient, nil +} diff --git a/plugin/list/list.go b/plugin/list/list.go index 71583081e..75a1d951c 100644 --- a/plugin/list/list.go +++ b/plugin/list/list.go @@ -31,12 +31,13 @@ import ( "github.com/google/osv-scalibr/plugin" cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/plugin/config" ) // FromCapabilities returns all plugins that can run under the specified // capabilities (OS, direct filesystem access, network access, etc.) of the // scanning environment. -func FromCapabilities(capabs *plugin.Capabilities, cfg *cpb.PluginConfig) ([]plugin.Plugin, error) { +func FromCapabilities(capabs *plugin.Capabilities, cfg *config.PluginConfig) ([]plugin.Plugin, error) { all, err := All(cfg) if err != nil { return nil, err @@ -45,10 +46,11 @@ func FromCapabilities(capabs *plugin.Capabilities, cfg *cpb.PluginConfig) ([]plu } // FromNames returns a deduplicated list of plugins from a list of names. -func FromNames(names []string, cfg *cpb.PluginConfig) ([]plugin.Plugin, error) { +func FromNames(names []string, cfg *config.PluginConfig) ([]plugin.Plugin, error) { if cfg == nil { - // Do the nil check here instead of in individual plugin initers. - cfg = &cpb.PluginConfig{} + cfg = config.DefaultPluginConfig() + } else if cfg.ProtoConfig == nil { + cfg.ProtoConfig = &cpb.PluginConfig{} } resultMap := make(map[string]plugin.Plugin) for _, name := range names { @@ -88,7 +90,7 @@ func FromNames(names []string, cfg *cpb.PluginConfig) ([]plugin.Plugin, error) { } // FromName returns a single plugin based on its exact name. -func FromName(name string, cfg *cpb.PluginConfig) (plugin.Plugin, error) { +func FromName(name string, cfg *config.PluginConfig) (plugin.Plugin, error) { plugins, err := FromNames([]string{name}, cfg) if err != nil { return nil, err @@ -103,10 +105,11 @@ func FromName(name string, cfg *cpb.PluginConfig) (plugin.Plugin, error) { // Note that these plugins have different capability Requirements and can't all // be run on the same host (e.g. some are Linux-only while others are Windows-only) // Prefer using FromCapabilities instead. -func All(cfg *cpb.PluginConfig) ([]plugin.Plugin, error) { +func All(cfg *config.PluginConfig) ([]plugin.Plugin, error) { if cfg == nil { - // Do the nil check here instead of in individual plugin initers. - cfg = &cpb.PluginConfig{} + cfg = config.DefaultPluginConfig() + } else if cfg.ProtoConfig == nil { + cfg.ProtoConfig = &cpb.PluginConfig{} } all := []plugin.Plugin{} for _, initers := range fl.All { diff --git a/plugin/list/list_test.go b/plugin/list/list_test.go index 4c691fa87..2aa38b1b2 100644 --- a/plugin/list/list_test.go +++ b/plugin/list/list_test.go @@ -19,13 +19,13 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/plugin/config" pl "github.com/google/osv-scalibr/plugin/list" ) func TestExtractorNamesUnique(t *testing.T) { - all, err := pl.All(&cpb.PluginConfig{}) + all, err := pl.All(config.DefaultPluginConfig()) if err != nil { t.Fatalf("pl.All(): %v", err) } @@ -47,7 +47,7 @@ func TestExtractorNamesUnique(t *testing.T) { } func TestDetectorNamesUnique(t *testing.T) { - all, err := pl.All(&cpb.PluginConfig{}) + all, err := pl.All(config.DefaultPluginConfig()) if err != nil { t.Fatalf("pl.All(): %v", err) } @@ -62,7 +62,7 @@ func TestDetectorNamesUnique(t *testing.T) { } func TestAnnotatorNamesUnique(t *testing.T) { - all, err := pl.All(&cpb.PluginConfig{}) + all, err := pl.All(config.DefaultPluginConfig()) if err != nil { t.Fatalf("pl.All(): %v", err) } @@ -77,7 +77,7 @@ func TestAnnotatorNamesUnique(t *testing.T) { } func TestEnricherNamesUnique(t *testing.T) { - all, err := pl.All(&cpb.PluginConfig{}) + all, err := pl.All(config.DefaultPluginConfig()) if err != nil { t.Fatalf("pl.All(): %v", err) } @@ -95,7 +95,7 @@ func TestFromCapabilities(t *testing.T) { capab := &plugin.Capabilities{OS: plugin.OSLinux} want := []string{"os/snap", "weakcredentials/etcshadow"} // Available for Linux dontWant := []string{"windows/dismpatch"} // Not available for Linux - plugins, err := pl.FromCapabilities(capab, &cpb.PluginConfig{}) + plugins, err := pl.FromCapabilities(capab, config.DefaultPluginConfig()) if err != nil { t.Fatalf("pl.FromCapabilities(%v): %v", capab, err) } @@ -148,7 +148,7 @@ func TestFromNames(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - got, err := pl.FromNames(tc.names, &cpb.PluginConfig{}) + got, err := pl.FromNames(tc.names, config.DefaultPluginConfig()) if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { t.Errorf("pl.FromNames(%v) error got diff (-want +got):\n%s", tc.names, diff) } @@ -190,7 +190,7 @@ func TestFromName(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - got, err := pl.FromName(tc.name, &cpb.PluginConfig{}) + got, err := pl.FromName(tc.name, config.DefaultPluginConfig()) if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { t.Errorf("pl.FromName(%v) error got diff (-want +got):\n%s", tc.name, diff) } diff --git a/scalibr.go b/scalibr.go index b12327a75..2cdd3b4d1 100644 --- a/scalibr.go +++ b/scalibr.go @@ -45,13 +45,12 @@ import ( "github.com/google/osv-scalibr/log" "github.com/google/osv-scalibr/packageindex" "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/plugin/config" pl "github.com/google/osv-scalibr/plugin/list" "github.com/google/osv-scalibr/result" "github.com/google/osv-scalibr/stats" "github.com/google/osv-scalibr/version" "go.uber.org/multierr" - - cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" ) var ( @@ -119,7 +118,7 @@ type ScanConfig struct { // isn't configured instead of enabling required plugins automatically. ExplicitPlugins bool // Optional: Configuration to apply to auto-enabled required plugins. - RequiredPluginConfig *cpb.PluginConfig + RequiredPluginConfig *config.PluginConfig } // EnableRequiredPlugins adds those plugins to the config that are required by enabled