feat: enable transitive extraction for Python requirements.txt (#1980)

This PR enables the transitive extractor for Python requirement files.
Considering that there is only the native registry client for PyPI, we
use that for requirements.txt transitive extractor.

Author: cuixq

cmd/osv-scanner/scan/source/__snapshots__/command_test.snap

@@ -825,7 +825,7 @@ func TestCommand_Licenses(t *testing.T) {
 	}
 }
 
-func TestCommand_MavenTransitive(t *testing.T) {
+func TestCommand_Transitive(t *testing.T) {
 	t.Parallel()
 
 	tests := []testcmd.Case{
@@ -862,10 +862,25 @@ func TestCommand_MavenTransitive(t *testing.T) {
 			Exit: 1,
 		},
 		{
-			Name: "resolve transitive dependencies with native data source",
+			Name: "resolves transitive dependencies with native data source",
 			Args: []string{"", "source", "--config=./fixtures/osv-scanner-empty-config.toml", "--data-source=native", "-L", "pom.xml:./fixtures/maven-transitive/registry.xml"},
 			Exit: 1,
 		},
+		{
+			Name: "uses native data source for requirements.txt",
+			Args: []string{"", "source", "--config=./fixtures/osv-scanner-empty-config.toml", "./fixtures/locks-requirements/requirements.txt"},
+			Exit: 1,
+		},
+		{
+			Name: "does not scan transitive dependencies for requirements.txt with no-resolve",
+			Args: []string{"", "source", "--config=./fixtures/osv-scanner-empty-config.toml", "--no-resolve", "./fixtures/locks-requirements/requirements.txt"},
+			Exit: 1,
+		},
+		{
+			Name: "does not scan transitive dependencies for requirements.txt with offline mode",
+			Args: []string{"", "source", "--config=./fixtures/osv-scanner-empty-config.toml", "--offline", "--download-offline-databases", "./fixtures/locks-requirements/requirements.txt"},
+			Exit: 1,
+		},
 	}
 
 	for _, tt := range tests {

cmd/osv-scanner/scan/source/command_test.go

@@ -1 +1 @@
-black==1.0.0
+black==25.1.0

cmd/osv-scanner/scan/source/fixtures/locks-requirements/requirements-dev.txt

@@ -1,3 +1,3 @@
 flask==1.0.0
-flask-cors==1.0.0
-pandas==0.23.4
+django==1.11.29
+requests==2.20.0

cmd/osv-scanner/scan/source/fixtures/locks-requirements/requirements.txt

@@ -1 +1 @@
-pytest==1.0.0
+pytest==2.0.0

cmd/osv-scanner/scan/source/fixtures/locks-requirements/the_requirements_for_test.txt

@@ -25,8 +25,6 @@ import (
 	"github.com/google/osv-scalibr/extractor/filesystem/language/python/pdmlock"
 	"github.com/google/osv-scalibr/extractor/filesystem/language/python/pipfilelock"
 	"github.com/google/osv-scalibr/extractor/filesystem/language/python/poetrylock"
-	"github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements"
-	"github.com/google/osv-scalibr/extractor/filesystem/language/python/requirementsnet"
 	"github.com/google/osv-scalibr/extractor/filesystem/language/python/uvlock"
 	"github.com/google/osv-scalibr/extractor/filesystem/language/python/wheelegg"
 	"github.com/google/osv-scalibr/extractor/filesystem/language/r/renvlock"
@@ -119,10 +117,6 @@ func build(name string) filesystem.Extractor {
 		return pdmlock.New()
 	case poetrylock.Name:
 		return poetrylock.New()
-	case requirements.Name:
-		return requirements.NewDefault()
-	case requirementsnet.Name:
-		return requirementsnet.NewDefault()
 	case requirementsenhancable.Name:
 		return requirementsenhancable.New()
 	case uvlock.Name:

internal/builders/extractors.go

@@ -29,7 +29,6 @@ import (
 	"github.com/google/osv-scalibr/extractor/filesystem/language/python/pdmlock"
 	"github.com/google/osv-scalibr/extractor/filesystem/language/python/pipfilelock"
 	"github.com/google/osv-scalibr/extractor/filesystem/language/python/poetrylock"
-	"github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements"
 	"github.com/google/osv-scalibr/extractor/filesystem/language/python/uvlock"
 	"github.com/google/osv-scalibr/extractor/filesystem/language/r/renvlock"
 	"github.com/google/osv-scalibr/extractor/filesystem/language/ruby/gemfilelock"
@@ -40,6 +39,7 @@ import (
 	"github.com/google/osv-scanner/v2/internal/scalibrextract"
 	"github.com/google/osv-scanner/v2/internal/scalibrextract/language/java/pomxmlenhanceable"
 	"github.com/google/osv-scanner/v2/internal/scalibrextract/language/osv/osvscannerjson"
+	"github.com/google/osv-scanner/v2/internal/scalibrextract/language/python/requirementsenhancable"
 )
 
 var lockfileExtractorMapping = map[string][]string{
@@ -54,7 +54,7 @@ var lockfileExtractorMapping = map[string][]string{
 	"poetry.lock":                 {poetrylock.Name},
 	"Pipfile.lock":                {pipfilelock.Name},
 	"pdm.lock":                    {pdmlock.Name},
-	"requirements.txt":            {requirements.Name},
+	"requirements.txt":            {requirementsenhancable.Name},
 	"uv.lock":                     {uvlock.Name},
 	"Cargo.lock":                  {cargolock.Name},
 	"composer.lock":               {composerlock.Name},

pkg/osvscanner/internal/scanners/lockfile.go

@@ -177,6 +177,9 @@ func initializeExternalAccessors(actions ScannerActions) (ExternalAccessors, err
 		externalAccessors.DependencyClients[osvschema.EcosystemMaven], err = resolution.NewMavenRegistryClient(actions.MavenRegistry, "")
 	}
 
+	// We only support native registry client for PyPI.
+	externalAccessors.DependencyClients[osvschema.EcosystemPyPI] = resolution.NewPyPIRegistryClient("")
+
 	if err != nil {
 		return ExternalAccessors{}, err
 	}

pkg/osvscanner/osvscanner.go

@@ -15,13 +15,16 @@ import (
 	"github.com/google/osv-scalibr/extractor"
 	"github.com/google/osv-scalibr/extractor/filesystem"
 	"github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxmlnet"
+	"github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements"
+	"github.com/google/osv-scalibr/extractor/filesystem/language/python/requirementsnet"
 	"github.com/google/osv-scalibr/fs"
 	"github.com/google/osv-scalibr/plugin"
 	"github.com/google/osv-scanner/v2/internal/builders"
 	"github.com/google/osv-scanner/v2/internal/imodels"
 	"github.com/google/osv-scanner/v2/internal/scalibrextract"
 	"github.com/google/osv-scanner/v2/internal/scalibrextract/filesystem/vendored"
 	"github.com/google/osv-scanner/v2/internal/scalibrextract/language/java/pomxmlenhanceable"
+	"github.com/google/osv-scanner/v2/internal/scalibrextract/language/python/requirementsenhancable"
 	"github.com/google/osv-scanner/v2/internal/scalibrextract/vcs/gitrepo"
 	"github.com/google/osv-scanner/v2/internal/testlogger"
 	"github.com/google/osv-scanner/v2/pkg/osvscanner/internal/scanners"
@@ -36,7 +39,12 @@ func configureExtractors(extractors []filesystem.Extractor, accessors ExternalAc
 				MavenRegistryAPIClient: accessors.MavenRegistryAPIClient,
 			})
 		}
-		// TODO: Enable requirementstxtnet extractor
+		if accessors.DependencyClients[osvschema.EcosystemPyPI] != nil {
+			requirementsenhancable.EnhanceIfPossible(tor, requirementsnet.Config{
+				Extractor: &requirements.Extractor{},
+				Client:    accessors.DependencyClients[osvschema.EcosystemPyPI],
+			})
+		}
 
 		// todo: the "disabled" aspect should probably be worked into the extractor being present in the first place
 		//  since "IncludeRootGit" is always true