Skip to content

Commit bd4fff4

Browse files
pradhyum6144pradhyum6144
committed
Fix resolve_dependencies pipeline for multiple manifest files (#1957)
- Modified resolve_pypi_packages() to accept multiple requirement files - Updated get_data_from_manifests() to batch-process PyPI manifests together - Maintained backward compatibility with existing single-file API - Added comprehensive tests for multiple files and backward compatibility This fix resolves issue #1957 where the pipeline failed when multiple manifest files (e.g., requirements.txt in different subfolders) were present in a project. The solution leverages python-inspector's ability to process multiple requirement files in a single call, which is more efficient and provides better dependency resolution context. Fixes #1957 Signed-off-by: pradhyum6144 <pradhyum314@gmail.com>
1 parent 8712d48 commit bd4fff4

2 files changed

Lines changed: 136 additions & 17 deletions

File tree

‎scanpipe/pipes/resolve.py‎

Lines changed: 84 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -97,22 +97,72 @@ def get_data_from_manifests(project, package_registry, manifest_resources, model
9797
)
9898
return []
9999

100+
# Group manifest resources by package type for batch processing
101+
manifests_by_type = {}
100102
for resource in manifest_resources:
101-
packages = resolve_manifest_resources(resource, package_registry)
102-
if packages:
103-
resolved_packages.extend(packages)
104-
if headers := get_manifest_headers(resource):
105-
sboms_headers[resource.name] = headers
106-
else:
107-
project.add_error(
108-
description="No packages could be resolved",
109-
model=model,
110-
object_instance=resource,
111-
)
103+
package_type = get_default_package_type(resource.location)
104+
if package_type:
105+
if package_type not in manifests_by_type:
106+
manifests_by_type[package_type] = []
107+
manifests_by_type[package_type].append(resource)
108+
109+
# Process PyPI manifests together in a single batch
110+
if "pypi" in manifests_by_type:
111+
pypi_resources = manifests_by_type["pypi"]
112+
pypi_locations = [resource.location for resource in pypi_resources]
113+
114+
resolver = package_registry.get("pypi")
115+
if resolver:
116+
try:
117+
packages = resolver(input_locations=pypi_locations)
118+
if packages:
119+
# Associate packages with their source resources
120+
# Since we're processing multiple files together, we need to
121+
# associate each package with all the manifest resources
122+
for package_data in packages:
123+
package_data["codebase_resources"] = pypi_resources
124+
resolved_packages.extend(packages)
125+
126+
# Collect headers for each manifest
127+
for resource in pypi_resources:
128+
if headers := get_manifest_headers(resource):
129+
sboms_headers[resource.name] = headers
130+
else:
131+
for resource in pypi_resources:
132+
project.add_error(
133+
description="No packages could be resolved",
134+
model=model,
135+
object_instance=resource,
136+
)
137+
except Exception as e:
138+
for resource in pypi_resources:
139+
project.add_error(
140+
description=f"Error resolving packages: {e}",
141+
model=model,
142+
object_instance=resource,
143+
)
144+
145+
# Remove pypi from the dict so we don't process it again below
146+
del manifests_by_type["pypi"]
147+
148+
# Process other manifest types individually (SPDX, CycloneDX, About files)
149+
for package_type, resources in manifests_by_type.items():
150+
for resource in resources:
151+
packages = resolve_manifest_resources(resource, package_registry)
152+
if packages:
153+
resolved_packages.extend(packages)
154+
if headers := get_manifest_headers(resource):
155+
sboms_headers[resource.name] = headers
156+
else:
157+
project.add_error(
158+
description="No packages could be resolved",
159+
model=model,
160+
object_instance=resource,
161+
)
112162

113-
dependencies = get_dependencies_from_manifest(resource)
114-
if dependencies:
115-
resolved_dependencies.extend(dependencies)
163+
dependencies = get_dependencies_from_manifest(resource)
164+
if dependencies:
165+
resolved_dependencies.extend(dependencies)
116166

117167
if sboms_headers:
118168
project.update_extra_data({"sboms_headers": sboms_headers})
@@ -222,13 +272,30 @@ def get_manifest_resources(project):
222272
return project.codebaseresources.filter(status=flag.APPLICATION_PACKAGE)
223273

224274

225-
def resolve_pypi_packages(input_location):
226-
"""Resolve the PyPI packages from the ``input_location`` requirements file."""
275+
def resolve_pypi_packages(input_location=None, input_locations=None):
276+
"""
277+
Resolve the PyPI packages from requirement file(s).
278+
279+
Args:
280+
input_location: Single requirement file path (for backward compatibility)
281+
input_locations: List of requirement file paths (for batch processing)
282+
283+
Returns:
284+
List of resolved package data dictionaries
285+
"""
286+
# Handle both single file and multiple files
287+
if input_locations:
288+
requirement_files = input_locations
289+
elif input_location:
290+
requirement_files = [input_location]
291+
else:
292+
raise ValueError("Either input_location or input_locations must be provided")
293+
227294
python_version = f"{sys.version_info.major}{sys.version_info.minor}"
228295
operating_system = "linux"
229296

230297
resolution_output = python_inspector.resolve_dependencies(
231-
requirement_files=[input_location],
298+
requirement_files=requirement_files,
232299
python_version=python_version,
233300
operating_system=operating_system,
234301
# Prefer source distributions over binary distributions,

‎scanpipe/tests/pipes/test_resolve.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -376,3 +376,55 @@ def test_scanpipe_resolve_get_manifest_headers(self):
376376
]
377377
headers = resolve.get_manifest_headers(resource)
378378
self.assertEqual(expected, list(headers.keys()))
379+
380+
@mock.patch("scanpipe.pipes.resolve.python_inspector.resolve_dependencies")
381+
def test_scanpipe_pipes_resolve_pypi_packages_multiple_files(self, mock_resolve):
382+
"""Test that resolve_pypi_packages can handle multiple requirement files."""
383+
# Generated with:
384+
# $ python-inspector --python-version 3.12 --operating-system linux \
385+
# --specifier pip==25.0.1 --json -
386+
inspector_output_location = (
387+
self.data / "resolve" / "python_inspector_resolve_dependencies.json"
388+
)
389+
with open(inspector_output_location) as f:
390+
inspector_output = json.loads(f.read())
391+
392+
mock_resolve.return_value = mock.Mock(packages=inspector_output["packages"])
393+
394+
# Test with multiple requirement files
395+
req_files = ["requirements1.txt", "requirements2.txt"]
396+
packages = resolve.resolve_pypi_packages(input_locations=req_files)
397+
398+
# Verify python_inspector was called with all files
399+
mock_resolve.assert_called_once()
400+
call_args = mock_resolve.call_args
401+
self.assertEqual(req_files, call_args.kwargs["requirement_files"])
402+
403+
# Verify packages were returned
404+
self.assertEqual(2, len(packages))
405+
self.assertEqual("pip", packages[0]["name"])
406+
407+
@mock.patch("scanpipe.pipes.resolve.python_inspector.resolve_dependencies")
408+
def test_scanpipe_pipes_resolve_pypi_packages_backward_compatibility(
409+
self, mock_resolve
410+
):
411+
"""Test that resolve_pypi_packages still works with single file (backward compatibility)."""
412+
inspector_output_location = (
413+
self.data / "resolve" / "python_inspector_resolve_dependencies.json"
414+
)
415+
with open(inspector_output_location) as f:
416+
inspector_output = json.loads(f.read())
417+
418+
mock_resolve.return_value = mock.Mock(packages=inspector_output["packages"])
419+
420+
# Test with single file (old API)
421+
packages = resolve.resolve_pypi_packages(input_location="requirements.txt")
422+
423+
# Verify python_inspector was called with single file in list
424+
mock_resolve.assert_called_once()
425+
call_args = mock_resolve.call_args
426+
self.assertEqual(["requirements.txt"], call_args.kwargs["requirement_files"])
427+
428+
# Verify packages were returned
429+
self.assertEqual(2, len(packages))
430+

0 commit comments

Comments
 (0)