Skip to content

Commit 3dc259a

Browse files
a-orenclaude
a-oren
and
claude
committed
fix(python): handle pip options, hashes, and line continuations in requirements.txt (TC-4527)
Add preprocessRequirementsLines() to PythonControllerBase that properly handles all requirements.txt line types: pip options (--extra-index-url, -r, -c, etc.), inline options (--hash, --config-settings), line continuations (\), PEP 508 direct references (name @ url), bare URLs, VCS URLs, and local paths. Applied in getDependenciesImpl, installingRequirementsOneByOne, and getIgnoredDependencies to fix parsing errors when requirements.txt contains non-package lines. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 95e2da5 commit 3dc259a

3 files changed

Lines changed: 203 additions & 10 deletions

File tree

‎src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,9 @@ public Content provideComponent() throws IOException {
9595

9696
@Override
9797
protected Set<PackageURL> getIgnoredDependencies(String manifestContent) {
98-
String[] lines = manifestContent.split(System.lineSeparator());
99-
return Arrays.stream(lines)
98+
List<String> rawLines = Arrays.asList(manifestContent.split("\\R"));
99+
List<String> preprocessed = PythonControllerBase.preprocessRequirementsLines(rawLines);
100+
return preprocessed.stream()
100101
.filter(this::containsIgnorePattern)
101102
.map(PythonPipProvider::extractDepFull)
102103
.map(this::splitToNameVersion)

‎src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java‎

Lines changed: 60 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -112,10 +112,9 @@ public final List<Map<String, Object>> getDependencies(
112112

113113
private void installingRequirementsOneByOne(String pathToRequirements) {
114114
try {
115-
List<String> requirementsRows = Files.readAllLines(Path.of(pathToRequirements));
115+
List<String> requirementsRows =
116+
preprocessRequirementsLines(Files.readAllLines(Path.of(pathToRequirements)));
116117
requirementsRows.stream()
117-
.filter((line) -> !line.trim().startsWith("#"))
118-
.filter((line) -> !line.trim().isEmpty())
119118
.forEach(
120119
(dependency) -> {
121120
String dependencyName = getDependencyName(dependency);
@@ -151,11 +150,7 @@ private List<Map<String, Object>> getDependenciesImpl(
151150
}
152151
List<String> linesOfRequirements;
153152
try {
154-
linesOfRequirements =
155-
Files.readAllLines(requirementsPath).stream()
156-
.filter((line) -> !line.trim().startsWith("#") && !line.trim().isEmpty())
157-
.map(String::trim)
158-
.collect(Collectors.toList());
153+
linesOfRequirements = preprocessRequirementsLines(Files.readAllLines(requirementsPath));
159154
} catch (IOException e) {
160155
log.warning(
161156
"Error while trying to read the requirements.txt file, will not be able to install"
@@ -377,6 +372,63 @@ protected String getDependencyNameShow(String pipShowOutput) {
377372
return versionToken.substring(0, endOfLine).trim();
378373
}
379374

375+
/**
376+
* Preprocesses raw requirements.txt lines by joining line continuations, stripping inline
377+
* options, and filtering out pip option lines, URLs, local paths, and empty/comment lines.
378+
*/
379+
public static List<String> preprocessRequirementsLines(List<String> rawLines) {
380+
// Join line continuations (trailing backslash, possibly followed by whitespace)
381+
List<String> joined = new ArrayList<>();
382+
StringBuilder current = new StringBuilder();
383+
for (String line : rawLines) {
384+
String stripped = line.stripTrailing();
385+
if (stripped.endsWith("\\")) {
386+
current.append(stripped, 0, stripped.length() - 1);
387+
} else {
388+
current.append(line);
389+
joined.add(current.toString());
390+
current = new StringBuilder();
391+
}
392+
}
393+
if (current.length() > 0) {
394+
joined.add(current.toString());
395+
}
396+
397+
List<String> result = new ArrayList<>();
398+
for (String raw : joined) {
399+
String line = raw.trim();
400+
if (line.isEmpty() || line.startsWith("#")) {
401+
continue;
402+
}
403+
// Filter out pip options (lines starting with -)
404+
if (line.startsWith("-")) {
405+
continue;
406+
}
407+
// Filter out local path requirements (./path, ../path, /abs/path)
408+
if (line.startsWith("./") || line.startsWith("../") || line.startsWith("/")) {
409+
continue;
410+
}
411+
// Strip PEP 508 direct references (name @ url -> name) before URL check
412+
int atIndex = line.indexOf(" @ ");
413+
if (atIndex != -1) {
414+
line = line.substring(0, atIndex).trim();
415+
}
416+
// Strip inline pip options (--hash=..., --config-settings=..., etc.)
417+
int optionIndex = line.indexOf(" --");
418+
if (optionIndex != -1) {
419+
line = line.substring(0, optionIndex).trim();
420+
}
421+
// Filter out bare URLs and VCS URLs (any line containing :// is not a package name)
422+
if (line.contains("://")) {
423+
continue;
424+
}
425+
if (!line.isEmpty()) {
426+
result.add(line);
427+
}
428+
}
429+
return result;
430+
}
431+
380432
public static String getDependencyName(String dep) {
381433
int markerSeparator = dep.indexOf(";");
382434
String requirement = markerSeparator == -1 ? dep : dep.substring(0, markerSeparator);

‎src/test/java/io/github/guacsec/trustifyda/utils/PythonControllerBaseTest.java‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2043,4 +2043,144 @@ void when_spliting_pip_show_dep_with_license() {
20432043
+ "Requires: \n"
20442044
+ "Required-by: cycler, gensim, gTTS, python-dateutil, tweepy\n");
20452045
}
2046+
2047+
@Test
2048+
void preprocessRequirementsLines_filters_extra_index_url() {
2049+
List<String> input =
2050+
List.of(
2051+
"--extra-index-url https://pypi.example.com/simple",
2052+
"requests==2.28.0",
2053+
"--index-url https://pypi.org/simple",
2054+
"flask==2.0.3");
2055+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2056+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2057+
}
2058+
2059+
@Test
2060+
void preprocessRequirementsLines_filters_short_pip_options() {
2061+
List<String> input =
2062+
List.of("-r other-requirements.txt", "-c constraints.txt", "numpy==1.24.0");
2063+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2064+
assertEquals(List.of("numpy==1.24.0"), result);
2065+
}
2066+
2067+
@Test
2068+
void preprocessRequirementsLines_strips_hashes() {
2069+
List<String> input =
2070+
List.of("requests==2.28.0 --hash=sha256:abc123 --hash=sha256:def456", "flask==2.0.3");
2071+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2072+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2073+
}
2074+
2075+
@Test
2076+
void preprocessRequirementsLines_strips_config_settings() {
2077+
List<String> input =
2078+
List.of(
2079+
"aiohappyeyeballs==2.6.1 --config-settings=KEY=VALUE --config-settings=OTHER=VAL",
2080+
"flask==2.0.3");
2081+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2082+
assertEquals(List.of("aiohappyeyeballs==2.6.1", "flask==2.0.3"), result);
2083+
}
2084+
2085+
@Test
2086+
void preprocessRequirementsLines_joins_line_continuations() {
2087+
List<String> input =
2088+
List.of(
2089+
"requests==2.28.0 \\",
2090+
" --hash=sha256:abc123 \\",
2091+
" --hash=sha256:def456",
2092+
"flask==2.0.3");
2093+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2094+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2095+
}
2096+
2097+
@Test
2098+
void preprocessRequirementsLines_joins_continuations_with_config_settings() {
2099+
List<String> input =
2100+
List.of(
2101+
"aiohappyeyeballs==2.6.1 \\",
2102+
" --config-settings=SAMPLE_TEXT=TEST_VALUE \\",
2103+
" --config-settings=ANOTHER_KEY=ANOTHER_VALUE",
2104+
"async-timeout==5.0.1 ; python_full_version < '3.11'");
2105+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2106+
assertEquals(
2107+
List.of("aiohappyeyeballs==2.6.1", "async-timeout==5.0.1 ; python_full_version < '3.11'"),
2108+
result);
2109+
}
2110+
2111+
@Test
2112+
void preprocessRequirementsLines_filters_comments_and_empty_lines() {
2113+
List<String> input = List.of("# this is a comment", "", " ", "requests==2.28.0");
2114+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2115+
assertEquals(List.of("requests==2.28.0"), result);
2116+
}
2117+
2118+
@Test
2119+
void preprocessRequirementsLines_strips_direct_references() {
2120+
List<String> input =
2121+
List.of(
2122+
"pip @ https://github.com/pypa/pip/archive/22.0.2.zip",
2123+
"requests[security] @ https://github.com/psf/requests/archive/main.zip",
2124+
"flask==2.0.3");
2125+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2126+
assertEquals(List.of("pip", "requests[security]", "flask==2.0.3"), result);
2127+
}
2128+
2129+
@Test
2130+
void preprocessRequirementsLines_handles_trailing_whitespace_after_backslash() {
2131+
List<String> input =
2132+
List.of("requests==2.28.0 \\ ", " --hash=sha256:abc123", "flask==2.0.3");
2133+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2134+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2135+
}
2136+
2137+
@Test
2138+
void preprocessRequirementsLines_filters_bare_urls() {
2139+
List<String> input =
2140+
List.of(
2141+
"https://example.com/packages/MyPackage-1.0.tar.gz",
2142+
"http://example.com/packages/other.whl",
2143+
"git+https://git.example.com/MyProject.git@v1.0",
2144+
"git+git://git.example.com/repo.git",
2145+
"hg+http://hg.example.com/repo",
2146+
"hg+ssh://hg.example.com/repo",
2147+
"svn+svn://svn.example.com/project/trunk",
2148+
"bzr+ftp://bzr.example.com/repo",
2149+
"requests==2.28.0");
2150+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2151+
assertEquals(List.of("requests==2.28.0"), result);
2152+
}
2153+
2154+
@Test
2155+
void preprocessRequirementsLines_filters_local_paths() {
2156+
List<String> input =
2157+
List.of(
2158+
"./local-package",
2159+
"../parent-package",
2160+
"./downloads/MyPackage-1.0.tar.gz",
2161+
"/absolute/path/to/package",
2162+
"requests==2.28.0");
2163+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2164+
assertEquals(List.of("requests==2.28.0"), result);
2165+
}
2166+
2167+
@Test
2168+
void preprocessRequirementsLines_combined_scenario() {
2169+
List<String> input =
2170+
List.of(
2171+
"--extra-index-url https://pypi.example.com/simple",
2172+
"# A comment",
2173+
"requests==2.28.0 \\ ",
2174+
" --hash=sha256:abc123",
2175+
"",
2176+
"--trusted-host pypi.example.com",
2177+
"flask==2.0.3 --hash=sha256:xyz789",
2178+
"pip @ https://github.com/pypa/pip/archive/22.0.2.zip",
2179+
"https://example.com/packages/other.tar.gz",
2180+
"git+git://git.example.com/repo.git",
2181+
"./local-package",
2182+
"numpy>=1.24.0");
2183+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2184+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3", "pip", "numpy>=1.24.0"), result);
2185+
}
20462186
}

0 commit comments

Comments
 (0)