Skip to content

Commit 2a2083f

Browse files
a-orenclaude
a-oren
and
claude
authored
fix(python): handle pip options, hashes, and line continuations in requirements.txt (guacsec#485)
## Summary - Adds `preprocessRequirementsLines()` to `PythonControllerBase` that properly handles all requirements.txt line types before dependency resolution - Fixes parsing errors when requirements.txt contains `--extra-index-url`, `--hash`, line continuations (`\`), and other non-package lines - Handles: pip options, inline options (--hash, --config-settings), line continuations, PEP 508 direct references (`name @ url`), bare URLs, VCS URLs, and local paths Fixes [TC-4527](https://redhat.atlassian.net/browse/TC-4527) Related: fabric8-analytics/fabric8-analytics-vscode-extension#843 ## Test plan - [x] 13 unit tests covering all requirements.txt line types - [x] Existing `getIgnoredDependencies_strips_environment_markers` test passes - [ ] Manual verification with requirements.txt files containing `--extra-index-url` and line continuations 🤖 Generated with [Claude Code](https://claude.com/claude-code) [TC-4527]: https://redhat.atlassian.net/browse/TC-4527?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ ## Summary by Sourcery Improve Python requirements.txt preprocessing to robustly handle pip options, hashes, line continuations, and non-package lines before dependency resolution. Bug Fixes: - Fix handling of requirements.txt entries that include pip options, hashes, line continuations, direct references, bare URLs, VCS URLs, and local paths so they no longer break dependency parsing and installation. Enhancements: - Introduce a reusable preprocessing routine for requirements.txt lines and apply it across Python dependency resolution and ignored-dependency detection to ensure consistent parsing behavior. Tests: - Add unit tests covering preprocessing of all supported requirements.txt line variants, including pip options, hashes, line continuations, direct references, URLs, local paths, and mixed scenarios. --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 95e2da5 commit 2a2083f

3 files changed

Lines changed: 230 additions & 9 deletions

File tree

‎src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ public Content provideComponent() throws IOException {
9595

9696
@Override
9797
protected Set<PackageURL> getIgnoredDependencies(String manifestContent) {
98-
String[] lines = manifestContent.split(System.lineSeparator());
98+
String[] lines = manifestContent.split("\\R");
9999
return Arrays.stream(lines)
100100
.filter(this::containsIgnorePattern)
101101
.map(PythonPipProvider::extractDepFull)

‎src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java‎

Lines changed: 70 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
import java.util.List;
3636
import java.util.Map;
3737
import java.util.logging.Logger;
38+
import java.util.regex.Matcher;
39+
import java.util.regex.Pattern;
3840
import java.util.stream.Collectors;
3941
import java.util.stream.Stream;
4042

@@ -112,10 +114,9 @@ public final List<Map<String, Object>> getDependencies(
112114

113115
private void installingRequirementsOneByOne(String pathToRequirements) {
114116
try {
115-
List<String> requirementsRows = Files.readAllLines(Path.of(pathToRequirements));
117+
List<String> requirementsRows =
118+
preprocessRequirementsLines(Files.readAllLines(Path.of(pathToRequirements)));
116119
requirementsRows.stream()
117-
.filter((line) -> !line.trim().startsWith("#"))
118-
.filter((line) -> !line.trim().isEmpty())
119120
.forEach(
120121
(dependency) -> {
121122
String dependencyName = getDependencyName(dependency);
@@ -151,11 +152,7 @@ private List<Map<String, Object>> getDependenciesImpl(
151152
}
152153
List<String> linesOfRequirements;
153154
try {
154-
linesOfRequirements =
155-
Files.readAllLines(requirementsPath).stream()
156-
.filter((line) -> !line.trim().startsWith("#") && !line.trim().isEmpty())
157-
.map(String::trim)
158-
.collect(Collectors.toList());
155+
linesOfRequirements = preprocessRequirementsLines(Files.readAllLines(requirementsPath));
159156
} catch (IOException e) {
160157
log.warning(
161158
"Error while trying to read the requirements.txt file, will not be able to install"
@@ -377,6 +374,71 @@ protected String getDependencyNameShow(String pipShowOutput) {
377374
return versionToken.substring(0, endOfLine).trim();
378375
}
379376

377+
private static final Pattern INLINE_OPTION_PATTERN = Pattern.compile("\\s--");
378+
private static final Pattern WINDOWS_DRIVE_PATH_PATTERN = Pattern.compile("^[a-zA-Z]:[/\\\\]");
379+
380+
/**
381+
* Preprocesses raw requirements.txt lines by joining line continuations, stripping inline
382+
* options, and filtering out pip option lines, URLs, local paths, and empty/comment lines.
383+
*/
384+
public static List<String> preprocessRequirementsLines(List<String> rawLines) {
385+
// Join line continuations (trailing backslash, possibly followed by whitespace)
386+
List<String> joined = new ArrayList<>();
387+
StringBuilder current = new StringBuilder();
388+
for (String line : rawLines) {
389+
String stripped = line.stripTrailing();
390+
if (stripped.endsWith("\\")) {
391+
current.append(stripped, 0, stripped.length() - 1);
392+
} else {
393+
current.append(line);
394+
joined.add(current.toString());
395+
current = new StringBuilder();
396+
}
397+
}
398+
if (current.length() > 0) {
399+
joined.add(current.toString());
400+
}
401+
402+
List<String> result = new ArrayList<>();
403+
for (String raw : joined) {
404+
String line = raw.trim();
405+
if (line.isEmpty() || line.startsWith("#")) {
406+
continue;
407+
}
408+
// Filter out pip options (lines starting with -)
409+
if (line.startsWith("-")) {
410+
continue;
411+
}
412+
// Filter out local path requirements (./path, ../path, /abs/path, C:\path, C:/path)
413+
if (line.startsWith("./")
414+
|| line.startsWith("../")
415+
|| line.startsWith("/")
416+
|| WINDOWS_DRIVE_PATH_PATTERN.matcher(line).find()) {
417+
continue;
418+
}
419+
// Strip PEP 508 direct references (name @ url -> name) before URL check
420+
int atIndex = line.indexOf(" @ ");
421+
if (atIndex != -1) {
422+
line = line.substring(0, atIndex).trim();
423+
}
424+
// Strip inline pip options (--hash=..., --config-settings=..., etc.)
425+
Matcher optionMatcher = INLINE_OPTION_PATTERN.matcher(line);
426+
if (optionMatcher.find()) {
427+
line = line.substring(0, optionMatcher.start()).trim();
428+
}
429+
// Filter out bare URLs and VCS URLs — check the requirement part (before any marker)
430+
// to avoid false positives from marker strings
431+
String requirementPart = line.contains(";") ? line.substring(0, line.indexOf(";")) : line;
432+
if (requirementPart.contains("://")) {
433+
continue;
434+
}
435+
if (!line.isEmpty()) {
436+
result.add(line);
437+
}
438+
}
439+
return result;
440+
}
441+
380442
public static String getDependencyName(String dep) {
381443
int markerSeparator = dep.indexOf(";");
382444
String requirement = markerSeparator == -1 ? dep : dep.substring(0, markerSeparator);

‎src/test/java/io/github/guacsec/trustifyda/utils/PythonControllerBaseTest.java‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2043,4 +2043,163 @@ void when_spliting_pip_show_dep_with_license() {
20432043
+ "Requires: \n"
20442044
+ "Required-by: cycler, gensim, gTTS, python-dateutil, tweepy\n");
20452045
}
2046+
2047+
@Test
2048+
void preprocessRequirementsLines_filters_extra_index_url() {
2049+
List<String> input =
2050+
List.of(
2051+
"--extra-index-url https://pypi.example.com/simple",
2052+
"requests==2.28.0",
2053+
"--index-url https://pypi.org/simple",
2054+
"flask==2.0.3");
2055+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2056+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2057+
}
2058+
2059+
@Test
2060+
void preprocessRequirementsLines_filters_short_pip_options() {
2061+
List<String> input =
2062+
List.of("-r other-requirements.txt", "-c constraints.txt", "numpy==1.24.0");
2063+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2064+
assertEquals(List.of("numpy==1.24.0"), result);
2065+
}
2066+
2067+
@Test
2068+
void preprocessRequirementsLines_strips_hashes() {
2069+
List<String> input =
2070+
List.of("requests==2.28.0 --hash=sha256:abc123 --hash=sha256:def456", "flask==2.0.3");
2071+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2072+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2073+
}
2074+
2075+
@Test
2076+
void preprocessRequirementsLines_strips_config_settings() {
2077+
List<String> input =
2078+
List.of(
2079+
"aiohappyeyeballs==2.6.1 --config-settings=KEY=VALUE --config-settings=OTHER=VAL",
2080+
"flask==2.0.3");
2081+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2082+
assertEquals(List.of("aiohappyeyeballs==2.6.1", "flask==2.0.3"), result);
2083+
}
2084+
2085+
@Test
2086+
void preprocessRequirementsLines_joins_line_continuations() {
2087+
List<String> input =
2088+
List.of(
2089+
"requests==2.28.0 \\",
2090+
" --hash=sha256:abc123 \\",
2091+
" --hash=sha256:def456",
2092+
"flask==2.0.3");
2093+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2094+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2095+
}
2096+
2097+
@Test
2098+
void preprocessRequirementsLines_joins_continuations_with_config_settings() {
2099+
List<String> input =
2100+
List.of(
2101+
"aiohappyeyeballs==2.6.1 \\",
2102+
" --config-settings=SAMPLE_TEXT=TEST_VALUE \\",
2103+
" --config-settings=ANOTHER_KEY=ANOTHER_VALUE",
2104+
"async-timeout==5.0.1 ; python_full_version < '3.11'");
2105+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2106+
assertEquals(
2107+
List.of("aiohappyeyeballs==2.6.1", "async-timeout==5.0.1 ; python_full_version < '3.11'"),
2108+
result);
2109+
}
2110+
2111+
@Test
2112+
void preprocessRequirementsLines_filters_comments_and_empty_lines() {
2113+
List<String> input = List.of("# this is a comment", "", " ", "requests==2.28.0");
2114+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2115+
assertEquals(List.of("requests==2.28.0"), result);
2116+
}
2117+
2118+
@Test
2119+
void preprocessRequirementsLines_strips_direct_references() {
2120+
List<String> input =
2121+
List.of(
2122+
"pip @ https://github.com/pypa/pip/archive/22.0.2.zip",
2123+
"requests[security] @ https://github.com/psf/requests/archive/main.zip",
2124+
"flask==2.0.3");
2125+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2126+
assertEquals(List.of("pip", "requests[security]", "flask==2.0.3"), result);
2127+
}
2128+
2129+
@Test
2130+
void preprocessRequirementsLines_handles_trailing_whitespace_after_backslash() {
2131+
List<String> input =
2132+
List.of("requests==2.28.0 \\ ", " --hash=sha256:abc123", "flask==2.0.3");
2133+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2134+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2135+
}
2136+
2137+
@Test
2138+
void preprocessRequirementsLines_filters_bare_urls() {
2139+
List<String> input =
2140+
List.of(
2141+
"https://example.com/packages/MyPackage-1.0.tar.gz",
2142+
"http://example.com/packages/other.whl",
2143+
"git+https://git.example.com/MyProject.git@v1.0",
2144+
"git+git://git.example.com/repo.git",
2145+
"hg+http://hg.example.com/repo",
2146+
"hg+ssh://hg.example.com/repo",
2147+
"svn+svn://svn.example.com/project/trunk",
2148+
"bzr+ftp://bzr.example.com/repo",
2149+
"requests==2.28.0");
2150+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2151+
assertEquals(List.of("requests==2.28.0"), result);
2152+
}
2153+
2154+
@Test
2155+
void preprocessRequirementsLines_filters_local_paths() {
2156+
List<String> input =
2157+
List.of(
2158+
"./local-package",
2159+
"../parent-package",
2160+
"./downloads/MyPackage-1.0.tar.gz",
2161+
"/absolute/path/to/package",
2162+
"requests==2.28.0");
2163+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2164+
assertEquals(List.of("requests==2.28.0"), result);
2165+
}
2166+
2167+
@Test
2168+
void preprocessRequirementsLines_filters_windows_paths() {
2169+
List<String> input =
2170+
List.of(
2171+
"C:\\Users\\dev\\my-package",
2172+
"c:/projects/my-lib",
2173+
"D:\\packages\\MyPackage-1.0.tar.gz",
2174+
"requests==2.28.0");
2175+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2176+
assertEquals(List.of("requests==2.28.0"), result);
2177+
}
2178+
2179+
@Test
2180+
void preprocessRequirementsLines_handles_final_line_ending_with_backslash() {
2181+
List<String> input = List.of("flask==2.0.3", "requests==2.28.0 \\");
2182+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2183+
assertEquals(List.of("flask==2.0.3", "requests==2.28.0"), result);
2184+
}
2185+
2186+
@Test
2187+
void preprocessRequirementsLines_combined_scenario() {
2188+
List<String> input =
2189+
List.of(
2190+
"--extra-index-url https://pypi.example.com/simple",
2191+
"# A comment",
2192+
"requests==2.28.0 \\ ",
2193+
" --hash=sha256:abc123",
2194+
"",
2195+
"--trusted-host pypi.example.com",
2196+
"flask==2.0.3 --hash=sha256:xyz789",
2197+
"pip @ https://github.com/pypa/pip/archive/22.0.2.zip",
2198+
"https://example.com/packages/other.tar.gz",
2199+
"git+git://git.example.com/repo.git",
2200+
"./local-package",
2201+
"numpy>=1.24.0");
2202+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2203+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3", "pip", "numpy>=1.24.0"), result);
2204+
}
20462205
}

0 commit comments

Comments
 (0)