Skip to content

Commit 7c1fa00

Browse files
a-orena-oren
authored
Merge branch 'main' into TC-4538
2 parents 3b45d0b + 2a2083f commit 7c1fa00

3 files changed

Lines changed: 230 additions & 9 deletions

File tree

‎src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ public Content provideComponent() throws IOException {
9595

9696
@Override
9797
protected Set<PackageURL> getIgnoredDependencies(String manifestContent) {
98-
String[] lines = manifestContent.split(System.lineSeparator());
98+
String[] lines = manifestContent.split("\\R");
9999
return Arrays.stream(lines)
100100
.filter(this::containsIgnorePattern)
101101
.map(PythonPipProvider::extractDepFull)

‎src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java‎

Lines changed: 70 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
import java.util.List;
3636
import java.util.Map;
3737
import java.util.logging.Logger;
38+
import java.util.regex.Matcher;
39+
import java.util.regex.Pattern;
3840
import java.util.stream.Collectors;
3941
import java.util.stream.Stream;
4042

@@ -112,10 +114,9 @@ public final List<Map<String, Object>> getDependencies(
112114

113115
private void installingRequirementsOneByOne(String pathToRequirements) {
114116
try {
115-
List<String> requirementsRows = Files.readAllLines(Path.of(pathToRequirements));
117+
List<String> requirementsRows =
118+
preprocessRequirementsLines(Files.readAllLines(Path.of(pathToRequirements)));
116119
requirementsRows.stream()
117-
.filter((line) -> !line.trim().startsWith("#"))
118-
.filter((line) -> !line.trim().isEmpty())
119120
.forEach(
120121
(dependency) -> {
121122
String dependencyName = getDependencyName(dependency);
@@ -151,11 +152,7 @@ private List<Map<String, Object>> getDependenciesImpl(
151152
}
152153
List<String> linesOfRequirements;
153154
try {
154-
linesOfRequirements =
155-
Files.readAllLines(requirementsPath).stream()
156-
.filter((line) -> !line.trim().startsWith("#") && !line.trim().isEmpty())
157-
.map(String::trim)
158-
.collect(Collectors.toList());
155+
linesOfRequirements = preprocessRequirementsLines(Files.readAllLines(requirementsPath));
159156
} catch (IOException e) {
160157
log.warning(
161158
"Error while trying to read the requirements.txt file, will not be able to install"
@@ -377,6 +374,71 @@ protected String getDependencyNameShow(String pipShowOutput) {
377374
return versionToken.substring(0, endOfLine).trim();
378375
}
379376

377+
private static final Pattern INLINE_OPTION_PATTERN = Pattern.compile("\\s--");
378+
private static final Pattern WINDOWS_DRIVE_PATH_PATTERN = Pattern.compile("^[a-zA-Z]:[/\\\\]");
379+
380+
/**
381+
* Preprocesses raw requirements.txt lines by joining line continuations, stripping inline
382+
* options, and filtering out pip option lines, URLs, local paths, and empty/comment lines.
383+
*/
384+
public static List<String> preprocessRequirementsLines(List<String> rawLines) {
385+
// Join line continuations (trailing backslash, possibly followed by whitespace)
386+
List<String> joined = new ArrayList<>();
387+
StringBuilder current = new StringBuilder();
388+
for (String line : rawLines) {
389+
String stripped = line.stripTrailing();
390+
if (stripped.endsWith("\\")) {
391+
current.append(stripped, 0, stripped.length() - 1);
392+
} else {
393+
current.append(line);
394+
joined.add(current.toString());
395+
current = new StringBuilder();
396+
}
397+
}
398+
if (current.length() > 0) {
399+
joined.add(current.toString());
400+
}
401+
402+
List<String> result = new ArrayList<>();
403+
for (String raw : joined) {
404+
String line = raw.trim();
405+
if (line.isEmpty() || line.startsWith("#")) {
406+
continue;
407+
}
408+
// Filter out pip options (lines starting with -)
409+
if (line.startsWith("-")) {
410+
continue;
411+
}
412+
// Filter out local path requirements (./path, ../path, /abs/path, C:\path, C:/path)
413+
if (line.startsWith("./")
414+
|| line.startsWith("../")
415+
|| line.startsWith("/")
416+
|| WINDOWS_DRIVE_PATH_PATTERN.matcher(line).find()) {
417+
continue;
418+
}
419+
// Strip PEP 508 direct references (name @ url -> name) before URL check
420+
int atIndex = line.indexOf(" @ ");
421+
if (atIndex != -1) {
422+
line = line.substring(0, atIndex).trim();
423+
}
424+
// Strip inline pip options (--hash=..., --config-settings=..., etc.)
425+
Matcher optionMatcher = INLINE_OPTION_PATTERN.matcher(line);
426+
if (optionMatcher.find()) {
427+
line = line.substring(0, optionMatcher.start()).trim();
428+
}
429+
// Filter out bare URLs and VCS URLs — check the requirement part (before any marker)
430+
// to avoid false positives from marker strings
431+
String requirementPart = line.contains(";") ? line.substring(0, line.indexOf(";")) : line;
432+
if (requirementPart.contains("://")) {
433+
continue;
434+
}
435+
if (!line.isEmpty()) {
436+
result.add(line);
437+
}
438+
}
439+
return result;
440+
}
441+
380442
public static String getDependencyName(String dep) {
381443
int markerSeparator = dep.indexOf(";");
382444
String requirement = markerSeparator == -1 ? dep : dep.substring(0, markerSeparator);

‎src/test/java/io/github/guacsec/trustifyda/utils/PythonControllerBaseTest.java‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2043,4 +2043,163 @@ void when_spliting_pip_show_dep_with_license() {
20432043
+ "Requires: \n"
20442044
+ "Required-by: cycler, gensim, gTTS, python-dateutil, tweepy\n");
20452045
}
2046+
2047+
@Test
2048+
void preprocessRequirementsLines_filters_extra_index_url() {
2049+
List<String> input =
2050+
List.of(
2051+
"--extra-index-url https://pypi.example.com/simple",
2052+
"requests==2.28.0",
2053+
"--index-url https://pypi.org/simple",
2054+
"flask==2.0.3");
2055+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2056+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2057+
}
2058+
2059+
@Test
2060+
void preprocessRequirementsLines_filters_short_pip_options() {
2061+
List<String> input =
2062+
List.of("-r other-requirements.txt", "-c constraints.txt", "numpy==1.24.0");
2063+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2064+
assertEquals(List.of("numpy==1.24.0"), result);
2065+
}
2066+
2067+
@Test
2068+
void preprocessRequirementsLines_strips_hashes() {
2069+
List<String> input =
2070+
List.of("requests==2.28.0 --hash=sha256:abc123 --hash=sha256:def456", "flask==2.0.3");
2071+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2072+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2073+
}
2074+
2075+
@Test
2076+
void preprocessRequirementsLines_strips_config_settings() {
2077+
List<String> input =
2078+
List.of(
2079+
"aiohappyeyeballs==2.6.1 --config-settings=KEY=VALUE --config-settings=OTHER=VAL",
2080+
"flask==2.0.3");
2081+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2082+
assertEquals(List.of("aiohappyeyeballs==2.6.1", "flask==2.0.3"), result);
2083+
}
2084+
2085+
@Test
2086+
void preprocessRequirementsLines_joins_line_continuations() {
2087+
List<String> input =
2088+
List.of(
2089+
"requests==2.28.0 \\",
2090+
" --hash=sha256:abc123 \\",
2091+
" --hash=sha256:def456",
2092+
"flask==2.0.3");
2093+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2094+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2095+
}
2096+
2097+
@Test
2098+
void preprocessRequirementsLines_joins_continuations_with_config_settings() {
2099+
List<String> input =
2100+
List.of(
2101+
"aiohappyeyeballs==2.6.1 \\",
2102+
" --config-settings=SAMPLE_TEXT=TEST_VALUE \\",
2103+
" --config-settings=ANOTHER_KEY=ANOTHER_VALUE",
2104+
"async-timeout==5.0.1 ; python_full_version < '3.11'");
2105+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2106+
assertEquals(
2107+
List.of("aiohappyeyeballs==2.6.1", "async-timeout==5.0.1 ; python_full_version < '3.11'"),
2108+
result);
2109+
}
2110+
2111+
@Test
2112+
void preprocessRequirementsLines_filters_comments_and_empty_lines() {
2113+
List<String> input = List.of("# this is a comment", "", " ", "requests==2.28.0");
2114+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2115+
assertEquals(List.of("requests==2.28.0"), result);
2116+
}
2117+
2118+
@Test
2119+
void preprocessRequirementsLines_strips_direct_references() {
2120+
List<String> input =
2121+
List.of(
2122+
"pip @ https://github.com/pypa/pip/archive/22.0.2.zip",
2123+
"requests[security] @ https://github.com/psf/requests/archive/main.zip",
2124+
"flask==2.0.3");
2125+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2126+
assertEquals(List.of("pip", "requests[security]", "flask==2.0.3"), result);
2127+
}
2128+
2129+
@Test
2130+
void preprocessRequirementsLines_handles_trailing_whitespace_after_backslash() {
2131+
List<String> input =
2132+
List.of("requests==2.28.0 \\ ", " --hash=sha256:abc123", "flask==2.0.3");
2133+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2134+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3"), result);
2135+
}
2136+
2137+
@Test
2138+
void preprocessRequirementsLines_filters_bare_urls() {
2139+
List<String> input =
2140+
List.of(
2141+
"https://example.com/packages/MyPackage-1.0.tar.gz",
2142+
"http://example.com/packages/other.whl",
2143+
"git+https://git.example.com/MyProject.git@v1.0",
2144+
"git+git://git.example.com/repo.git",
2145+
"hg+http://hg.example.com/repo",
2146+
"hg+ssh://hg.example.com/repo",
2147+
"svn+svn://svn.example.com/project/trunk",
2148+
"bzr+ftp://bzr.example.com/repo",
2149+
"requests==2.28.0");
2150+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2151+
assertEquals(List.of("requests==2.28.0"), result);
2152+
}
2153+
2154+
@Test
2155+
void preprocessRequirementsLines_filters_local_paths() {
2156+
List<String> input =
2157+
List.of(
2158+
"./local-package",
2159+
"../parent-package",
2160+
"./downloads/MyPackage-1.0.tar.gz",
2161+
"/absolute/path/to/package",
2162+
"requests==2.28.0");
2163+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2164+
assertEquals(List.of("requests==2.28.0"), result);
2165+
}
2166+
2167+
@Test
2168+
void preprocessRequirementsLines_filters_windows_paths() {
2169+
List<String> input =
2170+
List.of(
2171+
"C:\\Users\\dev\\my-package",
2172+
"c:/projects/my-lib",
2173+
"D:\\packages\\MyPackage-1.0.tar.gz",
2174+
"requests==2.28.0");
2175+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2176+
assertEquals(List.of("requests==2.28.0"), result);
2177+
}
2178+
2179+
@Test
2180+
void preprocessRequirementsLines_handles_final_line_ending_with_backslash() {
2181+
List<String> input = List.of("flask==2.0.3", "requests==2.28.0 \\");
2182+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2183+
assertEquals(List.of("flask==2.0.3", "requests==2.28.0"), result);
2184+
}
2185+
2186+
@Test
2187+
void preprocessRequirementsLines_combined_scenario() {
2188+
List<String> input =
2189+
List.of(
2190+
"--extra-index-url https://pypi.example.com/simple",
2191+
"# A comment",
2192+
"requests==2.28.0 \\ ",
2193+
" --hash=sha256:abc123",
2194+
"",
2195+
"--trusted-host pypi.example.com",
2196+
"flask==2.0.3 --hash=sha256:xyz789",
2197+
"pip @ https://github.com/pypa/pip/archive/22.0.2.zip",
2198+
"https://example.com/packages/other.tar.gz",
2199+
"git+git://git.example.com/repo.git",
2200+
"./local-package",
2201+
"numpy>=1.24.0");
2202+
List<String> result = PythonControllerBase.preprocessRequirementsLines(input);
2203+
assertEquals(List.of("requests==2.28.0", "flask==2.0.3", "pip", "numpy>=1.24.0"), result);
2204+
}
20462205
}

0 commit comments

Comments
 (0)