diff --git a/src/twyn/cli.py b/src/twyn/cli.py index 5ae569c8..4bb4c75c 100644 --- a/src/twyn/cli.py +++ b/src/twyn/cli.py @@ -47,6 +47,7 @@ def entry_point() -> None: @click.option( "--dependency-file", type=str, + multiple=True, help=( "Dependency file to analyze. By default, twyn will search in the current directory " "for supported files, but this option will override that behavior." @@ -112,7 +113,7 @@ def entry_point() -> None: ) def run( # noqa: C901 config: str, - dependency_file: Optional[str], + dependency_file: tuple[str], dependency: tuple[str], selector_method: str, v: bool, @@ -133,15 +134,16 @@ def run( # noqa: C901 "Only one of --dependency or --dependency-file can be set at a time.", ctx=click.get_current_context() ) - if dependency_file and not any(dependency_file.endswith(key) for key in DEPENDENCY_FILE_MAPPING): - raise click.UsageError("Dependency file name not supported.", ctx=click.get_current_context()) + for dep_file in dependency_file: + if dep_file and not any(dep_file.endswith(key) for key in DEPENDENCY_FILE_MAPPING): + raise click.UsageError(f"Dependency file name {dep_file} not supported.", ctx=click.get_current_context()) try: possible_typos = check_dependencies( selector_method=selector_method, dependencies=set(dependency) or None, config_file=config, - dependency_file=dependency_file, + dependency_file=set(dependency_file) or None, use_cache=not no_cache if no_cache is not None else no_cache, show_progress_bar=False if json else not no_track, load_config_from_file=True, diff --git a/src/twyn/config/config_handler.py b/src/twyn/config/config_handler.py index cf2b97db..5fc2a473 100644 --- a/src/twyn/config/config_handler.py +++ b/src/twyn/config/config_handler.py @@ -32,20 +32,20 @@ class TwynConfiguration: """Fully resolved configuration for Twyn.""" - dependency_file: Optional[str] selector_method: str allowlist: set[str] source: Optional[str] use_cache: bool package_ecosystem: Optional[PackageEcosystems] recursive: Optional[bool] + dependency_file: set[str] @dataclass class ReadTwynConfiguration: """Configuration for twyn as set by the user. It may have None values.""" - dependency_file: Optional[str] = None + dependency_file: Optional[set[str]] = field(default_factory=set) selector_method: Optional[str] = None allowlist: set[str] = field(default_factory=set) source: Optional[str] = None @@ -63,7 +63,7 @@ def __init__(self, file_handler: Optional[FileHandler] = None) -> None: def resolve_config( self, selector_method: Optional[str] = None, - dependency_file: Optional[str] = None, + dependency_files: Optional[set[str]] = None, use_cache: Optional[bool] = None, package_ecosystem: Optional[PackageEcosystems] = None, recursive: Optional[bool] = None, @@ -107,7 +107,7 @@ def resolve_config( final_recursive = DEFAULT_RECURSIVE return TwynConfiguration( - dependency_file=dependency_file or read_config.dependency_file, + dependency_file=dependency_files or read_config.dependency_file or set(), selector_method=final_selector_method, allowlist=read_config.allowlist, source=read_config.source, @@ -142,11 +142,13 @@ def _get_read_config(self, toml: TOMLDocument) -> ReadTwynConfiguration: """Read the twyn configuration from a provided toml document.""" twyn_config_data = toml.get("tool", {}).get("twyn", {}) return ReadTwynConfiguration( - dependency_file=twyn_config_data.get("dependency_file"), + dependency_file=set(twyn_config_data.get("dependency_file", set())), selector_method=twyn_config_data.get("selector_method"), allowlist=set(twyn_config_data.get("allowlist", set())), source=twyn_config_data.get("source"), use_cache=twyn_config_data.get("use_cache"), + package_ecosystem=twyn_config_data.get("package_ecosystem"), + recursive=twyn_config_data.get("recursive"), ) def _write_config(self, toml: TOMLDocument, config: ReadTwynConfiguration) -> None: @@ -155,12 +157,12 @@ def _write_config(self, toml: TOMLDocument, config: ReadTwynConfiguration) -> No All null values are simply omitted from the toml file. """ twyn_toml_data = asdict(config, dict_factory=lambda x: _serialize_config(x)) + if "tool" not in toml: toml.add("tool", table()) if "twyn" not in toml["tool"]: # type: ignore[operator] toml["tool"]["twyn"] = {} # type: ignore[index] toml["tool"]["twyn"] = twyn_toml_data # type: ignore[index] - self._write_toml(toml) def _write_toml(self, toml: TOMLDocument) -> None: diff --git a/src/twyn/dependency_parser/dependency_selector.py b/src/twyn/dependency_parser/dependency_selector.py index 595b151f..496004b1 100644 --- a/src/twyn/dependency_parser/dependency_selector.py +++ b/src/twyn/dependency_parser/dependency_selector.py @@ -42,7 +42,6 @@ def get_dependency_file_parsers_from_file_name(self) -> list[AbstractParser]: if self.dependency_file.endswith(known_dependency_file_name): file_parser = DEPENDENCY_FILE_MAPPING[known_dependency_file_name](self.dependency_file) parsers.append(file_parser) - if not parsers: raise NoMatchingParserError diff --git a/src/twyn/main.py b/src/twyn/main.py index efb3f16a..7d2dd2b4 100644 --- a/src/twyn/main.py +++ b/src/twyn/main.py @@ -37,7 +37,7 @@ def check_dependencies( selector_method: Union[SelectorMethod, None] = None, config_file: Optional[str] = None, - dependency_file: Optional[str] = None, + dependency_file: Optional[set[str]] = None, dependencies: Optional[set[str]] = None, use_cache: Optional[bool] = True, show_progress_bar: bool = False, @@ -68,7 +68,7 @@ def check_dependencies( load_config_from_file=load_config_from_file, config_file=config_file, selector_method=selector_method, - dependency_file=dependency_file, + dependency_files=dependency_file, use_cache=use_cache, package_ecosystem=package_ecosystem, recursive=recursive, @@ -104,7 +104,7 @@ def check_dependencies( maybe_cache_handler=maybe_cache_handler, allowlist=config.allowlist, show_progress_bar=show_progress_bar, - dependency_file=config.dependency_file, + dependency_files=config.dependency_file, ) @@ -153,7 +153,7 @@ def _analyze_packages_from_source( allowlist: set[str], selector_method: SelectorMethod, show_progress_bar: bool, - dependency_file: Optional[str], + dependency_files: Optional[set[str]], source: Optional[str], maybe_cache_handler: Optional[CacheHandler], ) -> TyposquatCheckResults: @@ -161,30 +161,31 @@ def _analyze_packages_from_source( It will return a list of the possible typos grouped by source, each source being a dependency file. """ - dependency_managers = _get_dependency_managers_and_parsers_mapping(dependency_file) - typos_by_file = TyposquatCheckResults() - for dependency_manager, parsers in dependency_managers.items(): - top_package_reference = dependency_manager.trusted_packages_source(source, maybe_cache_handler) - - packages_from_source = top_package_reference.get_packages() - trusted_packages = TrustedPackages( - names=packages_from_source, - algorithm=EditDistance(), - selector=selector_method, - threshold_class=SimilarityThreshold, - ) - results: list[TyposquatCheckResultFromSource] = [] - - for parser in parsers: - analyzed_dependencies = _analyze_dependencies( - top_package_reference, trusted_packages, parser.parse(), allowlist, show_progress_bar, parser.file_path + dependency_files = dependency_files or {""} + for dep_file in dependency_files: + dependency_managers = _get_dependency_managers_and_parsers_mapping(dep_file) + typos_by_file = TyposquatCheckResults() + for dependency_manager, parsers in dependency_managers.items(): + top_package_reference = dependency_manager.trusted_packages_source(source, maybe_cache_handler) + + packages_from_source = top_package_reference.get_packages() + trusted_packages = TrustedPackages( + names=packages_from_source, + algorithm=EditDistance(), + selector=selector_method, + threshold_class=SimilarityThreshold, ) - - if analyzed_dependencies: - results.append( - TyposquatCheckResultFromSource(source=str(parser.file_path), errors=analyzed_dependencies) + results: list[TyposquatCheckResultFromSource] = [] + for parser in parsers: + analyzed_dependencies = _analyze_dependencies( + top_package_reference, trusted_packages, parser.parse(), allowlist, show_progress_bar ) - typos_by_file.results += results + + if analyzed_dependencies: + results.append( + TyposquatCheckResultFromSource(source=str(parser.file_path), errors=analyzed_dependencies) + ) + typos_by_file.results += results return typos_by_file @@ -258,6 +259,7 @@ def _get_dependency_managers_and_parsers_mapping( dependency_managers: dict[type[BaseDependencyManager], list[AbstractParser]] = {} # No dependencies introduced via the CLI, so the dependecy file was either given or will be auto-detected + dependency_selector = DependencySelector(dependency_file) dependency_parsers = dependency_selector.get_dependency_parsers() @@ -274,7 +276,7 @@ def _get_config( load_config_from_file: bool, config_file: Optional[str], selector_method: Union[SelectorMethod, None], - dependency_file: Optional[str], + dependency_files: Optional[set[str]], use_cache: Optional[bool], package_ecosystem: Optional[PackageEcosystems], recursive: Optional[bool], @@ -286,7 +288,7 @@ def _get_config( config_file_handler = None return ConfigHandler(config_file_handler).resolve_config( selector_method=selector_method, - dependency_file=dependency_file, + dependency_files=dependency_files, use_cache=use_cache, package_ecosystem=package_ecosystem, recursive=recursive, diff --git a/tests/config/test_config_handler.py b/tests/config/test_config_handler.py index f24b2daa..e3afa516 100644 --- a/tests/config/test_config_handler.py +++ b/tests/config/test_config_handler.py @@ -1,7 +1,6 @@ import dataclasses from copy import deepcopy from pathlib import Path -from typing import NoReturn from unittest.mock import Mock, patch import pytest @@ -26,17 +25,14 @@ class TestConfigHandler: - def throw_exception(self) -> NoReturn: - raise PathNotFoundError - @patch("twyn.file_handler.file_handler.FileHandler.read") def test_no_enforce_file_on_non_existent_file(self, mock_is_file: Mock) -> None: """Resolving the config without enforcing the file to be present gives you defaults.""" - mock_is_file.side_effect = self.throw_exception + mock_is_file.side_effect = PathNotFoundError() config = ConfigHandler(FileHandler(DEFAULT_PROJECT_TOML_FILE)).resolve_config() assert config == TwynConfiguration( - dependency_file=None, + dependency_file=set(), selector_method="all", allowlist=set(), source=None, @@ -51,7 +47,7 @@ def test_config_raises_for_unknown_file(self) -> None: def test_read_config_values(self, pyproject_toml_file: Path) -> None: config = ConfigHandler(file_handler=FileHandler(pyproject_toml_file)).resolve_config() - assert config.dependency_file == "my_file.txt" + assert config.dependency_file == {"my_file.txt", "my_other_file.txt"} assert config.selector_method == "all" assert config.allowlist == {"boto4", "boto2"} assert config.use_cache is False @@ -62,7 +58,7 @@ def test_get_twyn_data_from_file(self, pyproject_toml_file: Path) -> None: toml = handler._read_toml() twyn_data = ConfigHandler(FileHandler(pyproject_toml_file))._get_read_config(toml) assert twyn_data == ReadTwynConfiguration( - dependency_file="my_file.txt", + dependency_file={"my_file.txt", "my_other_file.txt"}, selector_method="all", allowlist={"boto4", "boto2"}, source=None, @@ -75,7 +71,7 @@ def test_write_toml(self, pyproject_toml_file: Path) -> None: initial_config = handler.resolve_config() to_write = deepcopy(initial_config) - to_write = dataclasses.replace(to_write, allowlist={}) + to_write = dataclasses.replace(to_write, allowlist={}, dependency_file={}) handler._write_config(toml, to_write) @@ -100,7 +96,7 @@ def test_write_toml(self, pyproject_toml_file: Path) -> None: "scripts": {"twyn": "twyn.cli:entry_point"}, }, "twyn": { - "dependency_file": "my_file.txt", + "dependency_file": {}, "selector_method": "all", "allowlist": {}, "use_cache": False, @@ -141,7 +137,7 @@ def test_no_load_config_from_cache(self, pyproject_toml_file: Path) -> None: config = ConfigHandler().resolve_config() assert config.allowlist == set() - assert config.dependency_file is None + assert config.dependency_file == set() assert config.use_cache is True assert config.selector_method == DEFAULT_SELECTOR_METHOD assert config.source is None diff --git a/tests/conftest.py b/tests/conftest.py index bac4144d..61e296c1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -237,7 +237,7 @@ def pyproject_toml_file(tmp_path: Path) -> Iterator[Path]: twyn = "twyn.cli:entry_point" [tool.twyn] - dependency_file="my_file.txt" + dependency_file=["my_file.txt", "my_other_file.txt"] selector_method="all" logging_level="debug" allowlist=["boto4", "boto2"] diff --git a/tests/main/test_cli.py b/tests/main/test_cli.py index 72c28a7b..e833a334 100644 --- a/tests/main/test_cli.py +++ b/tests/main/test_cli.py @@ -88,7 +88,7 @@ def test_click_arguments_dependency_file(self, mock_check_dependencies: Mock) -> assert mock_check_dependencies.call_args_list == [ call( config_file="my-config", - dependency_file="requirements.txt", + dependency_file={"requirements.txt"}, dependencies=None, selector_method="first-letter", use_cache=None, @@ -113,7 +113,7 @@ def test_click_arguments_dependency_file_in_different_path(self, mock_check_depe assert mock_check_dependencies.call_args_list == [ call( config_file=None, - dependency_file="/path/requirements.txt", + dependency_file={"/path/requirements.txt"}, dependencies=None, selector_method=None, use_cache=None, @@ -328,7 +328,7 @@ def test_dependency_file_name_has_to_be_recognized(self) -> None: assert isinstance(result.exception, SystemExit) assert result.exit_code == 2 - assert "Dependency file name not supported." in result.output + assert "Dependency file name requirements-dev.txt not supported." in result.output @patch("twyn.cli.check_dependencies") def test_custom_twyn_error_is_caught_and_wrapped_in_cli_error(self, mock_check_dependencies, caplog): diff --git a/tests/main/test_main.py b/tests/main/test_main.py index fcb25e4e..e0bb2e8c 100644 --- a/tests/main/test_main.py +++ b/tests/main/test_main.py @@ -36,21 +36,21 @@ class TestCheckDependencies: ( { "selector_method": "first-letter", - "dependency_file": "requirements.txt", + "dependency_file": {"requirements.txt"}, "use_cache": True, "pypi_reference": "https://myurl.com", "recurisve": True, }, { "selector_method": "nearby-letter", - "dependency_file": "poetry.lock", + "dependency_file": ["poetry.lock"], "allowlist": ["boto4", "boto2"], # There is no allowlist option in the cli "use_cache": False, "pypi_reference": "https://mysecondurl.com", "recursive": False, }, TwynConfiguration( - dependency_file="requirements.txt", + dependency_file={"requirements.txt"}, selector_method="first-letter", allowlist={"boto4", "boto2"}, source=TopPyPiReference.DEFAULT_SOURCE, @@ -63,14 +63,14 @@ class TestCheckDependencies: {}, { "selector_method": "nearby-letter", - "dependency_file": "poetry.lock", + "dependency_file": ["poetry.lock"], "allowlist": ["boto4", "boto2"], "use_cache": False, "pypi_reference": "https://mysecondurl.com", "recursive": True, }, TwynConfiguration( - dependency_file="poetry.lock", + dependency_file={"poetry.lock"}, selector_method="nearby-letter", allowlist={"boto4", "boto2"}, source=TopPyPiReference.DEFAULT_SOURCE, @@ -83,7 +83,7 @@ class TestCheckDependencies: {}, {}, TwynConfiguration( - dependency_file=None, + dependency_file=set(), selector_method="all", allowlist=set(), source=TopPyPiReference.DEFAULT_SOURCE, @@ -114,7 +114,7 @@ def test_options_priorities_assignation( with patch.object(handler, "_read_toml", return_value=parse(dumps({"tool": {"twyn": file_config}}))): resolved = handler.resolve_config( selector_method=cli_config.get("selector_method"), - dependency_file=cli_config.get("dependency_file"), + dependency_files=cli_config.get("dependency_file", set()), use_cache=cli_config.get("use_cache"), ) @@ -131,7 +131,7 @@ def test_check_dependencies_detects_typosquats_from_file( mock_get_packages.return_value = {"requests"} error = check_dependencies( - dependency_file=str(uv_lock_file_with_typo), + dependency_file={str(uv_lock_file_with_typo)}, use_cache=False, ) @@ -153,7 +153,7 @@ def test_check_dependencies_detects_typosquats_from_file_and_language_is_set( mock_get_packages.return_value = {"requests"} error = check_dependencies( - dependency_file=str(uv_lock_file_with_typo), + dependency_file={str(uv_lock_file_with_typo)}, use_cache=False, package_ecosystem="pypi", ) @@ -181,7 +181,7 @@ def test_check_dependencies_detects_typosquats_and_autodetects_file( """Check that dependencies are loaded from file, retrieved from source, and a typosquat is detected.""" mock_get_packages.return_value = {"requests"} mock_config.return_value = TwynConfiguration( - str(uv_lock_file_with_typo), + dependency_file={str(uv_lock_file_with_typo)}, selector_method="all", allowlist=set(), source=None, @@ -216,7 +216,7 @@ def test_check_dependencies_detects_typosquats_and_autodetects_file_and_language """Check that dependencies are loaded from file, retrieved from source, and a typosquat is detected.""" mock_get_packages.return_value = {"requests"} mock_config.return_value = TwynConfiguration( - str(uv_lock_file_with_typo), + dependency_file={str(uv_lock_file_with_typo)}, selector_method="all", allowlist=set(), source=None, @@ -264,7 +264,7 @@ def test_check_dependencies_recursive_and_dependency_file_set( """Test that recursive and dependency file can be set at the same time, and then dependency file takes precedence while recurisve is ignored.""" mock_get_packages_from_cache.return_value = {"requests"} error = check_dependencies( - dependency_file=str(uv_lock_file_with_typo), + dependency_file={str(uv_lock_file_with_typo)}, package_ecosystem="pypi", ) @@ -286,7 +286,7 @@ def test_check_dependencies_with_input_loads_file_from_different_location( with create_tmp_file(tmp_file, "mypackag"): error = check_dependencies( config_file=None, - dependency_file=str(tmp_file), + dependency_file={str(tmp_file)}, dependencies=None, selector_method="first-letter", ) @@ -396,7 +396,7 @@ def test_check_dependencies_ignores_package_in_allowlist( # Verify that before the whitelist configuration the package is classified as an error. error = check_dependencies( - dependency_file=str(uv_lock_file_with_typo), + dependency_file={str(uv_lock_file_with_typo)}, ) assert mock_get_packages.call_count == 1 assert error == TyposquatCheckResults( @@ -410,7 +410,7 @@ def test_check_dependencies_ignores_package_in_allowlist( m_config = TwynConfiguration( allowlist={"reqests"}, - dependency_file=str(uv_lock_file_with_typo), + dependency_file={str(uv_lock_file_with_typo)}, selector_method="first-letter", use_cache=True, source=None, @@ -421,7 +421,7 @@ def test_check_dependencies_ignores_package_in_allowlist( # Check that the package is no longer an error with patch("twyn.main.ConfigHandler.resolve_config", return_value=m_config): error = check_dependencies( - dependency_file=str(uv_lock_file_with_typo), + dependency_file={str(uv_lock_file_with_typo)}, ) assert error == TyposquatCheckResults(results=[]) @@ -433,7 +433,7 @@ def test_check_dependencies_does_not_error_on_same_package( mock_get_packages.return_value = {"reqests"} # According to the source, this package is correct error = check_dependencies( - dependency_file=str(uv_lock_file_with_typo), + dependency_file={str(uv_lock_file_with_typo)}, ) assert error == TyposquatCheckResults(results=[]) @@ -444,7 +444,7 @@ def test_track_is_disabled_by_default_when_used_as_package( self, mock_config: Mock, mock_get_packages: Mock, uv_lock_file: Path ) -> None: mock_config.return_value = TwynConfiguration( - str(uv_lock_file), + dependency_file={str(uv_lock_file)}, selector_method="all", allowlist=set(), source=None, @@ -461,7 +461,7 @@ def test_track_is_disabled_by_default_when_used_as_package( @patch("twyn.main._get_config") def test_track_is_shown_when_enabled(self, mock_config: Mock, mock_get_packages: Mock, uv_lock_file: Path) -> None: mock_config.return_value = TwynConfiguration( - str(uv_lock_file), + dependency_file={str(uv_lock_file)}, selector_method="all", allowlist=set(), source=None, @@ -489,7 +489,7 @@ def test_check_dependency_file_invalid_language_error(self, m_packages: Mock, uv """Test that when a non valid package_ecosystem is given together with a depdendecy_file, package_ecosystem is ignored.""" m_packages.return_value = {"requests"} result = check_dependencies( - dependency_file=str(uv_lock_file_with_typo), package_ecosystem="npm", use_cache=False + dependency_file={str(uv_lock_file_with_typo)}, package_ecosystem="npm", use_cache=False ) assert bool(result) @@ -501,7 +501,7 @@ def test_check_dependencies_invalid_selector_method_error(self) -> None: def test_check_dependencies_npm_v3(self, package_lock_json_file_v3: Path) -> None: """Integration: check_dependencies works for npm v3 lockfile.""" with patch_npm_packages_download(["expres"]) as m_download: - result = check_dependencies(dependency_file=str(package_lock_json_file_v3), use_cache=False) + result = check_dependencies(dependency_file={str(package_lock_json_file_v3)}, use_cache=False) assert m_download.call_count == 1 assert len(result.results) == 1 # only one file was scanned @@ -525,7 +525,7 @@ def test_get_top_reference_from_name_no_matching_parser_error(self) -> None: def test_check_dependencies_yarn(self, yarn_lock_file_v1: Path) -> None: """Verifies that the yarn flow is working.""" with patch_npm_packages_download(["lodas"]) as mock_download: - result = check_dependencies(dependency_file=str(yarn_lock_file_v1), use_cache=False) + result = check_dependencies(dependency_file={str(yarn_lock_file_v1)}, use_cache=False) assert mock_download.call_count == 1 assert len(result.results) == 1 # only one file was scanned