feat: recursively look for lock files

Author: sdn4z

.gitignore

@@ -111,3 +111,6 @@ GitHub.sublime-settings
 .history
 /.ruff_cache/
 /.env.local
+
+# Test lock files
+yarn.lock

README.md

@@ -14,6 +14,7 @@
   - [Using `Twyn` as a cli tool](#using-twyn-as-a-cli-tool)
     - [Installation](#installation)
     - [Docker](#docker)
+    - [CLI Options Reference](#cli-options-reference)
     - [Run](#run)
     - [JSON Format](#json-format)
   - [Using `Twyn` as a library](#using-twyn-as-a-library)
@@ -61,21 +62,35 @@ docker pull elementsinteractive/twyn:latest
 docker run elementsinteractive/twyn --help
 ```
 
+##### CLI Options Reference
+
+| Option / Argument         | Type / Values                                      | Description                                                                                   |
+|--------------------------|----------------------------------------------------|-----------------------------------------------------------------------------------------------|
+| `--config`               | `str` (path)                                       | Path to configuration file (`twyn.toml` or `pyproject.toml` by default).                      |
+| `--dependency-file`      | `str` (path)                                       | Dependency file to analyze. Supported: `requirements.txt`, `poetry.lock`, `uv.lock`, etc.     |
+| `--dependency`           | `str` (multiple allowed)                           | Dependency to analyze directly. Can be specified multiple times.                              |
+| `--selector-method`      | `all`, `first-letter`, `nearby-letter`             | Method for selecting possible typosquats.                                                     |
+| `--package-ecosystem`    | `pypi`, `npm`                                      | Package ecosystem for analysis.                                                               |
+| `-v`                     | flag                                               | Enable info-level logging.                                                                    |
+| `-vv`                    | flag                                               | Enable debug-level logging.                                                                   |
+| `--no-cache`             | flag                                               | Disable use of trusted packages cache. Always fetch from the source.                          |
+| `--no-track`             | flag                                               | Do not show the progress bar while processing packages.                                       |
+| `--json`                 | flag                                               | Display results in JSON format. Implies `--no-track`.                                         |
+| `-r`, `--recursive`      | flag                                               | Scan directories recursively for dependency files.                                            |
 #### Run
 
-To run twyn simply type:
+**Usage Example:**
 
 ```sh
 twyn run <OPTIONS>
 ```
-
-For a list of all the available options as well as their expected arguments run:
+or get help with
 
 ```sh
 twyn run --help
 ```
 
-#### JSON format
+##### JSON format
 If you want your output in JSON format, you can run `Twyn` with the following flag:
 
 ```python
@@ -90,8 +105,8 @@ If `Twyn` was run by manually giving it dependencies (with `--dependency`), the
 
 In any other case (when dependencies are parsed from a file), the source will be the path to the dependencies file. One entry will be created for every source.
 
-### Using Twyn as a library
 
+### Using Twyn as a library
 
 #### Installation
 `Twyn` also supports being used as 3rd party library for you project. To install it, run:
@@ -133,16 +148,6 @@ logging.getLogger("twyn").setLevel(logging.INFO)
 pip install twyn
 ```
 
-Example usage in your code:
-
-```python
-from twyn import check_dependencies
-
-typos = check_dependencies()
-
-for typo in typos.errors:
-  print(f"Dependency:{typo.dependency}")
-  print(f"Did you mean any of [{','.join(typo.similars)}]")
 
 ```
 
@@ -268,5 +273,3 @@ To clear the cache, run:
 ```python
   twyn run cache clear
 ```
-
-

src/twyn/base/constants.py

@@ -35,6 +35,7 @@
 DEFAULT_PROJECT_TOML_FILE = "pyproject.toml"
 DEFAULT_TWYN_TOML_FILE = "twyn.toml"
 DEFAULT_USE_CACHE = True
+DEFAULT_RECURSIVE = False
 
 
 PackageEcosystems: TypeAlias = Literal["pypi", "npm"]

src/twyn/cli.py

@@ -103,6 +103,13 @@ def entry_point() -> None:
     default=False,
     help="Display the results in json format. It implies --no-track.",
 )
+@click.option(
+    "-r",
+    "--recursive",
+    is_flag=True,
+    default=False,
+    help="Display the results in json format. It implies --no-track.",
+)
 def run(  # noqa: C901
     config: str,
     dependency_file: Optional[str],
@@ -114,6 +121,7 @@ def run(  # noqa: C901
     no_track: bool,
     json: bool,
     package_ecosystem: Optional[str],
+    recursive: bool,
 ) -> int:
     if vv:
         logger.setLevel(logging.DEBUG)
@@ -138,6 +146,7 @@ def run(  # noqa: C901
             show_progress_bar=False if json else not no_track,
             load_config_from_file=True,
             package_ecosystem=package_ecosystem,
+            recursive=recursive,
         )
     except TwynError as e:
         raise CliError(str(e)) from e

src/twyn/config/config_handler.py

@@ -8,6 +8,7 @@
 
 from twyn.base.constants import (
     DEFAULT_PROJECT_TOML_FILE,
+    DEFAULT_RECURSIVE,
     DEFAULT_SELECTOR_METHOD,
     DEFAULT_TWYN_TOML_FILE,
     DEFAULT_USE_CACHE,
@@ -37,6 +38,7 @@ class TwynConfiguration:
     source: Optional[str]
     use_cache: bool
     package_ecosystem: Optional[PackageEcosystems]
+    recursive: Optional[bool]
 
 
 @dataclass
@@ -49,6 +51,7 @@ class ReadTwynConfiguration:
     source: Optional[str] = None
     use_cache: Optional[bool] = None
     package_ecosystem: Optional[PackageEcosystems] = None
+    recursive: Optional[bool] = None
 
 
 class ConfigHandler:
@@ -63,6 +66,7 @@ def resolve_config(
         dependency_file: Optional[str] = None,
         use_cache: Optional[bool] = None,
         package_ecosystem: Optional[PackageEcosystems] = None,
+        recursive: Optional[bool] = None,
     ) -> TwynConfiguration:
         """Resolve the configuration for Twyn.
 
@@ -95,13 +99,21 @@ def resolve_config(
         else:
             final_use_cache = DEFAULT_USE_CACHE
 
+        if recursive is not None:
+            final_recursive = use_cache
+        elif read_config.recursive is not None:
+            final_recursive = read_config.recursive
+        else:
+            final_recursive = DEFAULT_RECURSIVE
+
         return TwynConfiguration(
             dependency_file=dependency_file or read_config.dependency_file,
             selector_method=final_selector_method,
             allowlist=read_config.allowlist,
             source=read_config.source,
             use_cache=final_use_cache,
             package_ecosystem=package_ecosystem or read_config.package_ecosystem,
+            recursive=final_recursive,
         )
 
     def add_package_to_allowlist(self, package_name: str) -> None:

src/twyn/dependency_parser/dependency_selector.py

@@ -1,4 +1,5 @@
 import logging
+from pathlib import Path
 from typing import Optional
 
 from twyn.base.constants import DEPENDENCY_FILE_MAPPING
@@ -11,21 +12,29 @@
 
 
 class DependencySelector:
-    def __init__(self, dependency_file: Optional[str] = None) -> None:
+    def __init__(self, dependency_file: Optional[str] = None, root_path: str = ".") -> None:
         self.dependency_file = dependency_file or ""
+        self.root_path = root_path
 
     def auto_detect_dependency_file_parser(self) -> list[AbstractParser]:
         parsers: list[AbstractParser] = []
-        for dependency_parser in DEPENDENCY_FILE_MAPPING.values():
-            file_parser = dependency_parser()
-            if file_parser.file_exists():
-                parsers.append(file_parser)
-                logger.debug("Assigned %s parser for local dependencies file.", file_parser)
+        root = Path(self.root_path)
+        for path in root.rglob("*"):
+            if path.is_dir() and path.name == ".git":
+                # Skip .git directory and its contents
+                continue
+            if path.is_file():
+                for known_file, dependency_parser in DEPENDENCY_FILE_MAPPING.items():
+                    if path.name == known_file:
+                        file_parser = dependency_parser(str(path))
+                        if file_parser.file_exists():
+                            parsers.append(file_parser)
+                            logger.debug("Assigned %s parser for local dependencies file at %s.", file_parser, path)
 
         if not parsers:
             raise NoMatchingParserError
 
-        logger.debug("Dependencies file found")
+        logger.debug("Dependencies file(s) found: %s", [str(p.file_path) for p in parsers])
         return parsers
 
     def get_dependency_file_parsers_from_file_name(self) -> list[AbstractParser]:

src/twyn/main.py

@@ -43,6 +43,7 @@ def check_dependencies(
     show_progress_bar: bool = False,
     load_config_from_file: bool = False,
     package_ecosystem: Optional[PackageEcosystems] = None,
+    recursive: Optional[bool] = None,
 ) -> TyposquatCheckResults:
     """
     Check if the provided dependencies are potential typosquats of trusted packages.
@@ -70,6 +71,7 @@ def check_dependencies(
         dependency_file=dependency_file,
         use_cache=use_cache,
         package_ecosystem=package_ecosystem,
+        recursive=recursive,
     )
     maybe_cache_handler = CacheHandler() if config.use_cache else None
     selector_method_obj = _get_selector_method(config.selector_method)
@@ -259,6 +261,7 @@ def _get_config(
     dependency_file: Optional[str],
     use_cache: Optional[bool],
     package_ecosystem: Optional[PackageEcosystems],
+    recursive: Optional[bool],
 ) -> TwynConfiguration:
     """Given the arguments passed to the main function and the configuration loaded from the config file (if any), return a config object."""
     if load_config_from_file:
@@ -270,4 +273,5 @@ def _get_config(
         dependency_file=dependency_file,
         use_cache=use_cache,
         package_ecosystem=package_ecosystem,
+        recursive=recursive,
     )

tests/config/test_config_handler.py

@@ -42,6 +42,7 @@ def test_no_enforce_file_on_non_existent_file(self, mock_is_file: Mock) -> None:
             source=None,
             use_cache=True,
             package_ecosystem=None,
+            recursive=False,
         )
 
     def test_config_raises_for_unknown_file(self) -> None:
@@ -103,6 +104,7 @@ def test_write_toml(self, pyproject_toml_file: Path) -> None:
                     "selector_method": "all",
                     "allowlist": {},
                     "use_cache": False,
+                    "recursive": False,
                 },
             }
         }

tests/dependency_parser/test_dependency_selector.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from unittest.mock import Mock, patch
 
 import pytest
@@ -7,6 +8,8 @@
     NoMatchingParserError,
 )
 from twyn.dependency_parser.parsers.abstract_parser import AbstractParser
+from twyn.dependency_parser.parsers.package_lock_json import PackageLockJsonParser
+from twyn.dependency_parser.parsers.yarn_lock_parser import YarnLockParser
 
 
 class TestDependencySelector:
@@ -22,6 +25,8 @@ class TestDependencySelector:
             ("/some/path/poetry.lock", PoetryLockParser),
             ("/some/path/uv.lock", UvLockParser),
             ("/some/path/requirements.txt", RequirementsTxtParser),
+            ("/some/path/yarn.lock", YarnLockParser),
+            ("/some/path/package-lock.json", PackageLockJsonParser),
         ],
     )
     def test_get_dependency_parser(self, file_name: str, parser_class: type[AbstractParser]):
@@ -31,44 +36,19 @@ def test_get_dependency_parser(self, file_name: str, parser_class: type[Abstract
         assert isinstance(parser[0], parser_class)
         assert str(parser[0].file_handler.file_path).endswith(file_name)
 
-    @patch("twyn.dependency_parser.parsers.lock_parser.PoetryLockParser.file_exists")
-    @patch("twyn.dependency_parser.parsers.lock_parser.UvLockParser.file_exists")
-    @patch("twyn.dependency_parser.parsers.requirements_txt_parser.RequirementsTxtParser.file_exists")
-    def test_get_dependency_parser_auto_detect_requirements_file(
-        self, req_file_exists: Mock, uv_file_exists: Mock, poetry_file_exists: Mock
-    ):
-        poetry_file_exists.return_value = False
-        req_file_exists.return_value = True
-        uv_file_exists.return_value = False
-
-        parser = DependencySelector("").get_dependency_parsers()
+    def test_get_dependency_parser_auto_detect_requirements_file(self, requirements_txt_file: Path, tmp_path: Path):
+        parser = DependencySelector("", root_path=str(tmp_path)).get_dependency_parsers()
         assert isinstance(parser[0], RequirementsTxtParser)
 
-    @patch("twyn.dependency_parser.parsers.lock_parser.PoetryLockParser.file_exists")
-    @patch("twyn.dependency_parser.parsers.lock_parser.UvLockParser.file_exists")
-    @patch("twyn.dependency_parser.parsers.requirements_txt_parser.RequirementsTxtParser.file_exists")
     def test_get_dependency_parser_auto_detect_poetry_lock_file(
-        self, req_file_exists: Mock, uv_file_exists: Mock, poetry_file_exists: Mock
+        self, poetry_lock_file_ge_1_5: Path, tmp_path: Path
     ) -> None:
-        poetry_file_exists.return_value = True
-        req_file_exists.return_value = False
-        uv_file_exists.return_value = False
-
-        selector = DependencySelector("")
+        selector = DependencySelector("", root_path=str(tmp_path))
         parser = selector.get_dependency_parsers()
         assert isinstance(parser[0], PoetryLockParser)
 
-    @patch("twyn.dependency_parser.parsers.lock_parser.PoetryLockParser.file_exists")
-    @patch("twyn.dependency_parser.parsers.lock_parser.UvLockParser.file_exists")
-    @patch("twyn.dependency_parser.parsers.requirements_txt_parser.RequirementsTxtParser.file_exists")
-    def test_get_dependency_parser_auto_detect_uv_lock_file(
-        self, req_file_exists: Mock, uv_file_exists: Mock, poetry_file_exists: Mock
-    ) -> None:
-        poetry_file_exists.return_value = False
-        req_file_exists.return_value = False
-        uv_file_exists.return_value = True
-
-        parser = DependencySelector("").get_dependency_parsers()
+    def test_get_dependency_parser_auto_detect_uv_lock_file(self, uv_lock_file: Path, tmp_path: Path) -> None:
+        parser = DependencySelector("", root_path=str(tmp_path)).get_dependency_parsers()
         assert isinstance(parser[0], UvLockParser)
 
     @patch("twyn.dependency_parser.parsers.abstract_parser.AbstractParser.file_exists")
@@ -81,3 +61,29 @@ def test_auto_detect_dependency_file_parser_exceptions(self, file_exists: Mock)
     def test_get_dependency_file_parser_unknown_file_type(self, file_name: str) -> None:
         with pytest.raises(NoMatchingParserError):
             DependencySelector(file_name).get_dependency_file_parsers_from_file_name()
+
+    def test_auto_detect_dependency_file_parser_scans_subdirectories(self, tmp_path):
+        # Create nested directories and dependency files
+        subdir = tmp_path / "subdir"
+        subdir.mkdir()
+
+        req_file = subdir / "requirements.txt"
+        req_file.write_text("flask\n")
+        poetry_file = tmp_path / "poetry.lock"
+        poetry_file.write_text("[package]\nname = 'pytest'\n")
+
+        # Should not scan .git
+        git_dir = tmp_path / ".git"
+        git_dir.mkdir()
+        git_file = git_dir / "requirements.txt"
+        git_file.write_text("should not be found\n")
+
+        # Should find both files
+        selector = DependencySelector(root_path=str(tmp_path))
+        with patch(
+            "twyn.base.constants.DEPENDENCY_FILE_MAPPING",
+            {"requirements.txt": RequirementsTxtParser, "poetry.lock": PoetryLockParser},
+        ):
+            parsers = selector.auto_detect_dependency_file_parser()
+        found_files = {str(p.file_path.name) for p in parsers}
+        assert found_files == {"requirements.txt", "poetry.lock"}