feat: recursively look for lock files (#332)

Author: sdn4z

.gitignore

@@ -111,3 +111,6 @@ GitHub.sublime-settings
 .history
 /.ruff_cache/
 /.env.local
+
+# Test lock files
+yarn.lock

README.md

@@ -14,6 +14,7 @@
   - [Using `Twyn` as a cli tool](#using-twyn-as-a-cli-tool)
     - [Installation](#installation)
     - [Docker](#docker)
+    - [CLI Options Reference](#cli-options-reference)
     - [Run](#run)
     - [JSON Format](#json-format)
   - [Using `Twyn` as a library](#using-twyn-as-a-library)
@@ -61,15 +62,29 @@ docker pull elementsinteractive/twyn:latest
 docker run elementsinteractive/twyn --help
 ```
 
+##### CLI Options Reference
+
+| Option / Argument         | Type / Values                                      | Description                                                                                   |
+|--------------------------|----------------------------------------------------|-----------------------------------------------------------------------------------------------|
+| `--config`               | `str` (path)                                       | Path to configuration file (`twyn.toml` or `pyproject.toml` by default).                      |
+| `--dependency-file`      | `str` (path)                                       | Dependency file to analyze. Supported: `requirements.txt`, `poetry.lock`, `uv.lock`, etc.     |
+| `--dependency`           | `str` (multiple allowed)                           | Dependency to analyze directly. Can be specified multiple times.                              |
+| `--selector-method`      | `all`, `first-letter`, `nearby-letter`             | Method for selecting possible typosquats.                                                     |
+| `--package-ecosystem`    | `pypi`, `npm`                                      | Package ecosystem for analysis.                                                               |
+| `-v`                     | flag                                               | Enable info-level logging.                                                                    |
+| `-vv`                    | flag                                               | Enable debug-level logging.                                                                   |
+| `--no-cache`             | flag                                               | Disable use of trusted packages cache. Always fetch from the source.                          |
+| `--no-track`             | flag                                               | Do not show the progress bar while processing packages.                                       |
+| `--json`                 | flag                                               | Display results in JSON format. Implies `--no-track`.                                         |
+| `-r`, `--recursive`      | flag                                               | Scan directories recursively for dependency files.                                            |
 #### Run
 
-To run twyn simply type:
+**Usage Example:**
 
 ```sh
 twyn run <OPTIONS>
 ```
-
-For a list of all the available options as well as their expected arguments run:
+or get help with
 
 ```sh
 twyn run --help
@@ -90,41 +105,9 @@ If `Twyn` was run by manually giving it dependencies (with `--dependency`), the
 
 In any other case (when dependencies are parsed from a file), the source will be the path to the dependencies file. One entry will be created for every source.
 
-### Using Twyn as a library
-
-
-#### Installation
-`Twyn` also supports being used as 3rd party library for you project. To install it, run:
-
-
-```sh
-pip install twyn
-```
-
-Example usage in your code:
-
-```python
-from twyn import check_dependencies
-
-typos = check_dependencies()
-
-for typo in typos.errors:
-  print(f"Dependency:{typo.dependency}")
-  print(f"Did you mean any of [{','.join(typo.similars)}]")
-  
-```
-
-#### Logging level
-By default, logging is disabled when running as a 3rd party library. To override this behaviour, you can:
-
-```python
-logging.basicConfig(level=logging.INFO)
-logging.getLogger("twyn").setLevel(logging.INFO)
-```
 
 ### Using Twyn as a library
 
-
 #### Installation
 `Twyn` also supports being used as 3rd party library for you project. To install it, run:
 
@@ -154,6 +137,7 @@ logging.basicConfig(level=logging.INFO)
 logging.getLogger("twyn").setLevel(logging.INFO)
 ```
 
+
 ## Configuration
 
 ### Allowlist
@@ -268,5 +252,3 @@ To clear the cache, run:
 ```python
   twyn run cache clear
 ```
-
-

src/twyn/base/constants.py

@@ -35,6 +35,7 @@
 DEFAULT_PROJECT_TOML_FILE = "pyproject.toml"
 DEFAULT_TWYN_TOML_FILE = "twyn.toml"
 DEFAULT_USE_CACHE = True
+DEFAULT_RECURSIVE = False
 
 
 PackageEcosystems: TypeAlias = Literal["pypi", "npm"]

src/twyn/cli.py

@@ -103,6 +103,13 @@ def entry_point() -> None:
     default=False,
     help="Display the results in json format. It implies --no-track.",
 )
+@click.option(
+    "-r",
+    "--recursive",
+    is_flag=True,
+    default=False,
+    help="Recursively look for files when trying to locate them automatically. Ignored if --dependency-file is given.",
+)
 def run(  # noqa: C901
     config: str,
     dependency_file: Optional[str],
@@ -114,6 +121,7 @@ def run(  # noqa: C901
     no_track: bool,
     json: bool,
     package_ecosystem: Optional[str],
+    recursive: bool,
 ) -> int:
     if vv:
         logger.setLevel(logging.DEBUG)
@@ -138,6 +146,7 @@ def run(  # noqa: C901
             show_progress_bar=False if json else not no_track,
             load_config_from_file=True,
             package_ecosystem=package_ecosystem,
+            recursive=recursive,
         )
     except TwynError as e:
         raise CliError(str(e)) from e

src/twyn/config/config_handler.py

@@ -8,6 +8,7 @@
 
 from twyn.base.constants import (
     DEFAULT_PROJECT_TOML_FILE,
+    DEFAULT_RECURSIVE,
     DEFAULT_SELECTOR_METHOD,
     DEFAULT_TWYN_TOML_FILE,
     DEFAULT_USE_CACHE,
@@ -37,6 +38,7 @@ class TwynConfiguration:
     source: Optional[str]
     use_cache: bool
     package_ecosystem: Optional[PackageEcosystems]
+    recursive: Optional[bool]
 
 
 @dataclass
@@ -49,6 +51,7 @@ class ReadTwynConfiguration:
     source: Optional[str] = None
     use_cache: Optional[bool] = None
     package_ecosystem: Optional[PackageEcosystems] = None
+    recursive: Optional[bool] = None
 
 
 class ConfigHandler:
@@ -63,6 +66,7 @@ def resolve_config(
         dependency_file: Optional[str] = None,
         use_cache: Optional[bool] = None,
         package_ecosystem: Optional[PackageEcosystems] = None,
+        recursive: Optional[bool] = None,
     ) -> TwynConfiguration:
         """Resolve the configuration for Twyn.
 
@@ -95,13 +99,21 @@ def resolve_config(
         else:
             final_use_cache = DEFAULT_USE_CACHE
 
+        if recursive is not None:
+            final_recursive = use_cache
+        elif read_config.recursive is not None:
+            final_recursive = read_config.recursive
+        else:
+            final_recursive = DEFAULT_RECURSIVE
+
         return TwynConfiguration(
             dependency_file=dependency_file or read_config.dependency_file,
             selector_method=final_selector_method,
             allowlist=read_config.allowlist,
             source=read_config.source,
             use_cache=final_use_cache,
             package_ecosystem=package_ecosystem or read_config.package_ecosystem,
+            recursive=final_recursive,
         )
 
     def add_package_to_allowlist(self, package_name: str) -> None:

src/twyn/dependency_parser/dependency_selector.py

@@ -1,4 +1,5 @@
 import logging
+from pathlib import Path
 from typing import Optional
 
 from twyn.base.constants import DEPENDENCY_FILE_MAPPING
@@ -11,21 +12,28 @@
 
 
 class DependencySelector:
-    def __init__(self, dependency_file: Optional[str] = None) -> None:
+    def __init__(self, dependency_file: Optional[str] = None, root_path: str = ".") -> None:
         self.dependency_file = dependency_file or ""
+        self.root_path = root_path
 
     def auto_detect_dependency_file_parser(self) -> list[AbstractParser]:
         parsers: list[AbstractParser] = []
-        for dependency_parser in DEPENDENCY_FILE_MAPPING.values():
-            file_parser = dependency_parser()
-            if file_parser.file_exists():
-                parsers.append(file_parser)
-                logger.debug("Assigned %s parser for local dependencies file.", file_parser)
+        root = Path(self.root_path)
+        for path in root.rglob("*"):
+            if ".git" in path.parts:
+                continue
+            if path.is_file():
+                for known_file, dependency_parser in DEPENDENCY_FILE_MAPPING.items():
+                    if path.name == known_file:
+                        file_parser = dependency_parser(str(path))
+                        if file_parser.file_exists():
+                            parsers.append(file_parser)
+                            logger.debug("Assigned %s parser for local dependencies file at %s.", file_parser, path)
 
         if not parsers:
             raise NoMatchingParserError
 
-        logger.debug("Dependencies file found")
+        logger.debug("Dependencies file(s) found: %s", [str(p.file_path) for p in parsers])
         return parsers
 
     def get_dependency_file_parsers_from_file_name(self) -> list[AbstractParser]:

src/twyn/main.py

@@ -43,6 +43,7 @@ def check_dependencies(
     show_progress_bar: bool = False,
     load_config_from_file: bool = False,
     package_ecosystem: Optional[PackageEcosystems] = None,
+    recursive: Optional[bool] = None,
 ) -> TyposquatCheckResults:
     """
     Check if the provided dependencies are potential typosquats of trusted packages.
@@ -70,6 +71,7 @@ def check_dependencies(
         dependency_file=dependency_file,
         use_cache=use_cache,
         package_ecosystem=package_ecosystem,
+        recursive=recursive,
     )
     maybe_cache_handler = CacheHandler() if config.use_cache else None
     selector_method_obj = _get_selector_method(config.selector_method)
@@ -85,9 +87,17 @@ def check_dependencies(
             dependencies=dependencies,
         )
 
+    # The following checks do not result in an error to avoid inconsistencies.
+    # If the user has set in the config file a setting that would conflict with a cli provided option
+    # it would always result in an execution error rather than in overriding the behaviour.
     if config.package_ecosystem:
         logger.warning("`package_ecosystem` is not supported when reading dependencies from files. It will be ignored.")
 
+    if config.dependency_file and config.recursive:
+        logger.warning(
+            "`--recursive` has been set together with `--dependency-file`. `--dependency-file` will take precedence."
+        )
+
     return _analyze_packages_from_source(
         selector_method=selector_method_obj,
         source=config.source,
@@ -155,6 +165,7 @@ def _analyze_packages_from_source(
     typos_by_file = TyposquatCheckResults()
     for dependency_manager, parsers in dependency_managers.items():
         top_package_reference = dependency_manager.trusted_packages_source(source, maybe_cache_handler)
+
         packages_from_source = top_package_reference.get_packages()
         trusted_packages = TrustedPackages(
             names=packages_from_source,
@@ -259,6 +270,7 @@ def _get_config(
     dependency_file: Optional[str],
     use_cache: Optional[bool],
     package_ecosystem: Optional[PackageEcosystems],
+    recursive: Optional[bool],
 ) -> TwynConfiguration:
     """Given the arguments passed to the main function and the configuration loaded from the config file (if any), return a config object."""
     if load_config_from_file:
@@ -270,4 +282,5 @@ def _get_config(
         dependency_file=dependency_file,
         use_cache=use_cache,
         package_ecosystem=package_ecosystem,
+        recursive=recursive,
     )

tests/config/test_config_handler.py

@@ -42,6 +42,7 @@ def test_no_enforce_file_on_non_existent_file(self, mock_is_file: Mock) -> None:
             source=None,
             use_cache=True,
             package_ecosystem=None,
+            recursive=False,
         )
 
     def test_config_raises_for_unknown_file(self) -> None:
@@ -103,6 +104,7 @@ def test_write_toml(self, pyproject_toml_file: Path) -> None:
                     "selector_method": "all",
                     "allowlist": {},
                     "use_cache": False,
+                    "recursive": False,
                 },
             }
         }

tests/conftest.py

@@ -1,7 +1,8 @@
 import datetime
-from collections.abc import Iterator
+from collections.abc import Generator, Iterator
 from contextlib import contextmanager
 from pathlib import Path
+from typing import Any
 from unittest import mock
 
 import pytest
@@ -21,10 +22,8 @@ def patch_pypi_packages_download(packages: list[str]) -> Iterator[mock.Mock]:
     Replaces call with the output you would get from downloading the top PyPi packages list.
     """
     json_response = {"packages": packages, "date": datetime.datetime.now().isoformat()}
-
     with mock.patch("twyn.trusted_packages.TopPyPiReference._download") as mock_download:
         mock_download.return_value = json_response
-
         yield mock_download
 
 
@@ -447,3 +446,10 @@ def yarn_lock_file_v2(tmp_path: Path) -> Iterator[Path]:
         """
     with create_tmp_file(yarn_file, data) as tmp_file:
         yield tmp_file
+
+
+@pytest.fixture(autouse=True)
+def fail_on_requests_get(request) -> Generator[None, Any, None]:
+    with mock.patch("requests.get") as m_get:
+        m_get.side_effect = RuntimeError("`requests.get()` was called!")
+        yield