feat: support parsing multiple files (#326)

Author: sdn4z

README.md

@@ -34,7 +34,7 @@ In short, `Twyn` protects you against [typosquatting attacks](https://en.wikiped
 
 It works as follows:
 
-1. Either choose to scan the dependencies in a dependencies file you specify (`--dependency-file`) or some dependencies introduced through the CLI (`--dependency`). If no option was provided, it will try to find a dependencies file in your working path.
+1. Either choose to scan the dependencies in a dependencies file you specify (`--dependency-file`) or some dependencies introduced through the CLI (`--dependency`). If no option was provided, it will try to find a dependencies file in your working path. It will try to parse all the supported dependency files that it finds. To know which files are supported head to the [Dependency files](#dependency-files) section.
 2. If the name of your package name matches with the name of one of the most well known packages, the package is accepted.
 3. If the name of your package is similar to the name of one of the most used packages, `Twyn` will prompt an error.
 4. If your package name is not in the list of the most known ones and is not similar enough to any of those to be considered misspelled, the package is accepted. `Twyn` assumes that you're using either a not so popular package (therefore it can't verify its legitimacy) or a package created by yourself, therefore unknown for the rest.
@@ -84,8 +84,43 @@ If you want your output in JSON format, you can run `Twyn` with the following fl
 This will output:
 
  ```json
-  {"errors":[{"dependency":"reqests","similars":["requests","grequests"]}]}
+  {"results":[{"errors":[{"dependency":"my-package","similars":["mypackage"]}],"source":"manual_input"}]}
  ```
+If `Twyn` was run by manually giving it dependencies (with `--dependency`), the source will be `manual_input`. 
+
+In any other case (when dependencies are parsed from a file), the source will be the path to the dependencies file. One entry will be created for every source.
+
+### Using Twyn as a library
+
+
+#### Installation
+`Twyn` also supports being used as 3rd party library for you project. To install it, run:
+
+
+```sh
+pip install twyn
+```
+
+Example usage in your code:
+
+```python
+from twyn import check_dependencies
+
+typos = check_dependencies()
+
+for typo in typos.errors:
+  print(f"Dependency:{typo.dependency}")
+  print(f"Did you mean any of [{','.join(typo.similars)}]")
+  
+```
+
+#### Logging level
+By default, logging is disabled when running as a 3rd party library. To override this behaviour, you can:
+
+```python
+logging.basicConfig(level=logging.INFO)
+logging.getLogger("twyn").setLevel(logging.INFO)
+```
 
 ### Using Twyn as a library
 

dependencies/scripts/download_packages.py

@@ -61,7 +61,7 @@ def download(ecosystem: str) -> None:
     ):
         with attempt, httpx.Client(timeout=30) as client:
             logger.info("Attempting to download %s packages. Attempt #%d.", ecosystem, attempt.num)
-            response = client.get(ECOSYSTEMS[ecosystem]["url"])
+            response = client.get(str(ECOSYSTEMS[ecosystem]["url"]))
             response.raise_for_status()
 
     fpath = Path("dependencies") / f"{ecosystem}.json"

src/twyn/base/constants.py

@@ -11,6 +11,8 @@
     from twyn.dependency_parser.parsers.abstract_parser import AbstractParser
 
 
+MANUAL_INPUT_SOURCE = "manual_input"
+
 SELECTOR_METHOD_MAPPING: dict[str, type[selectors.AbstractSelector]] = {
     "first-letter": selectors.FirstLetterExact,
     "nearby-letter": selectors.FirstLetterNearbyInKeyboard,

src/twyn/cli.py

@@ -103,7 +103,7 @@ def entry_point() -> None:
     default=False,
     help="Display the results in json format. It implies --no-track.",
 )
-def run(
+def run(  # noqa: C901
     config: str,
     dependency_file: Optional[str],
     dependency: tuple[str],
@@ -146,14 +146,15 @@ def run(
 
     if json:
         click.echo(possible_typos.model_dump_json())
-        sys.exit(int(bool(possible_typos.errors)))
-    elif possible_typos.errors:
-        for possible_typosquats in possible_typos.errors:
-            click.echo(
-                click.style("Possible typosquat detected: ", fg="red") + f"`{possible_typosquats.dependency}`, "
-                f"did you mean any of [{', '.join(possible_typosquats.similars)}]?",
-                color=True,
-            )
+        sys.exit(int(bool(possible_typos)))
+    elif possible_typos:
+        for possible_typosquats in possible_typos.results:
+            for error in possible_typosquats.errors:
+                click.echo(
+                    click.style("Possible typosquat detected: ", fg="red") + f"`{error.dependency}`, "
+                    f"did you mean any of [{', '.join(error.similars)}]?",
+                    color=True,
+                )
         sys.exit(1)
     else:
         click.echo(click.style("No typosquats detected", fg="green"), color=True)

src/twyn/dependency_parser/dependency_selector.py

@@ -3,7 +3,6 @@
 
 from twyn.base.constants import DEPENDENCY_FILE_MAPPING
 from twyn.dependency_parser.exceptions import (
-    MultipleParsersError,
     NoMatchingParserError,
 )
 from twyn.dependency_parser.parsers.abstract_parser import AbstractParser
@@ -15,46 +14,36 @@ class DependencySelector:
     def __init__(self, dependency_file: Optional[str] = None) -> None:
         self.dependency_file = dependency_file or ""
 
-    @staticmethod
-    def _raise_for_selected_parsers(parsers: list[type[AbstractParser]]) -> None:
-        if len(parsers) > 1:
-            raise MultipleParsersError
+    def auto_detect_dependency_file_parser(self) -> list[AbstractParser]:
+        parsers: list[AbstractParser] = []
+        for dependency_parser in DEPENDENCY_FILE_MAPPING.values():
+            file_parser = dependency_parser()
+            if file_parser.file_exists():
+                parsers.append(file_parser)
+                logger.debug("Assigned %s parser for local dependencies file.", file_parser)
 
         if not parsers:
             raise NoMatchingParserError
 
-    def auto_detect_dependency_file_parser(self) -> type[AbstractParser]:
-        parsers: list[AbstractParser] = [
-            dependency_parser
-            for dependency_parser in DEPENDENCY_FILE_MAPPING.values()
-            if dependency_parser().file_exists()
-        ]
-        self._raise_for_selected_parsers(parsers)
-        self.dependency_file = parsers[0]().file_path
         logger.debug("Dependencies file found")
-        return parsers[0]
+        return parsers
 
-    def get_dependency_file_parser_from_file_name(
-        self,
-    ) -> type[AbstractParser]:
+    def get_dependency_file_parsers_from_file_name(self) -> list[AbstractParser]:
         parsers = []
         for known_dependency_file_name in DEPENDENCY_FILE_MAPPING:
             if self.dependency_file.endswith(known_dependency_file_name):
-                parsers.append(DEPENDENCY_FILE_MAPPING[known_dependency_file_name])
+                file_parser = DEPENDENCY_FILE_MAPPING[known_dependency_file_name](self.dependency_file)
+                parsers.append(file_parser)
 
-        self._raise_for_selected_parsers(parsers)
-        return parsers[0]
+        if not parsers:
+            raise NoMatchingParserError
 
-    def get_dependency_parser(self) -> AbstractParser:
+        return parsers
+
+    def get_dependency_parsers(self) -> list[AbstractParser]:
         if self.dependency_file:
             logger.debug("Dependency file provided. Assigning a parser.")
-            dependency_file_parser = self.get_dependency_file_parser_from_file_name()
-            file_parser = dependency_file_parser(self.dependency_file)
-        else:
-            logger.debug("No dependency file provided. Attempting to locate one.")
-            dependency_file_parser = self.auto_detect_dependency_file_parser()
-            file_parser = dependency_file_parser()
-
-        logger.debug("Assigned %s parser for local dependencies file.", file_parser)
+            return self.get_dependency_file_parsers_from_file_name()
 
-        return file_parser
+        logger.debug("No dependency file provided. Attempting to locate one.")
+        return self.auto_detect_dependency_file_parser()