Skip to content

Commit 27ff588

Browse files
committed
wip
1 parent 8139ee0 commit 27ff588

6 files changed

Lines changed: 250 additions & 34 deletions

File tree

src/twyn/config/config_handler.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ class TwynConfiguration:
3636
selector_method: str
3737
logging_level: AvailableLoggingLevels
3838
allowlist: set[str]
39-
pypi_reference: str
39+
package_reference: str
4040
use_cache: bool
4141

4242

@@ -48,7 +48,7 @@ class ReadTwynConfiguration:
4848
selector_method: Optional[str] = None
4949
logging_level: Optional[AvailableLoggingLevels] = None
5050
allowlist: set[str] = field(default_factory=set)
51-
pypi_reference: Optional[str] = None
51+
package_reference: Optional[str] = None
5252
use_cache: Optional[bool] = None
5353

5454

@@ -101,7 +101,7 @@ def resolve_config(
101101
selector_method=final_selector_method,
102102
logging_level=_get_logging_level(verbosity, read_config.logging_level),
103103
allowlist=read_config.allowlist,
104-
pypi_reference=read_config.pypi_reference or DEFAULT_TOP_PYPI_PACKAGES,
104+
package_reference=read_config.package_reference or DEFAULT_TOP_PYPI_PACKAGES,
105105
use_cache=final_use_cache,
106106
)
107107

@@ -135,7 +135,7 @@ def _get_read_config(self, toml: TOMLDocument) -> ReadTwynConfiguration:
135135
selector_method=twyn_config_data.get("selector_method"),
136136
logging_level=twyn_config_data.get("logging_level"),
137137
allowlist=set(twyn_config_data.get("allowlist", set())),
138-
pypi_reference=twyn_config_data.get("pypi_reference"),
138+
package_reference=twyn_config_data.get("package_reference"),
139139
use_cache=twyn_config_data.get("use_cache"),
140140
)
141141

src/twyn/main.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from twyn.similarity.algorithm import EditDistance, SimilarityThreshold
1515
from twyn.trusted_packages import TopPyPiReference
1616
from twyn.trusted_packages.cache_handler import CacheHandler
17+
from twyn.trusted_packages.references import AbstractPackageReference
1718
from twyn.trusted_packages.selectors import AbstractSelector
1819
from twyn.trusted_packages.trusted_packages import (
1920
TrustedPackages,
@@ -47,15 +48,17 @@ def check_dependencies(
4748

4849
cache_handler = CacheHandler() if config.use_cache else None
4950

51+
top_packages_reference = get_top_packages_reference(source=config.package_reference, cache_handler=cache_handler)
52+
5053
trusted_packages = TrustedPackages(
51-
names=TopPyPiReference(source=config.pypi_reference, cache_handler=cache_handler).get_packages(),
54+
names=top_packages_reference.get_packages(),
5255
algorithm=EditDistance(),
5356
selector=get_candidate_selector(config.selector_method),
5457
threshold_class=SimilarityThreshold,
5558
)
56-
normalized_allowlist_packages = TopPyPiReference.normalize_packages(config.allowlist)
59+
normalized_allowlist_packages = top_packages_reference.normalize_packages(config.allowlist)
5760
dependencies = dependencies if dependencies else get_parsed_dependencies_from_file(config.dependency_file)
58-
normalized_dependencies = TopPyPiReference.normalize_packages(dependencies)
61+
normalized_dependencies = top_packages_reference.normalize_packages(dependencies)
5962

6063
typos_list = TyposquatCheckResultList()
6164
dependencies_list = (
@@ -73,6 +76,10 @@ def check_dependencies(
7376
return typos_list
7477

7578

79+
def get_top_packages_reference(source: str, cache_handler: Optional[CacheHandler]) -> AbstractPackageReference:
80+
return TopPyPiReference(source=source, cache_handler=cache_handler)
81+
82+
7683
def get_config(
7784
load_config_from_file: bool,
7885
config_file: Optional[str],

src/twyn/trusted_packages/references.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,3 +118,34 @@ def normalize_packages(packages: set[str]) -> set[str]:
118118
raise PackageNormalizingError(f"Package name '{package}' does not match required pattern")
119119

120120
return renamed_packages
121+
122+
123+
class TopNpmReference(AbstractPackageReference):
124+
"""Top npm packages retrieved from an online source."""
125+
126+
@override
127+
@staticmethod
128+
def _parse(packages_info: dict[str, Any]) -> set[str]:
129+
try:
130+
names = {row["project"] for row in packages_info["rows"]}
131+
except KeyError as err:
132+
raise InvalidPyPiFormatError from err
133+
134+
if not names:
135+
raise EmptyPackagesListError
136+
137+
logger.debug("Successfully parsed trusted packages list")
138+
return names
139+
140+
@override
141+
@staticmethod
142+
def normalize_packages(packages: set[str]) -> set[str]:
143+
"""Normalize dependency names according to npm https://packaging.python.org/en/latest/specifications/name-normalization/."""
144+
renamed_packages = {re.sub(r"[-_.]+", "-", name).lower() for name in packages}
145+
146+
pattern = re.compile(r"^([a-z0-9]|[a-z0-9][a-z0-9._-]*[a-z0-9])\Z") # noqa: F821
147+
for package in renamed_packages:
148+
if not pattern.match(package):
149+
raise PackageNormalizingError(f"Package name '{package}' does not match required pattern")
150+
151+
return renamed_packages

tests/config/test_config_handler.py

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -31,21 +31,6 @@ class TestConfigHandler:
3131
def throw_exception(self) -> NoReturn:
3232
raise PathNotFoundError
3333

34-
@patch("twyn.file_handler.file_handler.FileHandler.read")
35-
def test_no_enforce_file_on_non_existent_file(self, mock_is_file: Mock) -> None:
36-
"""Resolving the config without enforcing the file to be present gives you defaults."""
37-
mock_is_file.side_effect = self.throw_exception
38-
config = ConfigHandler(FileHandler(DEFAULT_PROJECT_TOML_FILE)).resolve_config()
39-
40-
assert config == TwynConfiguration(
41-
dependency_file=None,
42-
selector_method="all",
43-
logging_level=AvailableLoggingLevels.warning,
44-
allowlist=set(),
45-
pypi_reference=DEFAULT_TOP_PYPI_PACKAGES,
46-
use_cache=True,
47-
)
48-
4934
def test_config_raises_for_unknown_file(self) -> None:
5035
with pytest.raises(TOMLError):
5136
ConfigHandler(FileHandler("non-existent-file.toml")).resolve_config()
@@ -68,7 +53,7 @@ def test_get_twyn_data_from_file(self, pyproject_toml_file: Path) -> None:
6853
selector_method="all",
6954
logging_level="debug",
7055
allowlist={"boto4", "boto2"},
71-
pypi_reference=None,
56+
package_reference=None,
7257
use_cache=False,
7358
)
7459

@@ -107,7 +92,7 @@ def test_write_toml(self, pyproject_toml_file: Path) -> None:
10792
"selector_method": "all",
10893
"logging_level": "debug",
10994
"allowlist": {},
110-
"pypi_reference": DEFAULT_TOP_PYPI_PACKAGES,
95+
"package_reference": DEFAULT_TOP_PYPI_PACKAGES,
11196
"use_cache": False,
11297
},
11398
}
@@ -149,7 +134,7 @@ def test_no_load_config_from_cache(self, pyproject_toml_file: Path) -> None:
149134
assert config.logging_level == AvailableLoggingLevels.warning
150135
assert config.use_cache is True
151136
assert config.selector_method == DEFAULT_SELECTOR_METHOD
152-
assert config.pypi_reference == DEFAULT_TOP_PYPI_PACKAGES
137+
assert config.package_reference == DEFAULT_TOP_PYPI_PACKAGES
153138

154139
def test_cannot_write_if_file_not_configured(self) -> None:
155140
with pytest.raises(ConfigFileNotConfiguredError, match="write operation"):

tests/core/test_config_handler.py

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
import dataclasses
2+
from copy import deepcopy
3+
from pathlib import Path
4+
from unittest.mock import patch
5+
6+
import pytest
7+
from tomlkit import TOMLDocument, dumps, parse
8+
from twyn.base.constants import DEFAULT_PROJECT_TOML_FILE, DEFAULT_TOP_PYPI_PACKAGES, AvailableLoggingLevels
9+
from twyn.config.config_handler import ConfigHandler, ReadTwynConfiguration, TwynConfiguration
10+
from twyn.config.exceptions import (
11+
AllowlistPackageAlreadyExistsError,
12+
AllowlistPackageDoesNotExistError,
13+
InvalidSelectorMethodError,
14+
TOMLError,
15+
)
16+
from twyn.file_handler.exceptions import PathNotFoundError
17+
from twyn.file_handler.file_handler import FileHandler
18+
19+
20+
class TestConfig:
21+
def throw_exception(self):
22+
raise PathNotFoundError
23+
24+
def test_config_raises_for_unknown_file(self):
25+
with pytest.raises(TOMLError):
26+
ConfigHandler(FileHandler("non-existent-file.toml")).resolve_config()
27+
28+
def test_read_config_values(self, pyproject_toml_file):
29+
config = ConfigHandler(file_handler=FileHandler(pyproject_toml_file)).resolve_config()
30+
assert config.dependency_file == "my_file.txt"
31+
assert config.selector_method == "all"
32+
assert config.logging_level == AvailableLoggingLevels.debug
33+
assert config.allowlist == {"boto4", "boto2"}
34+
35+
def test_get_twyn_data_from_file(self, pyproject_toml_file):
36+
handler = ConfigHandler(FileHandler(str(pyproject_toml_file)))
37+
38+
toml = handler._read_toml()
39+
twyn_data = ConfigHandler(FileHandler(pyproject_toml_file))._get_read_config(toml)
40+
assert twyn_data == ReadTwynConfiguration(
41+
dependency_file="my_file.txt",
42+
selector_method="all",
43+
logging_level="debug",
44+
allowlist={"boto4", "boto2"},
45+
package_reference=None,
46+
)
47+
48+
def test_write_toml(self, pyproject_toml_file):
49+
handler = ConfigHandler(FileHandler(pyproject_toml_file))
50+
toml = handler._read_toml()
51+
52+
initial_config = handler.resolve_config()
53+
to_write = deepcopy(initial_config)
54+
to_write = dataclasses.replace(to_write, allowlist={})
55+
56+
handler._write_config(toml, to_write)
57+
58+
new_config = handler.resolve_config()
59+
60+
assert new_config != initial_config
61+
assert new_config.allowlist == set()
62+
# Writing the config should result in changes in the twyn section but
63+
# nowhere else
64+
assert handler._read_toml() == {
65+
"tool": {
66+
"poetry": {
67+
"dependencies": {
68+
"python": "^3.11",
69+
"requests": "^2.28.2",
70+
"dparse": "^0.6.2",
71+
"click": "^8.1.3",
72+
"rich": "^13.3.1",
73+
"rapidfuzz": "^2.13.7",
74+
"regex": "^2022.10.31",
75+
},
76+
"scripts": {"twyn": "twyn.cli:entry_point"},
77+
},
78+
"twyn": {
79+
"dependency_file": "my_file.txt",
80+
"selector_method": "all",
81+
"logging_level": "debug",
82+
"allowlist": {},
83+
"package_reference": DEFAULT_TOP_PYPI_PACKAGES,
84+
},
85+
}
86+
}
87+
88+
89+
class TestAllowlistConfigHandler:
90+
@patch("twyn.file_handler.file_handler.FileHandler.write")
91+
@patch("twyn.config.config_handler.ConfigHandler._read_toml")
92+
def test_allowlist_add(self, mock_toml, mock_write_toml):
93+
mock_toml.return_value = TOMLDocument()
94+
95+
config = ConfigHandler(FileHandler("some-file"))
96+
97+
config.add_package_to_allowlist("mypackage")
98+
99+
final_toml = config._read_toml()
100+
101+
assert final_toml == {"tool": {"twyn": {"allowlist": ["mypackage"]}}}
102+
assert mock_write_toml.called
103+
104+
@patch("twyn.config.config_handler.ConfigHandler._write_toml")
105+
@patch("twyn.config.config_handler.ConfigHandler._read_toml")
106+
def test_allowlist_add_duplicate_error(self, mock_toml, mock_write_toml):
107+
mock_toml.return_value = parse(dumps({"tool": {"twyn": {"allowlist": ["mypackage"]}}}))
108+
109+
config = ConfigHandler(FileHandler("some-file"))
110+
with pytest.raises(
111+
AllowlistPackageAlreadyExistsError,
112+
match="Package 'mypackage' is already present in the allowlist. Skipping.",
113+
):
114+
config.add_package_to_allowlist("mypackage")
115+
116+
assert not mock_write_toml.called
117+
118+
@patch("twyn.config.config_handler.ConfigHandler._write_toml")
119+
@patch("twyn.config.config_handler.ConfigHandler._read_toml")
120+
def test_allowlist_remove_completely(self, mock_toml, mock_write_toml):
121+
mock_toml.return_value = parse(dumps({"tool": {"twyn": {"allowlist": ["mypackage"]}}}))
122+
123+
config = ConfigHandler(FileHandler("some-file"))
124+
125+
config.remove_package_from_allowlist("mypackage")
126+
assert config._read_toml() == {"tool": {"twyn": {}}}
127+
128+
@patch("twyn.config.config_handler.ConfigHandler._write_toml")
129+
@patch("twyn.config.config_handler.ConfigHandler._read_toml")
130+
def test_allowlist_remove(self, mock_toml, mock_write_toml):
131+
mock_toml.return_value = parse(dumps({"tool": {"twyn": {"allowlist": ["mypackage", "another-package"]}}}))
132+
133+
config = ConfigHandler(FileHandler("some-file"))
134+
135+
config.remove_package_from_allowlist("mypackage")
136+
assert config._read_toml() == {"tool": {"twyn": {"allowlist": ["another-package"]}}}
137+
138+
@patch("twyn.config.config_handler.ConfigHandler._write_toml")
139+
@patch("twyn.config.config_handler.ConfigHandler._read_toml")
140+
def test_allowlist_remove_non_existent_package_error(self, mock_toml, mock_write_toml):
141+
mock_toml.return_value = parse(dumps({"tool": {"twyn": {"allowlist": ["mypackage"]}}}))
142+
143+
config = ConfigHandler(FileHandler("some-file"))
144+
with pytest.raises(
145+
AllowlistPackageDoesNotExistError,
146+
match="Package 'mypackage2' is not present in the allowlist. Skipping.",
147+
):
148+
config.remove_package_from_allowlist("mypackage2")
149+
150+
assert not mock_write_toml.called
151+
152+
@pytest.mark.parametrize("valid_selector", ["first-letter", "nearby-letter", "all"])
153+
def test_valid_selector_methods_accepted(self, valid_selector: str, tmp_path: Path):
154+
"""Test that all valid selector methods are accepted."""
155+
pyproject_toml = tmp_path / "pyproject.toml"
156+
pyproject_toml.write_text("")
157+
config = ConfigHandler(FileHandler(str(pyproject_toml)))
158+
159+
# Should not raise any exception
160+
resolved_config = config.resolve_config(selector_method=valid_selector)
161+
assert resolved_config.selector_method == valid_selector
162+
163+
def test_invalid_selector_method_rejected(self, tmp_path: Path):
164+
"""Test that invalid selector methods are rejected with appropriate error."""
165+
pyproject_toml = tmp_path / "pyproject.toml"
166+
pyproject_toml.write_text("")
167+
config = ConfigHandler(FileHandler(str(pyproject_toml)))
168+
169+
with pytest.raises(InvalidSelectorMethodError) as exc_info:
170+
config.resolve_config(selector_method="random-selector")
171+
172+
error_message = str(exc_info.value)
173+
assert "Invalid selector_method 'random-selector'" in error_message
174+
assert "Must be one of: all, first-letter, nearby-letter" in error_message
175+
176+
def test_invalid_selector_method_from_config_file(self, tmp_path: Path):
177+
"""Test that invalid selector method from config file is rejected."""
178+
# Create a config file with invalid selector method
179+
pyproject_toml = tmp_path / "pyproject.toml"
180+
data = """
181+
[tool.twyn]
182+
selector_method="invalid-selector"
183+
"""
184+
pyproject_toml.write_text(data)
185+
186+
config = ConfigHandler(FileHandler(str(pyproject_toml)))
187+
188+
with pytest.raises(InvalidSelectorMethodError) as exc_info:
189+
config.resolve_config()
190+
191+
error_message = str(exc_info.value)
192+
assert "Invalid selector_method 'invalid-selector'" in error_message
193+
assert "Must be one of: all, first-letter, nearby-letter" in error_message

0 commit comments

Comments
 (0)