Skip to content

Commit ba70665

Browse files
FredHappyfaceFredHappyface
committed
add new uv resolver
1 parent 979ded3 commit ba70665

14 files changed

+414
-486
lines changed

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@
33
All major and minor version changes will be documented in this file. Details of
44
patch-level version changes can be found in [commit messages](../../commits/master).
55

6+
## 2024.3
7+
8+
- Use uv to parse dependencies before falling back to the native resolver
9+
- deprecate the native resolver as many 3rd party libs (uv/pip/poetry) have done better
10+
611
## 2024.2 - 2024/04/04
712

813
- Add html output using `markdown` lib https://github.com/FHPythonUtils/LicenseCheck/issues/77

‎licensecheck/get_deps.py‎

Lines changed: 28 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,13 @@
22

33
from __future__ import annotations
44

5-
import contextlib
6-
import re
7-
from importlib import metadata
85
from pathlib import Path
9-
from typing import Any
106

117
import tomli
12-
from packaging.requirements import Requirement
13-
from packaging.utils import canonicalize_name
148

159
from licensecheck import license_matrix, packageinfo
16-
from licensecheck.session import session
10+
from licensecheck.resolvers import native as res_native
11+
from licensecheck.resolvers import uv as res_uv
1712
from licensecheck.types import JOINS, License, PackageInfo, ucstr
1813

1914
USINGS = ["requirements", "poetry", "PEP631"]
@@ -46,143 +41,35 @@ def getReqs(using: str, skipDependencies: list[ucstr]) -> set[ucstr]:
4641
if using not in USINGS:
4742
using = "poetry"
4843

49-
pyproject = {}
50-
requirementsPaths = []
51-
52-
pyprojectPath = Path("pyproject.toml")
53-
if pyprojectPath.exists():
54-
pyproject = tomli.loads(pyprojectPath.read_text(encoding="utf-8"))
55-
56-
# Requirements
57-
if using == "requirements":
58-
requirementsPaths = [Path("requirements.txt")]
59-
if len(extras) > 0:
60-
requirementsPaths = [Path(x) for x in (extras)]
61-
62-
return do_get_reqs(using, skipDependencies, extras, pyproject, requirementsPaths)
63-
64-
65-
def do_get_reqs(
66-
using: str,
67-
skipDependencies: list[ucstr],
68-
extras: list[str],
69-
pyproject: dict[str, Any],
70-
requirementsPaths: list[Path],
71-
) -> set[ucstr]:
72-
"""Underlying machineary to get requirements.
73-
74-
Args:
75-
----
76-
using (str): use requirements, poetry or PEP631.
77-
skipDependencies (list[str]): list of dependencies to skip.
78-
extras (str | None): to-do
79-
pyproject (dict[str, Any]): to-do
80-
requirementsPaths (list[Path]): to-do
81-
82-
Returns:
83-
-------
84-
set[str]: set of requirement packages
85-
86-
"""
87-
reqs = set()
88-
extrasReqs = {}
89-
90-
def resolveReq(req: str, *, extra: bool = True) -> ucstr:
91-
requirement = Requirement(req)
92-
extras = {ucstr(extra) for extra in requirement.extras}
93-
name = ucstr(canonicalize_name(requirement.name))
94-
canonicalName = name
95-
if len(extras) > 0:
96-
canonicalName = ucstr(f"{name}[{next(iter(extras))}]")
97-
# Avoid overwriting the initial mapping in extrasReqs, only overwrite when extra is True
98-
if extra:
99-
extrasReqs[name] = extras
100-
return canonicalName if extra else name
101-
102-
def resolveExtraReq(extraReq: str) -> ucstr | None:
103-
match = re.search(r"extra\s*==\s*[\"'](.*?)[\"']", extraReq)
104-
if match is None:
105-
return None
106-
return ucstr(match.group(1))
107-
108-
if using == "poetry":
109-
try:
110-
project = pyproject["tool"]["poetry"]
111-
reqLists = [project["dependencies"]]
112-
except KeyError as error:
113-
msg = "Could not find specification of requirements (pyproject.toml)."
114-
raise RuntimeError(msg) from error
115-
for extra in extras:
116-
reqLists.append(
117-
project.get("group", {extra: {"dependencies": {}}})[extra]["dependencies"]
118-
)
119-
reqLists.append(project.get("dev-dependencies", {}))
120-
for reqList in reqLists:
121-
for req in reqList:
122-
reqs.add(resolveReq(req))
123-
# PEP631
124-
if using == "PEP631":
125-
try:
126-
project = pyproject["project"]
127-
reqLists = [project["dependencies"]]
128-
except KeyError as error:
129-
msg = "Could not find specification of requirements (pyproject.toml)."
130-
raise RuntimeError(msg) from error
131-
for extra in extras:
132-
reqLists.append(project["optional-dependencies"][extra])
133-
for reqList in reqLists:
134-
for req in reqList:
135-
reqs.add(resolveReq(req))
44+
# if using poetry or pep621
45+
requirementsPaths = ["pyproject.toml"]
13646

13747
# Requirements
13848
if using == "requirements":
139-
for reqPath in requirementsPaths:
140-
if not reqPath.exists():
141-
msg = f"Could not find specification of requirements ({reqPath})."
142-
raise RuntimeError(msg)
143-
144-
for _line in reqPath.read_text(encoding="utf-8").splitlines():
145-
line = _line.rstrip("\\").strip()
146-
if not line or line[0] in {"#", "-"}:
147-
continue
148-
reqs.add(resolveReq(line))
149-
150-
# Remove PYTHON if define as requirement
151-
with contextlib.suppress(KeyError):
152-
reqs.remove("PYTHON")
153-
# Remove skip dependencies
154-
for skipDependency in skipDependencies:
155-
with contextlib.suppress(KeyError):
156-
reqs.remove(skipDependency)
157-
158-
# Get Dependencies, 1 deep
159-
requirementsWithDeps = reqs.copy()
160-
161-
def update_dependencies(dependency: str) -> None:
162-
dep = resolveReq(dependency, extra=False)
163-
req = resolveReq(requirement, extra=False)
164-
extra = resolveExtraReq(dependency)
165-
if extra is not None:
166-
if req in extrasReqs and extra in extrasReqs.get(req, []):
167-
requirementsWithDeps.add(dep)
168-
else:
169-
requirementsWithDeps.add(dep)
170-
171-
for requirement in reqs:
172-
try:
173-
pkgMetadata = metadata.metadata(requirement)
174-
for dependency in pkgMetadata.get_all("Requires-Dist") or []:
175-
update_dependencies(dependency)
176-
except metadata.PackageNotFoundError: # noqa: PERF203
177-
request = session.get(
178-
f"https://pypi.org/pypi/{requirement.split('[')[0]}/json", timeout=60
179-
)
180-
response: dict = request.json()
181-
requires_dist: list = response.get("info", {}).get("requires_dist", []) or []
182-
for dependency in requires_dist:
183-
update_dependencies(dependency)
184-
185-
return {r.split("[")[0] for r in requirementsWithDeps}
49+
requirementsPaths = ["requirements.txt"] if len(extras) > 0 else extras
50+
extras = []
51+
52+
try:
53+
return res_uv.get_reqs(
54+
using=using,
55+
skipDependencies=skipDependencies,
56+
extras=extras,
57+
requirementsPaths=requirementsPaths,
58+
)
59+
60+
except RuntimeError:
61+
pyproject = {}
62+
if "pyproject.toml" in requirementsPaths:
63+
pyproject = tomli.loads(Path("pyproject.toml").read_text("utf-8"))
64+
65+
# Fallback to the old resolver (hopefully we can deprecate this asap!)
66+
return res_native.get_reqs(
67+
using=using,
68+
skipDependencies=skipDependencies,
69+
extras=extras,
70+
pyproject=pyproject,
71+
requirementsPaths=[Path(x) for x in requirementsPaths],
72+
)
18673

18774

18875
def getDepsWithLicenses(

‎licensecheck/resolvers/__init__.py‎

Whitespace-only changes.

‎licensecheck/resolvers/native.py‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
from __future__ import annotations
2+
3+
import contextlib
4+
import re
5+
from importlib import metadata
6+
from pathlib import Path
7+
from typing import Any
8+
9+
from packaging.requirements import Requirement
10+
from packaging.utils import canonicalize_name
11+
12+
from licensecheck.session import session
13+
from licensecheck.types import ucstr
14+
15+
16+
def get_reqs(
17+
using: str,
18+
skipDependencies: list[ucstr],
19+
extras: list[str],
20+
pyproject: dict[str, Any],
21+
requirementsPaths: list[Path],
22+
) -> set[ucstr]:
23+
"""Underlying machineary to get requirements.
24+
25+
Args:
26+
----
27+
using (str): use requirements, poetry or PEP631.
28+
skipDependencies (list[str]): list of dependencies to skip.
29+
extras (str | None): to-do
30+
pyproject (dict[str, Any]): to-do
31+
requirementsPaths (list[Path]): to-do
32+
33+
Returns:
34+
-------
35+
set[str]: set of requirement packages
36+
37+
"""
38+
reqs = set()
39+
extrasReqs = {}
40+
41+
def resolveReq(req: str, *, extra: bool = True) -> ucstr:
42+
requirement = Requirement(req)
43+
extras = {ucstr(extra) for extra in requirement.extras}
44+
name = ucstr(canonicalize_name(requirement.name))
45+
canonicalName = name
46+
if len(extras) > 0:
47+
canonicalName = ucstr(f"{name}[{next(iter(extras))}]")
48+
# Avoid overwriting the initial mapping in extrasReqs, only overwrite when extra is True
49+
if extra:
50+
extrasReqs[name] = extras
51+
return canonicalName if extra else name
52+
53+
def resolveExtraReq(extraReq: str) -> ucstr | None:
54+
match = re.search(r"extra\s*==\s*[\"'](.*?)[\"']", extraReq)
55+
if match is None:
56+
return None
57+
return ucstr(match.group(1))
58+
59+
if using == "poetry":
60+
try:
61+
project = pyproject["tool"]["poetry"]
62+
reqLists = [project["dependencies"]]
63+
except KeyError as error:
64+
msg = "Could not find specification of requirements (pyproject.toml)."
65+
raise RuntimeError(msg) from error
66+
for extra in extras:
67+
reqLists.append(
68+
project.get("group", {extra: {"dependencies": {}}})[extra]["dependencies"]
69+
)
70+
reqLists.append(project.get("dev-dependencies", {}))
71+
for reqList in reqLists:
72+
for req in reqList:
73+
reqs.add(resolveReq(req))
74+
# PEP631
75+
if using == "PEP631":
76+
try:
77+
project = pyproject["project"]
78+
reqLists = [project["dependencies"]]
79+
except KeyError as error:
80+
msg = "Could not find specification of requirements (pyproject.toml)."
81+
raise RuntimeError(msg) from error
82+
for extra in extras:
83+
reqLists.append(project["optional-dependencies"][extra])
84+
for reqList in reqLists:
85+
for req in reqList:
86+
reqs.add(resolveReq(req))
87+
88+
# Requirements
89+
if using == "requirements":
90+
for reqPath in requirementsPaths:
91+
if not reqPath.exists():
92+
msg = f"Could not find specification of requirements ({reqPath})."
93+
raise RuntimeError(msg)
94+
95+
for _line in reqPath.read_text(encoding="utf-8").splitlines():
96+
line = _line.rstrip("\\").strip()
97+
if not line or line[0] in {"#", "-"}:
98+
continue
99+
reqs.add(resolveReq(line))
100+
101+
# Remove PYTHON if define as requirement
102+
with contextlib.suppress(KeyError):
103+
reqs.remove("PYTHON")
104+
# Remove skip dependencies
105+
for skipDependency in skipDependencies:
106+
with contextlib.suppress(KeyError):
107+
reqs.remove(skipDependency)
108+
109+
# Get Dependencies, 1 deep
110+
requirementsWithDeps = reqs.copy()
111+
112+
def update_dependencies(dependency: str) -> None:
113+
dep = resolveReq(dependency, extra=False)
114+
req = resolveReq(requirement, extra=False)
115+
extra = resolveExtraReq(dependency)
116+
if extra is not None:
117+
if req in extrasReqs and extra in extrasReqs.get(req, []):
118+
requirementsWithDeps.add(dep)
119+
else:
120+
requirementsWithDeps.add(dep)
121+
122+
for requirement in reqs:
123+
try:
124+
pkgMetadata = metadata.metadata(requirement)
125+
for dependency in pkgMetadata.get_all("Requires-Dist") or []:
126+
update_dependencies(dependency)
127+
except metadata.PackageNotFoundError: # noqa: PERF203
128+
request = session.get(
129+
f"https://pypi.org/pypi/{requirement.split('[')[0]}/json", timeout=60
130+
)
131+
response: dict = request.json()
132+
requires_dist: list = response.get("info", {}).get("requires_dist", []) or []
133+
for dependency in requires_dist:
134+
update_dependencies(dependency)
135+
136+
return {r.split("[")[0] for r in requirementsWithDeps}

‎licensecheck/resolvers/uv.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
"""Use uv to get packages from project/ requirements.txt."""
2+
3+
from __future__ import annotations
4+
5+
import shutil
6+
import subprocess
7+
import tempfile
8+
from pathlib import Path
9+
10+
import requirements
11+
12+
from licensecheck.types import ucstr
13+
14+
15+
def get_reqs(
16+
using: str,
17+
skipDependencies: list[ucstr],
18+
extras: list[str],
19+
requirementsPaths: list[str],
20+
) -> set[ucstr]:
21+
if using == "requirements" and len(extras) > 0:
22+
msg = "You may not use extras with requirements.txt"
23+
raise RuntimeError(msg)
24+
25+
for idx, requirement in enumerate(requirementsPaths):
26+
if not Path(requirement).exists():
27+
msg = f"Could not find specification of requirements ({requirement})."
28+
raise RuntimeError(msg)
29+
30+
if not requirement.endswith("pyproject.toml") and requirement.endswith(".toml"):
31+
temp_dir_path = Path(tempfile.mkdtemp())
32+
destination_file = temp_dir_path / "pyproject.toml"
33+
shutil.copy(requirement, destination_file)
34+
requirementsPaths[idx] = destination_file.as_posix()
35+
36+
extras_cmd = [f"--extra {extra}" for extra in extras]
37+
command = f"uv pip compile {' '.join(requirementsPaths)} {' '.join(extras_cmd)}"
38+
39+
result = subprocess.run(command, shell=True, capture_output=True, text=True, check=False)
40+
41+
if result.returncode != 0:
42+
raise RuntimeError(result.stderr)
43+
44+
reqs = requirements.parse(result.stdout)
45+
reqs_out = [ucstr(x.name) for x in reqs]
46+
47+
return set(reqs_out) - set(skipDependencies)

0 commit comments

Comments
 (0)