Skip to content

Commit 4cc65cd

Browse files
spoorccspoorcc
committed
Report externals in svn subprojects
1 parent 8ee8d1d commit 4cc65cd

9 files changed

Lines changed: 673 additions & 45 deletions

‎CHANGELOG.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
Release 0.14.0 (unreleased)
22
===========================
33

4+
* Report SVN externals fetched during update (#1220)
45
* Use ``.cdx.json`` as the default extension for CycloneDX SBOM reports (#1118)
56
* Embed base64-encoded license text in SBOM ``licenses[].text`` when a license is successfully identified (#1112)
67
* Set SBOM ``licenses`` to the SPDX expression ``NOASSERTION`` when a license file is not found or cannot be classified (#1112)

‎dfetch/project/svnsubproject.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,32 @@ def _fetch_impl(self, version: Version) -> tuple[Version, list[Dependency]]:
152152
if self.ignore:
153153
self._remove_ignored_files()
154154

155-
return Version(tag=version.tag, branch=branch, revision=revision), []
155+
return (
156+
Version(tag=version.tag, branch=branch, revision=revision),
157+
self._fetch_externals(complete_path, revision),
158+
)
159+
160+
def _fetch_externals(self, complete_path: str, revision: str) -> list[Dependency]:
161+
"""Detect and log SVN externals that were exported with the project."""
162+
vcs_deps = []
163+
for external in SvnRepo.externals_from_url(complete_path, revision):
164+
path_display = "./" + external.path.lstrip("./")
165+
display_branch = external.branch or SvnRepo.DEFAULT_BRANCH
166+
self._log_project(
167+
f'Found & fetched external "{path_display}" '
168+
f"({external.url} @ {Version(tag=external.tag, branch=display_branch, revision=external.revision)})",
169+
)
170+
vcs_deps.append(
171+
Dependency(
172+
remote_url=external.url,
173+
destination=external.path,
174+
branch=external.branch,
175+
tag=external.tag,
176+
revision=external.revision,
177+
source_type="svn-external",
178+
)
179+
)
180+
return vcs_deps
156181

157182
@staticmethod
158183
def _parse_file_pattern(complete_path: str) -> tuple[str, str]:

‎dfetch/vcs/svn.py‎

Lines changed: 89 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -164,48 +164,97 @@ def externals(self) -> list[External]:
164164
"-R",
165165
],
166166
)
167-
168167
repo_root = SvnRepo.get_info_from_target()["Repository Root"]
168+
return SvnRepo._parse_externals(
169+
result.stdout.decode(), repo_root, toplevel=self._path
170+
)
169171

170-
externals: list[External] = []
171-
# Pattern matches: "path - ..." where path is the local directory
172-
path_pattern = r"([^\s^-]+)\s+-"
173-
for entry in result.stdout.decode().split(os.linesep * 2):
174-
match: re.Match[str] | None = None
175-
local_path: str = ""
176-
for match in re.finditer(path_pattern, entry):
177-
pass
178-
if match:
179-
local_path = match.group(1)
180-
entry = re.sub(path_pattern, "", entry)
181-
182-
# Pattern matches either:
183-
# - url@revision name (pinned)
184-
# - url name (unpinned)
185-
for match in re.finditer(
186-
r"([^-\s\d][^\s]+)(?:@)(\d+)\s+([^\s]+)|([^-\s\d][^\s]+)\s+([^\s]+)",
187-
entry,
188-
):
189-
url = match.group(1) or match.group(4)
190-
name = match.group(3) or match.group(5)
191-
rev = "" if not match.group(2) else match.group(2).strip()
192-
193-
url, branch, tag, src = SvnRepo._split_url(url, repo_root)
194-
195-
externals += [
196-
External(
197-
name=name,
198-
toplevel=self._path,
199-
path="/".join(os.path.join(local_path, name).split(os.sep)),
200-
revision=rev,
201-
url=url,
202-
branch=branch,
203-
tag=tag,
204-
src=src,
205-
)
206-
]
207-
208-
return externals
172+
@staticmethod
173+
def externals_from_url(url: str, revision: str = "") -> list[External]:
174+
"""Get list of externals from a remote SVN URL."""
175+
cmd = ["svn", "--non-interactive", "propget", "svn:externals", "-R"]
176+
if revision:
177+
cmd += ["--revision", revision]
178+
cmd += [url]
179+
result = run_on_cmdline(logger, cmd)
180+
repo_root = SvnRepo.get_info_from_target(url)["Repository Root"]
181+
normalized = SvnRepo._normalize_url_prefix(result.stdout.decode(), url)
182+
return SvnRepo._parse_externals(normalized, repo_root)
183+
184+
@staticmethod
185+
def _normalize_url_prefix(output: str, base_url: str) -> str:
186+
"""Convert URL-mode ``svn propget -R`` output to relative-path format.
187+
188+
When querying a remote URL, each entry is prefixed with the full SVN URL
189+
of the directory that owns the property instead of a relative path.
190+
Strip the base_url so the standard parser receives familiar relative paths.
191+
"""
192+
base = base_url.rstrip("/")
193+
entries = []
194+
for entry in output.split(os.linesep * 2):
195+
if entry.startswith(base + "/"):
196+
after = entry[len(base) + 1 :]
197+
sep = after.find(" -")
198+
if sep >= 0:
199+
rel = after[:sep] or "."
200+
entry = rel + after[sep:]
201+
elif entry.startswith(base + " -"):
202+
entry = "." + entry[len(base) :]
203+
entries.append(entry)
204+
return (os.linesep * 2).join(entries)
205+
206+
@staticmethod
207+
def _parse_externals(
208+
output: str, repo_root: str, toplevel: str = ""
209+
) -> list[External]:
210+
"""Parse svn propget svn:externals output into External objects.
211+
212+
Args:
213+
output: Raw stdout from ``svn propget svn:externals -R``.
214+
repo_root: Repository root URL (used to resolve ``^/`` relative URLs).
215+
toplevel: Local working-copy root to record in each External.
216+
"""
217+
externals: list[External] = []
218+
path_pattern = r"(.+?)\s+-"
219+
for entry in output.split(os.linesep * 2):
220+
match: re.Match[str] | None = None
221+
local_path: str = ""
222+
for match in re.finditer(path_pattern, entry):
223+
pass
224+
if match:
225+
local_path = match.group(1)
226+
entry = re.sub(path_pattern, "", entry)
227+
228+
for match in re.finditer(
229+
r"-r\s+(\d+)\s+(\S+?)(?:@\d+)?\s+(\S+)|([^@\s-][^@\s]*)(?:@(\d+))?\s+([^\s]+)",
230+
entry,
231+
):
232+
if match.group(1):
233+
rev = match.group(1)
234+
url = match.group(2)
235+
name = match.group(3)
236+
else:
237+
url = match.group(4)
238+
rev = match.group(5) or ""
239+
name = match.group(6)
240+
241+
url, branch, tag, src = SvnRepo._split_url(url, repo_root)
242+
243+
raw_path = "/".join(os.path.join(local_path, name).split(os.sep))
244+
externals += [
245+
External(
246+
name=name,
247+
toplevel=toplevel,
248+
path=raw_path[2:] if raw_path.startswith("./") else raw_path,
249+
revision=rev,
250+
url=url,
251+
branch=branch,
252+
tag=tag,
253+
src=src,
254+
)
255+
]
256+
257+
return externals
209258

210259
@staticmethod
211260
def _split_url(url: str, repo_root: str) -> tuple[str, str, str, str]:

‎features/fetch-svn-repo-with-external.feature‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
@update @report
2+
Feature: Fetch projects with nested SVN dependencies
3+
4+
When fetching an SVN project that defines svn:externals,
5+
the exported content already includes the external directories.
6+
DFetch should detect and report these externals so they are
7+
visible in the project report, mirroring the behaviour for
8+
Git submodules.
9+
10+
Background:
11+
Given a svn-server "ExternalLib" with the files
12+
| path |
13+
| README.md |
14+
And a svn-server "ParentProject" with the files
15+
| path |
16+
| README.md |
17+
And svn-server "ParentProject" has an external at "ext" from svn-server "ExternalLib"
18+
19+
Scenario: A project with an SVN external is fetched and the external is reported
20+
Given the manifest 'dfetch.yaml' in MyProject
21+
"""
22+
manifest:
23+
version: 0.0
24+
projects:
25+
- name: svn-project-with-external
26+
url: some-remote-server/ParentProject
27+
vcs: svn
28+
"""
29+
When I run "dfetch update" in MyProject
30+
Then the output shows
31+
"""
32+
Dfetch (0.13.0)
33+
svn-project-with-external:
34+
> Found & fetched external "./ext" (some-remote-server/ExternalLib @ trunk)
35+
> Fetched trunk - 2
36+
"""
37+
And 'MyProject' looks like:
38+
"""
39+
MyProject/
40+
dfetch.yaml
41+
svn-project-with-external/
42+
.dfetch_data.yaml
43+
README.md
44+
ext/
45+
README.md
46+
"""
47+
48+
Scenario: External changes are reported in the project report
49+
Given a fetched and committed MyProject with the manifest
50+
"""
51+
manifest:
52+
version: 0.0
53+
projects:
54+
- name: svn-project-with-external
55+
url: some-remote-server/ParentProject
56+
vcs: svn
57+
"""
58+
When I run "dfetch report" in MyProject
59+
Then the output shows
60+
"""
61+
Dfetch (0.13.0)
62+
svn-project-with-external:
63+
- remote : <none>
64+
remote url : some-remote-server/ParentProject
65+
branch : trunk
66+
tag : <none>
67+
last fetch :
68+
revision : 2
69+
patch : <none>
70+
licenses : <none>
71+
72+
dependencies :
73+
- path : ext
74+
url : some-remote-server/ExternalLib
75+
branch : <none>
76+
tag : <none>
77+
revision : <none>
78+
source-type : svn-external
79+
"""

‎features/fetch-svn-repo-with-nonstd-external.feature‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
@update @report
2+
Feature: Fetch projects with non-standard-layout SVN externals
3+
4+
SVN externals can point to a repository root that has no trunk/branches/tags
5+
layout. DFetch should detect and report these with an empty branch value
6+
(represented internally as a single space) rather than treating them as a
7+
standard branch.
8+
9+
Background:
10+
Given a non-standard svn-server "NonstdLib" with the files
11+
| path |
12+
| README.md |
13+
And a svn-server "ParentProject" with the files
14+
| path |
15+
| README.md |
16+
And svn-server "ParentProject" has an external at "nonstd-ext" from non-standard svn-server "NonstdLib"
17+
18+
Scenario: A project with a non-standard-layout SVN external is fetched and the external is reported
19+
Given the manifest 'dfetch.yaml' in MyProject
20+
"""
21+
manifest:
22+
version: 0.0
23+
projects:
24+
- name: svn-project-with-nonstd-external
25+
url: some-remote-server/ParentProject
26+
vcs: svn
27+
"""
28+
When I run "dfetch update" in MyProject
29+
Then the output shows
30+
"""
31+
Dfetch (0.13.0)
32+
svn-project-with-nonstd-external:
33+
> Found & fetched external "./nonstd-ext" (some-remote-server/NonstdLib @ )
34+
> Fetched trunk - 2
35+
"""
36+
And 'MyProject' looks like:
37+
"""
38+
MyProject/
39+
dfetch.yaml
40+
svn-project-with-nonstd-external/
41+
.dfetch_data.yaml
42+
README.md
43+
nonstd-ext/
44+
README.md
45+
"""
46+
47+
Scenario: Non-standard external is shown in the project report with no branch
48+
Given a fetched and committed MyProject with the manifest
49+
"""
50+
manifest:
51+
version: 0.0
52+
projects:
53+
- name: svn-project-with-nonstd-external
54+
url: some-remote-server/ParentProject
55+
vcs: svn
56+
"""
57+
When I run "dfetch report" in MyProject
58+
Then the output shows
59+
"""
60+
Dfetch (0.13.0)
61+
svn-project-with-nonstd-external:
62+
- remote : <none>
63+
remote url : some-remote-server/ParentProject
64+
branch : trunk
65+
tag : <none>
66+
last fetch :
67+
revision : 2
68+
patch : <none>
69+
licenses : <none>
70+
71+
dependencies :
72+
- path : nonstd-ext
73+
url : some-remote-server/NonstdLib
74+
branch :
75+
tag : <none>
76+
revision : <none>
77+
source-type : svn-external
78+
"""

‎features/import-from-svn.feature‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,15 @@ Feature: Importing externals from an existing svn repository
2626
- name: ext/cunit1
2727
revision: '176'
2828
src: Man
29-
dst: ./ext/cunit1
3029
repo-path: code
3130
3231
- name: ext/cunit2
3332
revision: '150'
3433
src: Man
35-
dst: ./ext/cunit2
3634
branch: mingw64
3735
repo-path: code
3836
3937
- name: ext/cunit3
40-
dst: ./ext/cunit3
4138
branch: ' '
4239
repo-path: code
4340

‎features/steps/svn_steps.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,3 +143,25 @@ def step_impl(_, pattern, directory):
143143
with in_directory(directory):
144144
subprocess.check_call(["svn", "propset", "svn:ignore", pattern, "."])
145145
commit_all(f"Ignore {pattern} files")
146+
147+
148+
@given(
149+
'svn-server "{name}" has an external at "{ext_path}" from svn-server "{source_name}"'
150+
)
151+
def step_impl(context, name, ext_path, source_name):
152+
source_url = (
153+
pathlib.Path(context.remotes_dir_path).joinpath(source_name, "trunk").as_uri()
154+
)
155+
with in_directory(name):
156+
with in_directory("trunk"):
157+
add_externals([{"url": source_url, "path": ext_path, "revision": ""}])
158+
159+
160+
@given(
161+
'svn-server "{name}" has an external at "{ext_path}" from non-standard svn-server "{source_name}"'
162+
)
163+
def step_impl(context, name, ext_path, source_name):
164+
source_url = pathlib.Path(context.remotes_dir_path).joinpath(source_name).as_uri()
165+
with in_directory(name):
166+
with in_directory("trunk"):
167+
add_externals([{"url": source_url, "path": ext_path, "revision": ""}])

0 commit comments

Comments
 (0)