Skip to content

Commit 9198601

Browse files
gavinmakgerrit-scoped@luci-project-accounts.iam.gserviceaccount.com
authored andcommitted
info: Parallelize project data gathering for JSON output
https://gerrit-review.googlesource.com/c/git-repo/+/581921 parallelized `repo info` for text output. This commit does the same for JSON output format. Benchmarked `repo info --format=json` on an Android workspace with ~3k projects (N=3): - Before (sequential): 1m 30s average - After (parallelized): 46s average (~2x speedup) Verified that the JSON output is identical before and after. Bug: 526685287 Change-Id: If573223aba584f8b932f87d29e34ed565c5c930a Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/601861 Commit-Queue: Gavin Mak <gavinmak@google.com> Reviewed-by: Brian Gan <brgan@google.com> Tested-by: Gavin Mak <gavinmak@google.com>
1 parent a27dbcd commit 9198601

2 files changed

Lines changed: 54 additions & 2 deletions

File tree

subcmds/info.py

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,8 @@ def _getSummaryData(self) -> Dict[str, Any]:
191191
"superproject_revision": srev,
192192
}
193193

194-
def _getProjectData(self, project) -> Dict[str, Any]:
194+
@classmethod
195+
def _getProjectData(cls, project) -> Dict[str, Any]:
195196
"""Gather project data as a dict."""
196197
data = {
197198
"name": project.name,
@@ -206,6 +207,12 @@ def _getProjectData(self, project) -> Dict[str, Any]:
206207
data["current_branch"] = currentBranch
207208
return data
208209

210+
@classmethod
211+
def _ProjectDataHelper(cls, project_idx: int) -> Dict[str, Any]:
212+
"""Helper to get project data in parallel."""
213+
project = cls.get_parallel_context()["projects"][project_idx]
214+
return cls._getProjectData(project)
215+
209216
def _ExecuteJson(self, opt, args) -> None:
210217
"""Output info as JSON."""
211218
result = {}
@@ -215,7 +222,22 @@ def _ExecuteJson(self, opt, args) -> None:
215222
projs = self.GetProjects(
216223
args, all_manifests=not opt.this_manifest_only
217224
)
218-
result["projects"] = [self._getProjectData(p) for p in projs]
225+
project_data = []
226+
227+
def _ProcessResults(_pool, _output, results):
228+
project_data.extend(results)
229+
230+
with self.ParallelContext():
231+
self.get_parallel_context()["projects"] = projs
232+
self.ExecuteInParallel(
233+
opt.jobs,
234+
self._ProjectDataHelper,
235+
range(len(projs)),
236+
callback=_ProcessResults,
237+
ordered=True,
238+
chunksize=1,
239+
)
240+
result["projects"] = project_data
219241

220242
json_settings = {
221243
# JSON style guide says Unicode characters are fully allowed.

tests/test_subcmds_info.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,3 +223,33 @@ def test_get_project_data_uses_head_revision() -> None:
223223
project.GetHeadRevisionId.return_value = None
224224
data = cmd._getProjectData(project)
225225
assert data["current_revision"] == "manifest_sha_54321"
226+
227+
228+
def test_json_with_projects(capsys) -> None:
229+
"""--format=json should emit project data."""
230+
cmd = _get_cmd()
231+
opts, args = cmd.OptionParser.parse_args(["--format=json"])
232+
opts.jobs = 1 # To avoid multiprocessing pickle issues with mocks
233+
234+
project = mock.MagicMock()
235+
project.name = "foo"
236+
project.worktree = "/path/to/foo"
237+
project.revisionExpr = "refs/heads/main"
238+
project.GetBranches.return_value = {"branch1": mock.MagicMock()}
239+
project.GetHeadRevisionId.return_value = "head_sha_12345"
240+
project.CurrentBranch = "branch1"
241+
242+
cmd.GetProjects = mock.MagicMock(return_value=[project])
243+
244+
cmd.Execute(opts, args)
245+
246+
data = json.loads(capsys.readouterr().out)
247+
assert "projects" in data
248+
assert len(data["projects"]) == 1
249+
project_data = data["projects"][0]
250+
assert project_data["name"] == "foo"
251+
assert project_data["mount_path"] == "/path/to/foo"
252+
assert project_data["current_revision"] == "head_sha_12345"
253+
assert project_data["manifest_revision"] == "refs/heads/main"
254+
assert project_data["local_branches"] == ["branch1"]
255+
assert project_data["current_branch"] == "branch1"

0 commit comments

Comments
 (0)