Merge pull request #3768 from OpenNeuroOrg/fix/get-repo-files

fix(worker): Compute rmet names and use git cat-file instead of ls-tree for get_repo_urls

Author: nellh

services/datalad/datalad_service/common/annex.py

@@ -126,12 +126,9 @@ def read_ls_tree_line(gitTreeLine, files, symlinkFilenames, symlinkObjects):
             )
 
 
-def compute_rmet(key, legacy=False):
+def compute_rmet(key):
     if len(key) == 40:
-        if legacy:
-            key = f'SHA1--{key}'
-        else:
-            key = f'GIT--{key}'
+        key = f'GIT--{key}'
     keyHash = hashlib.md5(key.encode()).hexdigest()
     return f'{keyHash[0:3]}/{keyHash[3:6]}/{key}.log.rmet'
 
@@ -192,21 +189,30 @@ def encode_remote_url(url):
 async def get_repo_urls(path, files):
     """For each file provided, obtain the rmet data and append URLs if possible."""
     # First obtain the git-annex branch objects
-    gitAnnexBranch = await asyncio.create_subprocess_exec(
+    rmet_queries = set(['remote.log', 'trust.log'])
+    for f in files:
+        if 'key' in f:
+            rmet_queries.add(compute_rmet(f['key']))
+
+    query_list = list(rmet_queries)
+    batch_input = '\n'.join(f'git-annex:{p}' for p in query_list)
+
+    process = await asyncio.create_subprocess_exec(
         'git',
-        'ls-tree',
-        '-l',
-        'git-annex',
+        'cat-file',
+        '--batch-check',
         cwd=path,
+        stdin=asyncio.subprocess.PIPE,
         stdout=asyncio.subprocess.PIPE,
     )
+    stdout, _ = await process.communicate(input=batch_input.encode('utf-8'))
+
     rmetObjects = {}
-    async for line in gitAnnexBranch.stdout:
-        filename, mode, obj_type, obj_hash, size = parse_ls_tree_line(
-            line.decode('utf-8').rstrip()
-        )
-        rmetObjects[filename] = obj_hash
-    await gitAnnexBranch.wait()
+    for i, line in enumerate(stdout.decode('utf-8').splitlines()):
+        parts = line.split()
+        if parts[-1] == 'missing':
+            continue
+        rmetObjects[query_list[i]] = parts[0]
 
     if 'remote.log' not in rmetObjects:
         # Skip searching for URLs if no remote.log is present
@@ -218,11 +224,6 @@ async def get_repo_urls(path, files):
             if rmetPath in rmetObjects:
                 # Keep a reference to the files so we can add URLs later
                 rmetFiles[rmetPath].append(f)
-            else:
-                # Check for alternate path used by older versions of git-annex
-                rmetPath = compute_rmet(f['key'], legacy=True)
-                if rmetPath in rmetObjects:
-                    rmetFiles[rmetPath].append(f)
     # Then read those objects with git cat-file --batch
     gitObjects = ''
     if 'trust.log' in rmetObjects:

services/datalad/tests/test_annex.py

@@ -117,14 +117,6 @@ def test_compute_rmet_git():
     )
 
 
-def test_compute_rmet_git_legacy():
-    # Test a git SHA1 key
-    assert (
-        compute_rmet('99fe93bfea62c16a10488593da870df25d09be81', legacy=True)
-        == '9e2/03e/SHA1--99fe93bfea62c16a10488593da870df25d09be81.log.rmet'
-    )
-
-
 def test_compute_rmet_annex():
     # Test a git annex MD5E key
     assert (