[IMP] runbot: batch show file0

Author: Xavier-Do

runbot/models/build.py

@@ -1126,15 +1126,21 @@ def _get_modules_to_test(self, modules_patterns=''):
     def _dependency_graph(self):
         dependency_graph = defaultdict(set)
         dependant_graph = defaultdict(set)
-        for commit, addons_path, module, manifest_file_name in self._list_available_modules():
-            manifest = commit._git_show_file(os.path.join(addons_path, module, manifest_file_name))
-            manifest_content = ast.literal_eval(manifest)
-            depends = manifest_content.get('depends', [])
-            if not depends and module != 'base':
-                depends = ['base']
-            for dep in depends:
-                dependency_graph[module].add(dep)
-                dependant_graph[dep].add(module)
+        for commit in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids:
+            file_paths = []
+            modules = []
+            for (addons_path, module, manifest_file_name) in commit._list_available_modules():
+                file_paths.append(os.path.join(addons_path, module, manifest_file_name))
+                modules.append(module)
+            contents = commit._git_show_files(file_paths)
+            for module, manifest in zip(modules, contents):
+                manifest_content = ast.literal_eval(manifest)
+                depends = manifest_content.get('depends', [])
+                if not depends and module != 'base':
+                    depends = ['base']
+                for dep in depends:
+                    dependency_graph[module].add(dep)
+                    dependant_graph[dep].add(module)
         return dependency_graph, dependant_graph
 
     def search_modules_graph(self, modules, graph, depth=None):

runbot/models/commit.py

@@ -163,12 +163,43 @@ def _read_source(self, file, mode='r'):
 
     @transactioncache
     def _git_show_file(self, file):
+        return self._git_show_files([file])[0]
+
+    def _git_show_files(self, files):
         self.ensure_one()
+        if not files:
+            return []
+
         self.repo_id._fetch(self.name)
+
+        queries = "\n".join([f"{self.name}:{f}" for f in files]) + "\n"
+
         try:
-            return self.repo_id._git(['show', '%s:%s' % (self.name, file)])
+            buffer = self.repo_id._git(
+                ['cat-file', '--batch'],
+                input_data=queries,
+                raw=True,
+            )
         except subprocess.CalledProcessError:
-            return False
+            return [False] * len(files)
+
+        results = []
+        offset = 0
+        buffer_len = len(buffer)
+        while offset < buffer_len:
+            newline_idx = buffer.find(b'\n', offset)
+            if newline_idx == -1:
+                break
+            header = buffer[offset:newline_idx].decode('utf-8')
+            offset = newline_idx + 1
+            try:
+                size_in_bytes = int(header.rsplit(' ',1)[-1])
+            except ValueError:  # most likely missing
+                 results.append(False)
+                 continue
+            results.append(buffer[offset : offset + size_in_bytes].decode('utf-8', errors='replace'))
+            offset += size_in_bytes + 1
+        return results
 
     def _source_path(self, *paths):
         if not self.tree_hash:

runbot/models/repo.py

@@ -503,11 +503,19 @@ def _get_git_command(self, cmd, errors='strict'):
         cmd = ['git', '-C', self.path] + config_args + cmd
         return cmd
 
-    def _git(self, cmd, errors='strict', quiet=False):
+    def _git(self, cmd, errors='strict', quiet=False, input_data=None, raw=False):
         cmd = self._get_git_command(cmd, errors)
         if not quiet:
             _logger.info("git command: %s", shlex.join(cmd))
-        return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode(errors=errors)
+        kwargs = {'stderr': subprocess.STDOUT}
+        if input_data is not None:
+            if isinstance(input_data, str):
+                input_data = input_data.encode('utf-8')
+            kwargs['input'] = input_data
+        output = subprocess.check_output(cmd, **kwargs)
+        if raw:
+            return output
+        return output.decode(errors=errors)
 
     def _fetch(self, sha):
         if not self._hash_exists(sha):