[IMP] runbot: allow to filter on dependencies

This commit adds the possibility to filter modules based on their
dependencies and dependants in the build configuration. This allows to
easily trigger tests on modules that are impacted by changes, even if
they are not directly modified.

To make it work the modules listing is now done only in the git
repository, without exporting sources, this should help to easily to
have faster builds when the only task is to create builds based on
modified modules. This should also and mostly help to test a dynamic
config without running it.

Author: Xavier-Do

runbot/controllers/frontend.py

@@ -323,7 +323,8 @@ def build(self, build_id, search=None, from_batch=None, **post):
     @route([
     '/runbot/build/search',
     ], website=True, auth='public', type='http', sitemap=False)
-    def builds(self, **kwargs):
+    def builds(self, limit=100, **kwargs):
+        limit = min(int(limit), 1000)
         domain = []
         for key in ('config_id', 'version_id', 'project_id', 'trigger_id', 'create_batch_id.bundle_id', 'create_batch_id'):  # allowed params
             value = kwargs.get(key)
@@ -337,10 +338,12 @@ def builds(self, **kwargs):
 
         for key in ('description',):
             if key in kwargs:
-                domain.append((f'{key}', 'ilike', kwargs.get(key)))
+                value = kwargs.get(key)
+                operator = 'ilike' if '%' in value else '='
+                domain.append((f'{key}', operator, value))
 
         context = {
-            'builds': request.env['runbot.build'].search(domain, limit=100),
+            'builds': request.env['runbot.build'].search(domain, limit=limit),
         }
 
         return request.render('runbot.build_search', context)

runbot/models/build.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+import ast
 import datetime
 import getpass
 import hashlib
@@ -17,7 +17,7 @@
 from psycopg2 import sql
 from psycopg2.extensions import TransactionRollbackError
 
-from ..common import dt2time, now, grep, local_pgadmin_cursor, dest_reg, os, list_local_dbs, pseudo_markdown, RunbotException, findall, sanitize, markdown_escape, tail
+from ..common import dt2time, now, grep, local_pgadmin_cursor, dest_reg, os, list_local_dbs, pseudo_markdown, RunbotException, findall, sanitize, markdown_escape, tail, transactioncache
 from ..container import docker_stop, docker_state, Command, docker_run, docker_pull
 from ..fields import JsonDictField
 
@@ -61,7 +61,6 @@ def remove_readonly(func, path_str, exinfo):
 def make_selection(array):
     return [(elem, elem.replace('_', ' ').capitalize()) if isinstance(elem, str) else elem for elem in array]
 
-
 class BuildParameters(models.Model):
     _name = 'runbot.build.params'
     _description = "Build parameters"
@@ -1091,25 +1090,28 @@ def _checkout(self):
 
         return exports
 
+    def _list_available_modules(self):
+        for commit in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids:
+            for (addons_path, module, manifest_file_name) in commit._list_available_modules():
+                yield commit, addons_path, module, manifest_file_name
+
     def _get_available_modules(self):
         all_modules = dict()
         available_modules = defaultdict(list)
         # repo_modules = []
-        for commit in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids:
-            for (addons_path, module, manifest_file_name) in commit._get_available_modules():
-                if module in all_modules:
-                    self._log(
-                        'Building environment',
-                        '%s is a duplicated modules (found in "%s", already defined in %s)' % (
-                            module,
-                            commit._source_path(addons_path, module, manifest_file_name),
-                            all_modules[module]._source_path(addons_path, module, manifest_file_name)),
-                        level='WARNING',
-                    )
-                else:
-                    available_modules[commit.repo_id].append(module)
-                    all_modules[module] = commit
-        # return repo_modules, available_modules
+        for commit, addons_path, module, manifest_file_name in self._list_available_modules():
+            if module in all_modules:
+                self._log(
+                    'Building environment',
+                    '%s is a duplicated modules (found in "%s", already defined in %s)' % (
+                        module,
+                        commit._source_path(addons_path, module, manifest_file_name),
+                        all_modules[module]._source_path(addons_path, module, manifest_file_name)),
+                    level='WARNING',
+                )
+            else:
+                available_modules[commit.repo_id].append(module)
+                all_modules[module] = commit
         return available_modules
 
     def _get_modules_to_test(self, modules_patterns=''):
@@ -1120,6 +1122,43 @@ def _get_modules_to_test(self, modules_patterns=''):
         modules_patterns = (modules_patterns or '').split(',')
         return trigger._filter_modules_to_test(modules, params_patterns + modules_patterns)  # we may switch params_patterns and modules_patterns order
 
+    @transactioncache
+    def _dependency_graph(self):
+        dependency_graph = defaultdict(set)
+        dependant_graph = defaultdict(set)
+        for commit, addons_path, module, manifest_file_name in self._list_available_modules():
+            manifest = commit._git_show_file(os.path.join(addons_path, module, manifest_file_name))
+            manifest_content = ast.literal_eval(manifest)
+            depends = manifest_content.get('depends', [])
+            if not depends and module != 'base':
+                depends = ['base']
+            for dep in depends:
+                dependency_graph[module].add(dep)
+                dependant_graph[dep].add(module)
+        return dependency_graph, dependant_graph
+
+    def search_modules_graph(self, modules, graph, depth=None):
+        def search(modules, depth=None, visited=None):
+            visited = visited or set()
+            modules = set(modules) - visited
+            visited |= modules
+            dependencies = set(modules)
+            if depth == 0 or not modules:
+                return dependencies
+            for module in modules:
+                dependencies |= search(graph[module], depth - 1 if depth is not None else None, visited)
+            return dependencies
+        return sorted(search(modules, depth))
+
+    def _get_modules_dependencies(self, modules, depth=None):
+        self.ensure_one()
+        dependency_graph, _ = self._dependency_graph()
+        return self.search_modules_graph(modules, dependency_graph, depth)
+
+    def _get_dependant_modules(self, modules, depth=None):
+        _, dependant_graph = self._dependency_graph()
+        return self.search_modules_graph(modules, dependant_graph, depth)
+
     def _local_pg_dropdb(self, dbname):
         msg = ''
         try:
@@ -1249,13 +1288,17 @@ def _modified_files(self, commit_link_links=None):
                 modified_files[commit_link] = files
         return modified_files
 
-    def _modified_modules(self, commit_link_links=None):
+    def _modified_modules(self, commit_link_links=None, defaults=None):
         modified_files = self._modified_files(commit_link_links)
         modified_modules = set()
         for commit_link, files in modified_files.items():
             commit = commit_link.commit_id
             for file in files:
-                modified_modules.add(commit.repo_id._get_module(file))
+                module = commit.repo_id._get_module(file)
+                if module:
+                    modified_modules.add(module)
+                elif defaults:
+                    modified_modules |= set(defaults)
         return modified_modules
 
     def _get_upgrade_path(self):