[IMP] runbot: allow to link a child to multiple builds

runbot/controllers/frontend.py

@@ -306,6 +306,7 @@ def build(self, build_id, search=None, from_batch=None, **post):
         if not build.exists():
             return request.not_found()
         siblings = (build.parent_id.children_ids if build.parent_id else from_batch.slot_ids.build_id if from_batch else build).sorted('id')
+        # TODO FIXME for linked builds. Sibling may depends on the context of the batch
         context = {
             'build': build,
             'from_batch': from_batch,

runbot/models/build.py

@@ -291,9 +291,9 @@ class BuildResult(models.Model):
     priority_level = fields.Integer('Priority', related='create_batch_id.priority_level', store=True, index=True)
 
     # state machine
-    global_state = fields.Selection(make_selection(state_order), string='Status', compute='_compute_global_state', store=True, recursive=True)
+    global_state = fields.Selection(make_selection(state_order), string='Status', default='pending')
     local_state = fields.Selection(make_selection(state_order), string='Build Status', default='pending', required=True, index=True)
-    global_result = fields.Selection(make_selection(result_order), string='Result', compute='_compute_global_result', store=True, recursive=True)
+    global_result = fields.Selection(make_selection(result_order), string='Result', default='ok')
     local_result = fields.Selection(make_selection(result_order), string='Build Result', default='ok')
 
     requested_action = fields.Selection([('wake_up', 'To wake up'), ('deathrow', 'To kill')], string='Action requested', index=True)
@@ -345,10 +345,15 @@ class BuildResult(models.Model):
                                   string='Build type')
 
     parent_id = fields.Many2one('runbot.build', 'Parent Build', index=True)
+    parent_link_ids = fields.One2many('runbot.build.link', 'child_id', string='Used in links')
+    linked_parent_ids = fields.Many2many('runbot.build', compute='_compute_linked_parent_ids', string='All parent builds')
+
+    child_link_ids = fields.One2many('runbot.build.link', 'parent_id', string='Child links')
+    linked_children_ids = fields.Many2many('runbot.build', compute='_compute_linked_child_ids', string='All child builds')
+
     parent_path = fields.Char('Parent path', index=True)
     top_parent = fields.Many2one('runbot.build', compute='_compute_top_parent')
     ancestors = fields.Many2many('runbot.build', compute='_compute_ancestors')
-    # should we add a has children stored boolean?
     children_ids = fields.One2many('runbot.build', 'parent_id')
 
     # config of top_build is inherithed from params, but subbuild will have different configs
@@ -371,6 +376,9 @@ class BuildResult(models.Model):
 
     access_token = fields.Char('Token', default=lambda self: uuid.uuid4().hex)
 
+    _global_state_idx = models.Index("(global_state) WHERE global_state != 'done'")
+
+
     @api.depends('description', 'params_id.config_id')
     def _compute_display_name(self):
         for build in self:
@@ -382,20 +390,46 @@ def _compute_host_id(self):
         for record in self:
             record.host_id = get_host(record.host)
 
-    @api.depends('children_ids.global_state', 'local_state')
-    def _compute_global_state(self):
+    def _compute_linked_parent_ids(self):
+        for build in self:
+            build.linked_parent_ids = build.parent_link_ids.parent_id
+
+    def _compute_linked_child_ids(self):
+        for build in self:
+            build.linked_child_ids = build.child_link_ids.child_id
+
+    def _update_global_state(self):
         for record in self:
             waiting_score = record._get_state_score('waiting')
-            children_ids = [child for child in record.children_ids if not child.orphan_result]
-            if record._get_state_score(record.local_state) > waiting_score and children_ids:  # if finish, check children
-                children_state = record._get_youngest_state([child.global_state for child in children_ids])
+            children_ids = [child.id for child in record.children_ids if not child.orphan_result]
+            children_ids += [link.child_id.id for link in record.child_link_ids if not link.orphan_result]
+            children = self.browse(children_ids)
+            if record._get_state_score(record.local_state) > waiting_score and children:  # if finish, check children
+                children_state = record._get_youngest_state([child.global_state for child in children])
                 if record._get_state_score(children_state) > waiting_score:
                     record.global_state = record.local_state
                 else:
                     record.global_state = 'waiting'
             else:
                 record.global_state = record.local_state
 
+    def _update_global_result(self):
+        for record in self:
+            if record.local_result and record._get_result_score(record.local_result) >= record._get_result_score('ko'):
+                record.global_result = record.local_result
+            else:
+                children_ids = [child.id for child in record.children_ids if not child.orphan_result]
+                children_ids += [link.child_id.id for link in record.child_link_ids if not link.orphan_result]
+                children = self.browse(children_ids)
+                if children:
+                    children_result = record._get_worst_result([child.global_result for child in children], max_res='ko')
+                    if record.local_result:
+                        record.global_result = record._get_worst_result([record.local_result, children_result])
+                    else:
+                        record.global_result = children_result
+                else:
+                    record.global_result = record.local_result
+
     @api.depends('message_ids')
     def _compute_to_kill(self):
         for record in self:
@@ -432,22 +466,6 @@ def _get_youngest_state(self, states):
     def _get_state_score(self, result):
         return state_order.index(result)
 
-    @api.depends('children_ids.global_result', 'local_result', 'children_ids.orphan_result')
-    def _compute_global_result(self):
-        for record in self:
-            if record.local_result and record._get_result_score(record.local_result) >= record._get_result_score('ko'):
-                record.global_result = record.local_result
-            else:
-                children_ids = [child for child in record.children_ids if not child.orphan_result]
-                if children_ids:
-                    children_result = record._get_worst_result([child.global_result for child in children_ids], max_res='ko')
-                    if record.local_result:
-                        record.global_result = record._get_worst_result([record.local_result, children_result])
-                    else:
-                        record.global_result = children_result
-                else:
-                    record.global_result = record.local_result
-
     @api.depends('build_error_link_ids')
     def _compute_build_error_ids(self):
         for record in self:
@@ -493,6 +511,17 @@ def copy_data(self, default=None):
         })
         return [values]
 
+    def create(self, vals):
+        records = super().create(vals)
+        parents = records.parent_id
+        # it doesn't make sense to create a build in another state than pending, and ok result,
+        # so we can assume that we don't need to update the created records globals,
+        # but we need to update the parents global state and result as the new build can impact them
+        if parents:
+            parents._update_global_state()
+            parents._update_global_result()
+        return records
+
     def write(self, values):
         # some validation to ensure db consistency
         if 'local_state' in values:
@@ -506,26 +535,37 @@ def write(self, values):
                     values.pop('local_result')
                 else:
                     raise ValidationError('Local result cannot be set to a less critical level')
-        init_global_results = self.mapped('global_result')
-        init_global_states = self.mapped('global_state')
-        init_local_states = self.mapped('local_state')
+        if "global_result" in values:
+            init_global_results = self.mapped('global_result')
+        if "global_state" in values:
+            init_global_states = self.mapped('global_state')
+        if "local_state" in values:
+            init_local_states = self.mapped('local_state')
 
         res = super(BuildResult, self).write(values)
-        for init_global_result, build in zip(init_global_results, self):
-            if init_global_result != build.global_result:
-                build._github_status()
+        if 'local_state' in values:
+            self._update_global_state()
+        if 'local_result' in values:
+            self._update_global_result()
+
+        if "global_result" in values:
+            for init_global_result, build in zip(init_global_results, self):
+                if init_global_result != build.global_result:
+                    build._github_status()
 
-        for init_local_state, build in zip(init_local_states, self):
-            if init_local_state not in ('done', 'running') and build.local_state in ('done', 'running'):
-                build.build_end = now()
+        if "local_state" in values:
+            for init_local_state, build in zip(init_local_states, self):
+                if init_local_state not in ('done', 'running') and build.local_state in ('done', 'running'):
+                    build.build_end = now()
 
-        for init_global_state, build in zip(init_global_states, self):
-            if init_global_state not in ('done', 'running') and build.global_state in ('done', 'running'):
-                build._github_status()
+        if "global_state" in values:
+            for init_global_state, build in zip(init_global_states, self):
+                if init_global_state not in ('done', 'running') and build.global_state in ('done', 'running'):
+                    build._github_status()
 
         return res
 
-    def _add_child(self, param_values, orphan=False, description=False, additionnal_commit_links=False):
+    def _add_child(self, param_values, orphan=False, description=False, additionnal_commit_links=False, link=False):
         build_values = {key: value for key, value in param_values.items() if key not in self.params_id._fields}
         param_values = {key: value for key, value in param_values.items() if key in self.params_id._fields}
 
@@ -538,17 +578,40 @@ def _add_child(self, param_values, orphan=False, description=False, additionnal_
             commit_link_ids |= additionnal_commit_links
             param_values['commit_link_ids'] = commit_link_ids
 
-        return self.create({
-            'params_id': self.params_id.copy(param_values).id,
-            'parent_id': self.id,
+        params = self.params_id.copy(param_values)
+
+        build_values = {
+            'params_id': params.id,
             'build_type': self.build_type,
             'priority_level': self.priority_level,
             'description': description,
             'orphan_result': orphan,
             'keep_host': self.keep_host,
             'host': self.host if self.keep_host else False,
             **build_values,
-        })
+        }
+
+        if link:
+            existing_builds = params.build_ids
+            if self.keep_host:
+                existing_builds = existing_builds.filtered(lambda b: b.host == self.host)
+            if existing_builds:
+                build = existing_builds.sorted('id')[-1]
+                build.killable = False
+
+            if build:
+                build = self.create(build_values)
+
+            self.env['runbot.build.link'].create({
+                'parent_id': self.id,
+                'child_id': link,
+            })
+            return build
+        else:
+            return self.create({
+                'parent_id': self.id,
+                **build_values,
+            })
 
     @api.depends('params_id.version_id.name')
     def _compute_dest(self):
@@ -611,6 +674,7 @@ def _compute_last_update(self):
     def _compute_load_time(self):
         for build in self:
             build.load_time = sum([build.build_time] + [child.load_time for child in build.children_ids])
+            # TODO check, we don't count linked childrent in load time to avoid duplicate, but we need to ensure we take them into account
 
     @api.depends('job_start')
     def _compute_build_age(self):
@@ -640,8 +704,17 @@ def _rebuild(self, message=None):
             self.orphan_result = True
 
         new_build = self.create(values)
+
+        if self.parent_link_ids:  # TODO check, should we forbid rebuild in some cases if we have multiple parents? Or just link to an active parent?
+            for link in self.parent_link_ids:
+                link.orphan_result = True
+                self.env['runbot.build.link'].create({
+                    'parent_id': link.parent_id.id,
+                    'child_id': new_build.id,
+                })
+
         if self.parent_id:
-            new_build._github_status()
+            new_build._github_status()  # not sure this is needed since creating a child should trigger an update of parent global state.
         user = self.env.user
         new_build._log('rebuild', 'Rebuild initiated by %s%s' % (user.name, (' :%s' % message) if message else ''))
 
@@ -1619,3 +1692,13 @@ def action_view_build_errors(self):
             "name": "Build errors",
             "view_mode": "list,form"
         }
+
+class BuildLink(models.Model):
+    _name = 'runbot.build.link'
+    _description = 'Runbot Build Link'
+    _order = 'id desc'
+
+    parent_id = fields.Many2one('runbot.build', string='Parent Build', required=True, ondelete='cascade')
+    child_id = fields.Many2one('runbot.build', string='Child Build', required=True, ondelete='cascade')
+    params_id = fields.Many2one('runbot.build.params', string='Params', related='child_id.params_id', store=True)
+    orphan_result = fields.Boolean(string='Orphan Result', help='If set, the result of the child build will not be taken into account for the parent build result')

runbot/models/build_config.py

@@ -1013,7 +1013,7 @@ def get_reference_builds_for_versions(versions):
                         'version_id': target.params_id.version_id.id,
                         'trigger_id': None,
                         'dockerfile_id': target.params_id.dockerfile_id.id,
-                    })
+                    }, link=True)
                     source_description = source.params_id.version_id.name
                     target_description = target.params_id.version_id.name
                     if source in build.create_batch_id.slot_ids.build_id:
@@ -1026,12 +1026,12 @@ def get_reference_builds_for_versions(versions):
                         db.name,
                     )
 
-                    if self.allow_similar_build_quick_result:
-                        existing_done_build = next((build for build in child.params_id.build_ids.sorted('id') if build.global_state == 'done' and build.local_result not in ('skipped', 'killed')), None)
-                        if existing_done_build:
-                            child._log('', 'A similar [build](%s) has been found, marking as done directly', existing_done_build.build_url, log_type='markdown')
-                            child.local_state = 'done'
-                            child.local_result = existing_done_build.local_result
+                    #if self.allow_similar_build_quick_result:
+                    #    existing_done_build = next((build for build in child.params_id.build_ids.sorted('id') if build.global_state == 'done' and build.local_result not in ('skipped', 'killed')), None)
+                    #    if existing_done_build:
+                    #        child._log('', 'A similar [build](%s) has been found, marking as done directly', existing_done_build.build_url, log_type='markdown')
+                    #        child.local_state = 'done'
+                    #        child.local_result = existing_done_build.local_result
 
     def _filter_upgrade_database(self, dbs, pattern):
         pat_list = pattern.split(',') if pattern else []

runbot/models/runbot.py

@@ -72,6 +72,10 @@ def _scheduler(self, host):
         self._commit()
         processed += self._assign_pending_builds(host, host.nb_worker and host.nb_worker + 1, [('build_type', '=', 'priority')])
         self._commit()
+        for build in host._get_builds([('global_state', 'in', ['pending', 'testing', 'waiting', 'running'])]):
+            build._update_global_state()
+            build._update_global_result()
+            self._commit()
         self._gc_running(host)
         self._commit()
         self._reload_nginx()
@@ -127,9 +131,22 @@ def _gc_testing(self, host):
         if available_slots > 0 or nb_pending == 0:
             return
 
+        killable_build = []
+
         for build in testing_builds:
             if build.top_parent.killable:
-                build.top_parent._ask_kill(message='Build automatically killed, new build found.')
+                killable_build.append(build)
+                continue
+            if not build.parent_id and build.parent_ids:
+                killable_build.append(build)
+                continue
+
+        for build in killable_build:
+            build._log('_ask_kill', "Build automatically killed, new build found.")
+            if build.local_state == 'pending':
+                build._skip()
+            elif build.local_state in ['testing', 'running']:
+                build.requested_action = 'deathrow'
 
     def _allocate_builds(self, host, nb_slots, domain=None):
         if nb_slots <= 0: