diff --git a/runbot/controllers/frontend.py b/runbot/controllers/frontend.py index fb55664ed..aa95a1416 100644 --- a/runbot/controllers/frontend.py +++ b/runbot/controllers/frontend.py @@ -306,6 +306,7 @@ def build(self, build_id, search=None, from_batch=None, **post): if not build.exists(): return request.not_found() siblings = (build.parent_id.children_ids if build.parent_id else from_batch.slot_ids.build_id if from_batch else build).sorted('id') + # TODO FIXME for linked builds. Sibling may depends on the context of the batch context = { 'build': build, 'from_batch': from_batch, diff --git a/runbot/models/build.py b/runbot/models/build.py index 12ecc007d..fd28f7f6e 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -291,9 +291,9 @@ class BuildResult(models.Model): priority_level = fields.Integer('Priority', related='create_batch_id.priority_level', store=True, index=True) # state machine - global_state = fields.Selection(make_selection(state_order), string='Status', compute='_compute_global_state', store=True, recursive=True) + global_state = fields.Selection(make_selection(state_order), string='Status', default='pending') local_state = fields.Selection(make_selection(state_order), string='Build Status', default='pending', required=True, index=True) - global_result = fields.Selection(make_selection(result_order), string='Result', compute='_compute_global_result', store=True, recursive=True) + global_result = fields.Selection(make_selection(result_order), string='Result', default='ok') local_result = fields.Selection(make_selection(result_order), string='Build Result', default='ok') requested_action = fields.Selection([('wake_up', 'To wake up'), ('deathrow', 'To kill')], string='Action requested', index=True) @@ -345,10 +345,15 @@ class BuildResult(models.Model): string='Build type') parent_id = fields.Many2one('runbot.build', 'Parent Build', index=True) + parent_link_ids = fields.One2many('runbot.build.link', 'child_id', string='Used in links') + linked_parent_ids = fields.Many2many('runbot.build', compute='_compute_linked_parent_ids', string='All parent builds') + + child_link_ids = fields.One2many('runbot.build.link', 'parent_id', string='Child links') + linked_children_ids = fields.Many2many('runbot.build', compute='_compute_linked_child_ids', string='All child builds') + parent_path = fields.Char('Parent path', index=True) top_parent = fields.Many2one('runbot.build', compute='_compute_top_parent') ancestors = fields.Many2many('runbot.build', compute='_compute_ancestors') - # should we add a has children stored boolean? children_ids = fields.One2many('runbot.build', 'parent_id') # config of top_build is inherithed from params, but subbuild will have different configs @@ -371,6 +376,9 @@ class BuildResult(models.Model): access_token = fields.Char('Token', default=lambda self: uuid.uuid4().hex) + _global_state_idx = models.Index("(global_state) WHERE global_state != 'done'") + + @api.depends('description', 'params_id.config_id') def _compute_display_name(self): for build in self: @@ -382,13 +390,22 @@ def _compute_host_id(self): for record in self: record.host_id = get_host(record.host) - @api.depends('children_ids.global_state', 'local_state') - def _compute_global_state(self): + def _compute_linked_parent_ids(self): + for build in self: + build.linked_parent_ids = build.parent_link_ids.parent_id + + def _compute_linked_child_ids(self): + for build in self: + build.linked_child_ids = build.child_link_ids.child_id + + def _update_global_state(self): for record in self: waiting_score = record._get_state_score('waiting') - children_ids = [child for child in record.children_ids if not child.orphan_result] - if record._get_state_score(record.local_state) > waiting_score and children_ids: # if finish, check children - children_state = record._get_youngest_state([child.global_state for child in children_ids]) + children_ids = [child.id for child in record.children_ids if not child.orphan_result] + children_ids += [link.child_id.id for link in record.child_link_ids if not link.orphan_result] + children = self.browse(children_ids) + if record._get_state_score(record.local_state) > waiting_score and children: # if finish, check children + children_state = record._get_youngest_state([child.global_state for child in children]) if record._get_state_score(children_state) > waiting_score: record.global_state = record.local_state else: @@ -396,6 +413,23 @@ def _compute_global_state(self): else: record.global_state = record.local_state + def _update_global_result(self): + for record in self: + if record.local_result and record._get_result_score(record.local_result) >= record._get_result_score('ko'): + record.global_result = record.local_result + else: + children_ids = [child.id for child in record.children_ids if not child.orphan_result] + children_ids += [link.child_id.id for link in record.child_link_ids if not link.orphan_result] + children = self.browse(children_ids) + if children: + children_result = record._get_worst_result([child.global_result for child in children], max_res='ko') + if record.local_result: + record.global_result = record._get_worst_result([record.local_result, children_result]) + else: + record.global_result = children_result + else: + record.global_result = record.local_result + @api.depends('message_ids') def _compute_to_kill(self): for record in self: @@ -432,22 +466,6 @@ def _get_youngest_state(self, states): def _get_state_score(self, result): return state_order.index(result) - @api.depends('children_ids.global_result', 'local_result', 'children_ids.orphan_result') - def _compute_global_result(self): - for record in self: - if record.local_result and record._get_result_score(record.local_result) >= record._get_result_score('ko'): - record.global_result = record.local_result - else: - children_ids = [child for child in record.children_ids if not child.orphan_result] - if children_ids: - children_result = record._get_worst_result([child.global_result for child in children_ids], max_res='ko') - if record.local_result: - record.global_result = record._get_worst_result([record.local_result, children_result]) - else: - record.global_result = children_result - else: - record.global_result = record.local_result - @api.depends('build_error_link_ids') def _compute_build_error_ids(self): for record in self: @@ -493,6 +511,17 @@ def copy_data(self, default=None): }) return [values] + def create(self, vals): + records = super().create(vals) + parents = records.parent_id + # it doesn't make sense to create a build in another state than pending, and ok result, + # so we can assume that we don't need to update the created records globals, + # but we need to update the parents global state and result as the new build can impact them + if parents: + parents._update_global_state() + parents._update_global_result() + return records + def write(self, values): # some validation to ensure db consistency if 'local_state' in values: @@ -506,26 +535,37 @@ def write(self, values): values.pop('local_result') else: raise ValidationError('Local result cannot be set to a less critical level') - init_global_results = self.mapped('global_result') - init_global_states = self.mapped('global_state') - init_local_states = self.mapped('local_state') + if "global_result" in values: + init_global_results = self.mapped('global_result') + if "global_state" in values: + init_global_states = self.mapped('global_state') + if "local_state" in values: + init_local_states = self.mapped('local_state') res = super(BuildResult, self).write(values) - for init_global_result, build in zip(init_global_results, self): - if init_global_result != build.global_result: - build._github_status() + if 'local_state' in values: + self._update_global_state() + if 'local_result' in values: + self._update_global_result() + + if "global_result" in values: + for init_global_result, build in zip(init_global_results, self): + if init_global_result != build.global_result: + build._github_status() - for init_local_state, build in zip(init_local_states, self): - if init_local_state not in ('done', 'running') and build.local_state in ('done', 'running'): - build.build_end = now() + if "local_state" in values: + for init_local_state, build in zip(init_local_states, self): + if init_local_state not in ('done', 'running') and build.local_state in ('done', 'running'): + build.build_end = now() - for init_global_state, build in zip(init_global_states, self): - if init_global_state not in ('done', 'running') and build.global_state in ('done', 'running'): - build._github_status() + if "global_state" in values: + for init_global_state, build in zip(init_global_states, self): + if init_global_state not in ('done', 'running') and build.global_state in ('done', 'running'): + build._github_status() return res - def _add_child(self, param_values, orphan=False, description=False, additionnal_commit_links=False): + def _add_child(self, param_values, orphan=False, description=False, additionnal_commit_links=False, link=False): build_values = {key: value for key, value in param_values.items() if key not in self.params_id._fields} param_values = {key: value for key, value in param_values.items() if key in self.params_id._fields} @@ -538,9 +578,10 @@ def _add_child(self, param_values, orphan=False, description=False, additionnal_ commit_link_ids |= additionnal_commit_links param_values['commit_link_ids'] = commit_link_ids - return self.create({ - 'params_id': self.params_id.copy(param_values).id, - 'parent_id': self.id, + params = self.params_id.copy(param_values) + + build_values = { + 'params_id': params.id, 'build_type': self.build_type, 'priority_level': self.priority_level, 'description': description, @@ -548,7 +589,29 @@ def _add_child(self, param_values, orphan=False, description=False, additionnal_ 'keep_host': self.keep_host, 'host': self.host if self.keep_host else False, **build_values, - }) + } + + if link: + existing_builds = params.build_ids + if self.keep_host: + existing_builds = existing_builds.filtered(lambda b: b.host == self.host) + if existing_builds: + build = existing_builds.sorted('id')[-1] + build.killable = False + + if build: + build = self.create(build_values) + + self.env['runbot.build.link'].create({ + 'parent_id': self.id, + 'child_id': link, + }) + return build + else: + return self.create({ + 'parent_id': self.id, + **build_values, + }) @api.depends('params_id.version_id.name') def _compute_dest(self): @@ -611,6 +674,7 @@ def _compute_last_update(self): def _compute_load_time(self): for build in self: build.load_time = sum([build.build_time] + [child.load_time for child in build.children_ids]) + # TODO check, we don't count linked childrent in load time to avoid duplicate, but we need to ensure we take them into account @api.depends('job_start') def _compute_build_age(self): @@ -640,8 +704,17 @@ def _rebuild(self, message=None): self.orphan_result = True new_build = self.create(values) + + if self.parent_link_ids: # TODO check, should we forbid rebuild in some cases if we have multiple parents? Or just link to an active parent? + for link in self.parent_link_ids: + link.orphan_result = True + self.env['runbot.build.link'].create({ + 'parent_id': link.parent_id.id, + 'child_id': new_build.id, + }) + if self.parent_id: - new_build._github_status() + new_build._github_status() # not sure this is needed since creating a child should trigger an update of parent global state. user = self.env.user new_build._log('rebuild', 'Rebuild initiated by %s%s' % (user.name, (' :%s' % message) if message else '')) @@ -1619,3 +1692,13 @@ def action_view_build_errors(self): "name": "Build errors", "view_mode": "list,form" } + +class BuildLink(models.Model): + _name = 'runbot.build.link' + _description = 'Runbot Build Link' + _order = 'id desc' + + parent_id = fields.Many2one('runbot.build', string='Parent Build', required=True, ondelete='cascade') + child_id = fields.Many2one('runbot.build', string='Child Build', required=True, ondelete='cascade') + params_id = fields.Many2one('runbot.build.params', string='Params', related='child_id.params_id', store=True) + orphan_result = fields.Boolean(string='Orphan Result', help='If set, the result of the child build will not be taken into account for the parent build result') diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index 9c3a5ccee..c994eb0ec 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -1013,7 +1013,7 @@ def get_reference_builds_for_versions(versions): 'version_id': target.params_id.version_id.id, 'trigger_id': None, 'dockerfile_id': target.params_id.dockerfile_id.id, - }) + }, link=True) source_description = source.params_id.version_id.name target_description = target.params_id.version_id.name if source in build.create_batch_id.slot_ids.build_id: @@ -1026,12 +1026,12 @@ def get_reference_builds_for_versions(versions): db.name, ) - if self.allow_similar_build_quick_result: - existing_done_build = next((build for build in child.params_id.build_ids.sorted('id') if build.global_state == 'done' and build.local_result not in ('skipped', 'killed')), None) - if existing_done_build: - child._log('', 'A similar [build](%s) has been found, marking as done directly', existing_done_build.build_url, log_type='markdown') - child.local_state = 'done' - child.local_result = existing_done_build.local_result + #if self.allow_similar_build_quick_result: + # existing_done_build = next((build for build in child.params_id.build_ids.sorted('id') if build.global_state == 'done' and build.local_result not in ('skipped', 'killed')), None) + # if existing_done_build: + # child._log('', 'A similar [build](%s) has been found, marking as done directly', existing_done_build.build_url, log_type='markdown') + # child.local_state = 'done' + # child.local_result = existing_done_build.local_result def _filter_upgrade_database(self, dbs, pattern): pat_list = pattern.split(',') if pattern else [] diff --git a/runbot/models/runbot.py b/runbot/models/runbot.py index b470e1df9..43b609cbe 100644 --- a/runbot/models/runbot.py +++ b/runbot/models/runbot.py @@ -72,6 +72,10 @@ def _scheduler(self, host): self._commit() processed += self._assign_pending_builds(host, host.nb_worker and host.nb_worker + 1, [('build_type', '=', 'priority')]) self._commit() + for build in host._get_builds([('global_state', 'in', ['pending', 'testing', 'waiting', 'running'])]): + build._update_global_state() + build._update_global_result() + self._commit() self._gc_running(host) self._commit() self._reload_nginx() @@ -127,9 +131,22 @@ def _gc_testing(self, host): if available_slots > 0 or nb_pending == 0: return + killable_build = [] + for build in testing_builds: if build.top_parent.killable: - build.top_parent._ask_kill(message='Build automatically killed, new build found.') + killable_build.append(build) + continue + if not build.parent_id and build.parent_ids: + killable_build.append(build) + continue + + for build in killable_build: + build._log('_ask_kill', "Build automatically killed, new build found.") + if build.local_state == 'pending': + build._skip() + elif build.local_state in ['testing', 'running']: + build.requested_action = 'deathrow' def _allocate_builds(self, host, nb_slots, domain=None): if nb_slots <= 0: diff --git a/runbot/tests/test_build.py b/runbot/tests/test_build.py index 3fc8cef24..776dfaf2a 100644 --- a/runbot/tests/test_build.py +++ b/runbot/tests/test_build.py @@ -618,6 +618,11 @@ def test_children(self): build1_1_2.local_state = 'done' + # simulate scheduler ran + build1_1._update_global_state() + build1_2._update_global_state() + build1._update_global_state() + self.assertEqual('done', build1.global_state) self.assertEqual('done', build1_1.global_state) self.assertEqual('done', build1_2.global_state) @@ -647,6 +652,14 @@ def test_rebuild_sub_sub_build(self): build1_1_1.local_result = 'ko' build1_1_1.local_state = 'done' + + + # simulate scheduler ran + build1_1._update_global_state() + build1._update_global_state() + build1_1._update_global_result() + build1._update_global_result() + self.assertEqual('done', build1.global_state) self.assertEqual('done', build1_1.global_state) self.assertEqual('done', build1_1_1.global_state) @@ -660,6 +673,12 @@ def test_rebuild_sub_sub_build(self): }) build1_1_1.orphan_result = True + # simulate scheduler ran + build1_1._update_global_state() + build1._update_global_state() + build1_1._update_global_result() + build1._update_global_result() + self.assertEqual('ok', build1.global_result) self.assertEqual('ok', build1_1.global_result) self.assertEqual('ko', build1_1_1.global_result) @@ -671,6 +690,13 @@ def test_rebuild_sub_sub_build(self): rebuild1_1_1.local_result = 'ok' rebuild1_1_1.local_state = 'done' + + # simulate scheduler ran + build1_1._update_global_state() + build1._update_global_state() + build1_1._update_global_result() + build1._update_global_result() + self.assertEqual('ok', build1.global_result) self.assertEqual('ok', build1_1.global_result) self.assertEqual('ko', build1_1_1.global_result) @@ -812,11 +838,13 @@ def github_status(build): with patch('odoo.addons.runbot.models.build.BuildResult._github_status', github_status): self.callcount = 0 self.build.local_state = 'testing' + self.assertEqual(self.build.global_state, 'testing') self.assertEqual(self.callcount, 0, "_github_status shouldn't have been called") self.callcount = 0 self.build.local_state = 'running' + self.assertEqual(self.build.global_state, 'running') self.assertEqual(self.callcount, 1, "_github_status should have been called") diff --git a/runbot/tests/test_build_config_step.py b/runbot/tests/test_build_config_step.py index 2e52171a3..a5889e061 100644 --- a/runbot/tests/test_build_config_step.py +++ b/runbot/tests/test_build_config_step.py @@ -315,6 +315,8 @@ def test_config_step_create_results(self): child_build.local_result = 'ko' self.assertEqual(child_build.global_result, 'ko') + # simulate sheduler ran + self.parent_build._update_global_result() self.assertEqual(self.parent_build.global_result, 'ko') diff --git a/runbot/tests/test_upgrade.py b/runbot/tests/test_upgrade.py index dbd7bb48d..22a4d256a 100644 --- a/runbot/tests/test_upgrade.py +++ b/runbot/tests/test_upgrade.py @@ -576,6 +576,10 @@ def docker_run_upgrade(cmd, *args, ro_volumes=False, **kwargs): self.assertEqual(current_build.global_state, 'done') # self.assertEqual(current_build.global_result, 'ok') + # simulate scheduler ran + from_version_builds._update_global_state() + to_version_builds._update_global_state() + self.assertEqual(from_version_builds.mapped('global_state'), ['done'] * 10) self.assertEqual(to_version_builds.mapped('global_state'), ['done'] * 5)