[IMP] runbot: allow to send status in a service

Currently the leader and status service are synchronous, meaning that if
either fetching a repo or sending a status is slow, the other is
impacted.

In some cases, a fetch may take a verry long time pausing the status
updates.
In other cases, github is slow to answer making the satus updates slow
taking some times multiple minutes to be sent.
Moreover, during the send of a status batch, new status could already
exists making the sent status outdated.

To solve these issues, we introduce a status service that will be
responsible for sending the status updates. This is optional and by
default the leader will still send the status updates.

Additionnally, we only send status on build older than 2 minutes to
avoid sending status that could be quickly outdated (style, cla,
instantly failling builds).

Author: Xavier-Do

runbot/models/commit.py

@@ -262,7 +262,7 @@ class CommitStatus(models.Model):
     to_process = fields.Boolean('Status was not processed yet', index=True)
 
     def _send_to_process(self):
-        commits_status = self.search([('to_process', '=', True)], order='create_date DESC, id DESC')
+        commits_status = self.search([('to_process', '=', True), ('build_id.create_date', '<', datetime.datetime.now() - datetime.timedelta(minutes=2))], order='create_date DESC, id DESC')
         if commits_status:
             _logger.info('Sending %s commit status', len(commits_status))
             commits_status._send()

runbot/models/host.py

@@ -49,6 +49,7 @@ class Host(models.Model):
     is_leader = fields.Boolean('Is leader', help='This host is the leader of the cluster', default=False)
     is_builder = fields.Boolean('Is builder', help='This host is a builder of the cluster', default=True)
     is_registry = fields.Boolean('Is docker registry', help='This host is a docker regisrty', default=False)
+    send_status = fields.Boolean('Send status', help='If leader, this host will send status updates, disable to use the status service', default=True)
 
     use_remote_docker_registry = fields.Boolean('Use remote Docker Registry', default=False, help="Use docker registry for pulling images")
     docker_registry_url = fields.Char('Registry Url', help="Override global registry URL for this host.")

runbot/models/runbot.py

@@ -271,8 +271,6 @@ def _fetch_loop_turn(self, host, pull_info_failures, default_sleep=1):
                         self._commit()
             self._commit()
 
-            self.env['runbot.commit.status']._send_to_process()
-            self._commit()
 
             # cleanup old pull_info_failures
             for pr_number, t in pull_info_failures.copy().items():

runbot/views/host_views.xml

@@ -13,6 +13,7 @@
                         <field name="active"/>
                         <field name="use_remote_docker_registry"/>
                         <field name="is_leader"/>
+                        <field name="send_status" invisible="not is_leader"/>
                         <field name="is_builder"/>
                         <field name="is_registry"/>
                         <field name="docker_registry_url"/>
@@ -58,7 +59,6 @@
                 <field name="use_remote_docker_registry" widget="boolean_toggle"/>
                 <field name="nb_worker"/>
                 <field name="nb_run_slot"/>
-
                 <field name="is_leader"/>
                 <field name="is_builder"/>
                 <field name="is_registry"/>

runbot_builder/leader.py

@@ -27,6 +27,11 @@ def loop_turn(self):
             self.env.cr.commit()
             self.git_gc()
             self.env.cr.commit()
+
+        if self.host.send_status:
+            self.env['runbot.commit.status']._send_to_process()
+            self.env.cr.commit()
+
         return self.env['runbot.runbot']._fetch_loop_turn(self.host, self.pull_info_failures)
 
 

runbot_builder/status.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python3
+from tools import RunbotClient, run
+
+
+class StatusClient(RunbotClient):
+
+    def loop_turn(self):
+        self.env['runbot.commit.status']._send_to_process()
+        self.env.cr.commit()
+        return 5
+
+
+if __name__ == '__main__':
+    run(StatusClient)