[IMP] runbot: limit the size of files to open

Author: d-fence

runbot/common.py

@@ -19,6 +19,8 @@
 from odoo.fields import Domain
 from odoo.tools.misc import DEFAULT_SERVER_DATETIME_FORMAT, file_open, html_escape, OrderedSet
 
+DEFAULT_MAX_FILE_SIZE = 500 * 1024 * 1024
+
 _logger = logging.getLogger(__name__)
 
 dest_reg = re.compile(r'^\d{5,}-.+$')

runbot/models/build.py

@@ -17,7 +17,7 @@
 from psycopg2 import sql
 from psycopg2.extensions import TransactionRollbackError
 
-from ..common import dt2time, now, grep, local_pgadmin_cursor, dest_reg, os, list_local_dbs, pseudo_markdown, RunbotException, findall, sanitize, markdown_escape, tail
+from ..common import dt2time, now, grep, local_pgadmin_cursor, dest_reg, os, list_local_dbs, pseudo_markdown, RunbotException, findall, sanitize, markdown_escape, tail, DEFAULT_MAX_FILE_SIZE
 from ..container import docker_stop, docker_state, Command, docker_run, docker_pull
 from ..fields import JsonDictField
 
@@ -1409,6 +1409,10 @@ def _is_file(self, file, mode='r'):
 
     def _read_file(self, file, mode='r'):
         file_path = self._path(file)
+        max_log_file_size = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_max_log_size', DEFAULT_MAX_FILE_SIZE)
+        if os.path.getsize(file_path) > max_log_file_size:
+            self._log('readfile', f"File size exceeds {max_log_file_size} limit", level="ERROR")
+            return False
         try:
             with file_open(file_path, mode) as f:
                 return f.read()

runbot/models/build_config.py

@@ -8,7 +8,7 @@
 import shlex
 import time
 from unidiff import PatchSet
-from ..common import now, grep, time2str, rfind, s2human, os, RunbotException, ReProxy, markdown_escape
+from ..common import now, grep, time2str, rfind, s2human, os, RunbotException, ReProxy, markdown_escape, DEFAULT_MAX_FILE_SIZE
 from ..container import docker_get_gateway_ip, Command
 from odoo import models, fields, api, tools
 from odoo.exceptions import UserError, ValidationError
@@ -1298,6 +1298,10 @@ def _check_log(self, build):
         if not os.path.isfile(log_path):
             build._log('_make_tests_results', "Log file not found at the end of test job", level="ERROR")
             return 'ko'
+        max_log_file_size = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_max_log_size', DEFAULT_MAX_FILE_SIZE)
+        if os.path.getsize(log_path) > max_log_file_size:
+            build._log('_make_tests_results', f"Log file exceeds {max_log_file_size} limit", level="ERROR")
+            return 'ko'
         return 'ok'
 
     def _check_module_loaded(self, build):

runbot/models/build_stat_regex.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import logging
 
-from ..common import os
+from ..common import os, DEFAULT_MAX_FILE_SIZE
 import re
 
 from odoo import models, fields, api
@@ -53,6 +53,10 @@ def _find_in_file(self, file_path):
         """
         if not os.path.exists(file_path):
             return {}
+        max_log_file_size = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_max_log_size', DEFAULT_MAX_FILE_SIZE)
+        if os.path.getsize(file_path) > max_log_file_size:
+            _logger.warning("Log file '%s' exceeds %s limit", file_path, max_log_file_size)
+            return 'ko'
         stats_matches = {}
         with file_open(file_path, "r") as log_file:
             data = log_file.read()

runbot/tests/test_build_config_step.py

@@ -8,7 +8,7 @@
 from odoo.exceptions import UserError
 from odoo.addons.runbot.common import RunbotException
 from .common import RunbotCase
-from ..common import markdown_unescape
+from ..common import markdown_unescape, DEFAULT_MAX_FILE_SIZE
 
 
 class TestBuildConfigStepCommon(RunbotCase):
@@ -1253,6 +1253,7 @@ def _log(build, func, message, level='INFO', log_type='runbot', path='runbot'):
             self.logs.append((level, message))
 
         self.start_patcher('log_patcher', 'odoo.addons.runbot.models.build.BuildResult._log', new=_log)
+        self.start_patcher('get_size', 'odoo.addons.runbot.models.build_config.os.path.getsize', return_value=100)
 
         self.build = self.Build.create({
             'params_id': self.base_params.id,
@@ -1474,3 +1475,10 @@ def make_warn(build):
         mock_make_odoo_results.side_effect = make_warn
         config_step._make_results(build)
         self.assertEqual(build.local_result, 'warn')
+
+    def test_make_result_large_file(self):
+        self.patchers['get_size'].return_value = DEFAULT_MAX_FILE_SIZE * 2
+        self.config_step._make_results(self.build)
+        self.assertEqual(str(self.build.job_end), '1970-01-01 02:00:00')
+        self.assertIn(('ERROR', 'Log file exceeds %s limit' % DEFAULT_MAX_FILE_SIZE), self.logs)
+        self.assertEqual(self.build.local_result, 'ko')

runbot/tests/test_upgrade.py

@@ -19,6 +19,7 @@ def setUp(self):
 
     def upgrade_flow_setup(self):
         self.start_patcher('find_patcher', 'odoo.addons.runbot.common.find', 0)
+        self.start_patcher('get_size', 'odoo.addons.runbot.models.build_config.os.path.getsize', return_value=100)
         self.additionnal_setup()
 
         self.master_bundle = self.branch_odoo.bundle_id