From 399c75e391b2e74e9a619c6a7700e8926426a47d Mon Sep 17 00:00:00 2001 From: Jonathan Christopher Date: Mon, 18 Jul 2022 21:28:48 +0800 Subject: [PATCH] Increase file cache chunk size from 16 KB to 1 MB To improve performance when connecting to worker instances over high latency network --- cms/db/filecacher.py | 6 +++--- .../unit_tests/server/file_middleware_test.py | 18 +++++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cms/db/filecacher.py b/cms/db/filecacher.py index 9bc25ce2a5..fee3ef1050 100644 --- a/cms/db/filecacher.py +++ b/cms/db/filecacher.py @@ -478,8 +478,7 @@ class FileCacher: """ # This value is very arbitrary, and in this case we want it to be a - # one-size-fits-all, since we use it for many conversions. It has - # been chosen arbitrarily based on performance tests on my machine. + # one-size-fits-all, since we use it for many conversions. # A few consideration on the value it could assume follow: # - The page size of large objects is LOBLKSIZE, which is BLCKSZ/4 # (BLCKSZ is the block size of the PostgreSQL database, which is @@ -489,7 +488,8 @@ class FileCacher: # - The `io' module defines a DEFAULT_BUFFER_SIZE constant, whose # value is 8192. # CHUNK_SIZE should be a multiple of these values. - CHUNK_SIZE = 16 * 1024 # 16 KiB + # Note that a too-small value can cause issues on high-latency networks. + CHUNK_SIZE = 1024 * 1024 # 1 MiB backend: FileCacherBackend def __init__(self, service: "Service | None" = None, path: str | None = None, null: bool = False): diff --git a/cmstestsuite/unit_tests/server/file_middleware_test.py b/cmstestsuite/unit_tests/server/file_middleware_test.py index 31523639be..577b328572 100755 --- a/cmstestsuite/unit_tests/server/file_middleware_test.py +++ b/cmstestsuite/unit_tests/server/file_middleware_test.py @@ -26,17 +26,20 @@ from werkzeug.wrappers import Response from werkzeug.wsgi import responder -from cms.db.filecacher import TombstoneError +from cms.db.filecacher import FileCacher, TombstoneError from cms.server.file_middleware import FileServerMiddleware from cmscommon.digest import bytes_digest +# Choose a size that is larger than FileCacher.CHUNK_SIZE. +TESTFILE_LEN = FileCacher.CHUNK_SIZE + 128 + + class TestFileByDigestMiddleware(unittest.TestCase): def setUp(self): - # Choose a size that is larger than FileCacher.CHUNK_SIZE. self.content = \ - bytes(random.getrandbits(8) for _ in range(17 * 1024)) + bytes(random.getrandbits(8) for _ in range(TESTFILE_LEN)) self.digest = bytes_digest(self.content) self.filename = "foobar.pdf" @@ -51,7 +54,7 @@ def setUp(self): self.provide_filename = True self.wsgi_app = \ - FileServerMiddleware(self.file_cacher,self.wrapped_wsgi_app) + FileServerMiddleware(self.file_cacher, self.wrapped_wsgi_app) self.environ_builder = EnvironBuilder("/some/url") self.client = Client(self.wsgi_app, Response) @@ -141,7 +144,7 @@ def test_range_request(self): self.assertEqual(response.content_range.units, "bytes") self.assertEqual(response.content_range.start, 256) self.assertEqual(response.content_range.stop, 768) - self.assertEqual(response.content_range.length, 17 * 1024) + self.assertEqual(response.content_range.length, TESTFILE_LEN) self.assertEqual(response.get_data(), self.content[256:768]) def test_range_request_end_overflows(self): @@ -151,12 +154,13 @@ def test_range_request_end_overflows(self): self.assertEqual(response.content_range.units, "bytes") self.assertEqual(response.content_range.start, 256) self.assertEqual(response.content_range.stop, 2048) - self.assertEqual(response.content_range.length, 17 * 1024) + self.assertEqual(response.content_range.length, TESTFILE_LEN) self.assertEqual(response.get_data(), self.content[256:2048]) def test_range_request_start_overflows(self): # Test a range that starts after the end of the file. - response = self.request(headers=[("Range", f"bytes={len(self.content) + 1}-")]) + response = self.request( + headers=[("Range", f"bytes={len(self.content) + 1}-")]) self.assertEqual(response.status_code, 416)