Merge pull request #118 from RedisLabsModules/add_rdb_preamble_flag

Add use rdb preamble flag
Fix caching in CI (use different keys for different platforms)

Author: alonre24

.github/workflows/ci.yml

@@ -2,6 +2,8 @@ name: CI
 
 on:
  push:
+   branches:
+     - master
  pull_request:
  schedule:
   - cron: "0 0 * * *"
@@ -10,7 +12,7 @@ jobs:
   build-ubuntu:
     name: Test on ${{ matrix.platform }} with Python ${{ matrix.python }}
     runs-on: ${{ matrix.platform }}
-    timeout-minutes: 10
+    timeout-minutes: 40
     strategy:
       matrix:
         platform: ['ubuntu-20.04', 'ubuntu-18.04', 'ubuntu-16.04']
@@ -37,23 +39,26 @@ jobs:
       with:
         path: ~/.cache/pip # This path is specific to Ubuntu
         # Look to see if there is a cache hit for the corresponding requirements file
-        key: ${{ runner.os }}-${{ matrix.python }}-pip-${{ hashFiles('requirements.txt') }}
+        key: ${{ matrix.platform }}-${{ matrix.python }}-pyproject.toml-${{ hashFiles('pyproject.toml') }}
         restore-keys: |
-          ${{ runner.os }}-${{ matrix.python }}-pip-
-          ${{ runner.os }}-${{ matrix.python }}-
+          ${{ matrix.platform }}-${{ matrix.python }}-pyproject.toml-${{hashFiles('pyproject.toml')}}}
 
     - name: Install Python dependencies
       run: |
         sudo apt-get install -y python-setuptools python3-setuptools
         pip install poetry
-        poetry install
+        poetry config virtualenvs.create false
+        poetry export --dev --without-hashes -o requirements-${{matrix.platform}}-${{matrix.python}}.txt
+        pip install -r requirements-${{matrix.platform}}-${{matrix.python}}.txt
 
     - name: Cache Redis
       id: cache-redis
       uses: actions/cache@v1
       with:
         path: redis
-        key: ${{ runner.os }}-redis
+        key: ${{ matrix.platform }}-${{ matrix.python }}-redis
+        restore-keys: |
+          ${{ matrix.platform }}-${{ matrix.python }}-redis
 
     - name: Install Redis Server test dependencies
       if: steps.cache-redis.outputs.cache-hit != 'true'
@@ -70,7 +75,7 @@ jobs:
         ./utils/gen-test-certs.sh
 
     - name: Unit Test with pytest
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
         TLS_CERT=./redis/tests/tls/redis.crt \
         TLS_KEY=./redis/tests/tls/redis.key \

README.md

@@ -39,7 +39,8 @@ usage: RLTest [-h] [--version] [--module MODULE] [--module-args MODULE_ARGS]
               [--env-only] [--clear-logs] [--log-dir LOG_DIR] [--use-slaves]
               [--shards-count SHARDS_COUNT] [--download-enterprise-binaries]
               [--proxy-binary-path PROXY_BINARY_PATH]
-              [--enterprise-lib-path ENTERPRISE_LIB_PATH] [-r] [--use-aof]
+              [--enterprise-lib-path ENTERPRISE_LIB_PATH] [-r]
+              [--use-aof] [--use-rdb-preamble]
               [--debug-print] [-V] [--vg-suppressions VG_SUPPRESSIONS]
               [--vg-options VG_OPTIONS] [--vg-no-leakcheck] [--vg-verbose]
               [--vg-no-fail-on-errors] [-i] [--debugger DEBUGGER] [-s]
@@ -120,6 +121,7 @@ optional arguments:
                         efforts, if the env can not be reused then it will be
                         taken down. (default: False)
   --use-aof             use aof instead of rdb (default: False)
+  --use-rdb-preamble    use rdb preamble when rewriting aof file (default: True)
   --debug-print         print debug messages (default: False)
   -V, --vg, --use-valgrind
                         running redis under valgrind (assuming valgrind is

RLTest/__main__.py

@@ -208,6 +208,10 @@ def do_normal_conn(self, line):
     '--use-aof', action='store_const', const=True, default=False,
     help='use aof instead of rdb')
 
+parser.add_argument(
+    '--use-rdb-preamble', action='store_const', const=True, default=True,
+    help='use rdb preamble when rewriting aof file')
+
 parser.add_argument(
     '--debug-print', action='store_const', const=True, default=False,
     help='print debug messages')

RLTest/env.py

@@ -109,6 +109,7 @@ class Defaults:
     re_libdir = None
     decode_responses = False
     use_aof = False
+    use_rdb_preamble = True
     use_TLS = False
     tls_cert_file = None
     tls_key_file = None
@@ -133,6 +134,7 @@ def getKwargs(self):
             'moduleArgs': self.module_args,
             'useSlaves': self.use_slaves,
             'useAof': self.use_aof,
+            'useRdbPreamble': self.use_rdb_preamble,
             'dbDirPath': self.logdir,
             'debugger': self.debugger,
             'noCatch': self.no_capture_output,
@@ -148,7 +150,8 @@ def getKwargs(self):
 
 class Env:
     RTestInstance = None
-    EnvCompareParams = ['module', 'moduleArgs', 'env', 'useSlaves', 'shardsCount', 'useAof', 'forceTcp']
+    EnvCompareParams = ['module', 'moduleArgs', 'env', 'useSlaves', 'shardsCount', 'useAof',
+                        'udsRdbPreamble', 'forceTcp']
 
     def compareEnvs(self, env):
         if env is None:
@@ -160,7 +163,7 @@ def compareEnvs(self, env):
 
     def __init__(self, testName=None, testDescription=None, module=None,
                  moduleArgs=None, env=None, useSlaves=None, shardsCount=None, decodeResponses=None,
-                 useAof=None, forceTcp=False, useTLS=False, tlsCertFile=None, tlsKeyFile=None,
+                 useAof=None, useRdbPreamble=None, forceTcp=False, useTLS=False, tlsCertFile=None, tlsKeyFile=None,
                  tlsCaCertFile=None, logDir=None, redisBinaryPath=None, dmcBinaryPath=None,
                  redisEnterpriseBinaryPath=None, noDefaultModuleArgs=False):
 
@@ -180,6 +183,7 @@ def __init__(self, testName=None, testDescription=None, module=None,
         self.shardsCount = shardsCount if shardsCount else Defaults.num_shards
         self.decodeResponses = decodeResponses if decodeResponses else Defaults.decode_responses
         self.useAof = useAof if useAof else Defaults.use_aof
+        self.useRdbPreamble = useRdbPreamble if useRdbPreamble is not None else Defaults.use_rdb_preamble
         self.verbose = Defaults.verbose
         self.logDir = logDir if logDir else Defaults.logdir
         self.forceTcp = forceTcp
@@ -278,6 +282,7 @@ def getEnvKwargs(self):
             'useSlaves': self.useSlaves,
             'decodeResponses': self.decodeResponses,
             'useAof': self.useAof,
+            'useRdbPreamble': self.useRdbPreamble,
             'dbDirPath': self.logDir,
             'debugger': Defaults.debugger,
             'noCatch': Defaults.no_capture_output,

RLTest/redis_std.py

@@ -20,7 +20,7 @@
 class StandardEnv(object):
     def __init__(self, redisBinaryPath, port=6379, modulePath=None, moduleArgs=None, outputFilesFormat=None,
                  dbDirPath=None, useSlaves=False, serverId=1, password=None, libPath=None, clusterEnabled=False, decodeResponses=False,
-                 useAof=False, debugger=None, noCatch=False, unix=False, verbose=False, useTLS=False, tlsCertFile=None,
+                 useAof=False, useRdbPreamble=True, debugger=None, noCatch=False, unix=False, verbose=False, useTLS=False, tlsCertFile=None,
                  tlsKeyFile=None, tlsCaCertFile=None):
         self.uuid = uuid.uuid4().hex
         self.redisBinaryPath = os.path.expanduser(redisBinaryPath) if redisBinaryPath.startswith(
@@ -35,6 +35,7 @@ def __init__(self, redisBinaryPath, port=6379, modulePath=None, moduleArgs=None,
         self.clusterEnabled = clusterEnabled
         self.decodeResponses = decodeResponses
         self.useAof = useAof
+        self.useRdbPreamble = useRdbPreamble
         self.envIsUp = False
         self.debugger = debugger
         self.noCatch = noCatch
@@ -187,9 +188,10 @@ def createCmdArgs(self, role):
             if self.useTLS:
                 cmdArgs += ['--tls-cluster', 'yes']
         if self.useAof:
-            cmdArgs += ['--appendonly yes']
+            cmdArgs += ['--appendonly', 'yes']
             cmdArgs += ['--appendfilename', self._getFileName(role, '.aof')]
-            cmdArgs += ['--aof-use-rdb-preamble', 'yes']
+            if not self.useRdbPreamble:
+                cmdArgs += ['--aof-use-rdb-preamble', 'no']
         if self.useTLS:
             cmdArgs += ['--tls-cert-file', self.getTLSCertFile()]
             cmdArgs += ['--tls-key-file', self.getTLSKeyFile()]

pyproject.toml

@@ -23,27 +23,27 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
+python = "^2.7,<2.8 || >= 3.5.0"
 distro = "^1.5.0"
 redis = "^3.5.3"
-redis-py-cluster = "^2.1.3"
+redis-py-cluster = "*"
 psutil = "^5.8.0"
+pytest-cov = "2.5"
 
 [tool.poetry.urls]
 repository = "https://github.com/RedisLabsModules/RLTest"
 
 [tool.poetry.scripts]
 RLTest = 'RLTest.__main__:main'
 
-
-
 [tool.poetry.dev-dependencies]
-codecov = "^2.1.11"
-flake8 = "^3.9.2"
+codecov = "*"
+flake8 = "*"
 rmtest = "^0.7.0"
 nose = "^1.3.7"
 ml2rt = "^0.2.0"
 pytest = "4.6"
-pytest-cov = "^2.12.1"
+
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"

tests/unit/test_redis_std.py

@@ -151,6 +151,18 @@ def test_create_cmd_args_modules_two_modules_array(self):
                 '--dbfilename', module_std_env._getFileName(role, '.rdb')] == cmd_args
         shutil.rmtree(directory_name)
 
+    def test_create_cmd_args_aof_without_rdb_preamble(self):
+        port = 8000
+        aof_std_env = StandardEnv(redisBinaryPath=REDIS_BINARY, outputFilesFormat='%s-test',
+                                  useAof=True, useRdbPreamble=False, port=8000)
+        role = 'master'
+        cmd_args = aof_std_env.createCmdArgs(role)
+        assert [REDIS_BINARY, '--port', '{}'.format(port), '--logfile',
+                aof_std_env._getFileName(role, '.log'), '--dbfilename',
+                aof_std_env._getFileName(role, '.rdb'), '--appendonly', 'yes',
+                '--appendfilename', aof_std_env._getFileName(role, '.aof'),
+                '--aof-use-rdb-preamble', 'no'] == cmd_args
+
     def test_wait_for_redis_to_start(self):
         pass