Adds a custom pytest option for archiving logs and conf files across different test executions

Author: ichristod

test/README.pytest

@@ -35,6 +35,58 @@ also raises the error log level of the tested modules.
 > pytest -vvv -k test_h2_004_01
 run the specific test with mod_http2 at log level TRACE2.
 
+There is an option to archive the results across different
+modules and httpd versions. The archiving will preserve
+error_log and access_log (cumulative for all tests in the module),
+and save a separate config file for each test case.
+> pytest -k test_core --archive=/path/to/archive/
+
+If you don't provide any specific module, pytest will execute all of them
+and the already archived folders and files will be replaced by the new ones.
+> pytest --archive=/path/to/archive
+
+Always use --archive=<path> (with =, not a space) to avoid pytest rootdir issues.
+
+Using the --archive option having installed a different httpd version
+will preserve any results from previous executions so you would
+end up with a different folder structure for each httpd version.
+You can also archive the results of different modules and that will
+work incrementally, which means it will add a new folder per module to the structure.
+
+
+What gets archived:
+- error_log and access_log for the module run
+- per-test configs: test.conf after each test runs
+- module configs: modules.conf, stop.conf
+- full server directory (conf, logs, htdocs)
+- shared infrastructure (CA, mod_md store, pebble files) in _shared/ to avoid duplication
+
+Example archive structure with 2 httpd versions:
+    ├── /path/to/archive
+    │   ├── 2.4.65
+    │   │   ├── _shared
+    │   │   │   ├── conf
+    │   │   │   │   ├── httpd.conf
+    │   │   │   │   └── mime.types
+    │   │   │   ├── ca
+    │   │   │   ├── md
+    │   │   │   ├── acme-ca.pem
+    │   │   │   └── eab.json
+    │   │   ├── core
+    │   │   │   ├── conf
+    │   │   │   │   ├── modules.conf
+    │   │   │   │   ├── test_core_001_01.conf
+    │   │   │   │   ├── test_core_002_01.conf
+    │   │   │   │   └── ...
+    │   │   │   ├── logs
+    │   │   │   │   ├── error_log
+    │   │   │   │   └── access_log
+    │   │   │   └── htdocs
+    │   │   ├── http1
+    │   │   └── http2
+    │   └── 2.4.66
+    │       └── core
+
 By default, test cases will configure httpd with mpm_event. You
 can change that with the invocation:
 > MPM=worker pytest test/modules/http2

test/conftest.py

@@ -1,5 +1,6 @@
 import sys
 import os
+import warnings
 
 import pytest
 
@@ -15,6 +16,8 @@ def pytest_addoption(parser):
     parser.addoption("--repeat", action="store", type=int, default=1,
                      help='Number of times to repeat each test')
     parser.addoption("--all", action="store_true")
+    parser.addoption("--archive", action="store", default=None,
+                     help='Archive the server directory after each test package to the specified folder')
 
 
 def pytest_generate_tests(metafunc):
@@ -29,6 +32,12 @@ def _function_scope(env, request):
     env.set_current_test_name(request.node.name)
     yield
     env.check_error_log()
+    archive_dir = request.config.getoption("--archive")
+    if archive_dir:
+        fspath = str(request.fspath)
+        if 'modules/' in fspath:
+            package_name = fspath.split('modules/')[1].split('/')[0]
+            env.archive_test_conf(request.node.name, package_name, archive_dir)
     env.set_current_test_name(None)
 
 
@@ -39,9 +48,18 @@ def _module_scope(env):
 
 
 @pytest.fixture(autouse=True, scope="package")
-def _package_scope(env):
+def _package_scope(env, request):
     env.httpd_error_log.clear_ignored_matches()
     env.httpd_error_log.clear_ignored_lognos()
     yield
     assert env.apache_stop() == 0
     env.check_error_log()
+
+    archive_dir = request.config.getoption("--archive")
+    if archive_dir == "":
+        warnings.warn("--archive option was empty, skipping archiving")
+    if archive_dir:
+        fspath = str(request.fspath)
+        parts = fspath.split('modules/')
+        package_name = parts[1].split('/')[0]
+        env.archive_logs(package_name, archive_dir)

test/pyhttpd/env.py

@@ -1,3 +1,4 @@
+import glob
 import importlib
 import inspect
 import logging
@@ -20,7 +21,6 @@
 from .nghttp import Nghttp
 from .result import ExecResult
 
-
 log = logging.getLogger(__name__)
 
 
@@ -29,7 +29,6 @@ class Dummy:
 
 
 class HttpdTestSetup:
-
     # the modules we want to load
     MODULES = [
         "log_config",
@@ -209,8 +208,8 @@ def _build_clients(self):
 
 
 class HttpdTestEnv:
-
     LIBEXEC_DIR = None
+    SHARED_SERVER_ENTRIES = {'ca', 'md', 'acme-ca.pem', 'eab.json'}
 
     @classmethod
     def has_python_package(cls, name: str) -> bool:
@@ -339,9 +338,60 @@ def setup_httpd(self, setup: HttpdTestSetup = None):
 
     def check_error_log(self):
         errors, warnings = self._error_log.get_missed()
-        assert (len(errors), len(warnings)) == (0, 0),\
-                f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\
-                "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings))
+        assert (len(errors), len(warnings)) == (0, 0), \
+            f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n" \
+            "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings))
+
+    # archives preserving metadata and avoiding duplication
+    def archive_logs(self, package_name, archive_dir):
+        version = self.get_httpd_version()
+        dest = os.path.join(archive_dir, version, package_name)
+        if os.path.isdir(dest):
+            shutil.rmtree(dest)
+
+        # use ignore argument to avoid duplication of files
+        shutil.copytree(self._server_dir, dest, ignore=self.ignore_files)
+        shared_dest = os.path.join(archive_dir, version, 'shared')
+
+        if os.path.isdir(shared_dest):
+            shutil.rmtree(shared_dest)
+        os.makedirs(shared_dest)
+
+        for entry in self.SHARED_SERVER_ENTRIES:
+            src = os.path.join(self._server_dir, entry)
+            entry_dest = os.path.join(shared_dest, entry)
+            if os.path.isdir(src):
+                shutil.copytree(src, entry_dest)
+            elif os.path.isfile(src):
+                shutil.copy2(src, entry_dest)
+
+        shared_conf_dest = os.path.join(shared_dest, 'conf')
+        os.makedirs(shared_conf_dest)
+
+        # copy them once
+        for f in ['httpd.conf', 'mime.types']:
+            src = os.path.join(self._server_conf_dir, f)
+            if os.path.isfile(src):
+                shutil.copy2(src, shared_conf_dest)
+
+    def archive_test_conf(self, test_name, package_name, archive_dir):
+        version = self.get_httpd_version()
+        dest = os.path.join(archive_dir, version, package_name, 'conf')
+        if not os.path.isdir(dest):
+            os.makedirs(dest)
+
+        test_conf = os.path.join(self._server_conf_dir, 'test.conf')
+        if os.path.isfile(test_conf):
+            final_name = test_name.replace('/', '_').replace('\\', '_')
+            dest_file = os.path.join(dest, f"{final_name}.conf")
+            shutil.copy(test_conf, dest_file)
+
+    # return files to ignore
+    def ignore_files(self, d, entries):
+        ignored = [e for e in entries if e.endswith('.sock') or e in self.SHARED_SERVER_ENTRIES]
+        if os.path.basename(d) == 'conf':
+            ignored += ['httpd.conf', 'mime.types']
+        return ignored
 
     @property
     def curl(self) -> str:
@@ -689,7 +739,7 @@ def apache_restart(self):
             timeout = timedelta(seconds=10)
             return 0 if self.is_live(self._http_base, timeout=timeout) else -1
         return r.exit_code
-        
+
     def apache_stop(self):
         r = self._run_apachectl("stop")
         if r.exit_code == 0:
@@ -778,6 +828,7 @@ def curl_parse_headerfile(self, headerfile: str, r: ExecResult = None) -> ExecRe
             r = ExecResult(args=[], exit_code=0, stdout=b'', stderr=b'')
 
         response = None
+
         def fin_response(response):
             if response:
                 r.add_response(response)
@@ -878,7 +929,7 @@ def curl_protocol_version(self, url, timeout=5, options=None):
         if r.exit_code == 0 and r.response:
             return r.response["body"].decode('utf-8').rstrip()
         return -1
-        
+
     def nghttp(self):
         return Nghttp(self._nghttp, connect_addr=self._httpd_addr,
                       tmp_dir=self.gen_dir, test_name=self._current_test)
@@ -920,4 +971,3 @@ def make_data_file(self, indir: str, fname: str, fsize: int) -> str:
                 s = f"{i:09d}-{s}\n"
                 fd.write(s[0:remain])
         return fpath
-