test: preserve failed integration logs

Copy CCM log and config files before integration cleanup clears or removes cluster directories, so failed CI jobs can publish Scylla-side diagnostics.

Author: dkropachev

tests/integration/__init__.py

@@ -31,6 +31,7 @@
 import time
 import traceback
 import platform
+from pathlib import Path
 from threading import Event
 from subprocess import call
 from itertools import groupby
@@ -72,10 +73,59 @@
 if not os.path.exists(path):
     os.mkdir(path)
 
+_failed_reports = []
+_log_artifact_dir = os.environ.get('INTEGRATION_LOG_ARTIFACT_DIR', 'integration-test-logs')
+_preserved_file_extensions = ('.conf', '.log', '.yaml', '.yml')
+
 cass_version = None
 cql_version = None
 
 
+def record_failed_test_report(report):
+    if report.failed:
+        _failed_reports.append(report)
+
+
+def preserve_ccm_logs_on_failure(cluster_name=None):
+    if not _failed_reports:
+        return
+
+    ccm_root = Path(path)
+    if cluster_name:
+        ccm_root = ccm_root / cluster_name
+    if not ccm_root.is_dir():
+        return
+
+    destination = Path(_log_artifact_dir).resolve()
+    ccm_destination = destination / 'ccm'
+    if cluster_name:
+        ccm_destination = ccm_destination / cluster_name
+    ccm_destination.mkdir(parents=True, exist_ok=True)
+
+    with (destination / 'failed-tests.txt').open('w') as failed_tests:
+        for report in _failed_reports:
+            failed_tests.write('%s %s\n' % (report.when, report.nodeid))
+
+    # Preserve CCM logs/configs before cleanup removes the cluster directories.
+    for source_file in ccm_root.rglob('*'):
+        if not source_file.is_file():
+            continue
+
+        rel_file = source_file.relative_to(ccm_root)
+        preserve_all = bool({'conf', 'logs'}.intersection(rel_file.parts))
+        if not preserve_all and not source_file.name.endswith(_preserved_file_extensions):
+            continue
+
+        destination_file = ccm_destination / rel_file
+        try:
+            destination_file.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(source_file, destination_file)
+        except OSError as exc:
+            log.warning("Failed to preserve file %s: %s", source_file, exc)
+
+    log.info("Preserved ccm logs for failed tests under %s", destination)
+
+
 def get_server_versions():
     """
     Probe system.local table to determine Cassandra and CQL version.
@@ -383,6 +433,7 @@ def remove_cluster():
         tries = 0
         while tries < 100:
             try:
+                preserve_ccm_logs_on_failure(CCM_CLUSTER.name)
                 CCM_CLUSTER.remove()
                 CCM_CLUSTER = None
                 return
@@ -455,6 +506,7 @@ def use_cluster(cluster_name, nodes, ipformat=None, start=True, workloads=None,
         try:
             CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
             log.debug("Found existing CCM cluster, {0}; clearing.".format(cluster_name))
+            preserve_ccm_logs_on_failure(cluster_name)
             CCM_CLUSTER.clear()
             CCM_CLUSTER.set_install_dir(**ccm_options)
             CCM_CLUSTER.set_configuration_options(configuration_options)
@@ -471,6 +523,7 @@ def use_cluster(cluster_name, nodes, ipformat=None, start=True, workloads=None,
             # Make sure we cleanup old cluster dir if it exists
             cluster_path = os.path.join(path, cluster_name)
             if os.path.exists(cluster_path):
+                preserve_ccm_logs_on_failure(cluster_name)
                 shutil.rmtree(cluster_path)
 
             if SCYLLA_VERSION:
@@ -553,6 +606,7 @@ def teardown_package():
         return
     # when multiple modules are run explicitly, this runs between them
     # need to make sure CCM_CLUSTER is properly cleared for that case
+    preserve_ccm_logs_on_failure()
     remove_cluster()
     for cluster_name in [CLUSTER_NAME, MULTIDC_CLUSTER_NAME]:
         try:

tests/integration/conftest.py

@@ -4,17 +4,23 @@
 import pytest
 from ccmlib.cluster_factory import ClusterFactory as CCMClusterFactory
 
-from tests.integration import teardown_package
+from tests.integration import record_failed_test_report, preserve_ccm_logs_on_failure, teardown_package
 
 from . import CLUSTER_NAME, SINGLE_NODE_CLUSTER_NAME, MULTIDC_CLUSTER_NAME
 from . import path as ccm_path
 
 
+def pytest_runtest_logreport(report):
+    record_failed_test_report(report)
+
+
 @pytest.fixture(scope="session", autouse=True)
 def cleanup_clusters():
 
     yield
 
+    preserve_ccm_logs_on_failure()
+
     if not os.environ.get('DISABLE_CLUSTER_CLEANUP'):
         for cluster_name in [CLUSTER_NAME, SINGLE_NODE_CLUSTER_NAME, MULTIDC_CLUSTER_NAME,
                              'cluster_tests', 'shared_aware', 'sni_proxy', 'test_ip_change', 'test_client_routes_replacement']:
@@ -29,4 +35,5 @@ def cleanup_clusters():
 def setup_and_teardown_packages():
     print('setup')
     yield
+    preserve_ccm_logs_on_failure()
     teardown_package()