test: preserve failed integration logs

Copy ccm log and config files before integration teardown removes clusters, so failed CI jobs can publish Scylla-side diagnostics.

Author: dkropachev

tests/integration/conftest.py

@@ -1,5 +1,6 @@
 import os
 import logging
+import shutil
 
 import pytest
 from ccmlib.cluster_factory import ClusterFactory as CCMClusterFactory
@@ -9,12 +10,56 @@
 from . import CLUSTER_NAME, SINGLE_NODE_CLUSTER_NAME, MULTIDC_CLUSTER_NAME
 from . import path as ccm_path
 
+_failed_reports = []
+_ccm_logs_preserved = False
+_log_artifact_dir = os.environ.get('INTEGRATION_LOG_ARTIFACT_DIR', 'integration-test-logs')
+_preserved_file_extensions = ('.conf', '.log', '.yaml', '.yml')
+
+
+def pytest_runtest_logreport(report):
+    if report.failed:
+        _failed_reports.append(report)
+
+
+def preserve_ccm_logs_on_failure():
+    global _ccm_logs_preserved
+    if _ccm_logs_preserved or not _failed_reports or not os.path.isdir(ccm_path):
+        return
+
+    destination = os.path.abspath(_log_artifact_dir)
+    ccm_destination = os.path.join(destination, 'ccm')
+    os.makedirs(ccm_destination, exist_ok=True)
+
+    with open(os.path.join(destination, 'failed-tests.txt'), 'w') as failed_tests:
+        for report in _failed_reports:
+            failed_tests.write('%s %s\n' % (report.when, report.nodeid))
+
+    for root, _, files in os.walk(ccm_path):
+        rel_root = os.path.relpath(root, ccm_path)
+        path_parts = set(rel_root.split(os.sep))
+        preserve_all = bool(path_parts.intersection(('conf', 'logs')))
+
+        for filename in files:
+            if not preserve_all and not filename.endswith(_preserved_file_extensions):
+                continue
+
+            source_file = os.path.join(root, filename)
+            rel_file = os.path.relpath(source_file, ccm_path)
+            destination_file = os.path.join(ccm_destination, rel_file)
+            os.makedirs(os.path.dirname(destination_file), exist_ok=True)
+            shutil.copy2(source_file, destination_file)
+
+    logging.info("Preserved ccm logs for failed tests under %s", destination)
+    _ccm_logs_preserved = True
+
 
 @pytest.fixture(scope="session", autouse=True)
 def cleanup_clusters():
 
     yield
 
+    preserve_ccm_logs_on_failure()
+
     if not os.environ.get('DISABLE_CLUSTER_CLEANUP'):
         for cluster_name in [CLUSTER_NAME, SINGLE_NODE_CLUSTER_NAME, MULTIDC_CLUSTER_NAME,
                              'cluster_tests', 'shared_aware', 'sni_proxy', 'test_ip_change', 'test_client_routes_replacement']:
@@ -29,4 +74,5 @@ def cleanup_clusters():
 def setup_and_teardown_packages():
     print('setup')
     yield
+    preserve_ccm_logs_on_failure()
     teardown_package()