Merge pull request #155 from fmount/compress

Avoid recompressing already-compressed SOS archives

Author: openshift-merge-bot[bot]

README.md

@@ -93,8 +93,14 @@ This is the list of available environmental variables:
   default (and preserves the default behavior required in a production environment).
   However, if set to 1, it dumps secrets and services config files without masking
   sensitive data.
+- `SOS_DECOMPRESS`: 0 or 1. When set to 1, SOS reports are extracted after
+  download. When set to 0 (default), they are kept as `.tar.xz` archives and
+  the final archiving step avoids recompressing them: non-SOS data is compressed
+  separately and bundled with the SOS archives into a plain `must-gather.tar`.
+  When set to 1 or when no SOS reports are collected, the final archive is a
+  single `must-gather.tar.xz` as before.
 - `COMPRESSED_PATH`: defines the path to store the compressed form of the
-  gathered data
+  gathered data.
 - `DELETE_AFTER_COMPRESSION`: 0 or 1. When set to 1 the uncompressed data is
   deleted after the archive is created. Defaulted to 0.
 - `SUPPORT_TOOLS`: The OpenShift support-tools container image. It allows to

collection-scripts/common.sh

@@ -8,9 +8,8 @@ export OMC=${OMC:-true}
 export OSP_NS="${OSP_NS-openstack}"
 export OSP_OPERATORS_NS="${OSP_OPERATORS_NS-openstack-operators}"
 
-# This option is used for CI purposes and
-# is enabled by default
-export SOS_DECOMPRESS=${SOS_DECOMPRESS:-1}
+# This option is used for CI purposes and is disabled by default
+export SOS_DECOMPRESS=${SOS_DECOMPRESS:-0}
 export BASE_COLLECTION_PATH="${BASE_COLLECTION_PATH:-/must-gather}"
 export SOS_PATH="${BASE_COLLECTION_PATH}/sos-reports"
 export SOS_PATH_NODES="${SOS_PATH}/_all_nodes"
@@ -98,6 +97,66 @@ declare -a OSP_SERVICES=(
 export OSP_SERVICES
 
 
+# Archive the collected data into a single downloadable file.
+# When SOS_DECOMPRESS=0, SOS reports are kept as .tar.xz archives and
+# recompressing them with XZ produces no size reduction while adding
+# significant time. In that case, non-SOS data is compressed separately
+# into an intermediate XZ archive, then bundled with the already-compressed
+# SOS files into a plain tar.
+# Otherwise (SOS_DECOMPRESS=1 or no SOS reports), a single XZ pass over
+# everything is used.
+function compress {
+    # The path to store the compressed result
+    local compressed_path=${COMPRESSED_PATH:-"${BASE_COLLECTION_PATH}"}
+    # whether to delete or keep the uncompressed files.
+    # Defaults to keep them.
+    local delete_after=${DELETE_AFTER_COMPRESSION:-0}
+    local archive
+
+    if [[ ${SOS_DECOMPRESS} -eq 0 ]] && \
+       [[ -n "$(find "${SOS_PATH}" -type f 2>/dev/null | head -1)" ]]; then
+        archive="${compressed_path}/must-gather.tar"
+        local rhoso_archive="${compressed_path}/rhoso-data.tar.xz"
+
+        # Compress only non-SOS data (CRDs, CMs, logs, DB dumps) into an
+        # intermediate XZ archive, excluding the SOS directory entirely.
+        tar \
+            --exclude='must-gather.tar' \
+            --exclude='rhoso-data.tar.xz' \
+            --exclude='sos-reports' \
+            --warning=no-file-changed --ignore-failed \
+            -cJf \
+            "${rhoso_archive}" "${BASE_COLLECTION_PATH}" || true
+
+        # Bundle the intermediate archive with the SOS reports into a plain
+        # tar (no compression) so already-compressed SOS data is not re-processed.
+        tar \
+            --exclude='must-gather.tar' \
+            --warning=no-file-changed --ignore-failed \
+            -cf \
+            "${archive}" "${rhoso_archive}" "${SOS_PATH}" || true
+
+        rm -f "${rhoso_archive}"
+    else
+        archive="${compressed_path}/must-gather.tar.xz"
+
+        tar \
+            --exclude='must-gather.tar.xz' \
+            --warning=no-file-changed --ignore-failed \
+            -cJf \
+            "${archive}" "${BASE_COLLECTION_PATH}" || true
+    fi
+
+    echo "The ${archive} now can be attached to the support case."
+
+    if [[ ${delete_after} -eq 1 ]]; then
+        find "${BASE_COLLECTION_PATH}" \
+            -mindepth 1 \
+            -not -path "*must-gather.tar*" \
+            -delete
+    fi
+}
+
 WEBHOOKS_COLLECTION_PATH=${BASE_COLLECTION_PATH}/webhooks
 export WEBHOOKS_COLLECTION_PATH
 

collection-scripts/gather_run

@@ -74,24 +74,5 @@ wait_bg
 source "${DIR_NAME}/gather_version"
 log_version
 
-# The path to store the compressed result
-export COMPRESSED_PATH=${COMPRESSED_PATH:-"${BASE_COLLECTION_PATH}"}
-# whether to delete or keep the uncompressed files.
-# Defaults to keep them.
-export DELETE_AFTER_COMPRESSION=${DELETE_AFTER_COMPRESSION:-0}
-
-# create an easy to download tar.xz from the whole content
-archive="${COMPRESSED_PATH}/must-gather.tar.xz"
-
-tar \
-    --exclude='must-gather.tar.xz' \
-    --warning=no-file-changed --ignore-failed  \
-    -cJf \
-    "${archive}" "${BASE_COLLECTION_PATH}" || true
-
-echo "The ${archive} now can be attached to the support case."
-
-if [[ ${DELETE_AFTER_COMPRESSION} -eq 1 ]]; then
-    find "${BASE_COLLECTION_PATH}" \
-        -mindepth 1 -not -path "*must-gather.tar.xz*" -delete
-fi
+# Compress and archive the collected data
+compress