diff --git a/README.md b/README.md index 29858bac..51928407 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,14 @@ This is the list of available environmental variables: default (and preserves the default behavior required in a production environment). However, if set to 1, it dumps secrets and services config files without masking sensitive data. +- `SOS_DECOMPRESS`: 0 or 1. When set to 1, SOS reports are extracted after + download. When set to 0 (default), they are kept as `.tar.xz` archives and + the final archiving step avoids recompressing them: non-SOS data is compressed + separately and bundled with the SOS archives into a plain `must-gather.tar`. + When set to 1 or when no SOS reports are collected, the final archive is a + single `must-gather.tar.xz` as before. - `COMPRESSED_PATH`: defines the path to store the compressed form of the - gathered data + gathered data. - `DELETE_AFTER_COMPRESSION`: 0 or 1. When set to 1 the uncompressed data is deleted after the archive is created. Defaulted to 0. - `SUPPORT_TOOLS`: The OpenShift support-tools container image. It allows to diff --git a/collection-scripts/common.sh b/collection-scripts/common.sh index 50b045be..eb7eda08 100644 --- a/collection-scripts/common.sh +++ b/collection-scripts/common.sh @@ -8,9 +8,8 @@ export OMC=${OMC:-true} export OSP_NS="${OSP_NS-openstack}" export OSP_OPERATORS_NS="${OSP_OPERATORS_NS-openstack-operators}" -# This option is used for CI purposes and -# is enabled by default -export SOS_DECOMPRESS=${SOS_DECOMPRESS:-1} +# This option is used for CI purposes and is disabled by default +export SOS_DECOMPRESS=${SOS_DECOMPRESS:-0} export BASE_COLLECTION_PATH="${BASE_COLLECTION_PATH:-/must-gather}" export SOS_PATH="${BASE_COLLECTION_PATH}/sos-reports" export SOS_PATH_NODES="${SOS_PATH}/_all_nodes" @@ -98,6 +97,66 @@ declare -a OSP_SERVICES=( export OSP_SERVICES +# Archive the collected data into a single downloadable file. +# When SOS_DECOMPRESS=0, SOS reports are kept as .tar.xz archives and +# recompressing them with XZ produces no size reduction while adding +# significant time. In that case, non-SOS data is compressed separately +# into an intermediate XZ archive, then bundled with the already-compressed +# SOS files into a plain tar. +# Otherwise (SOS_DECOMPRESS=1 or no SOS reports), a single XZ pass over +# everything is used. +function compress { + # The path to store the compressed result + local compressed_path=${COMPRESSED_PATH:-"${BASE_COLLECTION_PATH}"} + # whether to delete or keep the uncompressed files. + # Defaults to keep them. + local delete_after=${DELETE_AFTER_COMPRESSION:-0} + local archive + + if [[ ${SOS_DECOMPRESS} -eq 0 ]] && \ + [[ -n "$(find "${SOS_PATH}" -type f 2>/dev/null | head -1)" ]]; then + archive="${compressed_path}/must-gather.tar" + local rhoso_archive="${compressed_path}/rhoso-data.tar.xz" + + # Compress only non-SOS data (CRDs, CMs, logs, DB dumps) into an + # intermediate XZ archive, excluding the SOS directory entirely. + tar \ + --exclude='must-gather.tar' \ + --exclude='rhoso-data.tar.xz' \ + --exclude='sos-reports' \ + --warning=no-file-changed --ignore-failed \ + -cJf \ + "${rhoso_archive}" "${BASE_COLLECTION_PATH}" || true + + # Bundle the intermediate archive with the SOS reports into a plain + # tar (no compression) so already-compressed SOS data is not re-processed. + tar \ + --exclude='must-gather.tar' \ + --warning=no-file-changed --ignore-failed \ + -cf \ + "${archive}" "${rhoso_archive}" "${SOS_PATH}" || true + + rm -f "${rhoso_archive}" + else + archive="${compressed_path}/must-gather.tar.xz" + + tar \ + --exclude='must-gather.tar.xz' \ + --warning=no-file-changed --ignore-failed \ + -cJf \ + "${archive}" "${BASE_COLLECTION_PATH}" || true + fi + + echo "The ${archive} now can be attached to the support case." + + if [[ ${delete_after} -eq 1 ]]; then + find "${BASE_COLLECTION_PATH}" \ + -mindepth 1 \ + -not -path "*must-gather.tar*" \ + -delete + fi +} + WEBHOOKS_COLLECTION_PATH=${BASE_COLLECTION_PATH}/webhooks export WEBHOOKS_COLLECTION_PATH diff --git a/collection-scripts/gather_run b/collection-scripts/gather_run index 7d037431..d7e494bc 100755 --- a/collection-scripts/gather_run +++ b/collection-scripts/gather_run @@ -74,24 +74,5 @@ wait_bg source "${DIR_NAME}/gather_version" log_version -# The path to store the compressed result -export COMPRESSED_PATH=${COMPRESSED_PATH:-"${BASE_COLLECTION_PATH}"} -# whether to delete or keep the uncompressed files. -# Defaults to keep them. -export DELETE_AFTER_COMPRESSION=${DELETE_AFTER_COMPRESSION:-0} - -# create an easy to download tar.xz from the whole content -archive="${COMPRESSED_PATH}/must-gather.tar.xz" - -tar \ - --exclude='must-gather.tar.xz' \ - --warning=no-file-changed --ignore-failed \ - -cJf \ - "${archive}" "${BASE_COLLECTION_PATH}" || true - -echo "The ${archive} now can be attached to the support case." - -if [[ ${DELETE_AFTER_COMPRESSION} -eq 1 ]]; then - find "${BASE_COLLECTION_PATH}" \ - -mindepth 1 -not -path "*must-gather.tar.xz*" -delete -fi +# Compress and archive the collected data +compress