refactor: replacing modify_sources script with toml driven archive overlays

Author: Antonio Salinas

base/comps/apache-commons-compress/apache-commons-compress.comp.toml

@@ -17,25 +17,25 @@
 # will be skipped/excluded at build time if they reference these
 # fixtures.
 #
-# We replace Source0 (effectively) with a deterministically-repacked
-# tarball that is byte-identical to upstream except for the stripped
-# files. The repack is produced by
-# base/comps/apache-commons-compress/modify_source.sh, which is
-# reproducible — re-running it always yields the same SHA-512. We keep
-# the upstream filename (commons-compress-1.27.1-src.tar.gz) and use the
-# `replace-upstream` mechanism on the source-files entry below to swap it
-# in place in the Fedora `sources` manifest — no spec edit required.
-#
-# When bumping the apache-commons-compress version (or changing
-# REMOVE_PATHS):
-#   1. Edit VERSION and UPSTREAM_SHA512 in modify_source.sh.
-#   2. Re-run the script and copy the new SHA-512 into the source-files entry.
-#   3. Upload the new tarball to the modified-source lookaside (see script output).
+# Archive overlays strip the scanner-flagged fixtures from upstream Source0
+# during source preparation, avoiding a separate modified-source tarball.
+
+[[components.apache-commons-compress.overlays]]
+type = "file-remove"
+file = "commons-compress-1.27.1-src.tar.gz/src/test/resources/bla.encrypted.7z"
+description = "Remove encrypted 7-Zip test fixture flagged by AZL signing-pipeline AV scanner"
+
+[[components.apache-commons-compress.overlays]]
+type = "file-remove"
+file = "commons-compress-1.27.1-src.tar.gz/src/test/resources/password-encrypted.zip"
+description = "Remove encrypted ZIP test fixture flagged by AZL signing-pipeline AV scanner"
+
+[[components.apache-commons-compress.overlays]]
+type = "file-remove"
+file = "commons-compress-1.27.1-src.tar.gz/src/test/resources/COMPRESS-256.7z"
+description = "Remove crafted 7-Zip test fixture flagged by AZL signing-pipeline AV scanner"
 
-[[components.apache-commons-compress.source-files]]
-filename = "commons-compress-1.27.1-src.tar.gz"
-hash = "aeecee8776c60a549cbca9fc3c0312c8c98a953d024db64e5c480c643357be7b270193df69fc2172632e472feb9b9221eedf3b40dd933997b881a398dfb3a02b"
-hash-type = "SHA512"
-origin = { type = "download", uri = "https://azltempstaginglookaside.blob.core.windows.net/repo/pkgs_modified/apache-commons-compress/commons-compress-1.27.1-src.tar.gz/sha512/aeecee8776c60a549cbca9fc3c0312c8c98a953d024db64e5c480c643357be7b270193df69fc2172632e472feb9b9221eedf3b40dd933997b881a398dfb3a02b/commons-compress-1.27.1-src.tar.gz" }
-replace-upstream = true
-replace-reason = "AZL-repacked tarball with scanner-flagged encrypted and crafted-archive test fixtures stripped; see modify_source.sh REMOVE_PATHS"
+[[components.apache-commons-compress.overlays]]
+type = "file-remove"
+file = "commons-compress-1.27.1-src.tar.gz/src/test/resources/zip64support.tar.bz2"
+description = "Remove crafted tar.bz2 test fixture flagged by AZL signing-pipeline AV scanner"

base/comps/espeak-ng/espeak-ng.comp.toml

@@ -12,16 +12,11 @@
 # our spec does not reference it or ship it in any binary RPM, so stripping
 # this file is functionally inert.
 #
-# Replace upstream Source0 with a deterministically-repacked tarball produced
-# by base/comps/espeak-ng/modify_source.sh. The upstream filename is preserved
-# so `replace-upstream = true` swaps the entry in place in the Fedora `sources`
-# manifest without requiring a `Source0`/filename change.
+# Archive overlays strip this scanner-flagged fixture from upstream Source0
+# during source preparation, avoiding a separate modified-source tarball.
 [components.espeak-ng]
 
-[[components.espeak-ng.source-files]]
-filename = "espeak-ng-1.51.1.tar.gz"
-hash = "84685a24e93e743c4f0be73dd9d553a96ed95bc8c2c0c683d84935183e517ae039066de93e3f83617b2114b27b427ec18ff8169972188d2a81b55f839c7c726f"
-hash-type = "SHA512"
-origin = { type = "download", uri = "https://azltempstaginglookaside.blob.core.windows.net/repo/pkgs_modified/espeak-ng/espeak-ng-1.51.1.tar.gz/sha512/84685a24e93e743c4f0be73dd9d553a96ed95bc8c2c0c683d84935183e517ae039066de93e3f83617b2114b27b427ec18ff8169972188d2a81b55f839c7c726f/espeak-ng-1.51.1.tar.gz" }
-replace-upstream = true
-replace-reason = "Strips the `chromium_extension/index.php` demo file flagged as PHP/Webshell.NWM by anti-malware scanners on the AZL RPM-signing pipeline. See `modify_source.sh` next to this file."
+[[components.espeak-ng.overlays]]
+type = "file-remove"
+file = "espeak-ng-1.51.1.tar.gz/chromium_extension/index.php"
+description = "Remove PHP webshell demo flagged as PHP/Webshell.NWM by AZL signing-pipeline AV scanner"

base/comps/exfatprogs/exfatprogs.comp.toml

@@ -1,9 +1,6 @@
 [components.exfatprogs]
 
-[[components.exfatprogs.source-files]]
-filename = "exfatprogs-1.3.1.tar.xz"
-hash = "ded26326feab2b0013c1031fda7add53100e0581711555b8a64e8036448775a51304e4c5c368049cb26633dbf02e814b6f97bd804cafff324921a8e6a3a975d0"
-hash-type = "SHA512"
-origin = { type = "download", uri = "https://azltempstaginglookaside.blob.core.windows.net/repo/pkgs_modified/exfatprogs/exfatprogs-1.3.1.tar.xz/sha512/ded26326feab2b0013c1031fda7add53100e0581711555b8a64e8036448775a51304e4c5c368049cb26633dbf02e814b6f97bd804cafff324921a8e6a3a975d0/exfatprogs-1.3.1.tar.xz" }
-replace-upstream = true
-replace-reason = "The upstream `tests/` tree ships 19 deliberately-corrupted exFAT filesystem images (e.g. bad_bitmap, bad_dentries, bs_bad_csum, loop_chain) whose malformed metadata sends the malware scanner into runaway behaviour on the crafted FAT/dentry structures -- the shell harness alongside them (tests/upcase_table/, test_fsck.sh) is not itself problematic; it is removed as collateral because it becomes unused once the images are gone. The `tests/` tree is EXTRA_DIST-only, never built or installed, and the spec has no %check, so stripping it is functionally inert. See modify_source.sh."
+[[components.exfatprogs.overlays]]
+type = "file-remove"
+file = "exfatprogs-1.3.1.tar.xz/tests/**"
+description = "Remove tests/ directory containing deliberately-corrupted exFAT filesystem images that trip AZL signing-pipeline AV scanner"

base/comps/firefox/firefox.comp.toml

@@ -143,28 +143,19 @@ replacement = 'Release: %[1 + %{azl_release}]%{?pre_tag}%{?dist}'
 # The authoritative list of stripped files lives in
 # base/comps/firefox/modify_source.sh (REMOVE_PATHS).
 #
-# We replace Source0 with a deterministically-repacked tarball that is
-# byte-identical to the upstream one except for the stripped files.
-# The repack is produced by base/comps/firefox/modify_source.sh, which
-# is reproducible — re-running it always yields the same SHA-512. We keep
-# the upstream filename (firefox-<version>.source.tar.xz) and use the
-# `replace-upstream` mechanism on the source-files entry below to swap it
-# in place in the Fedora `sources` manifest — no spec edit required.
+# Archive overlays strip the scanner-flagged fixtures from upstream Source0
+# during source preparation, avoiding a separate modified-source tarball.
 #
 # When bumping the firefox version (or changing REMOVE_PATHS):
-#   1. Edit VERSION and UPSTREAM_SHA512 in modify_source.sh.
-#   2. Re-run the script and copy the new SHA-512 into the source-files entry.
-#   3. Upload the new tarball to the modified-source lookaside (see script output).
+#   1. Update the overlay paths below if upstream fixture paths change.
+#   2. Re-render and build to verify the archive overlays still apply.
 
-# Drop the upstream firefox-<version>.source.tar.xz entry from the Fedora
-# `sources` lookaside manifest so the build does not also try to fetch the
-# unmodified upstream tarball alongside our repacked one. This is handled
-# by `replace-upstream = true` on the source-files entry below.
+[[components.firefox.overlays]]
+type = "file-remove"
+file = "firefox-148.0.source.tar.xz/toolkit/components/mediasniffer/test/unit/data/ff-inst.exe"
+description = "Remove obfuscated Windows executable test fixture flagged by AZL signing-pipeline AV scanner"
 
-[[components.firefox.source-files]]
-filename = "firefox-148.0.source.tar.xz"
-hash = "c9c9f29fbd8f889bf3cf3d88776e1a62da7b2a65d386538d2bd048dd70caaaf8324adb5303a5fa9aa73c2cf6eb9f89cb4b34f9e67c4208d88636dd5376af90a9"
-hash-type = "SHA512"
-origin = { type = "download", uri = "https://azltempstaginglookaside.blob.core.windows.net/repo/pkgs_modified/firefox/firefox-148.0.source.tar.xz/sha512/c9c9f29fbd8f889bf3cf3d88776e1a62da7b2a65d386538d2bd048dd70caaaf8324adb5303a5fa9aa73c2cf6eb9f89cb4b34f9e67c4208d88636dd5376af90a9/firefox-148.0.source.tar.xz" }
-replace-upstream = true
-replace-reason = "AZL-repacked tarball with malware-flagged upstream test fixtures stripped (trips RPM signing pipeline); see modify_source.sh REMOVE_PATHS"
+[[components.firefox.overlays]]
+type = "file-remove"
+file = "firefox-148.0.source.tar.xz/dom/base/crashtests/607222.html"
+description = "Remove crashtest HTML fixture flagged by AZL signing-pipeline AV scanner"

base/comps/gdal/gdal.comp.toml

@@ -12,19 +12,14 @@
 # no-op (exits 0; gcore tests were OOM-killing the build POD), so stripping
 # this fixture is functionally inert.
 #
-# Replace upstream Source1 with a deterministically-repacked tarball produced
-# by base/comps/gdal/modify_source.sh. The upstream filename is preserved so
-# `replace-upstream = true` swaps the entry in place in the Fedora `sources`
-# manifest -- no spec edit required.
+# Archive overlays strip this scanner-flagged fixture from upstream Source1
+# during source preparation, avoiding a separate modified-source tarball.
 [components.gdal]
 
-[[components.gdal.source-files]]
-filename = "gdalautotest-3.11.5.tar.gz"
-hash = "a39cc826885a2336c80545203947c4693851aae3de02b980be25b4b579658e6314e058dc228660294b2370242df8f298c4fe64c6d3b1cd0bf5adf56b3e321b77"
-hash-type = "SHA512"
-origin = { type = "download", uri = "https://azltempstaginglookaside.blob.core.windows.net/repo/pkgs_modified/gdal/gdalautotest-3.11.5.tar.gz/sha512/a39cc826885a2336c80545203947c4693851aae3de02b980be25b4b579658e6314e058dc228660294b2370242df8f298c4fe64c6d3b1cd0bf5adf56b3e321b77/gdalautotest-3.11.5.tar.gz" }
-replace-upstream = true
-replace-reason = "Repacked source tarball without gcore/data/zero_5GB_sozip_of_sozip.zip which was flagged as a Trojan. See modify_source.sh."
+[[components.gdal.overlays]]
+type = "file-remove"
+file = "gdalautotest-3.11.5.tar.gz/gcore/data/zero_5GB_sozip_of_sozip.zip"
+description = "Remove large test fixture to reduce source size"
 
 [components.gdal.build]
 # Azure Linux does not support MinGW cross-compilation.

base/comps/kf6-karchive/kf6-karchive.comp.toml

@@ -12,16 +12,16 @@
 # upstream's CMake gates them on BUILD_TESTING=ON, which we don't set), so
 # stripping these test fixtures is functionally inert.
 #
-# Replace upstream Source0 with a deterministically-repacked tarball produced
-# by base/comps/kf6-karchive/modify_source.sh. The upstream filename is
-# preserved so `replace-upstream = true` swaps the entry in place in the
-# Fedora `sources` manifest -- no spec edit required.
+# Archive overlays strip these scanner-flagged fixtures from upstream Source0
+# during source preparation, avoiding a separate modified-source tarball.
 [components.kf6-karchive]
 
-[[components.kf6-karchive.source-files]]
-filename = "karchive-6.23.0.tar.xz"
-hash = "dc92a030772bfea4fd270e0bf5aa1e6b9f0bb45fed19ad8c2c992fd8f36238c4730efebe7ac2d950c6be91213cd062afd0f69f404733678e511c74b94d09ad4a"
-hash-type = "SHA512"
-origin = { type = "download", uri = "https://azltempstaginglookaside.blob.core.windows.net/repo/pkgs_modified/kf6-karchive/karchive-6.23.0.tar.xz/sha512/dc92a030772bfea4fd270e0bf5aa1e6b9f0bb45fed19ad8c2c992fd8f36238c4730efebe7ac2d950c6be91213cd062afd0f69f404733678e511c74b94d09ad4a/karchive-6.23.0.tar.xz" }
-replace-upstream = true
-replace-reason = "AZL-repacked tarball with autotest fixtures stripped that trip anti-malware scanning on the AZL RPM-signing pipeline: autotests/data/password_protected.7z (password-protected 7-Zip) and autotests/data/zip64_extra_zip64_size_first.zip.gz (ZIP64 edge-case fixture whose inner .zip the scanner rejects after decompressing the .gz wrapper). The autotests are not built or run in our spec (no %check, BUILD_TESTING is off), so removing these test fixtures is functionally inert. See modify_source.sh."
+[[components.kf6-karchive.overlays]]
+type = "file-remove"
+file = "karchive-6.23.0.tar.xz/autotests/data/password_protected.7z"
+description = "Remove password-protected 7-Zip test fixture flagged by AZL signing-pipeline AV scanner"
+
+[[components.kf6-karchive.overlays]]
+type = "file-remove"
+file = "karchive-6.23.0.tar.xz/autotests/data/zip64_extra_zip64_size_first.zip.gz"
+description = "Remove ZIP64 edge-case test fixture flagged by AZL signing-pipeline AV scanner"

base/comps/libabigail/libabigail.comp.toml

@@ -10,24 +10,18 @@
 #       across separated-debuginfo + dwz layouts. The scanner flags both
 #       .debug files as "packer_high_entropy:eod".
 #
-# Replace upstream Source0 with a deterministically-repacked tarball produced
-# by base/comps/libabigail/modify_source.sh, which strips the entire
-# PR30329/ fixture directory so nothing in-tree references the missing
-# files. The two corresponding `InOutSpec in_out_specs[]` entries in
+# Archive overlays strip the entire PR30329/ fixture directory from upstream
+# Source0 so nothing in-tree references the missing files. The two
+# corresponding `InOutSpec in_out_specs[]` entries in
 # tests/test-abidiff-exit.cc are dropped by a companion overlay patch
 # (`tests-drop-PR30329-fixture-entries.patch`, applied below), keeping
-# `make check` green. The upstream filename is preserved so
-# `replace-upstream = true` swaps the entry in place in the Fedora `sources`
-# manifest -- no spec edit required.
+# `make check` green. This avoids a separate modified-source tarball.
 [components.libabigail]
 
-[[components.libabigail.source-files]]
-filename = "libabigail-2.9.tar.xz"
-hash = "efa38b7de791d97910e292dc638537c98d920a68201110727bb5c2d6a6055b6da24beace05db5d540ef4349ce2b4f1592a6aceb4e4249e30a179a037bec2f5d4"
-hash-type = "SHA512"
-origin = { type = "download", uri = "https://azltempstaginglookaside.blob.core.windows.net/repo/pkgs_modified/libabigail/libabigail-2.9.tar.xz/sha512/efa38b7de791d97910e292dc638537c98d920a68201110727bb5c2d6a6055b6da24beace05db5d540ef4349ce2b4f1592a6aceb4e4249e30a179a037bec2f5d4/libabigail-2.9.tar.xz" }
-replace-upstream = true
-replace-reason = "Repacked source tarball with tests/data/test-abidiff-exit/PR30329/ removed (two libsqlite3.so.0.8.6.debug fixtures inside it were flagged as packer_high_entropy:eod by the AZL signing-pipeline AV scanner). The matching InOutSpec entries in tests/test-abidiff-exit.cc are dropped by the companion overlay patch tests-drop-PR30329-fixture-entries.patch. See modify_source.sh."
+[[components.libabigail.overlays]]
+type = "file-remove"
+file = "libabigail-2.9.tar.xz/tests/data/test-abidiff-exit/PR30329/**"
+description = "Remove PR30329 fixture directory (two libsqlite3.so.0.8.6.debug files flagged as packer_high_entropy:eod by AZL signing-pipeline AV scanner)"
 
 [[components.libabigail.overlays]]
 description = "Drop the two tests/test-abidiff-exit.cc InOutSpec entries that exercise the PR30329 fixture set (removed from the AZL-repacked Source0 because its two libsqlite3.so.0.8.6.debug files are flagged packer_high_entropy:eod by the AZL signing-pipeline AV scanner). Without this patch `make check` fails trying to open the missing fixtures."

base/comps/libkml/libkml.comp.toml

@@ -1,9 +1,18 @@
 [components.libkml]
 
-[[components.libkml.source-files]]
-filename = "libkml-1.3.0.tar.gz"
-hash = "6f93fcd390c6b21e307638df5d85b15dcb81af81c7409797a247b322df843fb1c36dc6c5eb7dc1346adbf228e09ec4ffdbf450dfe1f9b73cbc32e8803a098c58"
-hash-type = "SHA512"
-origin = { type = "download", uri = "https://azltempstaginglookaside.blob.core.windows.net/repo/pkgs_modified/libkml/libkml-1.3.0.tar.gz/sha512/6f93fcd390c6b21e307638df5d85b15dcb81af81c7409797a247b322df843fb1c36dc6c5eb7dc1346adbf228e09ec4ffdbf450dfe1f9b73cbc32e8803a098c58/libkml-1.3.0.tar.gz" }
-replace-upstream = true
-replace-reason = "Strips the scanner-flagged `testdata/kmz/bad-too-large.kmz` test fixture (a benign-by-intent crafted-malformed ZIP whose on-disk shape matches malicious-archive heuristics) and the matching `ZipFileTest.TestBadTooLarge` block from `tests/kml/base/zip_file_test.cc`. See `modify_source.sh` next to this file."
+# Remove decompression-bomb test fixture flagged by AV scanner
+[[components.libkml.overlays]]
+type = "file-remove"
+file = "libkml-1.3.0.tar.gz/testdata/kmz/bad-too-large.kmz"
+description = "Remove crafted decompression-bomb KMZ fixture flagged by AZL signing-pipeline AV scanner"
+
+# Surgically remove the TEST_F block that exercises the stripped fixture.
+# The sed range `TEST_F(ZipFileTest, TestBadTooLarge) {` through the next
+# column-0 `}` is safe because nested braces inside the test body are
+# always indented.
+[[components.libkml.overlays]]
+type = "file-search-replace"
+file = "libkml-1.3.0.tar.gz/tests/kml/base/zip_file_test.cc"
+regex = 'TEST_F\(ZipFileTest, TestBadTooLarge\) \{[^}]*\}\n'
+replacement = ''
+description = "Remove TestBadTooLarge test block that references stripped bad-too-large.kmz fixture"