Parallel strip class

classes/autoconf.yaml

@@ -8,7 +8,8 @@ privateEnvironment:
     APPLY_LIBTOOL_PATCH: "no"
 
 checkoutDeterministic: True
-checkoutTools: [autotools, m4]
+checkoutTools: [autotools]
+checkoutToolsWeak: [m4]
 checkoutSetup: |
     # Other classes can add paths to this array to pick up additional aclocal
     # m4 files.

classes/basement/rootrecipe.yaml

@@ -29,6 +29,9 @@ depends:
           # can pick them as they like. Make sure to update the basement::buildall
           # class to catch added tools too.
 
+          - name: utils::pxargs
+            use: [tools]
+            forward: True
           - name: devel::make
             use: [tools]
             forward: True

classes/install.yaml

@@ -66,12 +66,12 @@ packageSetup: |
 
     installStripBinary()
     {
-        stripBinary "$1"
+        stripBinary "$@"
     }
 
     installStripAll()
     {
-        stripAll "$1"
+        stripAll "$@"
     }
 
     # Copy files matching the given patterns,
@@ -159,7 +159,7 @@ packageSetup: |
                 done
             done
 
-            installStripAll .
+            installStripAll
         fi
     }
 
@@ -170,7 +170,8 @@ packageSetup: |
     installFixShebang()
     {
         local shebang
-        while IFS= read -r -d $'\0' f; do
+        find "${1:-.}" -type f -perm /111 -print0 \
+        | while IFS= read -r -d $'\0' f; do
             read -n 4096 -r shebang < "$f"
             if [[ "${shebang:0:2}" == "#!" ]] ; then
                 # Match part after "#!". Tabs and spaces before and after the
@@ -190,7 +191,7 @@ packageSetup: |
                     echo "WARNING: unrecognized shebang: $shebang"
                 fi
             fi
-       done < <(find "${1:-.}" -type f -perm /111 -print0)
+       done
     }
 
     # Everything except shared or static libraries or header files.
@@ -202,7 +203,7 @@ packageSetup: |
             "!/usr/lib/pkgconfig" \
             "!/usr/share/pkgconfig" \
             "!/usr/lib/cmake"
-        installStripAll .
+        installStripAll
         if [[ ${INSTALL_APPLY_SHEBANG_FIXUP:-yes} != no ]] ; then
             installFixShebang
         fi

classes/strip.yaml

@@ -1,12 +1,14 @@
+jobServer: True
+packageToolsWeak:
+    - name: pxargs
+      if: !expr |
+        "${BOB_HOST_PLATFORM}" == "linux"
 packageVars: [OBJCOPY, STRIP]
+packageVarsWeak: [MAKE_JOBS]
 packageSetup: |
     # $1: binary file
     stripBinary()
     {
-        if [[ $1 == *.o ]] ; then
-            return 0
-        fi
-
         local type="$(file -b "$1")"
         if [[ $type == *ELF*not\ stripped ]] ; then
             echo "Stripping ${1} ..."
@@ -33,11 +35,33 @@ packageSetup: |
         fi
     }
 
-    # $1: directory to process
-    stripAll()
-    {
-        find "$1" -type f -not -path '*/.debug/*' -print0 \
-        | while IFS= read -r -d $'\0' f ; do
-            stripBinary "$f"
-        done
-    }
+    # On Windows, pxargs is not available...
+    if [[ ${BOB_TOOL_PATHS[pxargs]+exists} ]] ; then
+        export -f stripBinary
+
+        # $*: directories to process
+        stripAll()
+        {
+            local JOBARGS=() BASH_ARGS=( -eu -o pipefail )
+            if [[ ! "${MAKEFLAGS+set}" ]] ; then
+                JOBARGS+=( -j "${MAKE_JOBS-$(nproc)}" )
+            fi
+            JOBARGS+=( -- )
+
+            if [[ $(set -o | awk '/xtrace/ { print $2 }') == on ]] ; then
+                BASH_ARGS+=( -x )
+            fi
+
+            find "$@" -type f -not -path '*/.debug/*' -not -name '*.o' -print0 \
+            | pxargs "${JOBARGS[@]}" bash "${BASH_ARGS[@]}" -c 'stripBinary "$@"' bash
+        }
+    else
+        # $*: directories to process
+        stripAll()
+        {
+            find "$@" -type f -not -path '*/.debug/*' -not -name '*.o' -print0 \
+            | while IFS= read -r -d $'\0' f ; do
+                stripBinary "$f"
+            done
+        }
+    fi

recipes/devel/autoconf-2.69.yaml

@@ -16,7 +16,7 @@ checkoutScript: |
     # Prevent that...
     touch man/*.1
 
-buildTools: [m4, perl]
+buildToolsWeak: [m4, perl]
 buildScript: |
     export EMACS="no"
     export HELP2MAN=false

recipes/devel/autoconf.yaml

@@ -16,7 +16,7 @@ checkoutScript: |
     # Prevent that...
     touch man/*.1
 
-buildTools: [m4, perl]
+buildToolsWeak: [m4, perl]
 buildScript: |
     export EMACS="no"
     export HELP2MAN=false
@@ -35,4 +35,4 @@ packageScript: |
 provideTools:
     autoconf:
         path: "usr/bin"
-        dependTools: ["perl"]
+        dependToolsWeak: ["perl"]

recipes/devel/automake.yaml

@@ -16,7 +16,8 @@ checkoutDeterministic: True
 checkoutScript: |
     patchApplySeries $<@automake/*.patch@>
 
-buildTools: [m4, help2man, perl]
+buildTools: [help2man]
+buildToolsWeak: [m4, perl]
 buildScript: |
     export PATH="${BOB_DEP_PATHS[devel::autoconf]}/usr/bin:$PATH"
     autotoolsNoarchBuild $1
@@ -27,4 +28,4 @@ packageScript: |
 provideTools:
     automake:
         path: "usr/bin"
-        dependTools: ["perl"]
+        dependToolsWeak: [perl]

recipes/devel/autotools-2.69.yaml

@@ -20,4 +20,4 @@ packageScript: |
 provideTools:
     autotools:
         path: "usr/bin"
-        dependTools: ["perl"]
+        dependToolsWeak: ["perl"]

recipes/devel/autotools.yaml

@@ -20,4 +20,4 @@ packageScript: |
 provideTools:
     autotools:
         path: "usr/bin"
-        dependTools: ["perl"]
+        dependToolsWeak: [m4, perl]

recipes/devel/binutils.yaml

@@ -15,7 +15,8 @@ checkoutScript: |
 
 # Some parts are compiled for the host during compilation. Hence we need the
 # host toolchain too.
-buildTools: [host-toolchain, bison, m4]
+buildTools: [host-toolchain, bison]
+buildToolsWeak: [m4]
 multiPackage:
     "":
         buildVars: [AUTOCONF_HOST, AUTOCONF_TARGET, BINUTILS_PREFIX]

recipes/devel/bootstrap-sandbox.yaml

@@ -12,6 +12,10 @@ depends:
         forward: True
 
     # some required host tools
+    -
+        name: utils::pxargs
+        use: [tools]
+        forward: True
     -
         name: devel::make
         use: [tools]

recipes/devel/compat/binutils.yaml

@@ -11,7 +11,8 @@ checkoutSCM:
 
 # Some parts are compiled for the host during compilation. Hence we need the
 # host toolchain too.
-buildTools: [host-toolchain, m4]
+buildTools: [host-toolchain]
+buildToolsWeak: [m4]
 
 buildVars: [AUTOCONF_HOST, AUTOCONF_TARGET, BINUTILS_PREFIX]
 buildScript: |

recipes/devel/compat/gcc.yaml

@@ -25,7 +25,8 @@ checkoutScript: |
         $<<gcc/buildroot-libtool-v2.2.patch>> \
         $<<gcc/0003-fix-canadian-cross-includes.patch>>
 
-buildTools: [host-toolchain, target-toolchain, m4]
+buildTools: [host-toolchain, target-toolchain]
+buildToolsWeak: [m4]
 buildVars: [AUTOCONF_BUILD, AUTOCONF_HOST, AUTOCONF_TARGET,
             GCC_TARGET_ARCH, GCC_TARGET_FLOAT_ABI, GCC_TARGET_FPU]
 buildScript: |
@@ -172,8 +173,7 @@ multiPackage:
             cp -a ${BOB_DEP_PATHS[devel::compat::binutils]}/* install/
 
         packageScript: |
-            installStripAll usr/bin
-            installStripAll usr/libexec
+            installStripAll usr/bin usr/libexec
 
         provideDeps:
             - libs::compat::glibc
@@ -199,5 +199,4 @@ multiPackage:
                 --enable-languages="${GCC_ENABLE_LANGUAGES:-c,c++}"
 
         packageScript: |
-            installStripAll usr/bin
-            installStripAll usr/libexec
+            installStripAll usr/bin usr/libexec

recipes/devel/flex.yaml

@@ -10,7 +10,8 @@ checkoutSCM:
     stripComponents: 1
 
 checkoutDeterministic: True
-checkoutTools: [gettext, m4]
+checkoutTools: [gettext]
+checkoutToolsWeak: [m4]
 checkoutScript: |
     patchApplySeries $<@flex/*.patch@>
     autoconfReconfigure
@@ -29,4 +30,4 @@ packageScript: |
 provideTools:
     flex:
         path: usr/bin
-        dependTools: [ m4 ]
+        dependToolsWeak: [ m4 ]

recipes/devel/gcc.yaml

@@ -71,7 +71,7 @@ checkoutScript: |
 buildVars: [AUTOCONF_BUILD, AUTOCONF_HOST, AUTOCONF_TARGET, GCC_TARGET_ABI,
             GCC_TARGET_ARCH, GCC_TARGET_FLOAT_ABI, GCC_TARGET_FPU,
             GCC_MULTILIB, GCC_EXTRA_OPTIONS]
-buildTools: [m4]
+buildToolsWeak: [m4]
 buildScript: |
     GCC_SRC=$1
     mkdir -p build install
@@ -262,8 +262,7 @@ multiPackage:
                 --enable-languages="${GCC_ENABLE_LANGUAGES:-c,c++}"
 
         packageScript: |
-            installStripAll usr/bin
-            installStripAll usr/libexec
+            installStripAll usr/bin usr/libexec
 
         provideDeps:
             - devel::binutils
@@ -331,7 +330,6 @@ multiPackage:
             cp -an ${BOB_TOOL_PATHS[target-toolchain]}/$TOOLCHAIN_SYSROOT/* "install/${TARGET_SYSROOT}"
 
         packageScript: |
-            installStripAll ./${GCC_PREFIX:-/usr}/bin
-            installStripAll ./${GCC_PREFIX:-/usr}/libexec
+            installStripAll ./${GCC_PREFIX:-/usr}/bin ./${GCC_PREFIX:-/usr}/libexec
 
         provideDeps: [ "*-tgt" ]

recipes/devel/help2man.yaml

@@ -8,7 +8,8 @@ checkoutSCM:
     digestSHA1: "3ed88430c97af3c5b57949f6f030b913044af507"
     stripComponents: 1
 
-buildTools: [host-toolchain, perl]
+buildTools: [host-toolchain]
+buildToolsWeak: [perl]
 buildScript: |
     autotoolsBuild $1
 

recipes/devel/host-compat-toolchain.yaml

@@ -12,6 +12,9 @@ depends:
     # The following tools are needed by the cross-toolchain build process.
     # Build them explicitly here to keep the basement::rootrecipe class
     # untainted.
+    - name: utils::pxargs
+      use: [tools]
+      forward: True
     - name: devel::make
       use: [tools]
       forward: True

recipes/devel/llvm.yaml

@@ -219,7 +219,7 @@ multiPackage:
                                 /usr/bin/{ld.lld,lld,lld-link,wasm-ld} \
                                 /usr/lib/ "/usr/lib/clang/***" \
                                 "!*"
-                            installStripAll .
+                            installStripAll
 
                         provideTools:
                             clang: "usr/bin"
@@ -229,7 +229,7 @@ multiPackage:
                             installCopy "$1/install/" \
                                 /usr/ /usr/bin/ /usr/bin/clangd \
                                 "!*"
-                            installStripAll .
+                            installStripAll
 
                         provideTools:
                             clangd: "usr/bin"
@@ -239,7 +239,7 @@ multiPackage:
                             installCopy "$1/install/" \
                                 /usr/ /usr/bin/ /usr/bin/{,git-}clang-format \
                                 "!*"
-                            installStripAll .
+                            installStripAll
 
                         provideTools:
                             clang-format: "usr/bin"
@@ -249,7 +249,7 @@ multiPackage:
                             installCopy "$1/install/" \
                                 /usr/ /usr/bin/ /usr/bin/{,run-}clang-tidy \
                                 "!*"
-                            installStripAll .
+                            installStripAll
 
                         provideTools:
                             clang-tidy: "usr/bin"

recipes/devel/ocaml.yaml

@@ -9,7 +9,7 @@ checkoutSCM:
     digestSHA256: eb9eab2f21758d3cfb1e78c7f83f0b4dd6302824316aba4abee047a5a4f85029
     stripComponents: 1
 
-buildTools: [m4]
+buildToolsWeak: [m4]
 buildVars: [STRIP]
 buildScript: |
     # Note: the configure script is broken - it generates the makefiles in the

recipes/devel/sandbox.yaml

@@ -21,6 +21,10 @@ depends:
         forward: True
 
     # some required host tools
+    -
+        name: utils::pxargs
+        use: [tools]
+        forward: True
     -
         name: devel::make
         use: [tools]

recipes/devel/texinfo.yaml

@@ -9,7 +9,8 @@ checkoutSCM:
     digestSHA1: "d39c2e35ddb0aff6ebdd323ce53729bd215534fa"
     stripComponents: 1
 
-buildTools: [host-toolchain, perl]
+buildTools: [host-toolchain]
+buildToolsWeak: [perl]
 buildScript: |
     autotoolsBuild $1
 

recipes/kernel/linux-libc-headers.yaml

@@ -19,7 +19,8 @@ checkoutSCM:
     digestSHA256: "d6ecff966f8c95ec4cb3bb303904f757b7de6a6bcfef0d0771cb852158e61c20"
     stripComponents: 1
 
-buildTools: [bison, flex, host-toolchain, m4]
+buildTools: [bison, flex, host-toolchain]
+buildToolsWeak: [m4]
 buildVars: [ARCH]
 buildScript: |
     # prevent timestamps in configuration

recipes/libs/compat/glibc.yaml

@@ -20,8 +20,8 @@ checkoutScript: |
     patchApplySeries $<@glibc/*.diff@>
 
 buildVars: [AUTOCONF_TARGET]
-buildTools: [host-toolchain, target-toolchain, bison, m4]
-buildToolsWeak: [python3]
+buildTools: [host-toolchain, target-toolchain, bison]
+buildToolsWeak: [python3, m4]
 buildScript: |
     EXTRA=
     [ -e $1/usr/include/selinux/selinux.h ] || EXTRA+=" --without-selinux"