diff --git a/classes/autoconf.yaml b/classes/autoconf.yaml index 8baca3d1..daf2df27 100644 --- a/classes/autoconf.yaml +++ b/classes/autoconf.yaml @@ -8,7 +8,8 @@ privateEnvironment: APPLY_LIBTOOL_PATCH: "no" checkoutDeterministic: True -checkoutTools: [autotools, m4] +checkoutTools: [autotools] +checkoutToolsWeak: [m4] checkoutSetup: | # Other classes can add paths to this array to pick up additional aclocal # m4 files. diff --git a/classes/basement/rootrecipe.yaml b/classes/basement/rootrecipe.yaml index b35bb72e..f2ff6889 100644 --- a/classes/basement/rootrecipe.yaml +++ b/classes/basement/rootrecipe.yaml @@ -29,6 +29,9 @@ depends: # can pick them as they like. Make sure to update the basement::buildall # class to catch added tools too. + - name: utils::pxargs + use: [tools] + forward: True - name: devel::make use: [tools] forward: True diff --git a/classes/install.yaml b/classes/install.yaml index fd26e6e9..ba91a865 100644 --- a/classes/install.yaml +++ b/classes/install.yaml @@ -66,12 +66,12 @@ packageSetup: | installStripBinary() { - stripBinary "$1" + stripBinary "$@" } installStripAll() { - stripAll "$1" + stripAll "$@" } # Copy files matching the given patterns, @@ -159,7 +159,7 @@ packageSetup: | done done - installStripAll . + installStripAll fi } @@ -170,7 +170,8 @@ packageSetup: | installFixShebang() { local shebang - while IFS= read -r -d $'\0' f; do + find "${1:-.}" -type f -perm /111 -print0 \ + | while IFS= read -r -d $'\0' f; do read -n 4096 -r shebang < "$f" if [[ "${shebang:0:2}" == "#!" ]] ; then # Match part after "#!". Tabs and spaces before and after the @@ -190,7 +191,7 @@ packageSetup: | echo "WARNING: unrecognized shebang: $shebang" fi fi - done < <(find "${1:-.}" -type f -perm /111 -print0) + done } # Everything except shared or static libraries or header files. @@ -202,7 +203,7 @@ packageSetup: | "!/usr/lib/pkgconfig" \ "!/usr/share/pkgconfig" \ "!/usr/lib/cmake" - installStripAll . + installStripAll if [[ ${INSTALL_APPLY_SHEBANG_FIXUP:-yes} != no ]] ; then installFixShebang fi diff --git a/classes/strip.yaml b/classes/strip.yaml index 8cee1c13..524ae15a 100644 --- a/classes/strip.yaml +++ b/classes/strip.yaml @@ -1,12 +1,14 @@ +jobServer: True +packageToolsWeak: + - name: pxargs + if: !expr | + "${BOB_HOST_PLATFORM}" == "linux" packageVars: [OBJCOPY, STRIP] +packageVarsWeak: [MAKE_JOBS] packageSetup: | # $1: binary file stripBinary() { - if [[ $1 == *.o ]] ; then - return 0 - fi - local type="$(file -b "$1")" if [[ $type == *ELF*not\ stripped ]] ; then echo "Stripping ${1} ..." @@ -33,11 +35,33 @@ packageSetup: | fi } - # $1: directory to process - stripAll() - { - find "$1" -type f -not -path '*/.debug/*' -print0 \ - | while IFS= read -r -d $'\0' f ; do - stripBinary "$f" - done - } + # On Windows, pxargs is not available... + if [[ ${BOB_TOOL_PATHS[pxargs]+exists} ]] ; then + export -f stripBinary + + # $*: directories to process + stripAll() + { + local JOBARGS=() BASH_ARGS=( -eu -o pipefail ) + if [[ ! "${MAKEFLAGS+set}" ]] ; then + JOBARGS+=( -j "${MAKE_JOBS-$(nproc)}" ) + fi + JOBARGS+=( -- ) + + if [[ $(set -o | awk '/xtrace/ { print $2 }') == on ]] ; then + BASH_ARGS+=( -x ) + fi + + find "$@" -type f -not -path '*/.debug/*' -not -name '*.o' -print0 \ + | pxargs "${JOBARGS[@]}" bash "${BASH_ARGS[@]}" -c 'stripBinary "$@"' bash + } + else + # $*: directories to process + stripAll() + { + find "$@" -type f -not -path '*/.debug/*' -not -name '*.o' -print0 \ + | while IFS= read -r -d $'\0' f ; do + stripBinary "$f" + done + } + fi diff --git a/recipes/devel/autoconf-2.69.yaml b/recipes/devel/autoconf-2.69.yaml index f6b6805d..66061d2f 100644 --- a/recipes/devel/autoconf-2.69.yaml +++ b/recipes/devel/autoconf-2.69.yaml @@ -16,7 +16,7 @@ checkoutScript: | # Prevent that... touch man/*.1 -buildTools: [m4, perl] +buildToolsWeak: [m4, perl] buildScript: | export EMACS="no" export HELP2MAN=false diff --git a/recipes/devel/autoconf.yaml b/recipes/devel/autoconf.yaml index a8a0cc53..266fb5d1 100644 --- a/recipes/devel/autoconf.yaml +++ b/recipes/devel/autoconf.yaml @@ -16,7 +16,7 @@ checkoutScript: | # Prevent that... touch man/*.1 -buildTools: [m4, perl] +buildToolsWeak: [m4, perl] buildScript: | export EMACS="no" export HELP2MAN=false @@ -35,4 +35,4 @@ packageScript: | provideTools: autoconf: path: "usr/bin" - dependTools: ["perl"] + dependToolsWeak: ["perl"] diff --git a/recipes/devel/automake.yaml b/recipes/devel/automake.yaml index ec2624a8..4924a933 100644 --- a/recipes/devel/automake.yaml +++ b/recipes/devel/automake.yaml @@ -16,7 +16,8 @@ checkoutDeterministic: True checkoutScript: | patchApplySeries $<@automake/*.patch@> -buildTools: [m4, help2man, perl] +buildTools: [help2man] +buildToolsWeak: [m4, perl] buildScript: | export PATH="${BOB_DEP_PATHS[devel::autoconf]}/usr/bin:$PATH" autotoolsNoarchBuild $1 @@ -27,4 +28,4 @@ packageScript: | provideTools: automake: path: "usr/bin" - dependTools: ["perl"] + dependToolsWeak: [perl] diff --git a/recipes/devel/autotools-2.69.yaml b/recipes/devel/autotools-2.69.yaml index 331b64fe..05e5d823 100644 --- a/recipes/devel/autotools-2.69.yaml +++ b/recipes/devel/autotools-2.69.yaml @@ -20,4 +20,4 @@ packageScript: | provideTools: autotools: path: "usr/bin" - dependTools: ["perl"] + dependToolsWeak: ["perl"] diff --git a/recipes/devel/autotools.yaml b/recipes/devel/autotools.yaml index 38be12d4..03d2051f 100644 --- a/recipes/devel/autotools.yaml +++ b/recipes/devel/autotools.yaml @@ -20,4 +20,4 @@ packageScript: | provideTools: autotools: path: "usr/bin" - dependTools: ["perl"] + dependToolsWeak: [m4, perl] diff --git a/recipes/devel/binutils.yaml b/recipes/devel/binutils.yaml index b61a9168..0b8d6d75 100644 --- a/recipes/devel/binutils.yaml +++ b/recipes/devel/binutils.yaml @@ -15,7 +15,8 @@ checkoutScript: | # Some parts are compiled for the host during compilation. Hence we need the # host toolchain too. -buildTools: [host-toolchain, bison, m4] +buildTools: [host-toolchain, bison] +buildToolsWeak: [m4] multiPackage: "": buildVars: [AUTOCONF_HOST, AUTOCONF_TARGET, BINUTILS_PREFIX] diff --git a/recipes/devel/bootstrap-sandbox.yaml b/recipes/devel/bootstrap-sandbox.yaml index 08060380..8285b67a 100644 --- a/recipes/devel/bootstrap-sandbox.yaml +++ b/recipes/devel/bootstrap-sandbox.yaml @@ -12,6 +12,10 @@ depends: forward: True # some required host tools + - + name: utils::pxargs + use: [tools] + forward: True - name: devel::make use: [tools] diff --git a/recipes/devel/compat/binutils.yaml b/recipes/devel/compat/binutils.yaml index 6db67d58..d80a61fb 100644 --- a/recipes/devel/compat/binutils.yaml +++ b/recipes/devel/compat/binutils.yaml @@ -11,7 +11,8 @@ checkoutSCM: # Some parts are compiled for the host during compilation. Hence we need the # host toolchain too. -buildTools: [host-toolchain, m4] +buildTools: [host-toolchain] +buildToolsWeak: [m4] buildVars: [AUTOCONF_HOST, AUTOCONF_TARGET, BINUTILS_PREFIX] buildScript: | diff --git a/recipes/devel/compat/gcc.yaml b/recipes/devel/compat/gcc.yaml index 5eb72531..5b9751b2 100644 --- a/recipes/devel/compat/gcc.yaml +++ b/recipes/devel/compat/gcc.yaml @@ -25,7 +25,8 @@ checkoutScript: | $<> \ $<> -buildTools: [host-toolchain, target-toolchain, m4] +buildTools: [host-toolchain, target-toolchain] +buildToolsWeak: [m4] buildVars: [AUTOCONF_BUILD, AUTOCONF_HOST, AUTOCONF_TARGET, GCC_TARGET_ARCH, GCC_TARGET_FLOAT_ABI, GCC_TARGET_FPU] buildScript: | @@ -172,8 +173,7 @@ multiPackage: cp -a ${BOB_DEP_PATHS[devel::compat::binutils]}/* install/ packageScript: | - installStripAll usr/bin - installStripAll usr/libexec + installStripAll usr/bin usr/libexec provideDeps: - libs::compat::glibc @@ -199,5 +199,4 @@ multiPackage: --enable-languages="${GCC_ENABLE_LANGUAGES:-c,c++}" packageScript: | - installStripAll usr/bin - installStripAll usr/libexec + installStripAll usr/bin usr/libexec diff --git a/recipes/devel/flex.yaml b/recipes/devel/flex.yaml index e3c87be5..07024bc1 100644 --- a/recipes/devel/flex.yaml +++ b/recipes/devel/flex.yaml @@ -10,7 +10,8 @@ checkoutSCM: stripComponents: 1 checkoutDeterministic: True -checkoutTools: [gettext, m4] +checkoutTools: [gettext] +checkoutToolsWeak: [m4] checkoutScript: | patchApplySeries $<@flex/*.patch@> autoconfReconfigure @@ -29,4 +30,4 @@ packageScript: | provideTools: flex: path: usr/bin - dependTools: [ m4 ] + dependToolsWeak: [ m4 ] diff --git a/recipes/devel/gcc.yaml b/recipes/devel/gcc.yaml index 57274803..4db55d2a 100644 --- a/recipes/devel/gcc.yaml +++ b/recipes/devel/gcc.yaml @@ -71,7 +71,7 @@ checkoutScript: | buildVars: [AUTOCONF_BUILD, AUTOCONF_HOST, AUTOCONF_TARGET, GCC_TARGET_ABI, GCC_TARGET_ARCH, GCC_TARGET_FLOAT_ABI, GCC_TARGET_FPU, GCC_MULTILIB, GCC_EXTRA_OPTIONS] -buildTools: [m4] +buildToolsWeak: [m4] buildScript: | GCC_SRC=$1 mkdir -p build install @@ -262,8 +262,7 @@ multiPackage: --enable-languages="${GCC_ENABLE_LANGUAGES:-c,c++}" packageScript: | - installStripAll usr/bin - installStripAll usr/libexec + installStripAll usr/bin usr/libexec provideDeps: - devel::binutils @@ -331,7 +330,6 @@ multiPackage: cp -an ${BOB_TOOL_PATHS[target-toolchain]}/$TOOLCHAIN_SYSROOT/* "install/${TARGET_SYSROOT}" packageScript: | - installStripAll ./${GCC_PREFIX:-/usr}/bin - installStripAll ./${GCC_PREFIX:-/usr}/libexec + installStripAll ./${GCC_PREFIX:-/usr}/bin ./${GCC_PREFIX:-/usr}/libexec provideDeps: [ "*-tgt" ] diff --git a/recipes/devel/help2man.yaml b/recipes/devel/help2man.yaml index e056af39..64b5b222 100644 --- a/recipes/devel/help2man.yaml +++ b/recipes/devel/help2man.yaml @@ -8,7 +8,8 @@ checkoutSCM: digestSHA1: "3ed88430c97af3c5b57949f6f030b913044af507" stripComponents: 1 -buildTools: [host-toolchain, perl] +buildTools: [host-toolchain] +buildToolsWeak: [perl] buildScript: | autotoolsBuild $1 diff --git a/recipes/devel/host-compat-toolchain.yaml b/recipes/devel/host-compat-toolchain.yaml index ab418758..ad965403 100644 --- a/recipes/devel/host-compat-toolchain.yaml +++ b/recipes/devel/host-compat-toolchain.yaml @@ -12,6 +12,9 @@ depends: # The following tools are needed by the cross-toolchain build process. # Build them explicitly here to keep the basement::rootrecipe class # untainted. + - name: utils::pxargs + use: [tools] + forward: True - name: devel::make use: [tools] forward: True diff --git a/recipes/devel/llvm.yaml b/recipes/devel/llvm.yaml index e6ddb84f..e584341c 100644 --- a/recipes/devel/llvm.yaml +++ b/recipes/devel/llvm.yaml @@ -219,7 +219,7 @@ multiPackage: /usr/bin/{ld.lld,lld,lld-link,wasm-ld} \ /usr/lib/ "/usr/lib/clang/***" \ "!*" - installStripAll . + installStripAll provideTools: clang: "usr/bin" @@ -229,7 +229,7 @@ multiPackage: installCopy "$1/install/" \ /usr/ /usr/bin/ /usr/bin/clangd \ "!*" - installStripAll . + installStripAll provideTools: clangd: "usr/bin" @@ -239,7 +239,7 @@ multiPackage: installCopy "$1/install/" \ /usr/ /usr/bin/ /usr/bin/{,git-}clang-format \ "!*" - installStripAll . + installStripAll provideTools: clang-format: "usr/bin" @@ -249,7 +249,7 @@ multiPackage: installCopy "$1/install/" \ /usr/ /usr/bin/ /usr/bin/{,run-}clang-tidy \ "!*" - installStripAll . + installStripAll provideTools: clang-tidy: "usr/bin" diff --git a/recipes/devel/ocaml.yaml b/recipes/devel/ocaml.yaml index 98c26c96..bcc9a04e 100644 --- a/recipes/devel/ocaml.yaml +++ b/recipes/devel/ocaml.yaml @@ -9,7 +9,7 @@ checkoutSCM: digestSHA256: eb9eab2f21758d3cfb1e78c7f83f0b4dd6302824316aba4abee047a5a4f85029 stripComponents: 1 -buildTools: [m4] +buildToolsWeak: [m4] buildVars: [STRIP] buildScript: | # Note: the configure script is broken - it generates the makefiles in the diff --git a/recipes/devel/sandbox.yaml b/recipes/devel/sandbox.yaml index ee8a34c0..13f61a99 100644 --- a/recipes/devel/sandbox.yaml +++ b/recipes/devel/sandbox.yaml @@ -21,6 +21,10 @@ depends: forward: True # some required host tools + - + name: utils::pxargs + use: [tools] + forward: True - name: devel::make use: [tools] diff --git a/recipes/devel/texinfo.yaml b/recipes/devel/texinfo.yaml index f3d8c78c..7612c117 100644 --- a/recipes/devel/texinfo.yaml +++ b/recipes/devel/texinfo.yaml @@ -9,7 +9,8 @@ checkoutSCM: digestSHA1: "d39c2e35ddb0aff6ebdd323ce53729bd215534fa" stripComponents: 1 -buildTools: [host-toolchain, perl] +buildTools: [host-toolchain] +buildToolsWeak: [perl] buildScript: | autotoolsBuild $1 diff --git a/recipes/kernel/linux-libc-headers.yaml b/recipes/kernel/linux-libc-headers.yaml index dc8973dd..466cc63f 100644 --- a/recipes/kernel/linux-libc-headers.yaml +++ b/recipes/kernel/linux-libc-headers.yaml @@ -19,7 +19,8 @@ checkoutSCM: digestSHA256: "d6ecff966f8c95ec4cb3bb303904f757b7de6a6bcfef0d0771cb852158e61c20" stripComponents: 1 -buildTools: [bison, flex, host-toolchain, m4] +buildTools: [bison, flex, host-toolchain] +buildToolsWeak: [m4] buildVars: [ARCH] buildScript: | # prevent timestamps in configuration diff --git a/recipes/libs/compat/glibc.yaml b/recipes/libs/compat/glibc.yaml index 0832a915..75008d68 100644 --- a/recipes/libs/compat/glibc.yaml +++ b/recipes/libs/compat/glibc.yaml @@ -20,8 +20,8 @@ checkoutScript: | patchApplySeries $<@glibc/*.diff@> buildVars: [AUTOCONF_TARGET] -buildTools: [host-toolchain, target-toolchain, bison, m4] -buildToolsWeak: [python3] +buildTools: [host-toolchain, target-toolchain, bison] +buildToolsWeak: [python3, m4] buildScript: | EXTRA= [ -e $1/usr/include/selinux/selinux.h ] || EXTRA+=" --without-selinux" diff --git a/recipes/libs/glibc.yaml b/recipes/libs/glibc.yaml index c73710a9..bf4498fb 100644 --- a/recipes/libs/glibc.yaml +++ b/recipes/libs/glibc.yaml @@ -17,8 +17,8 @@ checkoutSCM: stripComponents: 1 buildVars: [AS, CC, CXX, LD, AUTOCONF_HOST, GCC_MULTILIB, GLIBC_ENABLE_KERNEL] -buildTools: [host-toolchain, target-toolchain, bison, m4] -buildToolsWeak: [python3] +buildTools: [host-toolchain, target-toolchain, bison] +buildToolsWeak: [python3, m4] buildScript: | EXTRA= [ -e $1/usr/include/selinux/selinux.h ] || EXTRA+=" --without-selinux" diff --git a/recipes/libs/gmp.yaml b/recipes/libs/gmp.yaml index bf881dda..0c6f3871 100644 --- a/recipes/libs/gmp.yaml +++ b/recipes/libs/gmp.yaml @@ -19,7 +19,8 @@ checkoutScript: | updateConfigFile config.sub configfsf.sub autoconfReconfigure -buildTools: [host-toolchain, m4] +buildTools: [host-toolchain] +buildToolsWeak: [m4] buildScript: | autotoolsBuild $1 diff --git a/recipes/libs/openssl.yaml b/recipes/libs/openssl.yaml index 47a9f90e..5f9c0821 100644 --- a/recipes/libs/openssl.yaml +++ b/recipes/libs/openssl.yaml @@ -20,7 +20,8 @@ checkoutDeterministic: True checkoutScript: | patchApplySeries $<@openssl/*.patch@> -buildTools: [target-toolchain, perl] +buildTools: [target-toolchain] +buildToolsWeak: [perl] buildVars: [CC, AR, RANLIB, ARCH, AUTOCONF_HOST] buildScript: | mkdir -p install build diff --git a/recipes/libs/uclibc-l4re.yaml b/recipes/libs/uclibc-l4re.yaml index 3519b212..79cad9a9 100644 --- a/recipes/libs/uclibc-l4re.yaml +++ b/recipes/libs/uclibc-l4re.yaml @@ -16,7 +16,8 @@ checkoutDeterministic: True checkoutScript: | patchApplySeries $<@uclibc-l4re/*.patch@> -buildTools: [host-toolchain, target-toolchain, flex, bison, m4, perl] +buildTools: [host-toolchain, target-toolchain, flex, bison] +buildToolsWeak: [m4, perl] buildVars: [ARCH, CROSS_COMPILE] buildScript: | case $ARCH in diff --git a/recipes/net/curl.yaml b/recipes/net/curl.yaml index fe257819..fce3baeb 100644 --- a/recipes/net/curl.yaml +++ b/recipes/net/curl.yaml @@ -23,7 +23,7 @@ checkoutScript: | rm -f include/curl/curlbuild.h fi -buildTools: [perl] +buildToolsWeak: [perl] buildScript: | # Force configure script to use pkg-config, even when cross compiling. export PKGTEST=yes diff --git a/recipes/utils/pxargs.yaml b/recipes/utils/pxargs.yaml new file mode 100644 index 00000000..00fde105 --- /dev/null +++ b/recipes/utils/pxargs.yaml @@ -0,0 +1,15 @@ +# We don't inherit the cpackage class here on purpose! We want to keep the +# dependencies to a minimum! +buildTools: [target-toolchain] +buildVars: [CC, CPPFLAGS, CFLAGS, LDFLAGS] +buildScript: | + cp $<> main.c + cp $<> list.h + $CC -g ${CPPFLAGS:-} ${CFLAGS:-} ${LDFLAGS:-} main.c -o pxargs + +packageScript: | + mkdir -p usr/bin + cp "$1/pxargs" usr/bin/ + +provideTools: + pxargs: usr/bin diff --git a/recipes/utils/pxargs/list.h b/recipes/utils/pxargs/list.h new file mode 100644 index 00000000..83c63ea1 --- /dev/null +++ b/recipes/utils/pxargs/list.h @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2025 Jan Klötzke + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef LIST_H +#define LIST_H + +#include +#include + +struct list_node +{ + struct list_node *prev; + struct list_node *next; +}; + +#define LIST_HEAD(type__, anchor__) \ + struct { \ + struct list_node head; \ + type__ typevar[0]; \ + struct { \ + char off[offsetof(type__, anchor__)]; \ + } offsetvar[0]; \ + } + +#define DEFINE_LIST_HEAD(name, type, anchor) \ + LIST_HEAD(type, anchor) name = { { &(name).head, &(name).head } } + + +static inline void list_node_init(struct list_node *n) +{ + n->next = n->prev = n; +} + +static inline void list_node_append(struct list_node *l, struct list_node *e) +{ + e->next = l->next; + e->prev = l; + l->next->prev = e; + l->next = e; +} + +static inline void list_node_prepend(struct list_node *l, struct list_node *e) +{ + e->next = l; + e->prev = l->prev; + l->prev->next = e; + l->prev = e; +} + +static inline void list_node_del(struct list_node *e) +{ + e->prev->next = e->next; + e->next->prev = e->prev; + e->next = e->prev = e; +} + +static inline int list_node_in_list(struct list_node *e) +{ + return e->next != e; +} + + +#define list_init(l) \ + do { \ + (l)->head.prev = (struct list_node *)(l); \ + (l)->head.next = (struct list_node *)(l); \ + } while (0) + +#define list_empty(l) \ + ((l).head.next == &(l).head) + +#define list_add_front(head__, elem__) \ + do { \ + list_node_append(&((head__)->head), (struct list_node *)((uintptr_t)(elem__) + sizeof((head__)->offsetvar[0].off))); \ + } while (0) + +#define list_add_tail(head__, elem__) \ + do { \ + list_node_prepend(&((head__)->head), (struct list_node *)((uintptr_t)(elem__) + sizeof((head__)->offsetvar[0].off))); \ + } while (0) + +#define list_element_type(head__) \ + typeof((head__).typevar[0])* + +#define list_front(head__) \ + ((list_element_type((head__)))((uintptr_t)(head__).head.next - sizeof((head__).offsetvar[0].off))) + +#define list_pop_front(head__) \ + ({ \ + list_element_type(*head__) n = list_front(*head__); \ + list_node_del((head__)->head.next); \ + n; \ + }) + + +#define list_for_each(head__, var__) \ + for (typeof((head__).typevar[0]) *var__ = (void*)((uintptr_t)(head__).head.next - sizeof((head__).offsetvar[0].off)); \ + ((uintptr_t)var__ + sizeof((head__).offsetvar[0].off)) != (uintptr_t)&(head__).head; \ + var__ = (typeof((head__).typevar[0]) *)((uintptr_t)((struct list_node *)((uintptr_t)var__ + sizeof((head__).offsetvar[0].off)))->next - sizeof((head__).offsetvar[0].off))) + +#define list_iterate(head__, var__) \ + for (; \ + ((uintptr_t)var__ + sizeof((head__).offsetvar[0].off)) != (uintptr_t)&(head__).head; \ + var__ = (typeof((head__).typevar[0]) *)((uintptr_t)((struct list_node *)((uintptr_t)var__ + sizeof((head__).offsetvar[0].off)))->next - sizeof((head__).offsetvar[0].off))) + +#define list_for_each_safe(head__, var__) \ + for (typeof((head__).typevar[0]) *var__ = (void*)((uintptr_t)(head__).head.next - sizeof((head__).offsetvar[0].off)), *var__##_next; \ + (((uintptr_t)var__ + sizeof((head__).offsetvar[0].off)) != (uintptr_t)&(head__).head) && (var__##_next = (typeof((head__).typevar[0]) *)((uintptr_t)((struct list_node *)((uintptr_t)var__ + sizeof((head__).offsetvar[0].off)))->next - sizeof((head__).offsetvar[0].off)), 1); \ + var__ = var__##_next) + + +static inline void list_head_move__(struct list_node *dst, struct list_node *src) +{ + src->prev->next = dst; + src->next->prev = dst->prev; + dst->prev->next = src->next; + dst->prev = src->prev; + + list_node_init(src); +} + +#define list_move_tail(dst_head__, src_head__) \ + do { \ + if (!list_empty(src_head__)) \ + list_head_move__(&(dst_head__).head, &(src_head__).head); \ + } while (0) + +#endif diff --git a/recipes/utils/pxargs/main.c b/recipes/utils/pxargs/main.c new file mode 100644 index 00000000..a9ec3285 --- /dev/null +++ b/recipes/utils/pxargs/main.c @@ -0,0 +1,723 @@ +/* + * Small xargs like program that takes part of the make jobserver. + * + * Attention: assumes zero terminated input strings (-print0)! + * + * Copyright (c) 2025 Jan Klötzke + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "list.h" + +#define EXIT_ERROR (EXIT_FAILURE + 1) + +// Artificial token that represents our implicit token that we already have. +#define MY_TOKEN 0x100 + +struct input_file { + struct list_node node; + char *name; + dev_t dev; + ino_t ino; +}; + +struct running_child { + pid_t pid; + struct input_file *f; + int token; +}; + +int input_fd = 0; +char input_buf[PATH_MAX]; +unsigned input_len; + +DEFINE_LIST_HEAD(input_files, struct input_file, node); + +int jobs_pipe_rd = -1, jobs_pipe_wr = -1; +int my_token = MY_TOKEN; + +int jobs_argc; +char **jobs_argv; +int jobs_running, jobs_possible; + +volatile sig_atomic_t exit_status; +volatile sig_atomic_t zombies; + +struct running_child *children; + +char need_input, need_tokens; + +/** + * Signal termination. + * + * Will signal that we want to terminate. Errors take precedence above clean + * termination. + */ +static void set_done(int result) +{ + assert(result > 0); + if (exit_status < result) + exit_status = result; +} + +static int is_done(void) +{ + // We need to wait until the last child was reaped. + if (jobs_running) + return 0; + + if (exit_status) + // Premature termination + return exit_status; + else + // Regular termination if pipeline is idle + return input_fd < 0 && list_empty(input_files) && !jobs_running; +} + + +static struct input_file *new_input_file(char *fn) +{ + struct input_file *ret = calloc(1, sizeof(struct input_file)); + list_node_init(&ret->node); + ret->name = strdup(fn); + + struct stat sb; + if (lstat(fn, &sb) == 0) { + ret->dev = sb.st_dev; + ret->ino = sb.st_ino; + } else + perror(fn); + + return ret; +} + +static void delete_input_file(struct input_file *f) +{ + assert(!list_node_in_list(&f->node)); + free(f->name); + free(f); +} + + +static int get_next_token(void) +{ + // Always use our implicit job token first! + if (my_token) { + my_token = 0; + return MY_TOKEN; + } + + char buf; + switch (read(jobs_pipe_rd, &buf, 1)) { + case 1: + return buf; + case 0: + // That should not happen. The jobs token pipe was + // closed on the write end! + fprintf(stderr, "Broken jobs pipe"); + set_done(EXIT_ERROR); + return -1; + case -1: + if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) + return -1; + perror("jobs pipe read"); + set_done(EXIT_ERROR); + return -1; + } + + // unreachable + abort(); +} + +void return_token(int token) +{ + if (token == MY_TOKEN) { + my_token = MY_TOKEN; + } else { + char buf = token; + for (;;) { + ssize_t r = write(jobs_pipe_wr, &buf, 1); + if (r > 0) { + break; + } else if (r == 0) { + fprintf(stderr, "Broken jobs pipe"); + set_done(EXIT_ERROR); + } else if (r < 0 && errno != EINTR) { + perror("jobs pipe write"); + set_done(EXIT_ERROR); + } + } + } +} + + +static void add_child(pid_t pid, struct input_file *f, int token) +{ + // We know that there is enough room in the array + struct running_child *c = children; + while (c->pid) + ++c; + + c->pid = pid; + c->f = f; + c->token = token; +} + +static void remove_child(pid_t pid) +{ + struct running_child *c = children; + while (c->pid != pid) + ++c; + + c->pid = 0; + return_token(c->token); + delete_input_file(c->f); +} + +static int can_use_file(struct input_file *f) +{ + // Just in case the file couldn't be read... + if (f->dev == 0 && f->ino == 0) + return 1; + + for (int i = 0; i < jobs_possible; i++) { + if (children[i].pid == 0) + continue; + + // If a child is running that procsses the same file + // (hardlink), we postpone it. + if (children[i].f->dev == f->dev && children[i].f->ino == f->ino) + return 0; + } + + return 1; +} + + +static struct input_file *read_next_files() +{ + // Nothing to do if input is already exhausted or we're going down. + if (input_fd < 0 || exit_status) + return NULL; + + // We must not call read() with a length of zero. The post-read logic + // ensures that there is still room in the buffer. + ssize_t r = read(input_fd, &input_buf[input_len], sizeof(input_buf) - input_len); + if (r < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) + return NULL; + if (errno == EINTR) + return NULL; + + perror("stdin read"); + set_done(EXIT_ERROR); + return NULL; + } else if (r == 0) { + // End of input stream. + input_fd = -1; + return NULL; + } else + input_len += (unsigned)r; + + // Dissect input stream into individual files. + struct input_file *ret = NULL; + char *next = input_buf, *end; + unsigned len = input_len; + while (len && (end = (char *)memchr(next, 0, len))) { + struct input_file *nf = new_input_file(next); + list_add_tail(&input_files, nf); + + if (!ret) + ret = nf; + + len -= (end - next) + 1; + next = end + 1; + } + + // If the whole buffer was filled without a single, full file name, + // we're screwed. + if (len >= sizeof(input_buf)) { + fprintf(stderr, "Maximum path length reached!\n"); + set_done(EXIT_ERROR); + return NULL; + } + + input_len = len; + if (next != input_buf && len) + memmove(input_buf, next, len); + + return ret; +} + +/** + * Get next file from stdin. + * + * This will check for potential hard links of files that are already processed + * by a child. + */ +static struct input_file *get_next_file(void) +{ + // First let's see if any of the already ingested files can be used. + list_for_each(input_files, n) { + if (can_use_file(n)) { + list_node_del(&n->node); + return n; + } + } + + // Either we have no files or they were not usable. Read the next chunk + // of files... + struct input_file *n = read_next_files(); + if (!n) + return NULL; + + // Starting from the first read file name, try to find one that is + // usable... + list_iterate(input_files, n) { + if (can_use_file(n)) { + list_node_del(&n->node); + return n; + } + } + + return NULL; +} + +static void unget_next_file(struct input_file *f) +{ + list_add_front(&input_files, f); +} + +static void process_file(char *fn) +{ + jobs_argv[jobs_argc - 1] = fn; + execvp(jobs_argv[0], jobs_argv); + + perror("execve"); + fprintf(stderr, "Could not process %s\n", fn); + exit(EXIT_FAILURE); +} + +static pid_t fork_file_worker(char *fn) +{ + pid_t child_pid = fork(); + if (child_pid == 0) + process_file(fn); + else if (child_pid < 0) + perror("fork"); + + return child_pid; +} + + +static unsigned schedule(void) +{ + unsigned scheduled = 0; + + while (jobs_running < jobs_possible && !exit_status) { + struct input_file *next = get_next_file(); + if (!next) { + need_input = 1; + break; + } + + int token = get_next_token(); + if (token < 0) { + unget_next_file(next); + need_tokens = 1; + break; + } + + pid_t child = fork_file_worker(next->name); + if (child > 0) { + add_child(child, next, token); + jobs_running++; + } else { + set_done(EXIT_ERROR); + return_token(token); + delete_input_file(next); + break; + } + + scheduled++; + } + + return scheduled; +} + +static void wait_event(void) +{ + fd_set rfds; + + if (is_done()) + return; + + // Block SIGCHLD before going to sleep. We only want the handler to + // fire during pselect() after we checked! This is needed so that the + // waiting does not race with a concurrent SIGCHLD. + sigset_t block_set, orig_set; + sigemptyset(&block_set); + sigaddset(&block_set, SIGCHLD); + if (sigprocmask(SIG_BLOCK, &block_set, &orig_set) < 0) { + perror("sigprocmask"); + return; + } + + if (zombies) { + // The SIGCHLD just arrived. Don't sleep. + } else if (!exit_status) { + FD_ZERO(&rfds); + int nfds = 0; + if (need_input && input_fd >= 0) { + FD_SET(0, &rfds); + nfds++; + } + if (need_tokens) { + FD_SET(jobs_pipe_rd, &rfds); + nfds++; + } + pselect(nfds, &rfds, NULL, NULL, NULL, &orig_set); + need_input = need_tokens = 0; + } else if (jobs_running > 0) { + pselect(0, NULL, NULL, NULL, NULL, &orig_set); + } + + sigprocmask(SIG_SETMASK, &orig_set, NULL); +} + +static int reap_childs(void) +{ + int reaped = 0; + int status; + + zombies = 0; + + while (jobs_running > 0) { + pid_t pid = waitpid(-1, &status, WNOHANG); + if (pid > 0) { + reaped++; + jobs_running--; + remove_child(pid); + if (WIFEXITED(status)) { + if (WEXITSTATUS(status) != 0) + set_done(EXIT_FAILURE); + } else if (WIFSIGNALED(status)) { + set_done(EXIT_FAILURE); + } else { + fprintf(stderr, "unexpected waitpid status %d", status); + set_done(EXIT_ERROR); + } + } else if (pid == 0) { + break; + } else if (errno != EINTR) { + perror("waitpid"); + set_done(EXIT_ERROR); + break; + } + } + + return reaped; +} + +static void handle_sigchld(int signo) +{ + (void)signo; + zombies = 1; +} + +static void handle_sigint(int signo) +{ + (void)signo; + set_done(EXIT_FAILURE); +} + +static int handle_signal(int signo, void (*handler)(int)) +{ + struct sigaction act = { 0 }; + act.sa_handler = handler; + if (sigaction(signo, &act, NULL) < 0) { + perror("sigaction"); + return 0; + } + + return 1; +} + +static void parse_jobs_auth_named(char *path) +{ + jobs_pipe_rd = open(path, O_RDONLY | O_CLOEXEC | O_NONBLOCK); + if (jobs_pipe_rd < 0) { + perror(path); + return; + } + + jobs_pipe_wr = open(path, O_WRONLY | O_CLOEXEC); + if (jobs_pipe_wr < 0) { + perror(path); + close(jobs_pipe_rd); + jobs_pipe_rd = -1; + return; + } +} + +static void parse_jobs_auth_anon(char *numbers) +{ + char *end; + + errno = 0; + jobs_pipe_rd = strtol(numbers, &end, 0); + if (end == numbers || errno || jobs_pipe_rd < 0 || *end != ',') { + jobs_pipe_rd = jobs_pipe_wr = -1; + return; + } + + numbers = end + 1; + jobs_pipe_wr = strtol(numbers, &end, 0); + if (end == numbers || errno || jobs_pipe_wr < 0 || *end != '\0') { + jobs_pipe_rd = jobs_pipe_wr = -1; + return; + } +} + +// See https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html +static void parse_jobs_auth(char *arg) +{ + if (strncmp(arg, "fifo:", 5) == 0) + parse_jobs_auth_named(arg + 5); + else + parse_jobs_auth_anon(arg); +} + +#define OPT_JOBSERVER_AUTH 0x100 + +static int parse_options(int argc, char **argv, int from_cmd_line) +{ + optind = 1; + + static struct option long_options[] = { + {"jobserver-auth", required_argument, NULL, OPT_JOBSERVER_AUTH}, + {0, 0, 0, 0} + }; + + for (;;) { + int c = getopt_long(argc, argv, "j:", long_options, NULL); + if (c == -1) + break; + + switch (c) { + case 'j': + if (jobs_possible && from_cmd_line) { + fprintf(stderr, "Warning: override inherited job server!\n"); + if (jobs_pipe_rd >= 0) + close(jobs_pipe_rd); + if (jobs_pipe_wr >= 0) + close(jobs_pipe_wr); + jobs_pipe_rd = jobs_pipe_wr = -1; + unsetenv("MAKEFLAGS"); + } + jobs_possible = atoi(optarg); + break; + case OPT_JOBSERVER_AUTH: + if (!from_cmd_line) { + parse_jobs_auth(optarg); + break; + } + // fallthrough + case '?': + if (from_cmd_line) { + fprintf(stderr, "Unknown argument: %s", argv[optind]); + exit(EXIT_FAILURE); + } + break; + default: + abort(); + } + } + + return optind; +} + +static void parse_makeflags(char const *mf) +{ + size_t len = strlen(mf); + if (len == 0) + return; + + char *args = NULL; + + // If MAKEFLAGS starts with a space, there are no single letter + // options. Otherwise, we have to add a "-"... + if (isblank((unsigned char)*mf)) { + while (*mf && isblank((unsigned char)*mf)) + ++mf; + + if (!*mf) + return; + + args = strdup(mf); + } else { + // We need to add a "-" for single letter options. + args = malloc(len + 2); + args[0] = '-'; + strcpy(&args[1], mf); + } + + // The worst case are single letters with spaces in between. We need an + // additional argv[0] and a trailing NULL pointer. + char **argv = malloc(((len + 2) / 2 + 2) * sizeof(char *)); + argv[0] = ""; // dummy because getopt always starts at optind == 1 + int argc = 1; + + // Assume words separated by whitespace. There doesn't seem to be a + // formal definition for how whitespace in option arguments should be + // handled. + char *arg = args; + while (*arg) { + argv[argc++] = arg; + while (*arg && !isblank((unsigned char)*arg)) + ++arg; + if (!*arg) + break; + + *arg++ = '\0'; + + while (*arg && isblank((unsigned char)*arg)) + ++arg; + } + argv[argc] = NULL; + + parse_options(argc, argv, 0); + + free(argv); + free(args); +} + +int main(int argc, char **argv) +{ + // Parse make jobserver configuration. + char const *mf = getenv("MAKEFLAGS"); + if (mf) + parse_makeflags(mf); + + // Make is weird when it doesn't think the rule is a recursive "make". + // It will still pass the parallel make information but set the job + // token pipe file descriptors to a negative number. + if (jobs_possible > 1 && (jobs_pipe_rd < 0 || jobs_pipe_wr < 0)) { + fprintf(stderr, "Missing job server in recursive make! Add '+' to your Makefile rule.\n"); + jobs_possible = 0; + } + + // Parse command line options. + int used_args = parse_options(argc, argv, 1); + + jobs_argc = argc - used_args + 1; + if (jobs_argc < 1) { + fprintf(stderr, "usage: %s [-j N] [--] command [intial-args...]\n", + basename(argv[0])); + return 1; + } + + // Prepare argv[] of children + jobs_argv = malloc(sizeof(char *) * (jobs_argc + 1)); + for (int i = 0; i < jobs_argc - 1; i++) + jobs_argv[i] = argv[used_args + i]; + jobs_argv[jobs_argc] = NULL; + + // Make sure the job server is in the right state + if (jobs_possible <= 0) + jobs_possible = 1; + + // Create our own jobserver if we're not running under one already. + if (jobs_possible > 1 && (jobs_pipe_rd < 0 || jobs_pipe_wr < 0)) { + int pipefd[2]; + if (pipe(pipefd) < 0) { + perror("pipe2"); + return EXIT_FAILURE; + } + jobs_pipe_rd = pipefd[0]; + jobs_pipe_wr = pipefd[1]; + + for (int i = 0; i < jobs_possible - 1; i++) + write(jobs_pipe_wr, "a", 1); + + // The first whitespace in MAKEFLAGS is important! It means + // there are no single letter options. + char buf[64]; + int len = snprintf(buf, sizeof(buf), " -j%d --jobserver-auth=%d,%d", + jobs_possible, jobs_pipe_rd, jobs_pipe_wr); + if (len < 0 || (unsigned)len >= sizeof(buf)) { + fprintf(stderr, "Could not synthesize MAKEFLAGS\n"); + return EXIT_ERROR; + } + setenv("MAKEFLAGS", buf, 1); + } + + children = calloc(jobs_possible, sizeof(*children)); + + // Establish SIGCHLD handler. It will wake us from any blocking operation. + struct sigaction act = { 0 }; + act.sa_handler = &handle_sigchld; + act.sa_flags = SA_NOCLDSTOP; + if (sigaction(SIGCHLD, &act, NULL) < 0) { + perror("sigaction"); + return 2; + } + + // Catch SIGINT/SIGTERM/... to shut down cleanly on interruption. + if (!handle_signal(SIGINT, &handle_sigint)) + return EXIT_ERROR; + if (!handle_signal(SIGTERM, &handle_sigint)) + return EXIT_ERROR; + if (!handle_signal(SIGALRM, &handle_sigint)) + return EXIT_ERROR; + if (!handle_signal(SIGHUP, &handle_sigint)) + return EXIT_ERROR; + if (!handle_signal(SIGQUIT, &handle_sigint)) + return EXIT_ERROR; + + // Catch SIGPIPE as well, just in case we're part of a pipeline. + if (!handle_signal(SIGPIPE, &handle_sigint)) + return EXIT_ERROR; + + // Run as long as no error was encountered or if there is some + // subprocess running... + while (!is_done()) { + if (!reap_childs() && !schedule()) + wait_event(); + } + + return exit_status; +}