Merge pull request #89 from threefoldtech/development-zos-v4-new-bootstrap

v4: use latest version expected from spec and latest ubuntu as base image

Author: maxux

.github/workflows/kernel.yaml

@@ -2,7 +2,8 @@ name: Kernel
 on:
   push:
     branches:
-      - "*"
+      - "*-v4"
+      - "*-v4-*"
     tags-ignore:
       - v*
     paths-ignore:
@@ -28,9 +29,9 @@ env:
 
 jobs:
   kernel:
-    name: "Zero-OS Kernel Image"
+    name: "Zero-OS Kernel Image (v4)"
     runs-on: ubuntu-latest
-    container: ubuntu:20.04
+    container: ubuntu:24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v1
@@ -47,7 +48,7 @@ jobs:
       - name: "Install: go"
         uses: actions/setup-go@v1
         with:
-          go-version: 1.21
+          go-version: 1.23.4
         id: go
 
       - name: "Install: requirements"

autobuild/tf-build-deps-clean.sh

@@ -51,4 +51,7 @@ deps+=(musl musl-tools)
 # bmon dependencies
 deps+=(libconfuse-dev ncurses-bin)
 
+# xfsprogs dependencies
+deps+=(libinih-dev liburcu-dev)
+
 apt-get install -y ${deps[@]}

autobuild/tf-build-deps.sh

@@ -50,10 +50,13 @@ deps+=(musl musl-tools)
 # bmon dependencies
 deps+=(libconfuse-dev ncurses-bin)
 
+# xfsprogs dependencies
+deps+=(libinih-dev liburcu-dev)
+
 apt-get install -y ${deps[@]}
 
 # install go
-GOVER="1.21.0"
+GOVER="1.23.4"
 curl -L https://dl.google.com/go/go${GOVER}.linux-amd64.tar.gz > /tmp/go${GOVER}.linux-amd64.tar.gz
 tar -C /usr/local -xzf /tmp/go${GOVER}.linux-amd64.tar.gz
 mkdir -p /gopath

autobuild/tf-build-upload.sh

@@ -13,5 +13,5 @@ echo "[+] branch: ${linkname}"
 
 cp vmlinuz.efi "${kernel}"
 
-curl -b "token=${BOOTSTRAP_TOKEN}" -X POST -F "kernel=@${kernel}" "https://bootstrap.grid.tf/api/kernel"
-curl -b "token=${BOOTSTRAP_TOKEN}" "https://bootstrap.grid.tf/api/symlink/${linkname}/${kernel}"
+curl -b "token=${BOOTSTRAP_TOKEN}" -X POST -F "kernel=@${kernel}" "https://v4.bootstrap.grid.tf/api/kernel"
+curl -b "token=${BOOTSTRAP_TOKEN}" "https://v4.bootstrap.grid.tf/api/symlink/${linkname}/${kernel}"

config/build/busybox-config

@@ -1,7 +1,7 @@
 #
 # Automatically generated make config: don't edit
 # Busybox version: 1.36.1
-# Mon Aug 26 15:24:30 2024
+# Thu Jan 30 06:59:04 2025
 #
 CONFIG_HAVE_DOT_CONFIG=y
 
@@ -968,8 +968,8 @@ CONFIG_PSCAN=y
 CONFIG_ROUTE=y
 CONFIG_SLATTACH=y
 # CONFIG_SSL_CLIENT is not set
-CONFIG_TC=y
-CONFIG_FEATURE_TC_INGRESS=y
+# CONFIG_TC is not set
+# CONFIG_FEATURE_TC_INGRESS is not set
 CONFIG_TCPSVD=y
 CONFIG_UDPSVD=y
 CONFIG_TELNET=y

config/build/kernel-config-generic

@@ -1,18 +1,21 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/x86 6.6.51 Kernel Configuration
+# Linux/x86 6.6.74 Kernel Configuration
 #
-CONFIG_CC_VERSION_TEXT="gcc (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0"
+CONFIG_CC_VERSION_TEXT="gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0"
 CONFIG_CC_IS_GCC=y
-CONFIG_GCC_VERSION=90400
+CONFIG_GCC_VERSION=130300
 CONFIG_CLANG_VERSION=0
 CONFIG_AS_IS_GNU=y
-CONFIG_AS_VERSION=23400
+CONFIG_AS_VERSION=24200
 CONFIG_LD_IS_BFD=y
-CONFIG_LD_VERSION=23400
+CONFIG_LD_VERSION=24200
 CONFIG_LLD_VERSION=0
 CONFIG_CC_CAN_LINK=y
 CONFIG_CC_CAN_LINK_STATIC=y
+CONFIG_CC_HAS_ASM_GOTO_OUTPUT=y
+CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT=y
+CONFIG_TOOLS_SUPPORT_RELR=y
 CONFIG_CC_HAS_ASM_INLINE=y
 CONFIG_CC_HAS_NO_PROFILE_FN_ATTR=y
 CONFIG_PAHOLE_VERSION=0
@@ -180,6 +183,7 @@ CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH=y
 CONFIG_CC_HAS_INT128=y
 CONFIG_CC_IMPLICIT_FALLTHROUGH="-Wimplicit-fallthrough=5"
 CONFIG_GCC10_NO_ARRAY_BOUNDS=y
+CONFIG_CC_NO_ARRAY_BOUNDS=y
 CONFIG_ARCH_SUPPORTS_INT128=y
 CONFIG_NUMA_BALANCING=y
 CONFIG_NUMA_BALANCING_DEFAULT_ENABLED=y
@@ -502,7 +506,6 @@ CONFIG_PHYSICAL_ALIGN=0x1000000
 CONFIG_DYNAMIC_MEMORY_LAYOUT=y
 CONFIG_RANDOMIZE_MEMORY=y
 CONFIG_RANDOMIZE_MEMORY_PHYSICAL_PADDING=0xa
-# CONFIG_ADDRESS_MASKING is not set
 CONFIG_HOTPLUG_CPU=y
 # CONFIG_COMPAT_VDSO is not set
 CONFIG_LEGACY_VSYSCALL_XONLY=y
@@ -516,6 +519,7 @@ CONFIG_HAVE_LIVEPATCH=y
 # CONFIG_LIVEPATCH is not set
 # end of Processor type and features
 
+CONFIG_CC_HAS_SLS=y
 CONFIG_CC_HAS_RETURN_THUNK=y
 CONFIG_CC_HAS_ENTRY_PADDING=y
 CONFIG_FUNCTION_PADDING_CFI=11
@@ -534,6 +538,7 @@ CONFIG_CALL_DEPTH_TRACKING=y
 CONFIG_CPU_IBPB_ENTRY=y
 CONFIG_CPU_IBRS_ENTRY=y
 CONFIG_CPU_SRSO=y
+# CONFIG_SLS is not set
 # CONFIG_GDS_FORCE_MITIGATION is not set
 CONFIG_MITIGATION_RFDS=y
 CONFIG_MITIGATION_SPECTRE_BHI=y
@@ -1041,11 +1046,11 @@ CONFIG_ZSWAP_COMPRESSOR_DEFAULT_LZO=y
 # CONFIG_ZSWAP_COMPRESSOR_DEFAULT_ZSTD is not set
 CONFIG_ZSWAP_COMPRESSOR_DEFAULT="lzo"
 CONFIG_ZSWAP_ZPOOL_DEFAULT_ZBUD=y
-# CONFIG_ZSWAP_ZPOOL_DEFAULT_Z3FOLD is not set
+# CONFIG_ZSWAP_ZPOOL_DEFAULT_Z3FOLD_DEPRECATED is not set
 # CONFIG_ZSWAP_ZPOOL_DEFAULT_ZSMALLOC is not set
 CONFIG_ZSWAP_ZPOOL_DEFAULT="zbud"
 CONFIG_ZBUD=y
-CONFIG_Z3FOLD=y
+# CONFIG_Z3FOLD_DEPRECATED is not set
 CONFIG_ZSMALLOC=y
 # CONFIG_ZSMALLOC_STAT is not set
 CONFIG_ZSMALLOC_CHAIN_SIZE=8
@@ -2316,6 +2321,7 @@ CONFIG_ZRAM_DEF_COMP_LZORLE=y
 # CONFIG_ZRAM_DEF_COMP_842 is not set
 CONFIG_ZRAM_DEF_COMP="lzo-rle"
 CONFIG_ZRAM_WRITEBACK=y
+# CONFIG_ZRAM_TRACK_ENTRY_ACTIME is not set
 # CONFIG_ZRAM_MEMORY_TRACKING is not set
 # CONFIG_ZRAM_MULTI_COMP is not set
 CONFIG_BLK_DEV_LOOP=m
@@ -5058,6 +5064,7 @@ CONFIG_HID_GENERIC=m
 CONFIG_HID_SENSOR_HUB=m
 # CONFIG_HID_SENSOR_CUSTOM_SENSOR is not set
 # CONFIG_HID_ALPS is not set
+# CONFIG_HID_MCP2200 is not set
 # CONFIG_HID_MCP2221 is not set
 # end of Special HID drivers
 
@@ -7014,9 +7021,9 @@ CONFIG_TMPFS_XATTR=y
 # CONFIG_TMPFS_INODE64 is not set
 # CONFIG_TMPFS_QUOTA is not set
 CONFIG_HUGETLBFS=y
+# CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON is not set
 CONFIG_HUGETLB_PAGE=y
 CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP=y
-# CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON is not set
 CONFIG_ARCH_HAS_GIGANTIC_PAGE=y
 CONFIG_CONFIGFS_FS=y
 CONFIG_EFIVAR_FS=y
@@ -7240,6 +7247,9 @@ CONFIG_ENCRYPTED_KEYS=m
 # CONFIG_USER_DECRYPTED_DATA is not set
 CONFIG_KEY_DH_OPERATIONS=y
 # CONFIG_SECURITY_DMESG_RESTRICT is not set
+CONFIG_PROC_MEM_ALWAYS_FORCE=y
+# CONFIG_PROC_MEM_FORCE_PTRACE is not set
+# CONFIG_PROC_MEM_NO_FORCE is not set
 CONFIG_SECURITY=y
 CONFIG_SECURITYFS=y
 # CONFIG_SECURITY_NETWORK is not set
@@ -7272,9 +7282,16 @@ CONFIG_LSM="lockdown,yama,loadpin,safesetid,integrity"
 #
 # Memory initialization
 #
+CONFIG_CC_HAS_AUTO_VAR_INIT_PATTERN=y
+CONFIG_CC_HAS_AUTO_VAR_INIT_ZERO_BARE=y
+CONFIG_CC_HAS_AUTO_VAR_INIT_ZERO=y
 CONFIG_INIT_STACK_NONE=y
+# CONFIG_INIT_STACK_ALL_PATTERN is not set
+# CONFIG_INIT_STACK_ALL_ZERO is not set
 # CONFIG_INIT_ON_ALLOC_DEFAULT_ON is not set
 # CONFIG_INIT_ON_FREE_DEFAULT_ON is not set
+CONFIG_CC_HAS_ZERO_CALL_USED_REGS=y
+# CONFIG_ZERO_CALL_USED_REGS is not set
 # end of Memory initialization
 
 #
@@ -7779,6 +7796,8 @@ CONFIG_HAVE_ARCH_KGDB=y
 CONFIG_ARCH_HAS_UBSAN_SANITIZE_ALL=y
 # CONFIG_UBSAN is not set
 CONFIG_HAVE_ARCH_KCSAN=y
+CONFIG_HAVE_KCSAN_COMPILER=y
+# CONFIG_KCSAN is not set
 # end of Generic Kernel Debugging Instruments
 
 #

initramfs.sh

@@ -41,7 +41,7 @@ fi
 #
 # Flags
 #
-OPTS=$(getopt -o adbtckMelmnzrh --long all,download,busybox,tools,cores,kernel,modules,extensions,clean,mrproper,nomirror,compact,release,help -n 'parse-options' -- "$@")
+OPTS=$(getopt -o adbtfckMelmnzrh --long all,download,busybox,tools,firmwares,cores,kernel,modules,extensions,clean,mrproper,nomirror,compact,release,help -n 'parse-options' -- "$@")
 if [ $? != 0 ]; then
     echo "Failed parsing options." >&2
     exit 1
@@ -56,6 +56,7 @@ if [ "$OPTS" != " --" ] && [ "$OPTS" != " --release --" ]; then
     DO_DOWNLOAD=0
     DO_BUSYBOX=0
     DO_TOOLS=0
+    DO_FIRMWARES=0
     DO_CORES=0
     DO_KERNEL=0
     DO_KMODULES=0
@@ -73,6 +74,7 @@ while true; do
         -d | --download)   DO_DOWNLOAD=1;       shift ;;
         -b | --busybox)    DO_BUSYBOX=1;        shift ;;
         -t | --tools)      DO_TOOLS=1;          shift ;;
+        -f | --firmwares)  DO_FIRMWARES=1;      shift ;;
         -c | --cores)      DO_CORES=1;          shift ;;
         -k | --kernel)     DO_KERNEL=1;         shift ;;
         -M | --modules)    DO_KMODULES=1;       shift ;;
@@ -88,6 +90,7 @@ while true; do
             echo " -d --download    only download and extract archives"
             echo " -b --busybox     only (re)build busybox"
             echo " -t --tools       only (re)build tools (ssl, fuse, ...)"
+            echo " -f --firmwares   only (re)build linux-firmwares"
             echo " -c --cores       only (re)build core0 and coreX"
             echo " -k --kernel      only (re)build kernel (vmlinuz, produce final image)"
             echo " -M --modules     only (re)build kernel modules"
@@ -496,12 +499,28 @@ optimize_size() {
 
     echo "[+] optimizing binaries size"
 
-    for file in $(find ./bin ./sbin ./libexec ./usr/bin ./usr/sbin ./usr/libexec ./usr/lib -type f); do
+    excludes=("./usr/lib/libc.so.6" "./usr/lib/libz.so.1.2.11" "./usr/lib/libzstd.so.1")
+
+    for file in $(find ./bin ./sbin ./usr/bin ./usr/sbin ./usr/libexec ./usr/lib -type f); do
+        skipfile=0
+
+        for ex in "${excludes[@]}"; do
+            if [ "${file}" == "${ex}" ]; then
+                echo "[+] strip: skipping ${file}"
+                skipfile=1
+            fi
+        done
+
+        if [ $skipfile -eq 1 ]; then
+            continue
+        fi
+
         # dumping 4 first bytes
         header=$(dd if=$file bs=1 count=4 2> /dev/null | hexdump -e '/1 "%02X"')
 
         # checking if it's a ELF file
         if [ "$header" == "7F454C46" ]; then
+            # echo "[+] strip: ${file}"
             strip --strip-debug $file || true
         fi
     done
@@ -580,7 +599,8 @@ zero_os_root() {
     cp -a "${CONFDIR}"/debugfs/ssh-add-github-key "${ROOTDIR}"/usr/sbin/
 
     # Ensure ncurses terminfo are available (needed for bmon)
-    cp -ar /lib/terminfo ${ROOTDIR}/lib/
+    cp -ar /usr/share/terminfo ${ROOTDIR}/lib/
+    cp -ar /usr/share/terminfo ${ROOTDIR}/usr/share/
 
     # Copy debugging helpers
     if [ "${BUILDMODE}" = "debug" ]; then
@@ -720,7 +740,6 @@ main() {
         build_netcat
         build_redis
         build_ethtool
-        build_rtinfo
         build_seektime
         build_curl
         build_zflist
@@ -730,7 +749,6 @@ main() {
         build_bcache
         build_tcpdump
         build_rscoreutils
-        build_firmware
         build_xfsprogs
         build_bmon
 
@@ -745,6 +763,11 @@ main() {
         ## disabled build
         # build_qemu
         # build_restic
+        # build_rtinfo
+    fi
+
+    if [[ $DO_ALL == 1 ]] || [[ $DO_FIRMWARES == 1 ]]; then
+        build_firmware
     fi
 
     if [[ $DO_ALL == 1 ]] || [[ $DO_CORES == 1 ]]; then

packages/curl.sh

@@ -1,27 +1,28 @@
-CURL_VERSION="7_65_1"
-CURL_CHECKSUM="651706b87c501317030ec317dd84f3ef"
-CURL_LINK="https://github.com/curl/curl/archive/curl-${CURL_VERSION}.tar.gz"
+CURL_VERSION="8.11.1"
+CURL_CHECKSUM="25e65a5156ca4928060b61cb051813db"
+CURL_LINK="https://curl.se/download/curl-${CURL_VERSION}.tar.xz"
 
 download_curl() {
     download_file $CURL_LINK $CURL_CHECKSUM
 }
 
 extract_curl() {
-    # curl-curl is not a script mistake
-    if [ ! -d "curl-curl-${CURL_VERSION}" ]; then
+    if [ ! -d "curl-${CURL_VERSION}" ]; then
         echo "[+] extracting: curl-${CURL_VERSION}"
-        tar -xf ${DISTFILES}/curl-${CURL_VERSION}.tar.gz -C .
+        tar -xf ${DISTFILES}/curl-${CURL_VERSION}.tar.xz -C .
     fi
 }
 
 prepare_curl() {
     echo "[+] configuring curl"
+
     autoreconf -f -i -s
-    ./configure --disable-debug --enable-optimize --disable-curldebug --disable-symbol-hiding --disable-rt \
+    ./configure --prefix=${ROOTDIR}/usr \
+        --disable-debug --enable-optimize --disable-curldebug --disable-symbol-hiding --disable-rt \
         --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict \
         --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher \
-        --disable-manual --disable-libcurl-option --disable-sspi --disable-ntlm-wb --without-brotli --without-librtmp --without-winidn \
-        --disable-threaded-resolver \
+        --disable-manual --disable-libcurl-option --disable-sspi --without-brotli --without-librtmp --without-winidn \
+        --disable-threaded-resolver --without-libpsl --without-zstd \
         --with-openssl
 }
 
@@ -34,8 +35,7 @@ install_curl() {
 }
 
 build_curl() {
-    # curl-curl is not a script mistake
-    pushd "${WORKDIR}/curl-curl-${CURL_VERSION}"
+    pushd "${WORKDIR}/curl-${CURL_VERSION}"
 
     prepare_curl
     compile_curl

packages/iproute2.sh

@@ -1,5 +1,5 @@
-IPROUTE2_VERSION="5.4.0"
-IPROUTE2_CHECKSUM="54d86cadb4cd1d19fd7114b4e53adf51"
+IPROUTE2_VERSION="6.13.0"
+IPROUTE2_CHECKSUM="1603d25120d03feeaba9b360d03ffaec"
 IPROUTE2_LINK="https://www.kernel.org/pub/linux/utils/net/iproute2/iproute2-${IPROUTE2_VERSION}.tar.xz"
 
 download_iproute2() {

packages/libfuse.sh

@@ -1,5 +1,5 @@
-FUSE_VERSION="2.9.7"
-FUSE_CHECKSUM="91c97e5ae0a40312115dfecc4887bd9d"
+FUSE_VERSION="2.9.9"
+FUSE_CHECKSUM="23009734faca2f62d337e3a59be4c280"
 FUSE_LINK="https://github.com/libfuse/libfuse/archive/fuse-${FUSE_VERSION}.tar.gz"
 
 download_fuse() {
@@ -16,6 +16,12 @@ extract_fuse() {
 prepare_fuse() {
     echo "[+] preparing fuse"
 
+    if [ ! -f .patched_fuse-2.9.9-closefrom-glibc-2-34.patch ]; then
+        echo "[+] applying patch"
+        patch -p1 < ${PATCHESDIR}/fuse-2.9.9-closefrom-glibc-2-34.patch
+        touch .patched_fuse-2.9.9-closefrom-glibc-2-34.patch
+    fi
+
     ./makeconf.sh
     ./configure --prefix /usr
 }