Merge commit '6e22be2514d02aeee3e58ae72028842c83584849'

Author: RuoshiZhang

lib/mmseqs/.cirrus.yml

@@ -5,13 +5,10 @@ task:
   name: FreeBSD
   freebsd_instance:
     image_family: freebsd-14-2-snap
-  install_script: |
-    # pkg update -f
-    pkg install -y cmake git samtools rust
-    export PATH=/usr/local/bin:$PATH
+  install_script: pkg install -y cmake git samtools
   compile_script: |
     mkdir build && cd build
-    cmake -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=1 -DHAVE_SSE4_1=1 -DRust_COMPILER=/usr/local/bin/rustc ..
+    cmake -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=1 -DHAVE_SSE4_1=1 ..
     make -j $(sysctl -n hw.ncpu)
   test_script: ./util/regression/run_regression.sh ./build/src/mmseqs SCRATCH
 
@@ -27,9 +24,7 @@ task:
         echo "deb [trusted=yes] http://archive.debian.org/debian jessie main" > /etc/apt/sources.list
         echo "deb [trusted=yes] http://archive.debian.org/debian-security jessie/updates main" >> /etc/apt/sources.list
         apt-get update --yes --force-yes || true
-        apt-get install -y --no-install-suggests --no-install-recommends wget clang-4.0 libc++-dev make git ca-certificates curl
-        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-        export PATH=/root/.cargo/bin:$PATH
+        apt-get install wget clang-4.0 libc++-dev make git ca-certificates --yes --force-yes --no-install-suggests --no-install-recommends
       env:
         CC: clang-4.0
         CXX: clang++-4.0
@@ -38,24 +33,20 @@ task:
         echo "deb [trusted=yes] http://archive.debian.org/debian jessie main" > /etc/apt/sources.list
         echo "deb [trusted=yes] http://archive.debian.org/debian-security jessie/updates main" >> /etc/apt/sources.list
         apt-get update --yes --force-yes || true
-        apt-get install -y --no-install-suggests --no-install-recommends wget gcc-4.9 g++-4.9 make git ca-certificates curl
-        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-        export PATH=/root/.cargo/bin:$PATH
+        apt-get install wget gcc-4.9 g++-4.9 make git ca-certificates --yes --force-yes --no-install-suggests --no-install-recommends
       env:
         CC: gcc-4.9
         CXX: g++-4.9
-
   compile_script: |
     mkdir -p /opt/cmake-3.31.0
     wget https://cmake.org/files/v3.31/cmake-3.31.0-linux-x86_64.sh
     chmod +x cmake-3.31.0-linux-x86_64.sh
     ./cmake-3.31.0-linux-x86_64.sh --skip-license --prefix=/opt/cmake-3.31.0
     export PATH=/opt/cmake-3.31.0/bin/:$PATH
-    export PATH=/root/.cargo/bin:$PATH
     mkdir build && cd build
-    cmake -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=0 -DHAVE_SSE4_1=1 -DREQUIRE_OPENMP=0 -DRust_COMPILER=/root/.cargo/bin/rustc ..
+    cmake -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=0 -DHAVE_SSE4_1=1 -DREQUIRE_OPENMP=0 ..
     make -j $(nproc --all)
   test_script: |
     export PATH=/opt/cmake-3.31.0/bin/:$PATH
-    export PATH=/root/.cargo/bin:$PATH
     MMSEQS_NUM_THREADS=4 ./util/regression/run_regression.sh ./build/src/mmseqs SCRATCH SEARCH
+

lib/mmseqs/.github/mmseqs2_logo.png

@@ -7,12 +7,10 @@ RUN mkdir -p /work/build && cd /work/build; \
         -DCMAKE_TOOLCHAIN_FILE=/opt/toolchain.cmake \
         -DOpenMP_C_FLAGS="-Xpreprocessor -fopenmp -I${LIBOMP_AARCH64}" -DOpenMP_C_LIB_NAMES=omp -DOpenMP_CXX_FLAGS="-Xpreprocessor -fopenmp -I${LIBOMP_AARCH64}" -DOpenMP_CXX_LIB_NAMES=omp -DOpenMP_omp_LIBRARY=${LIBOMP_AARCH64}/libomp.a \
         -DCMAKE_POLICY_DEFAULT_CMP0074=NEW -DCMAKE_POLICY_DEFAULT_CMP0144=NEW \
-        -DRust_TOOLCHAIN=stable-x86_64-unknown-linux-gnu \
-        -DRust_CARGO_TARGET=aarch64-unknown-linux-gnu \
         -DFORCE_STATIC_DEPS=1 -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" ..; \
     cmake --build . -j$(nproc --all) -v;
 
-RUN if readelf -a /work/build/src/mmseqs | grep -i "Shared library" | grep -P -v "(linux-vdso|ld-linux-aarch64|libc|libm|libdl|librt|libpthread|libutil).so" | grep -q .; then \
+RUN if readelf -a /work/build/src/mmseqs | grep -i "Shared library" | grep -P -v "(linux-vdso|ld-linux-aarch64|libc|libm|libdl|librt|libpthread).so" | grep -q .; then \
         echo "Error: unwanted libraries found"; \
         readelf -a /work/build/src/mmseqs | grep -i "Shared library"; \
         exit 1; \

lib/mmseqs/.github/workflows/Dockerfile.GPU-buster-cross-sbsa

@@ -14,7 +14,7 @@ RUN mkdir -p /work/build && cd /work/build; \
         -DFORCE_STATIC_DEPS=1 -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" ..; \
     cmake --build . -j$(nproc --all) -v;
 
-RUN if ldd /work/build/src/mmseqs | grep -P -v "linux-vdso.so|/lib64/(ld-linux-x86-64|libc|libm|libdl|librt|libpthread|libutil).so" | grep -q .; then \
+RUN if ldd /work/build/src/mmseqs | grep -P -v "linux-vdso.so|/lib64/(ld-linux-x86-64|libc|libm|libdl|librt|libpthread).so" | grep -q .; then \
         echo "Error: unwanted libraries found"; \
         ldd /work/build/src/mmseqs; \
         exit 1; \

lib/mmseqs/.github/workflows/Dockerfile.GPU-manylinux2014

@@ -128,9 +128,10 @@ jobs:
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
           context: .
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           push: true
-          no-cache: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
           build-args: |
             GPU=1
           tags: |

lib/mmseqs/.github/workflows/docker.yml

@@ -20,7 +20,6 @@ jobs:
       - name: Build
         run: |
           mkdir -p build
-          rustup update
           cd build
           LIBOMP=$(brew --prefix libomp)
           cmake \

lib/mmseqs/.github/workflows/mac-arm64.yml

@@ -18,7 +18,6 @@ set(HAVE_ARM8 0 CACHE BOOL "Have ARMv8 CPU")
 set(HAVE_S390X 0 CACHE BOOL "Have s390x architecture")
 set(NATIVE_ARCH 1 CACHE BOOL "Assume native architecture for SIMD. Use one of the HAVE_* options or set CMAKE_CXX_FLAGS to the appropriate flags if you disable this.")
 set(USE_SYSTEM_ZSTD 0 CACHE BOOL "Use zstd provided by system instead of bundled version")
-set(IGNORE_RUST_VERSION 0 CACHE BOOL "Ignore Rust version check")
 set(ENABLE_CUDA 0 CACHE BOOL "Enable CUDA")
 set(FORCE_STATIC_DEPS 0 CACHE BOOL "Force static linking of deps")
 
@@ -197,53 +196,6 @@ set(MMSEQS_CXX_FLAGS "${MMSEQS_CXX_FLAGS} -std=c++1y")
 #     set(MMSEQS_CXX_FLAGS "${MMSEQS_CXX_FLAGS} -stdlib=libc++")
 # endif ()
 
-set(ENV{CARGO_NET_OFFLINE} true)
-add_subdirectory(lib/corrosion)
-# don't try to link to gcc_s, its not needed anyway
-list(REMOVE_ITEM Rust_CARGO_TARGET_LINK_NATIVE_LIBS "gcc_s")
-set(Rust_CARGO_TARGET_LINK_NATIVE_LIBS ${Rust_CARGO_TARGET_LINK_NATIVE_LIBS} CACHE INTERNAL "")
-
-if(Rust_VERSION VERSION_LESS "1.68.0" AND NOT IGNORE_RUST_VERSION)
-    message(FATAL_ERROR "Rust >= 1.68.0 is required")
-endif()
-
-set(RUST_FEATURE "" CACHE INTERNAL "")
-if(HAVE_AVX2)
-    set(RUST_FEATURE simd_avx2 CACHE INTERNAL "")
-elseif(HAVE_SSE4_1 OR HAVE_SSE2)
-    set(RUST_FEATURE simd_sse2 CACHE INTERNAL "")
-elseif(HAVE_ARM8)
-    set(RUST_FEATURE simd_neon CACHE INTERNAL "")
-endif()
-
-if (NATIVE_ARCH AND (RUST_FEATURE STREQUAL ""))
-    if(EMSCRIPTEN)
-        set(RUST_FEATURE simd_wasm CACHE INTERNAL "")
-    elseif(ARM)
-        set(RUST_FEATURE simd_neon CACHE INTERNAL "")
-    elseif(X86 OR X64)
-        if (HAVE_AVX2_EXTENSIONS)
-            set(RUST_FEATURE simd_avx2 CACHE INTERNAL "")
-        elseif (HAVE_SSE2_EXTENSIONS)
-            set(RUST_FEATURE simd_sse2 CACHE INTERNAL "")
-        endif()
-    endif()
-endif()
-if(RUST_FEATURE STREQUAL "")
-    message(WARNING "No SIMD support for this architecture. Alignment performance will be bad")
-    set(RUST_FEATURE no_simd CACHE INTERNAL "")
-endif()
-message("-- Rust Feature: ${RUST_FEATURE}")
-corrosion_import_crate(
-    MANIFEST_PATH lib/block-aligner/c/Cargo.toml
-    FEATURES "${RUST_FEATURE}"
-    CRATE_TYPES staticlib
-)
-include_directories(lib/block-aligner/c)
-if(EMSCRIPTEN)
-    corrosion_add_target_local_rustflags(block_aligner_c "-Clink-args=--no-entry -sRELOCATABLE=1")
-endif()
-
 if (USE_SYSTEM_ZSTD)
     include(FindPackageHandleStandardArgs)
     find_path(ZSTD_INCLUDE_DIRS NAMES zstd.h REQUIRED)

lib/mmseqs/CMakeLists.txt

@@ -1,62 +1,78 @@
-ARG TARGETARCH
 ARG APP=mmseqs
-ARG GPU=0
-
-FROM --platform=$BUILDPLATFORM ghcr.io/steineggerlab/build-containers:main-sbsa AS builder-arm64
-
-FROM --platform=$BUILDPLATFORM ghcr.io/steineggerlab/build-containers:main-x86_64 AS builder-amd64
-
-FROM builder-${TARGETARCH} AS builder
+FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS builder
 ARG TARGETARCH
 ARG APP
 ARG GPU
 
+RUN dpkg --add-architecture $TARGETARCH \
+    && apt-get update \
+    && apt-get install -y \
+      build-essential cmake xxd git wget \
+      zlib1g-dev libbz2-dev libatomic1 \
+      crossbuild-essential-$TARGETARCH zlib1g-dev:$TARGETARCH libbz2-dev:$TARGETARCH; \
+    if [ "$GPU" = "1" ]; then \
+      wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb; \
+      dpkg -i cuda-keyring_1.1-1_all.deb; \
+      apt-get update && apt-get install -y cuda-nvcc-12-6 cuda-cudart-dev-12-6 ninja-build; \
+    fi; \
+    rm -rf /var/lib/apt/lists/*;
+
 WORKDIR /opt/build
 ADD . .
-RUN set -x; \
-    mkdir -p build/src; \
-    cd /opt/build/build; \
-    if [ "$TARGETARCH" = "arm64" ]; then \
-      /usr/local/bin/cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=1 -DHAVE_ARM8=1 \
-          -DCMAKE_TOOLCHAIN_FILE=/opt/toolchain.cmake \
-          -DOpenMP_C_FLAGS="-Xpreprocessor -fopenmp -I${LIBOMP_AARCH64} -pthread" -DOpenMP_C_LIB_NAMES=omp \
-          -DOpenMP_CXX_FLAGS="-Xpreprocessor -fopenmp -I${LIBOMP_AARCH64} -pthread" -DOpenMP_CXX_LIB_NAMES=omp \
-          -DOpenMP_C_LIB_NAMES="libomp;dl" \
-          -DOpenMP_CXX_LIB_NAMES="libomp;dl" \
-          -DOpenMP_libomp_LIBRARY=${LIBOMP_AARCH64}/libomp.a \
-          -DOpenMP_dl_LIBRARY=dl \
-          -DRust_TOOLCHAIN=stable-x86_64-unknown-linux-gnu \
-          -DRust_CARGO_TARGET=aarch64-unknown-linux-gnu \
-          -DCMAKE_POLICY_DEFAULT_CMP0074=NEW -DCMAKE_POLICY_DEFAULT_CMP0144=NEW \
-          -DFORCE_STATIC_DEPS=1 -DENABLE_CUDA=${GPU} -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" ..; \
-      /usr/local/bin/cmake --build . -j$(nproc --all) -v; \
+
+RUN if [ "$TARGETARCH" = "arm64" ]; then \
+      mkdir -p build_$TARGETARCH/src; \
+      cd /opt/build/build_$TARGETARCH; \
+      CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ cmake -DHAVE_ARM8=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+      make -j $(nproc --all); \
+      mv src/${APP} /opt/build/${APP}_arch; \
+      touch /opt/build/${APP}_sse2 /opt/build/${APP}_sse41 /opt/build/${APP}_avx2; \
     else \
-      if [ -e "${LIBGCC}/libgomp.so" ]; then \
-          mv -f -- "${LIBGCC}/libgomp.so" "${LIBGCC}/libgomp.so.disabled"; \
+      if [ "$GPU" = "1" ]; then \
+        export CUDACXX=/usr/local/cuda/bin/nvcc; \
+        mkdir -p build_avx2/src; \
+        cd /opt/build/build_avx2; \
+        LIBGOMP=/usr/lib/gcc/x86_64-linux-gnu/12/; \
+        cmake -GNinja -DHAVE_AVX2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DFORCE_STATIC_DEPS=1 \
+        -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" \
+        -DOpenMP_C_FLAGS="-fopenmp -I${LIBGOMP}" -DOpenMP_C_LIB_NAMES=gomp -DOpenMP_CXX_FLAGS="-fopenmp -I${LIBGOMP}" -DOpenMP_CXX_LIB_NAMES=gomp -DOpenMP_gomp_LIBRARY=${LIBGOMP}/libgomp.a \
+        -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+        cmake --build . -j$(nproc --all); \
+        mv src/${APP} /opt/build/${APP}_avx2; \
+        touch /opt/build/${APP}_arch /opt/build/${APP}_sse41 /opt/build/${APP}_sse2; \
+      else \
+        mkdir -p build_sse2/src && mkdir -p build_sse41/src && mkdir -p build_avx2/src; \
+        cd /opt/build/build_sse2; \
+        cmake -DHAVE_SSE2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+        make -j $(nproc --all); \
+        mv src/${APP} /opt/build/${APP}_sse2; \
+        cd /opt/build/build_sse41; \
+        cmake -DHAVE_SSE4_1=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+        make -j $(nproc --all); \
+        mv src/${APP} /opt/build/${APP}_sse41; \
+        cd /opt/build/build_avx2; \
+        cmake -DHAVE_AVX2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+        make -j $(nproc --all); \
+        mv src/${APP} /opt/build/${APP}_avx2; \
+        touch /opt/build/${APP}_arch; \
       fi; \
-      /usr/local/bin/cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=1 -DHAVE_AVX2=1 \
-          -DOpenMP_C_FLAGS="-fopenmp -I${LIBGCC} -L${LIBGCC}" -DOpenMP_C_LIB_NAMES=gomp \
-          -DOpenMP_CXX_FLAGS="-fopenmp -I${LIBGCC} -L${LIBGCC}" -DOpenMP_CXX_LIB_NAMES=gomp \
-          -DOpenMP_gomp_LIBRARY="${LIBGCC}/libgomp.a" \
-          -DATOMIC_LIB_OVERRIDE="${LIBGCC}/libatomic.a" \
-          -DCMAKE_POLICY_DEFAULT_CMP0074=NEW -DCMAKE_POLICY_DEFAULT_CMP0144=NEW \
-          -DZLIB_ROOT=/deps -DBZIP2_ROOT=/deps \
-          -DFORCE_STATIC_DEPS=1 -DENABLE_CUDA=${GPU} -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" ..; \
-      /usr/local/bin/cmake --build . -j$(nproc --all) -v; \
-    fi; \
-    mv src/${APP} /opt/build/${APP};
+    fi
 
-FROM debian:trixie-slim
+FROM debian:bookworm-slim
 ARG TARGETARCH
 ARG APP
 ARG GPU
 
-COPY --from=builder /opt/build/${APP} /usr/local/bin/
+COPY --from=builder /opt/build/${APP}_arch /opt/build/${APP}_sse2 /opt/build/${APP}_sse41 /opt/build/${APP}_avx2 /usr/local/bin/
+ADD util/${APP}_wrapper.sh /usr/local/bin/entrypoint
 
 RUN apt-get update && apt-get install -y \
-  gawk bash grep wget tar aria2 \
-  && rm -rf /var/lib/apt/lists/*;
+      gawk bash grep libstdc++6 libgomp1 libatomic1 zlib1g libbz2-1.0 wget tar aria2 \
+    && rm -rf /var/lib/apt/lists/*; \
+    if [ "$TARGETARCH" = "arm64" ]; then \
+      rm -f /usr/local/bin/entrypoint; ln -s /usr/local/bin/${APP}_arch /usr/local/bin/entrypoint; \
+    elif [ "$GPU" = "1" ]; then \
+      rm -f /usr/local/bin/entrypoint; ln -s /usr/local/bin/${APP}_avx2 /usr/local/bin/entrypoint; \
+    fi
 
-RUN echo "#!/bin/sh\nexec /usr/local/bin/$APP \"\${@}\"" > /usr/local/bin/entrypoint && chmod +x /usr/local/bin/entrypoint
-RUN cat /usr/local/bin/entrypoint
 ENTRYPOINT ["/usr/local/bin/entrypoint"]