Merge pull request #38 from COMBINE-lab/main

merge main back into dev

Author: rob-p

.cargo/config-portable.toml

@@ -0,0 +1,14 @@
+# inspired by https://curiouscoding.nl/posts/release-flow/#avx2
+
+# .cargo/config-portable.toml
+[target.x86_64-unknown-linux-gnu]
+rustflags = ["-C", "target-cpu=x86-64-v3", "-C", "target-feature=+avx2"]
+
+[target.aarch64-unknown-linux-gnu]
+rustflags = ["-C", "target-cpu=neoverse-n1"]
+
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "target-cpu=apple-a14"]
+
+[target.x86_64-apple-darwin]
+rustflags = ["-C", "target-cpu=x86-64-v3", "-C", "target-feature=+avx2"]

.cargo/config.toml

@@ -0,0 +1,4 @@
+# taken from https://curiouscoding.nl/posts/release-flow/#avx2
+[build]
+# By default, we want maximum performance rather than portability.
+rustflags = ["-C", "target-cpu=native"]

.github/build-setup.yml

@@ -0,0 +1,19 @@
+- name: Use portable Cargo config for CI builds
+  shell: bash
+  run: |
+    set -euo pipefail
+    cp .cargo/config-portable.toml .cargo/config.toml
+
+- name: Force AVX2 for x86_64 (via RUSTFLAGS)
+  if: contains(join(matrix.targets, ','), 'x86_64-')
+  shell: bash
+  run: |
+    set -euo pipefail
+    echo 'RUSTFLAGS=-C target-cpu=x86-64-v3 -C target-feature=+avx2' >> "$GITHUB_ENV"
+
+- name: Show effective x86_64 rustflags (proof)
+  if: contains(join(matrix.targets, ','), 'x86_64-')
+  shell: bash
+  run: |
+    set -euo pipefail
+    echo "RUSTFLAGS=${RUSTFLAGS-}"

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-# This file was autogenerated by dist: https://opensource.axo.dev/cargo-dist/
+# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist
 #
 # Copyright 2022-2024, axodotdev
 # SPDX-License-Identifier: MIT or Apache-2.0
@@ -47,7 +47,7 @@ on:
 jobs:
   # Run 'dist plan' (or host) to determine what tasks we need to do
   plan:
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     outputs:
       val: ${{ steps.plan.outputs.manifest }}
       tag: ${{ !github.event.pull_request && github.ref_name || '' }}
@@ -56,16 +56,17 @@ jobs:
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
+          persist-credentials: false
           submodules: recursive
       - name: Install dist
         # we specify bash to get pipefail; it guards against the `curl` command
         # failing. otherwise `sh` won't catch that `curl` returned non-0
         shell: bash
-        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.24.1/cargo-dist-installer.sh | sh"
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh | sh"
       - name: Cache dist
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: cargo-dist-cache
           path: ~/.cargo/bin/dist
@@ -81,7 +82,7 @@ jobs:
           cat plan-dist-manifest.json
           echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
       - name: "Upload dist-manifest.json"
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: artifacts-plan-dist-manifest
           path: plan-dist-manifest.json
@@ -107,21 +108,47 @@ jobs:
       # - N "local" tasks that build each platform's binaries and platform-specific installers
       matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
     runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.container && matrix.container.image || null }}
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
     steps:
       - name: enable windows longpaths
         run: |
           git config --global core.longpaths true
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
+          persist-credentials: false
           submodules: recursive
+      - name: Install Rust non-interactively if not already installed
+        if: ${{ matrix.container }}
+        run: |
+          if ! command -v cargo > /dev/null 2>&1; then
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+            echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+          fi
+      - name: "Use portable Cargo config for CI builds"
+        run: |
+          set -euo pipefail
+          cp .cargo/config-portable.toml .cargo/config.toml
+        shell: "bash"
+      - name: "Force AVX2 for x86_64 (via RUSTFLAGS)"
+        if: "contains(join(matrix.targets, ','), 'x86_64-')"
+        run: |
+          set -euo pipefail
+          echo 'RUSTFLAGS=-C target-cpu=x86-64-v3 -C target-feature=+avx2' >> "$GITHUB_ENV"
+        shell: "bash"
+      - name: "Show effective x86_64 rustflags (proof)"
+        if: "contains(join(matrix.targets, ','), 'x86_64-')"
+        run: |
+          set -euo pipefail
+          echo "RUSTFLAGS=${RUSTFLAGS-}"
+        shell: "bash"
       - name: Install dist
-        run: ${{ matrix.install_dist }}
+        run: ${{ matrix.install_dist.run }}
       # Get the dist-manifest
       - name: Fetch local artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           pattern: artifacts-*
           path: target/distrib/
@@ -143,12 +170,12 @@ jobs:
         run: |
           # Parse out what we just built and upload it to scratch storage
           echo "paths<<EOF" >> "$GITHUB_OUTPUT"
-          jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
+          dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT"
           echo "EOF" >> "$GITHUB_OUTPUT"
 
           cp dist-manifest.json "$BUILD_MANIFEST_NAME"
       - name: "Upload artifacts"
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: artifacts-build-local-${{ join(matrix.targets, '_') }}
           path: |
@@ -160,23 +187,24 @@ jobs:
     needs:
       - plan
       - build-local-artifacts
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
+          persist-credentials: false
           submodules: recursive
       - name: Install cached dist
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: cargo-dist-cache
           path: ~/.cargo/bin/
       - run: chmod +x ~/.cargo/bin/dist
       # Get all the local artifacts for the global tasks to use (for e.g. checksums)
       - name: Fetch local artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           pattern: artifacts-*
           path: target/distrib/
@@ -194,7 +222,7 @@ jobs:
 
           cp dist-manifest.json "$BUILD_MANIFEST_NAME"
       - name: "Upload artifacts"
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: artifacts-build-global
           path: |
@@ -206,26 +234,27 @@ jobs:
       - plan
       - build-local-artifacts
       - build-global-artifacts
-    # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine)
-    if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
+    # Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine)
+    if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     outputs:
       val: ${{ steps.host.outputs.manifest }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
+          persist-credentials: false
           submodules: recursive
       - name: Install cached dist
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: cargo-dist-cache
           path: ~/.cargo/bin/
       - run: chmod +x ~/.cargo/bin/dist
       # Fetch artifacts from scratch-storage
       - name: Fetch artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           pattern: artifacts-*
           path: target/distrib/
@@ -238,14 +267,14 @@ jobs:
           cat dist-manifest.json
           echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
       - name: "Upload dist-manifest.json"
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           # Overwrite the previous copy
           name: artifacts-dist-manifest
           path: dist-manifest.json
       # Create a GitHub Release while uploading all files to it
       - name: "Download GitHub Artifacts"
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           pattern: artifacts-*
           path: artifacts
@@ -274,10 +303,11 @@ jobs:
     # still allowing individual publish jobs to skip themselves (for prereleases).
     # "host" however must run to completion, no skipping allowed!
     if: ${{ always() && needs.host.result == 'success' }}
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
+          persist-credentials: false
           submodules: recursive

.github/workflows/sanitize-cargo.yml

@@ -0,0 +1,38 @@
+name: Sanitize Cargo
+on: 
+  push:
+    branches:
+      - main
+
+permissions:
+  "contents": "write"
+
+jobs:
+  sanitize_cargo_file:
+    if: "contains(github.event.head_commit.message, '[do_tag]')"
+    runs-on: ubuntu-latest
+    # This is optional; it exposes the plan to your job as an environment variable
+    #env:
+    #  PLAN: ${{ inputs.plan }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.PAT_TOK }}
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Install cargo-sanitize
+        run: |
+          cargo install cargo-sanitize
+      - name: Sanitize Cargo.toml file
+        run: |
+          cp Cargo.toml Cargo.toml.cs_orig
+          cargo-sanitize -i Cargo.toml.cs_orig -o Cargo.toml
+      - name: Tag and push new commit
+        run: |
+          export VERSION_TAG=`cargo read-manifest | jq ".version" | tr -d '"'`
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add Cargo.toml.cs_orig
+          git add Cargo.toml
+          git commit -m "create sanitized release"
+          git tag --force -a v${VERSION_TAG} -m "version ${VERSION_TAG}"
+          git push --force origin v${VERSION_TAG}

.gitignore

@@ -8,3 +8,14 @@ Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk
+
+# Local cargo config (contains dev-only [patch] overrides)
+# .cargo/config.toml
+radtk/
+gencode_v49/
+gencode_v49_new/
+piscem-cpp/
+sshash/
+sample_quant_new/
+sample_quant/
+