diff --git a/.dockerignore b/.dockerignore index e31713c71..057931f43 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,10 @@ target moon moon/** +# Not in workspace image; local targets/deps can be very large and slow `docker build` context transfer. +tools +tools/** + docs scripts @@ -18,6 +22,13 @@ tests # node **/node_modules/ +**/.next/ +**/.turbo/ +**/dist/ +**/build/ +**/out/ +**/coverage/ +**/.cargo/ # IDE configurations .idea @@ -40,6 +51,7 @@ buck-out # local docker build cache (huge, never needed in image) .buildx-cache .buildx-cache/** +.buildx-cache*/ # Dockderfile docker @@ -47,3 +59,6 @@ docker # git / ci metadata (not needed for docker builds, reduces build context) .git/ .github/ + +# logs +*.log diff --git a/.github/workflows/orion-release.yml b/.github/workflows/orion-release.yml new file mode 100644 index 000000000..e636acd34 --- /dev/null +++ b/.github/workflows/orion-release.yml @@ -0,0 +1,208 @@ +name: Orion Release + +on: + push: + tags: + - "orion-v*" + workflow_dispatch: + inputs: + version: + description: "Override version tag (e.g. v0.1.1). Required for manual runs." + required: true + type: string + +permissions: + contents: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + build: + name: Build orion (linux-amd64) + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + version: ${{ steps.meta.outputs.version }} + bundle_name: ${{ steps.meta.outputs.bundle_name }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Resolve version + id: meta + run: | + set -euo pipefail + if [[ "${GITHUB_REF}" == refs/tags/orion-* ]]; then + VERSION="${GITHUB_REF#refs/tags/orion-}" + else + VERSION="${{ inputs.version }}" + fi + if [[ -z "$VERSION" ]]; then + echo "::error::Could not resolve version. Push an orion-vX.Y.Z tag or supply 'version'." + exit 1 + fi + SHORT_SHA=$(git rev-parse --short=8 HEAD) + BUNDLE_NAME="orion-${VERSION}-linux-amd64" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT" + echo "bundle_name=$BUNDLE_NAME" >> "$GITHUB_OUTPUT" + + - name: Install system build deps + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends \ + build-essential \ + clang \ + fuse3 \ + libfuse3-dev \ + libssl-dev \ + pkg-config \ + protobuf-compiler + + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-unknown-linux-gnu + + - name: Cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: orion-release-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + cache-on-failure: true + + - name: Build orion (release) + run: cargo build --release -p orion --bin orion --target x86_64-unknown-linux-gnu + + - name: Stage bundle + env: + BUNDLE_NAME: ${{ steps.meta.outputs.bundle_name }} + VERSION: ${{ steps.meta.outputs.version }} + SHORT_SHA: ${{ steps.meta.outputs.short_sha }} + run: | + set -euo pipefail + BUILT_AT=$(date -u +%FT%TZ) + BUNDLE_ROOT="dist/${BUNDLE_NAME}" + mkdir -p "${BUNDLE_ROOT}/runner-config" "${BUNDLE_ROOT}/systemd" + + install -m 0755 \ + target/x86_64-unknown-linux-gnu/release/orion \ + "${BUNDLE_ROOT}/orion" + install -m 0644 \ + orion/runner-config/.env.prod \ + "${BUNDLE_ROOT}/runner-config/.env.prod" + install -m 0644 \ + orion/runner-config/scorpio.toml \ + "${BUNDLE_ROOT}/runner-config/scorpio.toml" + install -m 0755 \ + orion/runner-config/run.sh \ + "${BUNDLE_ROOT}/runner-config/run.sh" + install -m 0755 \ + orion/runner-config/preflight.sh \ + "${BUNDLE_ROOT}/runner-config/preflight.sh" + install -m 0755 \ + orion/runner-config/cleanup.sh \ + "${BUNDLE_ROOT}/runner-config/cleanup.sh" + install -m 0644 \ + orion/systemd/orion-runner.service \ + "${BUNDLE_ROOT}/systemd/orion-runner.service" + install -m 0644 \ + orion/systemd/orion-runner.env.example \ + "${BUNDLE_ROOT}/systemd/orion-runner.env.example" + + cat > "${BUNDLE_ROOT}/VERSION" < "${BUNDLE_NAME}.tar.gz.sha256") + + ls -lh dist/ + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.meta.outputs.bundle_name }} + include-hidden-files: true + path: | + dist/${{ steps.meta.outputs.bundle_name }}.tar.gz + dist/${{ steps.meta.outputs.bundle_name }}.tar.gz.sha256 + if-no-files-found: error + retention-days: 7 + + release: + name: Publish GitHub Release + needs: build + if: startsWith(github.ref, 'refs/tags/orion-') + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ needs.build.outputs.bundle_name }} + path: dist + + - name: Render release notes + id: notes + env: + VERSION: ${{ needs.build.outputs.version }} + BUNDLE_NAME: ${{ needs.build.outputs.bundle_name }} + run: | + set -euo pipefail + SHA256=$(awk '{print $1}' "dist/${BUNDLE_NAME}.tar.gz.sha256") + { + echo "## orion ${VERSION}" + echo + echo "Linux amd64 runner bundle for the Orion Buck2 worker." + echo "The bundle contains the release binary, runner scripts," + echo "Scorpio config, and the systemd unit used by orion-scheduler." + echo + echo "### Download" + echo + echo '```bash' + echo "curl -LO https://github.com/${{ github.repository }}/releases/download/orion-${VERSION}/${BUNDLE_NAME}.tar.gz" + echo "curl -LO https://github.com/${{ github.repository }}/releases/download/orion-${VERSION}/${BUNDLE_NAME}.tar.gz.sha256" + echo "sha256sum -c ${BUNDLE_NAME}.tar.gz.sha256" + echo "tar -xzf ${BUNDLE_NAME}.tar.gz" + echo '```' + echo + echo "Bundle layout:" + echo + echo '```text' + echo "${BUNDLE_NAME}/" + echo "├── orion" + echo "├── runner-config/" + echo "├── systemd/" + echo "└── VERSION" + echo '```' + echo + echo "### Checksum" + echo + echo '```' + echo "${SHA256} ${BUNDLE_NAME}.tar.gz" + echo '```' + } > release-notes.md + echo "path=release-notes.md" >> "$GITHUB_OUTPUT" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + name: orion ${{ needs.build.outputs.version }} + body_path: ${{ steps.notes.outputs.path }} + draft: false + prerelease: ${{ contains(needs.build.outputs.version, '-') }} + files: | + dist/${{ needs.build.outputs.bundle_name }}.tar.gz + dist/${{ needs.build.outputs.bundle_name }}.tar.gz.sha256 diff --git a/.github/workflows/orion-scheduler-release.yml b/.github/workflows/orion-scheduler-release.yml new file mode 100644 index 000000000..005d8a02e --- /dev/null +++ b/.github/workflows/orion-scheduler-release.yml @@ -0,0 +1,183 @@ +name: Orion Scheduler Release + +on: + push: + tags: + - "orion-scheduler-v*" + workflow_dispatch: + inputs: + version: + description: "Override version tag (e.g. v0.1.0). Required for manual runs." + required: true + type: string + +permissions: + contents: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + build: + name: Build orion-scheduler (linux-amd64) + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + version: ${{ steps.meta.outputs.version }} + bundle_name: ${{ steps.meta.outputs.bundle_name }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Resolve version + id: meta + run: | + set -euo pipefail + if [[ "${GITHUB_REF}" == refs/tags/orion-scheduler-* ]]; then + VERSION="${GITHUB_REF#refs/tags/orion-scheduler-}" + else + VERSION="${{ inputs.version }}" + fi + if [[ -z "$VERSION" ]]; then + echo "::error::Could not resolve version. Push an orion-scheduler-vX.Y.Z tag or supply 'version'." + exit 1 + fi + SHORT_SHA=$(git rev-parse --short=8 HEAD) + BUNDLE_NAME="orion-scheduler-${VERSION}-linux-amd64" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT" + echo "bundle_name=$BUNDLE_NAME" >> "$GITHUB_OUTPUT" + + - name: Install system build deps + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends \ + build-essential pkg-config + + - name: Install Rust stable + uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-unknown-linux-gnu + + - name: Cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: orion-scheduler-release-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + cache-on-failure: true + + - name: Build orion-scheduler (release) + run: cargo build --release -p orion-scheduler --target x86_64-unknown-linux-gnu + + - name: Stage bundle + id: stage + env: + BUNDLE_NAME: ${{ steps.meta.outputs.bundle_name }} + VERSION: ${{ steps.meta.outputs.version }} + SHORT_SHA: ${{ steps.meta.outputs.short_sha }} + run: | + set -euo pipefail + BUILT_AT=$(date -u +%FT%TZ) + BUNDLE_ROOT="dist/${BUNDLE_NAME}" + mkdir -p "${BUNDLE_ROOT}"/{bin,etc,systemd} + + install -m 0755 \ + target/x86_64-unknown-linux-gnu/release/orion-scheduler \ + "${BUNDLE_ROOT}/bin/orion-scheduler" + install -m 0644 \ + orion-scheduler/target_config.json.template \ + "${BUNDLE_ROOT}/etc/target_config.json.template" + install -m 0644 \ + orion-scheduler/systemd/orion-scheduler.service \ + "${BUNDLE_ROOT}/systemd/orion-scheduler.service" + install -m 0755 \ + orion-scheduler/install.sh \ + "${BUNDLE_ROOT}/install.sh" + + cat > "${BUNDLE_ROOT}/VERSION" < "${BUNDLE_NAME}.tar.gz.sha256") + + ls -lh dist/ + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.meta.outputs.bundle_name }} + path: | + dist/${{ steps.meta.outputs.bundle_name }}.tar.gz + dist/${{ steps.meta.outputs.bundle_name }}.tar.gz.sha256 + if-no-files-found: error + retention-days: 7 + + release: + name: Publish GitHub Release + needs: build + if: startsWith(github.ref, 'refs/tags/orion-scheduler-') + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ needs.build.outputs.bundle_name }} + path: dist + + - name: Render release notes + id: notes + env: + VERSION: ${{ needs.build.outputs.version }} + BUNDLE_NAME: ${{ needs.build.outputs.bundle_name }} + run: | + set -euo pipefail + SHA256=$(awk '{print $1}' "dist/${BUNDLE_NAME}.tar.gz.sha256") + { + echo "## orion-scheduler ${VERSION}" + echo + echo "Linux amd64 bundle for the qlean/KVM micro-VM controller. Drop the" + echo "tarball on a Linux host with KVM + QEMU and run \`install.sh\`." + echo + echo "### Install" + echo + echo "\`\`\`bash" + echo "curl -LO https://github.com/${{ github.repository }}/releases/download/orion-scheduler-${VERSION}/${BUNDLE_NAME}.tar.gz" + echo "curl -LO https://github.com/${{ github.repository }}/releases/download/orion-scheduler-${VERSION}/${BUNDLE_NAME}.tar.gz.sha256" + echo "sha256sum -c ${BUNDLE_NAME}.tar.gz.sha256" + echo "tar -xzf ${BUNDLE_NAME}.tar.gz" + echo "sudo bash ${BUNDLE_NAME}/install.sh" + echo "\`\`\`" + echo + echo "Edit \`/etc/orion-scheduler/target_config.json\` before starting the service." + echo "See [\`orion-scheduler/README.md\`](https://github.com/${{ github.repository }}/blob/main/orion-scheduler/README.md) for the full prerequisites (KVM, \`qlbr0\` bridge, custom image)." + echo + echo "### Checksum" + echo + echo "\`\`\`" + echo "${SHA256} ${BUNDLE_NAME}.tar.gz" + echo "\`\`\`" + } > release-notes.md + echo "path=release-notes.md" >> "$GITHUB_OUTPUT" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + name: orion-scheduler ${{ needs.build.outputs.version }} + body_path: ${{ steps.notes.outputs.path }} + draft: false + prerelease: ${{ contains(needs.build.outputs.version, '-') }} + files: | + dist/${{ needs.build.outputs.bundle_name }}.tar.gz + dist/${{ needs.build.outputs.bundle_name }}.tar.gz.sha256 diff --git a/Cargo.lock b/Cargo.lock index b30ac2fcd..3712b8b12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,9 +42,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66bd29a732b644c0431c6140f370d097879203d79b80c94a6747ba0872adaef8" +checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" dependencies = [ "cipher 0.5.2", "cpubits", @@ -73,7 +73,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da8c919c118108f144adecad74b425b804ad075580d605d9b33c2d6d1c62a2f8" dependencies = [ "aead 0.6.0-rc.10", - "aes 0.9.0", + "aes 0.9.1", "cipher 0.5.2", "ctr 0.10.1", "ghash 0.6.0", @@ -676,18 +676,6 @@ dependencies = [ "sha2 0.11.0", ] -[[package]] -name = "bellatrix" -version = "0.1.0" -dependencies = [ - "anyhow", - "api-model", - "common", - "reqwest 0.13.4", - "serde", - "tracing", -] - [[package]] name = "better_default" version = "1.0.5" @@ -756,7 +744,7 @@ dependencies = [ "quote", "regex", "rustc-hash", - "shlex", + "shlex 1.3.0", "syn 2.0.117", ] @@ -777,15 +765,15 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitcoin-io" -version = "0.1.4" +version = "0.1.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953" +checksum = "11301df0b06f22dea7bb1916403fdd88a371031e495c49b8f96931b28189e175" [[package]] name = "bitcoin_hashes" -version = "0.14.2" +version = "0.14.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ed83caece3afc59919481b33b472e1432d1abc4641ed9100be142ef5110b406" +checksum = "0c9901a56e133a1fc86eeb1113e2591f45f4682451ca893bff494d2f88918e3f" dependencies = [ "bitcoin-io", "hex-conservative", @@ -974,9 +962,9 @@ dependencies = [ [[package]] name = "brotli" -version = "8.0.2" +version = "8.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -985,9 +973,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.0" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1174,14 +1162,14 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.62" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex", + "shlex 2.0.1", ] [[package]] @@ -1256,7 +1244,6 @@ dependencies = [ "async-recursion", "async-trait", "axum", - "bellatrix", "bytes", "callisto", "chrono", @@ -1266,6 +1253,7 @@ dependencies = [ "hex", "io-orbit", "jupiter", + "orion-client", "pgp", "rand 0.10.1", "regex", @@ -1275,7 +1263,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.11.0", - "sysinfo 0.39.2", + "sysinfo 0.39.3", "tempfile", "tokio", "tokio-stream", @@ -1448,9 +1436,9 @@ dependencies = [ [[package]] name = "cmov" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "colorchoice" @@ -2325,9 +2313,9 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -3010,9 +2998,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab9e9188e97a93276e1fe7b56401b851e2b45a46d045ca658100c1303ada649" +checksum = "fb130435a959a8d525e6bca66ff6c40981a300ee96d70e3ef56f046556d614a3" dependencies = [ "generic-array 0.14.7", "rustversion", @@ -3560,9 +3548,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -3643,7 +3631,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.6.4", "system-configuration", "tokio", "tower-service", @@ -4022,9 +4010,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.27" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "392c70591e8749fe235ddaf513e6f58b26bce3dcc16524cecc8936f75afa161e" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" dependencies = [ "jiff-static", "log", @@ -4035,9 +4023,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.27" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b605b0c050d845fc355bb11eb3f9a8deddc218ea60c76e61aa1f2adfb2c96a" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", @@ -4308,7 +4296,7 @@ dependencies = [ "quoted_printable", "rustls", "rustls-platform-verifier", - "socket2 0.6.3", + "socket2 0.6.4", "tokio", "tokio-rustls", "url", @@ -4450,14 +4438,14 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" dependencies = [ "bitflags 2.11.1", "libc", "plain", - "redox_syscall 0.7.5", + "redox_syscall 0.8.0", ] [[package]] @@ -4716,9 +4704,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "memmap2" @@ -4804,9 +4792,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -4869,7 +4857,6 @@ dependencies = [ "axum", "axum-extra", "base64", - "bellatrix", "bytes", "callisto", "cedar-policy", @@ -4888,6 +4875,7 @@ dependencies = [ "lettre", "mimalloc", "once_cell", + "orion-client", "percent-encoding", "qlean", "rand 0.10.1", @@ -5408,6 +5396,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "orion-client" +version = "0.1.0" +dependencies = [ + "anyhow", + "api-model", + "common", + "reqwest 0.13.4", + "serde", + "tracing", +] + [[package]] name = "orion-scheduler" version = "0.1.0" @@ -6031,7 +6031,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "279a91971a1d8eb1260a30938eae3be9cb67b472dffecb222fbbbe2fd2dc1453" dependencies = [ - "aes 0.9.0", + "aes 0.9.1", "cbc 0.2.1", "der 0.8.0", "pbkdf2 0.13.0", @@ -6237,7 +6237,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.11+spec-1.1.0", + "toml_edit 0.25.12+spec-1.1.0", ] [[package]] @@ -6502,7 +6502,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -6540,7 +6540,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.6.4", "tracing", "windows-sys 0.60.2", ] @@ -6705,9 +6705,9 @@ dependencies = [ [[package]] name = "redis" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d32a1ac9123f0d84fda64bfc02a271d9868483162dd2d9099b5c362ece064c" +checksum = "a12e6b5f4d8ef33944e833e2b1859ad478deab6e431d7337b30ee2efe21f7543" dependencies = [ "arc-swap", "arcstr", @@ -6726,7 +6726,7 @@ dependencies = [ "rustls-native-certs", "ryu", "sha1_smol", - "socket2 0.6.3", + "socket2 0.6.4", "tokio", "tokio-rustls", "tokio-util", @@ -6736,14 +6736,14 @@ dependencies = [ [[package]] name = "redis-test" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c44e98700033fcce3c3774ab8e4f31b9cebb2761df3298600f4be30a95d2f2a" +checksum = "804d36862e4323b69f96440cbb13c9894fc90176abdeaf91264e21d5d77f6aca" dependencies = [ "futures", "rand 0.9.4", "redis", - "socket2 0.6.3", + "socket2 0.6.4", "tempfile", ] @@ -6767,9 +6767,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4666a1a60d8412eab19d94f6d13dcc9cea0a5ef4fdf6a5db306537413c661b1b" +checksum = "7c7591fa2c6b601dfcfe5f043f65a1c39fcdf50efefcd7f1572e538c1f4b398d" dependencies = [ "bitflags 2.11.1", ] @@ -7162,7 +7162,7 @@ dependencies = [ "enum_dispatch", "flate2", "futures", - "generic-array 1.4.1", + "generic-array 1.4.2", "getrandom 0.2.17", "hex-literal 0.4.1", "hmac 0.12.1", @@ -7205,7 +7205,7 @@ version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f67013f080c226e5a34db1c71f2567f44d95a6300005bb6cd4e2c8fe3c326d1b" dependencies = [ - "aes 0.9.0", + "aes 0.9.1", "aws-lc-rs", "bitflags 2.11.1", "block-padding 0.4.2", @@ -7226,7 +7226,7 @@ dependencies = [ "enum_dispatch", "flate2", "futures", - "generic-array 1.4.1", + "generic-array 1.4.2", "getrandom 0.2.17", "ghash 0.6.0", "hex-literal 1.1.0", @@ -8293,6 +8293,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -8431,9 +8437,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -8701,7 +8707,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10db6f219196a8528f9ec904d9d45cdad692d65b0e57e72be4dedd1c5fddce36" dependencies = [ "aead 0.6.0-rc.10", - "aes 0.9.0", + "aes 0.9.1", "aes-gcm 0.11.0-rc.4", "cbc 0.2.1", "chacha20 0.10.0", @@ -8983,9 +8989,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.39.2" +version = "0.39.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14311e7e9a03114cd4b65eedd54e8fed2945e17f08586ae97ef53bc0669f9581" +checksum = "21d0d938c10fcda3e897e28aaddf4ab462375d411f4378cd63b1c945f69aba96" dependencies = [ "libc", "memchr", @@ -9264,7 +9270,7 @@ dependencies = [ "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.3", + "socket2 0.6.4", "tokio-macros", "windows-sys 0.61.2", ] @@ -9448,9 +9454,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.11+spec-1.1.0" +version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", @@ -9812,9 +9818,9 @@ checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" @@ -10044,9 +10050,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.23.1" +version = "1.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -11095,18 +11101,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 5be3caaa0..00fdf6000 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" members = [ "api-model", "ceres", + "clients/orion-client", "common", "context", "io-orbit", @@ -33,7 +34,7 @@ callisto = { path = "jupiter/callisto" } vault = { path = "vault" } saturn = { path = "saturn" } orion = { path = "orion" } -bellatrix = { path = "orion-server/bellatrix" } +orion-client = { path = "clients/orion-client" } context = { path = "context" } @@ -94,7 +95,7 @@ idgenerator = "2.0.0" config = "0.15.23" reqwest = "0.13.4" lazy_static = "1.5.0" -uuid = "1.23.1" +uuid = "1.23.2" regex = "1.12.3" ed25519-dalek = "2.2.0" ctrlc = "3.5.2" @@ -110,7 +111,7 @@ tempfile = "3.27.0" dashmap = "6.2.1" once_cell = "1.21.4" serial_test = "3.4.0" -sysinfo = "0.39.2" +sysinfo = "0.39.3" http = "1.4.1" url = "2.5.8" jemallocator = "0.5.4" @@ -123,8 +124,8 @@ bs58 = "0.5.1" indexmap = "2.14" envsubst = "0.2.1" directories = "6.0.0" -redis = "1.2.1" -redis-test = "1.0.3" +redis = "1.2.2" +redis-test = "1.0.4" rustls = "0.23" object_store = "0.13.2" parse-display = "0.11.0" @@ -135,4 +136,13 @@ percent-encoding = "2.3" [profile.release] -debug = true +debug = false +strip = "symbols" + +[profile.demo] +inherits = "release" +debug = false +strip = "symbols" +codegen-units = 16 +lto = false +incremental = false diff --git a/ceres/Cargo.toml b/ceres/Cargo.toml index ac75962ea..4e813713b 100644 --- a/ceres/Cargo.toml +++ b/ceres/Cargo.toml @@ -16,7 +16,7 @@ common = { workspace = true } jupiter = { workspace = true } callisto = { workspace = true } git-internal = { workspace = true } -bellatrix = { workspace = true } +orion-client = { workspace = true } saturn = { workspace = true } anyhow = { workspace = true } diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index 61c1fd749..a2cac1a44 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -45,7 +45,6 @@ use std::{ use api_model::common::Pagination; use async_trait::async_trait; -use bellatrix::Bellatrix; use bytes::Bytes; use callisto::{ mega_cl, mega_refs, mega_tag, mega_tree, @@ -84,6 +83,7 @@ use jupiter::{ }, utils::converter::{FromMegaModel, IntoMegaModel, generate_git_keep_with_timestamp}, }; +use orion_client::OrionBuildClient; use regex::Regex; use tracing::debug; @@ -1542,13 +1542,13 @@ impl MonoApiService { username: &str, ) -> Result<(), GitError> { let config = self.storage.config(); - let bellatrix = Bellatrix::new(config.build.clone()); + let orion_client = OrionBuildClient::new(config.build.clone()); let git_cache = self.git_object_cache.clone(); editor .trigger_build_and_check( self.storage.clone(), git_cache, - Arc::new(bellatrix), + Arc::new(orion_client), cl, username, ) @@ -1560,8 +1560,8 @@ impl MonoApiService { /// Triggers a build for Buck upload completion fn trigger_build_for_buck_upload(&self, response: &CompleteResponse, username: &str) { let config = self.storage.config(); - let bellatrix = Arc::new(Bellatrix::new(config.build.clone())); - if !bellatrix.enable_build() { + let orion_client = Arc::new(OrionBuildClient::new(config.build.clone())); + if !orion_client.enable_build() { return; } let storage = self.storage.clone(); @@ -1578,7 +1578,8 @@ impl MonoApiService { context.ref_type = Some("branch".to_string()); tokio::spawn(async move { if let Err(e) = - BuildTriggerService::build_by_context(storage, git_cache, bellatrix, context).await + BuildTriggerService::build_by_context(storage, git_cache, orion_client, context) + .await { tracing::error!("Failed to create build trigger for buck upload: {}", e); } diff --git a/ceres/src/build_trigger/dispatcher.rs b/ceres/src/build_trigger/dispatcher.rs index e4ed64c07..0a8ddf70c 100644 --- a/ceres/src/build_trigger/dispatcher.rs +++ b/ceres/src/build_trigger/dispatcher.rs @@ -1,16 +1,16 @@ use std::sync::Arc; use api_model::buck2::{api::TaskBuildRequest, status::Status, types::ProjectRelativePath}; -use bellatrix::Bellatrix; use common::errors::MegaError; use jupiter::storage::Storage; +use orion_client::OrionBuildClient; use crate::build_trigger::{BuildTrigger, BuildTriggerPayload}; -/// Handles dispatching build triggers to the build execution layer (Bellatrix/Orion). +/// Handles dispatching build triggers to the build execution layer (Orion). pub struct BuildDispatcher { storage: Storage, - bellatrix: Arc, + orion_client: Arc, } fn payload_to_task_request(payload: &BuildTriggerPayload) -> Result { @@ -40,8 +40,11 @@ fn payload_to_task_request(payload: &BuildTriggerPayload) -> Result) -> Self { - Self { storage, bellatrix } + pub fn new(storage: Storage, orion_client: Arc) -> Self { + Self { + storage, + orion_client, + } } pub async fn dispatch(&self, trigger: BuildTrigger) -> Result { @@ -51,12 +54,12 @@ impl BuildDispatcher { })?; // Determine task_id based on whether build system is enabled - let task_id: Option = if self.bellatrix.enable_build() { + let task_id: Option = if self.orion_client.enable_build() { let req = payload_to_task_request(&trigger.payload)?; - let task_id_str = self.bellatrix.on_post_receive(req).await.map_err(|e| { - tracing::error!("Failed to dispatch build to Bellatrix: {}", e); - MegaError::Other(format!("Failed to dispatch build to Bellatrix: {}", e)) + let task_id_str = self.orion_client.on_post_receive(req).await.map_err(|e| { + tracing::error!("Failed to dispatch build to Orion: {}", e); + MegaError::Other(format!("Failed to dispatch build to Orion: {}", e)) })?; let task_uuid = uuid::Uuid::parse_str(&task_id_str).map_err(|e| { @@ -179,12 +182,12 @@ mod tests { spawn_mock_orion(worker_tx, expected_task_id.clone()).await; tokio::time::sleep(Duration::from_millis(50)).await; - let bellatrix = Arc::new(Bellatrix::new(BuildConfig { + let orion_client = Arc::new(OrionBuildClient::new(BuildConfig { enable_build: true, orion_server: orion_base, ..Default::default() })); - let dispatcher = BuildDispatcher::new(storage.clone(), bellatrix); + let dispatcher = BuildDispatcher::new(storage.clone(), orion_client); let trigger = web_edit_trigger(repo, cl_link, changes.clone()); let trigger_id = dispatcher @@ -254,12 +257,12 @@ mod tests { async fn test_dispatch_skips_orion_when_build_disabled_and_persists_trigger() { let temp_dir = tempdir().expect("create temp dir"); let storage = jupiter::tests::test_storage(temp_dir.path()).await; - let bellatrix = Arc::new(Bellatrix::new(BuildConfig { + let orion_client = Arc::new(OrionBuildClient::new(BuildConfig { enable_build: false, orion_server: "http://127.0.0.1:0".to_string(), ..Default::default() })); - let dispatcher = BuildDispatcher::new(storage.clone(), bellatrix); + let dispatcher = BuildDispatcher::new(storage.clone(), orion_client); let trigger = web_edit_trigger( "/project/buck2_test", diff --git a/ceres/src/build_trigger/mod.rs b/ceres/src/build_trigger/mod.rs index b8ca6bea3..868845506 100644 --- a/ceres/src/build_trigger/mod.rs +++ b/ceres/src/build_trigger/mod.rs @@ -1,9 +1,9 @@ use std::{collections::HashMap, sync::Arc}; use async_trait::async_trait; -use bellatrix::Bellatrix; use common::errors::MegaError; use jupiter::storage::Storage; +use orion_client::OrionBuildClient; use crate::api_service::cache::GitObjectCache; @@ -50,11 +50,11 @@ impl TriggerRegistry { pub fn new( storage: Storage, git_object_cache: Arc, - bellatrix: Arc, + orion_client: Arc, ) -> Self { let mut registry = Self { handlers: HashMap::new(), - dispatcher: Arc::new(BuildDispatcher::new(storage.clone(), bellatrix)), + dispatcher: Arc::new(BuildDispatcher::new(storage.clone(), orion_client)), }; // Register core handlers (Git Push, Manual, Retry) diff --git a/ceres/src/build_trigger/service.rs b/ceres/src/build_trigger/service.rs index 71d1ea247..ec50ea244 100644 --- a/ceres/src/build_trigger/service.rs +++ b/ceres/src/build_trigger/service.rs @@ -28,9 +28,9 @@ use std::sync::Arc; use api_model::common::Pagination; -use bellatrix::Bellatrix; use common::errors::MegaError; use jupiter::storage::Storage; +use orion_client::OrionBuildClient; use crate::{ api_service::cache::GitObjectCache, @@ -48,7 +48,7 @@ use crate::{ pub struct BuildTriggerService { storage: Storage, registry: TriggerRegistry, - bellatrix: Arc, + orion_client: Arc, } impl BuildTriggerService { @@ -65,18 +65,19 @@ impl BuildTriggerService { pub fn new( storage: Storage, git_object_cache: Arc, - bellatrix: Arc, + orion_client: Arc, ) -> Self { - let registry = TriggerRegistry::new(storage.clone(), git_object_cache, bellatrix.clone()); + let registry = + TriggerRegistry::new(storage.clone(), git_object_cache, orion_client.clone()); Self { storage, registry, - bellatrix, + orion_client, } } pub fn is_enabled(&self) -> bool { - self.bellatrix.enable_build() + self.orion_client.enable_build() } fn check_build_enabled(&self) -> Result<(), MegaError> { @@ -113,13 +114,13 @@ impl BuildTriggerService { pub async fn build_by_context( storage: Storage, git_cache: Arc, - bellatrix: Arc, + orion_client: Arc, context: TriggerContext, ) -> Result, MegaError> { - if !bellatrix.enable_build() { + if !orion_client.enable_build() { return Ok(None); } - let registry = TriggerRegistry::new(storage, git_cache, bellatrix); + let registry = TriggerRegistry::new(storage, git_cache, orion_client); let id = registry.trigger_build(context).await?; Ok(Some(id)) diff --git a/ceres/src/code_edit/model.rs b/ceres/src/code_edit/model.rs index cf020a642..2aa9f5892 100644 --- a/ceres/src/code_edit/model.rs +++ b/ceres/src/code_edit/model.rs @@ -1,6 +1,5 @@ use std::sync::Arc; -use bellatrix::Bellatrix; use callisto::{entity_ext::generate_link, mega_cl, mega_refs, sea_orm_active_enums::ConvTypeEnum}; use common::errors::MegaError; use git_internal::internal::object::commit::Commit; @@ -9,6 +8,7 @@ use jupiter::{ storage::{Storage, mono_storage::MonoStorage}, utils::converter::FromMegaModel, }; +use orion_client::OrionBuildClient; use crate::{ api_service::{ApiHandler, cache::GitObjectCache}, @@ -73,7 +73,7 @@ pub(crate) trait TriggerContextBuilder { &self, storage: Storage, git_cache: Arc, - bellatrix: Arc, + orion_client: Arc, cl: &mega_cl::Model, username: &str, ) -> Result<(), MegaError> { @@ -81,7 +81,7 @@ pub(crate) trait TriggerContextBuilder { let username = username.to_string(); let context = self.get_context(&cl_model, &username).await?; tokio::spawn(async move { - BuildTriggerService::build_by_context(storage, git_cache, bellatrix, context).await + BuildTriggerService::build_by_context(storage, git_cache, orion_client, context).await }); Ok(()) } @@ -364,12 +364,12 @@ impl< &self, storage: Storage, git_cache: Arc, - bellatrix: Arc, + orion_client: Arc, cl: &mega_cl::Model, username: &str, ) -> Result<(), MegaError> { self.builder - .trigger_build(storage, git_cache, bellatrix, cl, username) + .trigger_build(storage, git_cache, orion_client, cl, username) .await } @@ -394,11 +394,11 @@ impl< &self, storage: Storage, git_cache: Arc, - bellatrix: Arc, + orion_client: Arc, cl: &mega_cl::Model, username: &str, ) -> Result<(), MegaError> { - self.trigger_build(storage.clone(), git_cache, bellatrix, cl, username) + self.trigger_build(storage.clone(), git_cache, orion_client, cl, username) .await?; self.trigger_check(storage, username, cl).await?; Ok(()) diff --git a/ceres/src/code_edit/on_edit.rs b/ceres/src/code_edit/on_edit.rs index 56829aac1..d4722b53c 100644 --- a/ceres/src/code_edit/on_edit.rs +++ b/ceres/src/code_edit/on_edit.rs @@ -84,7 +84,7 @@ impl model::TriggerContextBuilder for OneditTrigerBuilder { &self, storage: Storage, git_cache: std::sync::Arc, - bellatrix: std::sync::Arc, + orion_client: std::sync::Arc, cl: &mega_cl::Model, username: &str, ) -> Result<(), MegaError> { @@ -112,7 +112,7 @@ impl model::TriggerContextBuilder for OneditTrigerBuilder { Some(cl_model.id), Some(username), ); - BuildTriggerService::build_by_context(storage, git_cache, bellatrix, context).await + BuildTriggerService::build_by_context(storage, git_cache, orion_client, context).await }); Ok(()) } diff --git a/ceres/src/code_edit/on_push.rs b/ceres/src/code_edit/on_push.rs index 41ba5b0fe..56da28edc 100644 --- a/ceres/src/code_edit/on_push.rs +++ b/ceres/src/code_edit/on_push.rs @@ -1,9 +1,9 @@ use std::sync::Arc; -use bellatrix::Bellatrix; use callisto::{mega_cl, mega_refs}; use common::errors::MegaError; use jupiter::storage::Storage; +use orion_client::OrionBuildClient; use crate::{ api_service::{cache::GitObjectCache, mono_api_service::MonoApiService}, @@ -59,7 +59,7 @@ impl model::TriggerContextBuilder for OnpushTrigerBuilder { &self, storage: Storage, git_cache: Arc, - bellatrix: Arc, + orion_client: Arc, cl: &mega_cl::Model, username: &str, ) -> Result<(), MegaError> { @@ -89,7 +89,7 @@ impl model::TriggerContextBuilder for OnpushTrigerBuilder { Some(cl_model.id), Some(username), ); - BuildTriggerService::build_by_context(storage, git_cache, bellatrix, context).await + BuildTriggerService::build_by_context(storage, git_cache, orion_client, context).await }); Ok(()) diff --git a/ceres/src/pack/monorepo.rs b/ceres/src/pack/monorepo.rs index 8fe8ded85..89016a9a4 100644 --- a/ceres/src/pack/monorepo.rs +++ b/ceres/src/pack/monorepo.rs @@ -12,7 +12,6 @@ use std::{ use api_model::common::Pagination; use async_recursion::async_recursion; use async_trait::async_trait; -use bellatrix::Bellatrix; use callisto::{ entity_ext::generate_link, mega_cl, mega_code_review_anchor, mega_commit, mega_refs, @@ -36,6 +35,7 @@ use git_internal::{ }; use io_orbit::object_storage::MultiObjectByteStream; use jupiter::{sea_orm::DatabaseTransaction, storage::Storage, utils::converter::FromMegaModel}; +use orion_client::OrionBuildClient; use tokio::sync::{RwLock, mpsc}; use tokio_stream::wrappers::ReceiverStream; @@ -58,7 +58,7 @@ pub struct MonoRepo { // When only a branch is updated and the pack file is empty, this value will be None. pub current_commit: Arc>>, pub cl_link: Arc>>, - pub bellatrix: Arc, + pub orion_client: Arc, pub username: Option, /// Ref commands for this push (same role as on [`ImportRepo`](crate::pack::import_repo::ImportRepo)). pub command_list: Mutex>, @@ -530,12 +530,12 @@ impl MonoRepo { .update_or_create_cl(&self.storage, &self.from_hash, &self.to_hash, &username) .await?; self.traverses_tree_and_update_filepath().await?; - if self.bellatrix.enable_build() { + if self.orion_client.enable_build() { editor .trigger_build_and_check( self.storage.clone(), self.git_object_cache.clone(), - self.bellatrix.clone(), + self.orion_client.clone(), &cl, &username, ) diff --git a/ceres/src/protocol/mod.rs b/ceres/src/protocol/mod.rs index 8c628428d..53fdf30c1 100644 --- a/ceres/src/protocol/mod.rs +++ b/ceres/src/protocol/mod.rs @@ -6,7 +6,6 @@ use std::{ sync::{Arc, Mutex}, }; -use bellatrix::Bellatrix; use callisto::sea_orm_active_enums::RefTypeEnum; use common::{ errors::{MegaError, ProtocolError}, @@ -14,6 +13,7 @@ use common::{ }; use import_refs::RefCommand; use jupiter::redis::lock::RedLock; +use orion_client::OrionBuildClient; use repo::Repo; use tokio::sync::RwLock; @@ -210,7 +210,7 @@ impl SmartSession { to_hash: String::new(), current_commit: Arc::new(RwLock::new(None)), cl_link: Arc::new(RwLock::new(None)), - bellatrix: Arc::new(Bellatrix::new(config.build.clone())), + orion_client: Arc::new(OrionBuildClient::new(config.build.clone())), username: self.auth.username.clone(), command_list: Mutex::new(commands.clone()), }; diff --git a/orion-server/bellatrix/Cargo.toml b/clients/orion-client/Cargo.toml similarity index 51% rename from orion-server/bellatrix/Cargo.toml rename to clients/orion-client/Cargo.toml index 5369817f0..2b4b2881f 100644 --- a/orion-server/bellatrix/Cargo.toml +++ b/clients/orion-client/Cargo.toml @@ -1,16 +1,17 @@ [package] -name = "bellatrix" +name = "orion-client" version = "0.1.0" edition.workspace = true +description = "Client for triggering Orion builds via orion-server's task API." [lib] -name = "bellatrix" +name = "orion_client" path = "src/lib.rs" [dependencies] api-model = { workspace = true } common = { workspace = true } -reqwest = { workspace = true, features = ["json"]} +reqwest = { workspace = true, features = ["json"] } anyhow = { workspace = true } serde = { workspace = true } -tracing = { workspace = true } \ No newline at end of file +tracing = { workspace = true } diff --git a/orion-server/bellatrix/src/orion_client/mod.rs b/clients/orion-client/src/http_client.rs similarity index 86% rename from orion-server/bellatrix/src/orion_client/mod.rs rename to clients/orion-client/src/http_client.rs index e7dff23bd..fff834e70 100644 --- a/orion-server/bellatrix/src/orion_client/mod.rs +++ b/clients/orion-client/src/http_client.rs @@ -1,4 +1,4 @@ -pub use api_model::buck2::{api::TaskBuildRequest, status::Status, types::ProjectRelativePath}; +use api_model::buck2::api::TaskBuildRequest; use serde::Deserialize; /// Response from Orion task handler containing the assigned task ID. @@ -8,14 +8,16 @@ pub struct TaskResponse { } #[derive(Clone)] -pub(crate) struct OrionClient { +pub(crate) struct OrionTaskHttpClient { base_url: String, client: reqwest::Client, } -impl OrionClient { +impl OrionTaskHttpClient { pub fn new(base_url: impl Into) -> Self { let base_url = base_url.into(); + // Loopback connections must bypass any system/HTTP_PROXY env so tests + // and local dev don't get black-holed by a corporate proxy. let use_direct_connection = base_url.starts_with("http://127.0.0.1") || base_url.starts_with("https://127.0.0.1") || base_url.starts_with("http://localhost") @@ -34,7 +36,6 @@ impl OrionClient { Self { base_url, client } } - /// Trigger a build on Orion and return the assigned task ID. pub async fn trigger_build(&self, req: TaskBuildRequest) -> anyhow::Result { let url = format!("{}/v2/task", self.base_url); tracing::info!("Try to trigger build with params:{:?}", req); diff --git a/clients/orion-client/src/lib.rs b/clients/orion-client/src/lib.rs new file mode 100644 index 000000000..cd259dcc0 --- /dev/null +++ b/clients/orion-client/src/lib.rs @@ -0,0 +1,36 @@ +//! Client for triggering Orion builds via `orion-server`'s task API. +//! +//! `OrionBuildClient` wraps an HTTP transport (`http_client::OrionTaskHttpClient`) +//! and exposes a small, opinionated surface tailored for `ceres`/`mono` callers +//! (e.g. `enable_build()`, `on_post_receive()`). Keep the abstraction narrow: +//! only methods that callers actually need belong here. + +mod http_client; + +use api_model::buck2::api::TaskBuildRequest; +use common::config::BuildConfig; +pub use http_client::TaskResponse; + +use crate::http_client::OrionTaskHttpClient; + +#[derive(Clone)] +pub struct OrionBuildClient { + http: OrionTaskHttpClient, + build_config: BuildConfig, +} + +impl OrionBuildClient { + pub fn new(build_config: BuildConfig) -> Self { + let http = OrionTaskHttpClient::new(build_config.orion_server.clone()); + Self { http, build_config } + } + + pub fn enable_build(&self) -> bool { + self.build_config.enable_build + } + + pub async fn on_post_receive(&self, req: TaskBuildRequest) -> anyhow::Result { + let task_id = self.http.trigger_build(req).await?; + Ok(task_id) + } +} diff --git a/jupiter/src/service/buck_service.rs b/jupiter/src/service/buck_service.rs index c9d9dcfef..2aa6a1e81 100644 --- a/jupiter/src/service/buck_service.rs +++ b/jupiter/src/service/buck_service.rs @@ -959,7 +959,7 @@ impl BuckService { // TODO: Buck Upload completion flow - remaining steps (not implemented): // 1. Output CL creation and diff logs (need to calculate file diffs) // 2. Notify change-detector (need to implement change-detector client) - // 3. Buck2 build flow (need to integrate bellatrix and dependency analysis): + // 3. Buck2 build flow (need to integrate orion-client and dependency analysis): // - Analyze affected targets (based on BUCK dependency graph) // - Return build target list (affected_targets) // - Start build tasks (only build affected_targets) diff --git a/mono/Cargo.toml b/mono/Cargo.toml index 75b2a2dce..ae8ffc48c 100644 --- a/mono/Cargo.toml +++ b/mono/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main.rs" [dependencies] api-model = { workspace = true } -bellatrix = { workspace = true } +orion-client = { workspace = true } common = { workspace = true } callisto = { workspace = true } jupiter = { workspace = true } diff --git a/mono/Dockerfile b/mono/Dockerfile index 6ddfea3df..4b29325a9 100644 --- a/mono/Dockerfile +++ b/mono/Dockerfile @@ -1,22 +1,17 @@ # ────── Stage 0: Chef ────── # Downloads and installs cargo-chef (prebuilt binary, no Rust compilation needed). -FROM rust:1.95.0 AS chef +FROM rust:1.96.0 AS chef -ARG TARGETARCH=amd64 -ARG CARGO_CHEF_VERSION=0.1.73 +ARG CARGO_CHEF_VERSION=0.1.77 WORKDIR /opt/mega -# Prebuilt cargo-chef (avoids compiling it on every cold planner layer). +# Install cargo-chef from the release installer script. RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \ && rm -rf /var/lib/apt/lists/* \ - && case "$TARGETARCH" in \ - amd64) chef_arch=x86_64-unknown-linux-musl ;; \ - arm64) chef_arch=aarch64-unknown-linux-gnu ;; \ - *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;; \ - esac \ - && curl -fsSL "https://github.com/LukeMathWalker/cargo-chef/releases/download/v${CARGO_CHEF_VERSION}/cargo-chef-${chef_arch}.tar.gz" \ - | tar xz -C /usr/local/cargo/bin + && curl --proto '=https' --tlsv1.2 -LsSf \ + "https://github.com/LukeMathWalker/cargo-chef/releases/download/v${CARGO_CHEF_VERSION}/cargo-chef-installer.sh" \ + | sh # ────── Stage 1: Planner ────── # Copies all Cargo.toml files; cargo-chef emits recipe.json describing the full dependency graph. @@ -24,6 +19,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates FROM chef AS planner WORKDIR /opt/mega +ARG TARGETARCH COPY Cargo.toml ./ COPY api-model/Cargo.toml api-model/ @@ -40,13 +36,13 @@ COPY orion/td_util/Cargo.toml orion/td_util/ COPY orion/buck/Cargo.toml orion/buck/ COPY orion-scheduler/Cargo.toml orion-scheduler/ COPY orion-server/Cargo.toml orion-server/ -COPY orion-server/bellatrix/Cargo.toml orion-server/bellatrix/ +COPY clients/orion-client/Cargo.toml clients/orion-client/ COPY saturn/Cargo.toml saturn/ COPY vault/Cargo.toml vault/ # Named cache mounts let the builder stage inherit this cache via `from=planner`. -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=mono-registry \ - --mount=type=cache,target=/usr/local/cargo/git,id=mono-git \ +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=mega-cargo-registry-${TARGETARCH},sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,id=mega-cargo-git-${TARGETARCH},sharing=locked \ cargo chef prepare --bin mono --recipe-path recipe.json # ────── Stage 2: Builder ────── @@ -54,6 +50,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,id=mono-registry \ FROM chef AS builder WORKDIR /opt/mega +ARG TARGETARCH # Install system dependencies required for building RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -71,19 +68,19 @@ ENV RUSTFLAGS="-C link-arg=-fuse-ld=mold" # Inherit registry/git caches from the planner stage. COPY --from=planner /opt/mega/recipe.json recipe.json -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=mono-registry,from=planner \ - --mount=type=cache,target=/usr/local/cargo/git,id=mono-git,from=planner \ - cargo chef cook --release --recipe-path recipe.json +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=mega-cargo-registry-${TARGETARCH},sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,id=mega-cargo-git-${TARGETARCH},sharing=locked \ + cargo chef cook --profile demo --recipe-path recipe.json COPY . . # Persist built binary into the image layer (buildkit cache mounts evaporate after build). -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=mono-registry,from=planner \ - --mount=type=cache,target=/usr/local/cargo/git,id=mono-git,from=planner \ +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=mega-cargo-registry-${TARGETARCH},sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,id=mega-cargo-git-${TARGETARCH},sharing=locked \ --mount=type=cache,target=/opt/mega/target-cache \ - CARGO_TARGET_DIR=/opt/mega/target-cache cargo build --release -p mono && \ - mkdir -p /opt/mega/target/release && \ - cp /opt/mega/target-cache/release/mono /opt/mega/target/release/mono + CARGO_TARGET_DIR=/opt/mega/target-cache cargo build --profile demo -p mono && \ + mkdir -p /opt/mega/target/demo && \ + cp /opt/mega/target-cache/demo/mono /opt/mega/target/demo/mono # ────── Stage 3: Runtime ────── FROM debian:trixie-slim @@ -96,7 +93,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Copy the compiled binary and startup script -COPY --from=builder /opt/mega/target/release/mono /usr/local/bin/mono +COPY --from=builder /opt/mega/target/demo/mono /usr/local/bin/mono COPY --from=builder /opt/mega/mono/start-mono.sh /usr/local/bin/start-mono.sh # Ensure binaries are executable diff --git a/mono/src/api/mod.rs b/mono/src/api/mod.rs index 41ff71be8..695f896ae 100644 --- a/mono/src/api/mod.rs +++ b/mono/src/api/mod.rs @@ -4,7 +4,6 @@ use std::{ }; use axum::extract::FromRef; -use bellatrix::Bellatrix; use ceres::{ api_service::{ ApiHandler, cache::GitObjectCache, import_api_service::ImportApiService, @@ -23,6 +22,7 @@ use jupiter::{ user_storage::UserStorage, webhook_storage::WebhookStorage, }, }; +use orion_client::OrionBuildClient; use saturn::entitystore::EntityStore; use tower_sessions::MemoryStore; @@ -43,7 +43,7 @@ pub struct MonoApiServiceState { pub session_store: Option, pub listen_addr: String, pub entity_store: EntityStore, - pub bellatrix: Arc, + pub orion_client: Arc, } impl FromRef for MemoryStore { @@ -137,7 +137,7 @@ impl MonoApiServiceState { BuildTriggerService::new( self.storage.clone(), self.git_object_cache.clone(), - self.bellatrix.clone(), + self.orion_client.clone(), ) } diff --git a/mono/src/server/http_server.rs b/mono/src/server/http_server.rs index eb557e03b..b610752f9 100644 --- a/mono/src/server/http_server.rs +++ b/mono/src/server/http_server.rs @@ -10,12 +10,12 @@ use axum::{ response::Response, routing::any, }; -use bellatrix::Bellatrix; use ceres::api_service::{cache::GitObjectCache, state::ProtocolApiState}; use common::errors::ProtocolError; use context::AppContext; use http::{HeaderName, HeaderValue, Method}; use jupiter::service::artifact_service::ArtifactService; +use orion_client::OrionBuildClient; use saturn::entitystore::EntityStore; use time::Duration; use tokio::task::JoinHandle; @@ -395,7 +395,7 @@ pub async fn app(ctx: AppContext, host: String, port: u16) -> Router { listen_addr: format!("http://{host}:{port}"), entity_store: EntityStore::new(), git_object_cache, - bellatrix: Arc::new(Bellatrix::new(storage.config().build.clone())), + orion_client: Arc::new(OrionBuildClient::new(storage.config().build.clone())), }; let origins: Vec = oauth_config diff --git a/moon/apps/sync-server/Dockerfile b/moon/apps/sync-server/Dockerfile index 6e22e0337..b831accda 100644 --- a/moon/apps/sync-server/Dockerfile +++ b/moon/apps/sync-server/Dockerfile @@ -1,22 +1,22 @@ # syntax = docker/dockerfile:1 -FROM node:22-slim as base +FROM node:22-slim AS base # configure pnpm ENV PNPM_HOME="/pnpm" -ENV PATH="$PNPM_HOME:$PATH" -RUN corepack enable +ENV PATH="$PNPM_HOME/bin:$PNPM_HOME:$PATH" +RUN corepack enable && corepack prepare pnpm@9.7.1 --activate # ENV SENTRY_PROPERTIES=sentry.properties # Throw-away build stage to reduce size of final image -FROM base as builder +FROM base AS builder # Install packages needed to build node modules RUN apt-get update -qq && \ apt-get install -y build-essential pkg-config python-is-python3 ca-certificates WORKDIR /app -RUN pnpm install turbo@2.0.9 --global +RUN pnpm install turbo@2.9.16 --global COPY . . RUN turbo prune @gitmono/sync-server --docker WORKDIR /app @@ -56,6 +56,6 @@ COPY --from=installer /app /app # Start the server by default, this can be overwritten at runtime EXPOSE 9000 -ENV PORT 9000 +ENV PORT=9000 ENTRYPOINT [ "node", "dist/index.mjs" ] diff --git a/moon/apps/web/Dockerfile b/moon/apps/web/Dockerfile index 5131ab1b6..7acf664e0 100644 --- a/moon/apps/web/Dockerfile +++ b/moon/apps/web/Dockerfile @@ -7,10 +7,8 @@ ENV APP_ENV=${APP_ENV} # configure pnpm ENV PNPM_HOME="/pnpm" -ENV PATH="$PNPM_HOME:$PATH" -RUN npm install -g pnpm - -ARG TIPTAP_PRIVATE_REGISTRY_KEY +ENV PATH="$PNPM_HOME/bin:$PNPM_HOME:$PATH" +RUN corepack enable && corepack prepare pnpm@9.7.1 --activate # Throw-away build stage to reduce size of final image @@ -19,27 +17,29 @@ FROM base AS builder RUN apt-get update -qq && \ apt-get install -y build-essential pkg-config python-is-python3 ca-certificates WORKDIR /app -RUN pnpm install turbo@2.1.2 --global +RUN pnpm install turbo@2.9.16 --global COPY . . RUN turbo prune @gitmono/web --docker WORKDIR /app FROM base AS installer +ARG TARGETARCH RUN apt-get update -qq && \ apt-get install -y build-essential pkg-config python-is-python3 ca-certificates WORKDIR /app COPY --from=builder /app/out/json/ . COPY --from=builder /app/out/pnpm-lock.yaml ./pnpm-lock.yaml -COPY --from=builder /app/package.json ./ COPY --from=builder /app/patches ./patches COPY --from=builder /app/pnpm-workspace.yaml ./ RUN pnpm config set "@tiptap-pro:registry" https://registry.tiptap.dev/ RUN pnpm config set "//registry.tiptap.dev/:_authToken" CjqFil0n7z4GGur+RPric21fBBeSB4R4FoNdRYOE1bQEz6AXLCoANCc+o9rLIg6Q -RUN pnpm install +RUN --mount=type=cache,id=mega-ui-pnpm-store-${TARGETARCH},target=/pnpm/store,sharing=locked \ + pnpm config set store-dir /pnpm/store && \ + pnpm install --frozen-lockfile RUN npx patch-package COPY --from=builder /app/out/full/ . @@ -50,9 +50,6 @@ RUN rm -f /app/apps/web/.env.local \ RUN npx turbo run build --filter=@gitmono/web -RUN apt-get update -y && apt-get install -y ca-certificates - - FROM base AS runner diff --git a/moon/packages/editor/package.json b/moon/packages/editor/package.json index 63798e5b9..fb8cb6c02 100644 --- a/moon/packages/editor/package.json +++ b/moon/packages/editor/package.json @@ -61,6 +61,6 @@ "vitest": "catalog:" }, "peerDependencies": { - "prosemirror-model": "^1.22.3" + "prosemirror-model": "^1.25.1" } } diff --git a/moon/pnpm-lock.yaml b/moon/pnpm-lock.yaml index 1cd678bae..3e8d85566 100644 --- a/moon/pnpm-lock.yaml +++ b/moon/pnpm-lock.yaml @@ -400,8 +400,8 @@ catalogs: specifier: ^14.2.3 version: 14.2.3 eslint-config-turbo: - specifier: ^2.0.9 - version: 2.0.9 + specifier: ^2.9.16 + version: 2.9.16 eslint-plugin-react: specifier: ^7.34.1 version: 7.34.1 @@ -658,8 +658,8 @@ catalogs: specifier: ^8.0.2 version: 8.0.2 turbo: - specifier: ^2.1.2 - version: 2.1.2 + specifier: ^2.9.16 + version: 2.9.16 typescript: specifier: ^5.4.3 version: 5.4.3 @@ -714,7 +714,7 @@ importers: version: 12.0.4(encoding@0.1.13) turbo: specifier: 'catalog:' - version: 2.1.2 + version: 2.9.16 typescript: specifier: 'catalog:' version: 5.4.3 @@ -1411,8 +1411,8 @@ importers: specifier: 'catalog:' version: 14.1.0 prosemirror-model: - specifier: ^1.22.3 - version: 1.22.3 + specifier: ^1.25.1 + version: 1.25.1 refractor: specifier: 'catalog:' version: 4.8.1 @@ -1473,7 +1473,7 @@ importers: version: 14.2.3(eslint@8.57.0)(typescript@5.4.3) eslint-config-turbo: specifier: 'catalog:' - version: 2.0.9(eslint@8.57.0) + version: 2.9.16(eslint@8.57.0)(turbo@2.9.16) eslint-plugin-react: specifier: 'catalog:' version: 7.34.1(eslint@8.57.0) @@ -5617,6 +5617,36 @@ packages: '@tsconfig/node16@1.0.4': resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + '@turbo/darwin-64@2.9.16': + resolution: {integrity: sha512-jLjApWTSNd7JZ5JaLYfelW1ytnGQOvB7ivl+2RD1xQvJTbi8I9gBjzcga7tDZVPyaxpl10YTfJt3BrYXR18KDw==} + cpu: [x64] + os: [darwin] + + '@turbo/darwin-arm64@2.9.16': + resolution: {integrity: sha512-YPgrn+5HIGzrx0O2a631SV4MBQUe4W/DafMFUuBVgaU32PW9/OTT0ehviF0QSxTXuRJlHvW2eUTemddF5/spmw==} + cpu: [arm64] + os: [darwin] + + '@turbo/linux-64@2.9.16': + resolution: {integrity: sha512-vAEf1H6l26lTpl9FJ/peQo1NUB8RC0sbEJJz5mPcUhHA2bPDup2x3CZPgo/bH8S4cUcBLm4FN3UHd5iUO2RAew==} + cpu: [x64] + os: [linux] + + '@turbo/linux-arm64@2.9.16': + resolution: {integrity: sha512-xDBLR2PZg4BrQOchfG6svgpv5FCNJ2TOtT2psLdEJcdKo1BH+pnPs9Xj6pvUjgfkHbuvBOfeE4R6tvxMoQKDHQ==} + cpu: [arm64] + os: [linux] + + '@turbo/windows-64@2.9.16': + resolution: {integrity: sha512-NBAJnaUiGdgkSzQwUIdOvkCkcpTSu58G/sBGa0mvBtzfvFOOgrQwepKOOQ8cp6sWM6OcKDNFj2p1dsZA1OWjPg==} + cpu: [x64] + os: [win32] + + '@turbo/windows-arm64@2.9.16': + resolution: {integrity: sha512-Y7SJppD0Z8wjO3Ec0ZGd9KQ4Yv0BMnA8CIowj5Vp+OEVsosXDG2weK6/t1RRLfJmc2Ozrnd6y4DOgQys+mn3WQ==} + cpu: [arm64] + os: [win32] + '@types/aria-query@5.0.4': resolution: {integrity: sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==} @@ -7680,10 +7710,11 @@ packages: typescript: optional: true - eslint-config-turbo@2.0.9: - resolution: {integrity: sha512-FoIMElI8md/dR5DxjB5Om52KJfi7Qf7RInXeE+PGU6lN388rumppwyqEJsZ7vnR5GhGa9cLPt0vNZwEK9iXtKg==} + eslint-config-turbo@2.9.16: + resolution: {integrity: sha512-tIbnu6/rIZQzSFcGvocqP0LD97jirVVg235ohRfEiyjmoT8EoGgqvkvZKksmNyInjHB91er9TtDzPBtAMb3Bvw==} peerDependencies: eslint: '>6.6.0' + turbo: '>2.0.0' eslint-import-resolver-node@0.3.9: resolution: {integrity: sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g==} @@ -7750,10 +7781,11 @@ packages: peerDependencies: eslint: '>=6' - eslint-plugin-turbo@2.0.9: - resolution: {integrity: sha512-q4s4mg6JcXzz5zK4LC3c6FcWehGAWjGj7kIM76ZvG0KiR9Ks0znzjnAKW0NoiDP4s/gt3r4YPOpI357qWt167Q==} + eslint-plugin-turbo@2.9.16: + resolution: {integrity: sha512-NOITpZwuq+c/urW2aGD7QruJ/1frWyci2SF9lX8IaFFGwu2iNSI/90XqOiiWbQe5Ur96C8rspnhDSKzh+EXzIg==} peerDependencies: eslint: '>6.6.0' + turbo: '>2.0.0' eslint-plugin-unused-imports@3.2.0: resolution: {integrity: sha512-6uXyn6xdINEpxE1MtDjxQsyXB37lfyO2yKGVVgtD7WEWQGORSOZjgrD6hBhvGv4/SO+TOlS+UnC6JppRqbuwGQ==} @@ -10314,9 +10346,6 @@ packages: prosemirror-menu@1.2.4: resolution: {integrity: sha512-S/bXlc0ODQup6aiBbWVsX/eM+xJgCTAfMq/nLqaO5ID/am4wS0tTCIkzwytmao7ypEtjj39i7YbJjAgO20mIqA==} - prosemirror-model@1.22.3: - resolution: {integrity: sha512-V4XCysitErI+i0rKFILGt/xClnFJaohe/wrrlT2NSZ+zk8ggQfDH4x2wNK7Gm0Hp4CIoWizvXFP7L9KMaCuI0Q==} - prosemirror-model@1.25.1: resolution: {integrity: sha512-AUvbm7qqmpZa5d9fPKMvH1Q5bqYQvAZWOGRvxsB6iFLyycvC9MwNemNVjHVrWgjaoxAfY8XVg7DbvQ/qxvI9Eg==} @@ -11684,38 +11713,8 @@ packages: tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - turbo-darwin-64@2.1.2: - resolution: {integrity: sha512-3TEBxHWh99h2yIzkuIigMEOXt/ItYQp0aPiJjPd1xN4oDcsKK5AxiFKPH9pdtfIBzYsY59kQhZiFj0ELnSP7Bw==} - cpu: [x64] - os: [darwin] - - turbo-darwin-arm64@2.1.2: - resolution: {integrity: sha512-he0miWNq2WxJzsH82jS2Z4MXpnkzn9SH8a79iPXiJkq25QREImucscM4RPasXm8wARp91pyysJMq6aasD45CeA==} - cpu: [arm64] - os: [darwin] - - turbo-linux-64@2.1.2: - resolution: {integrity: sha512-fKUBcc0rK8Vdqv5a/E3CSpMBLG1bzwv+Q0Q83F8fG2ZfNCNKGbcEYABdonNZkkx141Rj03cZQFCgxu3MVEGU+A==} - cpu: [x64] - os: [linux] - - turbo-linux-arm64@2.1.2: - resolution: {integrity: sha512-sV8Bpmm0WiuxgbhxymcC7wSsuxfBBieI98GegSwbr/bs1ANAgzCg93urIrdKdQ3/b31zZxQwcaP4FBF1wx1Qdg==} - cpu: [arm64] - os: [linux] - - turbo-windows-64@2.1.2: - resolution: {integrity: sha512-wcmIJZI9ORT9ykHGliFE6kWRQrlH930QGSjSgWC8uFChFFuOyUlvC7ttcxuSvU9VqC7NF4C+GVAcFJQ8lTjN7g==} - cpu: [x64] - os: [win32] - - turbo-windows-arm64@2.1.2: - resolution: {integrity: sha512-zdnXjrhk7YO6CP+Q5wPueEvOCLH4lDa6C4rrwiakcWcPgcQGbVozJlo4uaQ6awo8HLWQEvOwu84RkWTdLAc/Hw==} - cpu: [arm64] - os: [win32] - - turbo@2.1.2: - resolution: {integrity: sha512-Jb0rbU4iHEVQ18An/YfakdIv9rKnd3zUfSE117EngrfWXFHo3RndVH96US3GsT8VHpwTncPePDBT2t06PaFLrw==} + turbo@2.9.16: + resolution: {integrity: sha512-NqgRQy6j6dPYcdSdv0q1g9QsZg7SWg87RERM8otw/1AtKU2yTFVClOM7cbwKzOonZr/Ek1blTBucw64L9H0Bwg==} hasBin: true tweetnacl@1.0.3: @@ -17712,6 +17711,24 @@ snapshots: '@tsconfig/node16@1.0.4': optional: true + '@turbo/darwin-64@2.9.16': + optional: true + + '@turbo/darwin-arm64@2.9.16': + optional: true + + '@turbo/linux-64@2.9.16': + optional: true + + '@turbo/linux-arm64@2.9.16': + optional: true + + '@turbo/windows-64@2.9.16': + optional: true + + '@turbo/windows-arm64@2.9.16': + optional: true + '@types/aria-query@5.0.4': {} '@types/babel__core@7.20.5': @@ -20159,10 +20176,11 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-config-turbo@2.0.9(eslint@8.57.0): + eslint-config-turbo@2.9.16(eslint@8.57.0)(turbo@2.9.16): dependencies: eslint: 8.57.0 - eslint-plugin-turbo: 2.0.9(eslint@8.57.0) + eslint-plugin-turbo: 2.9.16(eslint@8.57.0)(turbo@2.9.16) + turbo: 2.9.16 eslint-import-resolver-node@0.3.9: dependencies: @@ -20280,10 +20298,11 @@ snapshots: - supports-color - typescript - eslint-plugin-turbo@2.0.9(eslint@8.57.0): + eslint-plugin-turbo@2.9.16(eslint@8.57.0)(turbo@2.9.16): dependencies: dotenv: 16.0.3 eslint: 8.57.0 + turbo: 2.9.16 eslint-plugin-unused-imports@3.2.0(@typescript-eslint/eslint-plugin@7.11.0(@typescript-eslint/parser@7.11.0(eslint@8.57.0)(typescript@5.4.3))(eslint@8.57.0)(typescript@5.4.3))(eslint@8.57.0): dependencies: @@ -23248,10 +23267,6 @@ snapshots: prosemirror-history: 1.4.1 prosemirror-state: 1.4.3 - prosemirror-model@1.22.3: - dependencies: - orderedmap: 2.1.1 - prosemirror-model@1.25.1: dependencies: orderedmap: 2.1.1 @@ -24976,32 +24991,14 @@ snapshots: dependencies: safe-buffer: 5.2.1 - turbo-darwin-64@2.1.2: - optional: true - - turbo-darwin-arm64@2.1.2: - optional: true - - turbo-linux-64@2.1.2: - optional: true - - turbo-linux-arm64@2.1.2: - optional: true - - turbo-windows-64@2.1.2: - optional: true - - turbo-windows-arm64@2.1.2: - optional: true - - turbo@2.1.2: + turbo@2.9.16: optionalDependencies: - turbo-darwin-64: 2.1.2 - turbo-darwin-arm64: 2.1.2 - turbo-linux-64: 2.1.2 - turbo-linux-arm64: 2.1.2 - turbo-windows-64: 2.1.2 - turbo-windows-arm64: 2.1.2 + '@turbo/darwin-64': 2.9.16 + '@turbo/darwin-arm64': 2.9.16 + '@turbo/linux-64': 2.9.16 + '@turbo/linux-arm64': 2.9.16 + '@turbo/windows-64': 2.9.16 + '@turbo/windows-arm64': 2.9.16 tweetnacl@1.0.3: {} diff --git a/moon/pnpm-workspace.yaml b/moon/pnpm-workspace.yaml index b2f549b27..e7ce2915f 100644 --- a/moon/pnpm-workspace.yaml +++ b/moon/pnpm-workspace.yaml @@ -151,7 +151,7 @@ catalog: easymde: '2' eslint: ^8.57.0 eslint-config-next: ^14.2.3 - eslint-config-turbo: ^2.0.9 + eslint-config-turbo: ^2.9.16 eslint-plugin-react: ^7.34.1 eslint-plugin-react-hooks: ^4.6.2 eslint-plugin-storybook: ^0.8.0 @@ -252,7 +252,7 @@ catalog: tippy.js: ^6.3.7 tsup: ^8.0.2 typescript: ^5.4.3 - turbo: ^2.1.2 + turbo: ^2.9.16 use-debounce: ^9.0.4 use-sound: ^4.0.1 uuid: ^9.0.1 diff --git a/orion-scheduler/README.md b/orion-scheduler/README.md index 6c93fec40..07e545288 100644 --- a/orion-scheduler/README.md +++ b/orion-scheduler/README.md @@ -275,6 +275,88 @@ ssh -i ~/.ssh/orion_vm_access root@ --- +## Release 与部署 + +orion-scheduler 是宿主机上的 KVM/QEMU 管理 daemon(依赖 `/dev/kvm`、`qlbr0` 网桥、QEMU 子进程长期存活),不适合做成容器镜像运行。发布方式是「**Linux 原生二进制 + systemd**」,由 GitHub Action 构建,作为 **GitHub Release** 资产发布。 + +### 触发一次 release + +向仓库推一个 `orion-scheduler-vX.Y.Z` 形式的 tag: + +```bash +git tag orion-scheduler-v0.1.0 +git push origin orion-scheduler-v0.1.0 +``` + +[`.github/workflows/orion-scheduler-release.yml`](../.github/workflows/orion-scheduler-release.yml) 会: + +1. 在 `ubuntu-latest` 上 `cargo build --release -p orion-scheduler --target x86_64-unknown-linux-gnu` +2. 打包 `orion-scheduler-vX.Y.Z-linux-amd64.tar.gz`(含二进制、systemd unit、`install.sh`、`target_config.json.template`、`VERSION` 元数据) +3. 生成 `*.tar.gz.sha256` +4. 创建 GitHub Release 并上传 tarball + 校验文件 + +也可以在 Actions 页面手动 `workflow_dispatch` 触发(需要填 `version`),手动触发的产物只放在 Actions artifacts 里,不会发布为 Release。 + +Release bundle 内部结构: + +```text +orion-scheduler-vX.Y.Z-linux-amd64/ +├── bin/orion-scheduler +├── etc/target_config.json.template +├── systemd/orion-scheduler.service +├── install.sh +└── VERSION +``` + +### Host 端安装 + +主机预先满足前置要求(KVM 内核模块、`qlbr0` 网桥、QEMU、自定义镜像,见 [前置要求](#前置要求)),然后: + +```bash +VERSION=v0.1.0 +BUNDLE=orion-scheduler-${VERSION}-linux-amd64 +REPO=web3infra-foundation/mega + +curl -LO https://github.com/${REPO}/releases/download/orion-scheduler-${VERSION}/${BUNDLE}.tar.gz +curl -LO https://github.com/${REPO}/releases/download/orion-scheduler-${VERSION}/${BUNDLE}.tar.gz.sha256 +sha256sum -c ${BUNDLE}.tar.gz.sha256 +tar -xzf ${BUNDLE}.tar.gz + +sudo bash ${BUNDLE}/install.sh +sudo $EDITOR /etc/orion-scheduler/target_config.json # 首次安装会留下模板 +sudo systemctl start orion-scheduler +sudo journalctl -u orion-scheduler -f +``` + +[`install.sh`](install.sh) 完成的工作: + +| 工作 | 路径 / 命令 | +| --- | --- | +| 创建 service 用户 | `orion` 用户,`-G kvm` 加入 KVM 组 | +| 落二进制 | `/opt/orion-scheduler/bin/orion-scheduler` | +| 落配置 | `/etc/orion-scheduler/target_config.json`(已存在则保留) | +| qlean 状态目录 | `/var/lib/orion-scheduler/qlean/{images,runs}`,软链到 `~orion/.local/share/qlean` | +| 日志 & 缓存 | `/var/log/orion-scheduler`、`/var/cache/orion-scheduler` | +| systemd | `/etc/systemd/system/orion-scheduler.service`,`daemon-reload` + `enable` | + +可用环境变量覆盖默认值:`PREFIX` / `ETC_DIR` / `STATE_DIR` / `LOG_DIR` / `CACHE_DIR` / `SERVICE_USER` / `SERVICE_GROUP` / `SKIP_ENABLE=1`。 + +升级时,下载新 tarball 再跑一次 `install.sh` 即可——配置不会被覆盖,systemd unit 会重启。 + +### 升级 / 回滚 + +| 操作 | 命令 | +| --- | --- | +| 升级 | 下载新版本 tarball → `sudo bash /install.sh` → `sudo systemctl restart orion-scheduler` | +| 回滚 | 下载旧版本 tarball → 同上 | +| 检查当前版本 | `cat /opt/orion-scheduler/bin/orion-scheduler --version`(若启用了 clap version)或 `cat ~/.../VERSION`(解压后) | +| 停服并保留 VM | `sudo systemctl stop orion-scheduler`;VM 不会被关闭 | +| 完全停服 | 先 `POST /shutdown` 关闭 VM,再 `systemctl stop` | + +> **注意**:必须从 Linux x86_64 host 上跑。orion-scheduler 通过 `qlean` 依赖 `kvm-ioctls`,不可在 macOS / ARM64 上运行。 + +--- + ## 演进方向 当前 `orion` 二进制路径通过 `target_config.json` 的 `orion_binary_path` 配置,镜像路径通过 `image_path` 或 `image_url`(API 参数)配置。 diff --git a/orion-scheduler/install.sh b/orion-scheduler/install.sh new file mode 100755 index 000000000..58aff2c29 --- /dev/null +++ b/orion-scheduler/install.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Install orion-scheduler from an extracted release bundle. +# +# Layout expected in the same directory as this script: +# bin/orion-scheduler +# etc/target_config.json.template +# systemd/orion-scheduler.service +# +# Env vars (all optional): +# PREFIX install prefix (default /opt/orion-scheduler) +# ETC_DIR config dir (default /etc/orion-scheduler) +# STATE_DIR qlean state dir (default /var/lib/orion-scheduler) +# LOG_DIR default /var/log/orion-scheduler +# CACHE_DIR default /var/cache/orion-scheduler +# SERVICE_USER default orion +# SERVICE_GROUP default orion +# SKIP_ENABLE if "1", install files only, do not enable/start unit +# +# Run as root (or via sudo). + +set -euo pipefail + +PREFIX="${PREFIX:-/opt/orion-scheduler}" +ETC_DIR="${ETC_DIR:-/etc/orion-scheduler}" +STATE_DIR="${STATE_DIR:-/var/lib/orion-scheduler}" +LOG_DIR="${LOG_DIR:-/var/log/orion-scheduler}" +CACHE_DIR="${CACHE_DIR:-/var/cache/orion-scheduler}" +SERVICE_USER="${SERVICE_USER:-orion}" +SERVICE_GROUP="${SERVICE_GROUP:-orion}" +SKIP_ENABLE="${SKIP_ENABLE:-0}" + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) + +if [[ $EUID -ne 0 ]]; then + echo "[install] must run as root (use sudo)" >&2 + exit 1 +fi + +for f in "$SCRIPT_DIR/bin/orion-scheduler" \ + "$SCRIPT_DIR/etc/target_config.json.template" \ + "$SCRIPT_DIR/systemd/orion-scheduler.service"; do + if [[ ! -f "$f" ]]; then + echo "[install] missing bundle file: $f" >&2 + exit 1 + fi +done + +echo "[install] ensuring group $SERVICE_GROUP, user $SERVICE_USER (member of kvm)" +if ! getent group "$SERVICE_GROUP" >/dev/null; then + groupadd --system "$SERVICE_GROUP" +fi +if ! id -u "$SERVICE_USER" >/dev/null 2>&1; then + # qlean needs $HOME for ~/.local/share/qlean; we point HOME at $STATE_DIR. + useradd --system \ + --gid "$SERVICE_GROUP" \ + --home-dir "$STATE_DIR" \ + --shell /usr/sbin/nologin \ + "$SERVICE_USER" +fi +if getent group kvm >/dev/null; then + usermod -aG kvm "$SERVICE_USER" +else + echo "[install] WARN: 'kvm' group not present on this host; install qemu-kvm first" >&2 +fi + +echo "[install] creating directories" +install -d -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0755 \ + "$PREFIX/bin" "$STATE_DIR" "$LOG_DIR" "$CACHE_DIR" +install -d -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0750 "$ETC_DIR" + +# qlean's QleanDirs hardcodes ~/.local/share/qlean; symlink that into +# $STATE_DIR so VM images/runs persist outside the user's home. +install -d -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0755 \ + "$STATE_DIR/.local" "$STATE_DIR/.local/share" +if [[ ! -e "$STATE_DIR/.local/share/qlean" ]]; then + ln -s "$STATE_DIR/qlean" "$STATE_DIR/.local/share/qlean" +fi +install -d -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0755 \ + "$STATE_DIR/qlean" "$STATE_DIR/qlean/images" "$STATE_DIR/qlean/runs" + +echo "[install] installing binary to $PREFIX/bin/orion-scheduler" +install -o root -g root -m 0755 \ + "$SCRIPT_DIR/bin/orion-scheduler" "$PREFIX/bin/orion-scheduler" + +echo "[install] installing config to $ETC_DIR (only if missing)" +if [[ ! -f "$ETC_DIR/target_config.json" ]]; then + install -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0640 \ + "$SCRIPT_DIR/etc/target_config.json.template" \ + "$ETC_DIR/target_config.json" + echo "[install] wrote template — edit $ETC_DIR/target_config.json before starting" +else + echo "[install] $ETC_DIR/target_config.json already exists, keeping" +fi + +echo "[install] installing systemd unit" +install -o root -g root -m 0644 \ + "$SCRIPT_DIR/systemd/orion-scheduler.service" \ + /etc/systemd/system/orion-scheduler.service +systemctl daemon-reload + +if [[ "$SKIP_ENABLE" == "1" ]]; then + echo "[install] SKIP_ENABLE=1, leaving unit disabled" + echo "[install] Done." + exit 0 +fi + +if [[ -f "$ETC_DIR/target_config.json" ]] \ + && grep -q '"/path/to/' "$ETC_DIR/target_config.json"; then + echo "[install] config still contains template placeholders;" + echo "[install] enabling unit but NOT starting it. Edit $ETC_DIR/target_config.json" + echo "[install] then run: sudo systemctl start orion-scheduler" + systemctl enable orion-scheduler.service +else + systemctl enable --now orion-scheduler.service + systemctl --no-pager --lines=0 status orion-scheduler.service || true +fi + +echo "[install] Done." diff --git a/orion-scheduler/systemd/orion-scheduler.service b/orion-scheduler/systemd/orion-scheduler.service new file mode 100644 index 000000000..3b83be148 --- /dev/null +++ b/orion-scheduler/systemd/orion-scheduler.service @@ -0,0 +1,44 @@ +[Unit] +Description=Orion Scheduler (qlean-based KVM micro-VM controller) +Documentation=https://github.com/web3infra-foundation/mega/tree/main/orion-scheduler +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +# The service user must be a member of the `kvm` group so QEMU can open +# /dev/kvm. install.sh creates the user and adds it to `kvm` for you. +User=orion +Group=orion +SupplementaryGroups=kvm + +WorkingDirectory=/opt/orion-scheduler +Environment=RUST_LOG=info +Environment=CONFIG_PATH=/etc/orion-scheduler/target_config.json +# Override these in /etc/systemd/system/orion-scheduler.service.d/*.conf if needed. + +ExecStart=/opt/orion-scheduler/bin/orion-scheduler + +# qlean writes images/runs under the service user's HOME; the install +# script provisions /var/lib/orion-scheduler/qlean and symlinks +# ~orion/.local/share/qlean to it so state survives reinstalls. +ReadWritePaths=/var/log/orion-scheduler /var/cache/orion-scheduler /var/lib/orion-scheduler + +# Give the process room for many concurrent VMs/SSH connections. +LimitNOFILE=65536 + +# QEMU bridge helper needs CAP_NET_ADMIN. Keep the rest of the sandbox +# loose: aggressive ProtectSystem/PrivateDevices flags break bridge +# networking and /dev/kvm access. +AmbientCapabilities=CAP_NET_ADMIN + +Restart=on-failure +RestartSec=5s + +# Give qlean a chance to power off running VMs gracefully before SIGKILL. +TimeoutStopSec=60s +KillMode=mixed +KillSignal=SIGTERM + +[Install] +WantedBy=multi-user.target diff --git a/orion-server/Dockerfile b/orion-server/Dockerfile index 79ac46221..a7c332d4c 100644 --- a/orion-server/Dockerfile +++ b/orion-server/Dockerfile @@ -1,11 +1,29 @@ # ────── Stage 0: Chef ────── -FROM rust:1.95.0 AS chef -RUN cargo install cargo-chef --version 0.1.73 -WORKDIR /app +# Downloads and installs cargo-chef (prebuilt binary, no Rust compilation needed). +# NOTE: This stage is intentionally identical to `mono/Dockerfile`'s chef stage +# (same base image, WORKDIR, cargo-chef version) so BuildKit can share the +# resulting layer between the two demo images. +FROM rust:1.96.0 AS chef + +ARG CARGO_CHEF_VERSION=0.1.77 + +WORKDIR /opt/mega + +# Install cargo-chef from the release installer script. +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* \ + && curl --proto '=https' --tlsv1.2 -LsSf \ + "https://github.com/LukeMathWalker/cargo-chef/releases/download/v${CARGO_CHEF_VERSION}/cargo-chef-installer.sh" \ + | sh # ────── Stage 1: Planner ────── +# Copies all Cargo.toml files; cargo-chef emits recipe.json describing the full dependency graph. +# This stage is cached unless a Cargo.toml changes. FROM chef AS planner +WORKDIR /opt/mega +ARG TARGETARCH + COPY Cargo.toml ./ COPY api-model/Cargo.toml api-model/ COPY ceres/Cargo.toml ceres/ @@ -19,59 +37,69 @@ COPY orion/Cargo.toml orion/ COPY orion/audit/Cargo.toml orion/audit/ COPY orion/td_util/Cargo.toml orion/td_util/ COPY orion/buck/Cargo.toml orion/buck/ +COPY orion-scheduler/Cargo.toml orion-scheduler/ COPY orion-server/Cargo.toml orion-server/ -COPY orion-server/bellatrix/Cargo.toml orion-server/bellatrix/ +COPY clients/orion-client/Cargo.toml clients/orion-client/ COPY saturn/Cargo.toml saturn/ COPY vault/Cargo.toml vault/ -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=orion-registry \ - --mount=type=cache,target=/usr/local/cargo/git,id=orion-git \ +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=mega-cargo-registry-${TARGETARCH},sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,id=mega-cargo-git-${TARGETARCH},sharing=locked \ cargo chef prepare --bin orion-server --recipe-path recipe.json # ────── Stage 2: Builder ────── +# Installs system libs, builds all dependencies (cached), then builds the orion-server binary. FROM chef AS builder -RUN apt-get update && \ - apt-get install -y --no-install-recommends libclang-dev && \ - rm -rf /var/lib/apt/lists/* +WORKDIR /opt/mega +ARG TARGETARCH +# Install system dependencies required for building. +# `libclang-dev` is required by some bindgen-using crates pulled in via orion-server. +RUN apt-get update && apt-get install -y --no-install-recommends \ + libssl-dev \ + ca-certificates \ + clang \ + libclang-dev \ + mold \ + protobuf-compiler \ + libprotobuf-dev \ + && rm -rf /var/lib/apt/lists/* -COPY --from=planner /app/recipe.json recipe.json +# Faster linking for release builds. +ENV RUSTFLAGS="-C link-arg=-fuse-ld=mold" -# -- Step 1: cook dependencies with cache -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=orion-registry \ - --mount=type=cache,target=/usr/local/cargo/git,id=orion-git \ - cargo chef cook --release --recipe-path recipe.json +COPY --from=planner /opt/mega/recipe.json recipe.json + +# Step 1: cook dependencies with shared cargo cache. +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=mega-cargo-registry-${TARGETARCH},sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,id=mega-cargo-git-${TARGETARCH},sharing=locked \ + cargo chef cook --profile demo --recipe-path recipe.json COPY . . -# -- Step 2: build binary into normal target directory -# NOTE: build output must be persisted into the image layer for the runtime stage `COPY`. -# A buildkit cache mount is not committed into the layer, so we build into a cached dir -# then copy the final binary into `/app/target/...` (regular filesystem). -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=orion-registry \ - --mount=type=cache,target=/usr/local/cargo/git,id=orion-git \ - --mount=type=cache,target=/app/target-cache \ - CARGO_TARGET_DIR=/app/target-cache cargo build -p orion-server --release && \ - mkdir -p /app/target/release && \ - cp /app/target-cache/release/orion-server /app/target/release/orion-server +# Step 2: build binary, then copy out of the cache mount so the runtime stage can COPY it. +# A buildkit cache mount is not committed into the image layer, so we build into a cached dir +# and persist the final binary into a regular filesystem path before the layer is sealed. +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=mega-cargo-registry-${TARGETARCH},sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,id=mega-cargo-git-${TARGETARCH},sharing=locked \ + --mount=type=cache,target=/opt/mega/target-cache \ + CARGO_TARGET_DIR=/opt/mega/target-cache cargo build --profile demo -p orion-server && \ + mkdir -p /opt/mega/target/demo && \ + cp /opt/mega/target-cache/demo/orion-server /opt/mega/target/demo/orion-server # ────── Stage 3: Runtime ────── -FROM debian:trixie-slim +FROM debian:trixie-slim -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ libssl3t64 \ && rm -rf /var/lib/apt/lists/* # Copy built executable -COPY --from=builder /app/target/release/orion-server /usr/local/bin/orion-server - -# Reduce image size (optional) -RUN strip /usr/local/bin/orion-server || true +COPY --from=builder /opt/mega/target/demo/orion-server /usr/local/bin/orion-server EXPOSE 8004 -CMD ["orion-server"] \ No newline at end of file +CMD ["orion-server"] diff --git a/orion-server/bellatrix/src/lib.rs b/orion-server/bellatrix/src/lib.rs deleted file mode 100644 index 79a7d295a..000000000 --- a/orion-server/bellatrix/src/lib.rs +++ /dev/null @@ -1,31 +0,0 @@ -pub mod orion_client; - -use api_model::buck2::api::TaskBuildRequest; -use common::config::BuildConfig; - -use crate::orion_client::OrionClient; - -#[derive(Clone)] -pub struct Bellatrix { - orion: OrionClient, - build_config: BuildConfig, -} - -impl Bellatrix { - pub fn new(build_config: BuildConfig) -> Self { - let orion = OrionClient::new(build_config.orion_server.clone()); - Self { - orion, - build_config, - } - } - - pub fn enable_build(&self) -> bool { - self.build_config.enable_build - } - - pub async fn on_post_receive(&self, req: TaskBuildRequest) -> anyhow::Result { - let task_id = self.orion.trigger_build(req).await?; - Ok(task_id) - } -} diff --git a/orion/.env.example b/orion/.env.example deleted file mode 100644 index 23a4b7006..000000000 --- a/orion/.env.example +++ /dev/null @@ -1,71 +0,0 @@ -# ============================================================================= -# Mega Dev Image - Environment Configuration (Client Toolchain) -# ============================================================================= -# Copy this file to .env and modify as needed. -# -# This image provides: -# - Scorpio daemon (FUSE + HTTP API) -# - Orion Worker (optional; connects to external Orion Server) -# - Buck2 CLI -# -# NOTE: Scorpio needs a running Mono (Mega main service) to fetch repository content. -# Configure `SCORPIO_BASE_URL` / `SCORPIO_LFS_URL` to point to Mono. - -# --------------------------------------------------------------------------- -# Common -# --------------------------------------------------------------------------- -RUST_LOG=info - -# --------------------------------------------------------------------------- -# Scorpio (daemon) Configuration -# --------------------------------------------------------------------------- -# Mono/Mega service URL (required for real mounts) -SCORPIO_BASE_URL=http://host.docker.internal:8000 -SCORPIO_LFS_URL=http://host.docker.internal:8000 - -# Scorpio HTTP port -SCORPIO_PORT=2725 - -# Optional: override Scorpio bind address inside container (newer scorpio supports `--http-addr`) -# If you set this, ensure your port mappings / SCORPIO_API_BASE_URL match. -# SCORPIO_HTTP_ADDR=0.0.0.0:2725 - -# Store path for Scorpio data (inside container) -SCORPIO_STORE_PATH=/data/scorpio/store - -# Workspace root (inside container) -SCORPIO_WORKSPACE=/workspace/mount - -# Git author info used by Scorpio operations -SCORPIO_GIT_AUTHOR=MEGA -SCORPIO_GIT_EMAIL=admin@mega.org - -# DicFuse configuration -SCORPIO_DICFUSE_READABLE=true -SCORPIO_LOAD_DIR_DEPTH=3 -SCORPIO_FETCH_FILE_THREAD=10 - -# --------------------------------------------------------------------------- -# Orion Worker (optional) Configuration -# --------------------------------------------------------------------------- -# Orion Server WebSocket URL (external service) -SERVER_WS=ws://host.docker.internal:8004/ws - -# Optional fixed worker id (otherwise generated per session) -# ORION_WORKER_ID= - -# Worker local paths (inside container) -BUCK_PROJECT_ROOT=/workspace -BUILD_TMP=/tmp/orion-builds - -# Scorpio API base URL used by the worker -# - When ORION_WORKER_START_SCORPIO=true, this is usually the embedded Scorpio. -SCORPIO_API_BASE_URL=http://127.0.0.1:2725 - -# Start embedded Scorpio inside the worker container (recommended when the worker needs mounts) -ORION_WORKER_START_SCORPIO=true - -# --------------------------------------------------------------------------- -# Buck2 Configuration -# --------------------------------------------------------------------------- - diff --git a/orion/Dockerfile b/orion/Dockerfile deleted file mode 100644 index 8ebf71a31..000000000 --- a/orion/Dockerfile +++ /dev/null @@ -1,196 +0,0 @@ -# ============================================================================= -# Mega Dev Image - Multi-stage Dockerfile -# ============================================================================= -# This Dockerfile builds a unified development image containing: -# - Orion Worker (build execution client) -# - Buck2 (build tool) -# -# Note: Scorpio (FUSE filesystem) has been moved to: -# https://github.com/web3infra-foundation/scorpiofs -# -# Supports: linux/amd64, linux/arm64 -# Final stage: runtime (slim, default) -# NOTE: This Dockerfile assumes the Docker build context is the repo root. -# Example: `docker build -f orion/Dockerfile .` -# ============================================================================= - -# ----------------------------------------------------------------------------- -# Build Arguments -# ----------------------------------------------------------------------------- -# Use "bookworm" for latest stable, or specify version like "1.83" -ARG RUST_VERSION=1.92-bookworm -ARG BUCK2_VERSION=2025-06-01 - -# ============================================================================= -# Stage 1: Chef Base - Tooling and system dependencies -# ============================================================================= -FROM rust:${RUST_VERSION} AS chef-base - -# Build arguments for version tracking -ARG GIT_COMMIT=unknown -ARG BUILD_DATE=unknown - -# Platform args (provided by buildx/BuildKit). Used to scope cache keys. -ARG TARGETARCH -ARG TARGETOS -ARG TARGETPLATFORM - -WORKDIR /build - -RUN mkdir -p /build/bin - -# Install build dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - binutils \ - pkg-config \ - cmake \ - clang \ - llvm-dev \ - libclang-dev \ - libssl-dev \ - libfuse3-dev \ - protobuf-compiler \ - && rm -rf /var/lib/apt/lists/* - -# Install cargo-chef for dependency layer caching -RUN cargo install cargo-chef --version 0.1.73 - -# ============================================================================= -# Stage 1a: Planner - Generate dependency recipe -# ============================================================================= -FROM chef-base AS planner - -COPY Cargo.toml ./ -COPY api-model/Cargo.toml api-model/ -COPY ceres/Cargo.toml ceres/ -COPY common/Cargo.toml common/ -COPY context/Cargo.toml context/ -COPY io-orbit/Cargo.toml io-orbit/ -COPY jupiter/Cargo.toml jupiter/ -COPY jupiter/callisto/Cargo.toml jupiter/callisto/ -COPY mono/Cargo.toml mono/ -COPY orion/Cargo.toml orion/ -COPY orion/audit/Cargo.toml orion/audit/ -COPY orion/td_util/Cargo.toml orion/td_util/ -COPY orion/buck/Cargo.toml orion/buck/ -COPY orion-server/Cargo.toml orion-server/ -COPY orion-server/bellatrix/Cargo.toml orion-server/bellatrix/ -COPY saturn/Cargo.toml saturn/ -COPY vault/Cargo.toml vault/ - -RUN cargo chef prepare --recipe-path recipe.json - -# ============================================================================= -# Stage 1b: Builder - Build cached deps, then compile binaries -# ============================================================================= -FROM chef-base AS builder - -COPY --from=planner /build/recipe.json /build/recipe.json - -RUN --mount=type=cache,id=mega-cargo-registry-${TARGETARCH},target=/usr/local/cargo/registry,sharing=locked \ - --mount=type=cache,id=mega-cargo-git-${TARGETARCH},target=/usr/local/cargo/git,sharing=locked \ - --mount=type=cache,id=mega-cargo-target-${TARGETARCH},target=/build/target,sharing=locked \ - CARGO_TARGET_DIR=/build/target \ - CARGO_INCREMENTAL=0 \ - CARGO_PROFILE_RELEASE_DEBUG=0 \ - cargo chef cook --release --recipe-path recipe.json - -COPY . . - -RUN --mount=type=cache,id=mega-cargo-registry-${TARGETARCH},target=/usr/local/cargo/registry,sharing=locked \ - --mount=type=cache,id=mega-cargo-git-${TARGETARCH},target=/usr/local/cargo/git,sharing=locked \ - --mount=type=cache,id=mega-cargo-target-${TARGETARCH},target=/build/target,sharing=locked \ - CARGO_TARGET_DIR=/build/target \ - CARGO_INCREMENTAL=0 \ - CARGO_PROFILE_RELEASE_DEBUG=0 \ - cargo build --release --package orion \ - && strip /build/target/release/orion \ - && install -m 0755 /build/target/release/orion /build/bin/orion - -# ============================================================================= -# Stage 2: Buck2 Downloader - Fetch Buck2 binary -# ============================================================================= -FROM debian:bookworm-slim AS buck2-downloader - -ARG BUCK2_VERSION -ARG TARGETARCH - -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - zstd \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /buck2 - -# Download Buck2 based on architecture -# Using official GitHub releases from facebook/buck2 -RUN set -eux; \ - case "${TARGETARCH}" in \ - amd64) BUCK2_ARCH="x86_64-unknown-linux-musl" ;; \ - arm64) BUCK2_ARCH="aarch64-unknown-linux-musl" ;; \ - *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ - esac; \ - BUCK2_URL="https://github.com/facebook/buck2/releases/download/${BUCK2_VERSION}/buck2-${BUCK2_ARCH}.zst"; \ - echo "Downloading Buck2 from: ${BUCK2_URL}"; \ - curl -fsSL -o buck2.zst "${BUCK2_URL}"; \ - zstd -d buck2.zst -o buck2; \ - chmod +x buck2; \ - ./buck2 --version - -# ============================================================================= -# Stage 3: Runtime Base - Minimal production image -# ============================================================================= -FROM debian:bookworm-slim AS runtime-base - -# Build arguments for labels -ARG GIT_COMMIT=unknown -ARG BUILD_DATE=unknown -ARG BUCK2_VERSION - -# Install runtime dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - libssl3 \ - libfuse3-3 \ - gettext-base \ - netcat-openbsd \ - && rm -rf /var/lib/apt/lists/* - -# Create necessary directories -RUN mkdir -p \ - /app/bin \ - /app/config \ - /workspace - -# Copy Buck2 binary -COPY --from=buck2-downloader /buck2/buck2 /usr/local/bin/ - -# Copy entrypoint script -COPY orion/entrypoint.sh /app/ - -# Copy scorpio configuration template -COPY orion/scorpio.toml.template /app/config/ - -# Write a small metadata file that the entrypoint can print for debugging. -RUN printf "GIT_COMMIT=%s\nBUILD_DATE=%s\nBUCK2_VERSION=%s\n" "${GIT_COMMIT}" "${BUILD_DATE}" "${BUCK2_VERSION}" > /etc/mega-image-info - -# Make entrypoint executable -RUN chmod +x /app/entrypoint.sh - -# Set environment variables -ENV PATH="/app/bin:/usr/local/bin:${PATH}" \ - RUST_LOG=info - -WORKDIR /workspace - -ENTRYPOINT ["/app/entrypoint.sh"] -CMD ["help"] - -# ============================================================================= -# Stage 3a: Runtime - Copy binaries built with cache -# ============================================================================= -FROM runtime-base AS runtime - -COPY --from=builder /build/bin/orion /app/bin/ diff --git a/orion/README.md b/orion/README.md index 14514d1c1..cb8031150 100644 --- a/orion/README.md +++ b/orion/README.md @@ -14,3 +14,60 @@ Orion is a Rust-based Buck build task WebSocket client. It communicates with a s The repository includes a ready-to-install unit file at `orion/systemd/orion-runner.service`. Create `/etc/orion/orion-runner.env` based on `orion/systemd/orion-runner.env.example` and make sure at least `SERVER_WS` and `BUCK_PROJECT_ROOT` are set. + +## GitHub Release + +Orion is released as a native Linux amd64 runner bundle, not as a Docker runtime image. Push an +`orion-vX.Y.Z` tag to create a GitHub Release: + +```bash +git tag -a orion-v0.1.1 -m "Release orion v0.1.1" +git push origin orion-v0.1.1 +``` + +The `.github/workflows/orion-release.yml` workflow builds: + +```bash +cargo build --release -p orion --bin orion --target x86_64-unknown-linux-gnu +``` + +and uploads these GitHub Release assets: + +```text +orion-vX.Y.Z-linux-amd64.tar.gz +orion-vX.Y.Z-linux-amd64.tar.gz.sha256 +``` + +The tarball layout is: + +```text +orion-vX.Y.Z-linux-amd64/ +├── orion +├── runner-config/ +│ ├── .env.prod +│ ├── cleanup.sh +│ ├── preflight.sh +│ ├── run.sh +│ └── scorpio.toml +├── systemd/ +│ ├── orion-runner.env.example +│ └── orion-runner.service +└── VERSION +``` + +Download and verify: + +```bash +VERSION=v0.1.1 +BUNDLE=orion-${VERSION}-linux-amd64 +REPO=web3infra-foundation/mega + +curl -LO https://github.com/${REPO}/releases/download/orion-${VERSION}/${BUNDLE}.tar.gz +curl -LO https://github.com/${REPO}/releases/download/orion-${VERSION}/${BUNDLE}.tar.gz.sha256 +sha256sum -c ${BUNDLE}.tar.gz.sha256 +tar -xzf ${BUNDLE}.tar.gz +``` + +The bundle shape intentionally matches the path layout expected by `orion-scheduler` artifact +distribution: binary at bundle root, runtime scripts under `runner-config/`, and systemd files under +`systemd/`. diff --git a/orion/bellatrix/Cargo.toml b/orion/bellatrix/Cargo.toml deleted file mode 100644 index ea775758b..000000000 --- a/orion/bellatrix/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "simple-bellatrix" -version = "0.1.0" -edition = "2024" - -[lib] -name = "simple_bellatrix" -path = "src/lib.rs" - -[dependencies] -reqwest = { workspace = true, features = ["json"]} -anyhow = { workspace = true } -serde = { workspace = true } -tracing = { workspace = true } \ No newline at end of file diff --git a/orion/bellatrix/src/lib.rs b/orion/bellatrix/src/lib.rs deleted file mode 100644 index af5f2240b..000000000 --- a/orion/bellatrix/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod orion_client; \ No newline at end of file diff --git a/orion/bellatrix/src/orion_client/mod.rs b/orion/bellatrix/src/orion_client/mod.rs deleted file mode 100644 index 22f213976..000000000 --- a/orion/bellatrix/src/orion_client/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -use serde::Serialize; - -#[derive(Serialize, Debug)] -pub struct OrionBuildRequest { - pub change: String, - pub mr: String, - pub repo: String, -} - -#[derive(Clone)] -pub struct OrionClient { - base_url: String, - client: reqwest::Client, -} - -impl OrionClient { - pub fn new() -> Self { - Self { - base_url: "http://127.0.0.1:8004".into(), - client: reqwest::Client::new(), - } - } - - pub async fn trigger_build(&self, req: OrionBuildRequest) -> anyhow::Result<()> { - tracing::info!("Try to trigger build with params:{:?}", req); - let res = self.client.post(&self.base_url).json(&req).send().await?; - if res.status().is_success() { - Ok(()) - } else { - tracing::error!("Failed to trigger build: {}", res.status()); - Err(anyhow::anyhow!("Failed to trigger build: {}", res.status())) - } - } -} diff --git a/orion/docker-compose.yml b/orion/docker-compose.yml deleted file mode 100644 index 113426936..000000000 --- a/orion/docker-compose.yml +++ /dev/null @@ -1,89 +0,0 @@ -# ============================================================================= -# Mega Dev Image - Docker Compose (Client Toolchain) -# ============================================================================= -# This compose file provides a lightweight environment for: -# - Scorpio (FUSE filesystem daemon + HTTP API) -# - Orion Worker (optional; connects to an external Orion Server) -# - Buck2 tooling (available in the image; typically run via `docker exec` into scorpio) -# -# Notes: -# - Orion Server is NOT started here. Configure `SERVER_WS` to point to your Orion Server. -# - Scorpio performs real mounts and therefore requires Linux FUSE support. -# -# Profiles: -# fuse - Scorpio daemon (requires privileged mode) -# worker - Orion Worker (requires privileged mode; embeds Scorpio by default) -# ============================================================================= - -services: - # --------------------------------------------------------------------------- - # Scorpio - FUSE Filesystem Service (daemon) - # --------------------------------------------------------------------------- - scorpio: - image: ${ORION_CLIENT_IMAGE:-orion-client:latest} - container_name: mega-scorpio - command: scorpio - profiles: - - fuse - privileged: true - environment: - SCORPIO_BASE_URL: ${SCORPIO_BASE_URL:-http://host.docker.internal:8000} - SCORPIO_LFS_URL: ${SCORPIO_LFS_URL:-http://host.docker.internal:8000} - SCORPIO_STORE_PATH: /data/scorpio/store - SCORPIO_WORKSPACE: /workspace/mount - SCORPIO_GIT_AUTHOR: ${SCORPIO_GIT_AUTHOR:-MEGA} - SCORPIO_GIT_EMAIL: ${SCORPIO_GIT_EMAIL:-admin@mega.org} - RUST_LOG: ${RUST_LOG:-info} - ports: - - "${SCORPIO_PORT:-2725}:2725" - volumes: - - scorpio-data:/data/scorpio - - workspace:/workspace - restart: unless-stopped - networks: - - mega-network - extra_hosts: - - "host.docker.internal:host-gateway" - - # --------------------------------------------------------------------------- - # Orion Worker - Build Execution Client (optional) - # --------------------------------------------------------------------------- - # The worker can run without an Orion Server, but it will keep retrying the - # WebSocket connection and won't execute builds until tasks are assigned. - orion-worker: - image: ${ORION_CLIENT_IMAGE:-orion-client:latest} - container_name: mega-orion-worker - command: orion - profiles: - - worker - privileged: true - environment: - SERVER_WS: ${SERVER_WS:-ws://host.docker.internal:8004/ws} - ORION_WORKER_ID: ${ORION_WORKER_ID:-} - BUCK_PROJECT_ROOT: ${BUCK_PROJECT_ROOT:-/workspace} - BUILD_TMP: ${BUILD_TMP:-/tmp/orion-builds} - SCORPIO_API_BASE_URL: ${SCORPIO_API_BASE_URL:-http://127.0.0.1:2725} - ORION_WORKER_START_SCORPIO: ${ORION_WORKER_START_SCORPIO:-true} - SCORPIO_BASE_URL: ${SCORPIO_BASE_URL:-http://host.docker.internal:8000} - SCORPIO_LFS_URL: ${SCORPIO_LFS_URL:-http://host.docker.internal:8000} - SCORPIO_STORE_PATH: /data/scorpio/store - SCORPIO_WORKSPACE: /workspace/mount - RUST_LOG: ${RUST_LOG:-info} - volumes: - - scorpio-data:/data/scorpio - - workspace:/workspace - restart: unless-stopped - networks: - - mega-network - extra_hosts: - - "host.docker.internal:host-gateway" - -volumes: - scorpio-data: - name: mega-scorpio-data - workspace: - name: mega-workspace - -networks: - mega-network: - name: mega-network diff --git a/orion/entrypoint.sh b/orion/entrypoint.sh deleted file mode 100644 index 28359cb77..000000000 --- a/orion/entrypoint.sh +++ /dev/null @@ -1,248 +0,0 @@ -#!/bin/bash -# ============================================================================= -# Mega Dev Image - Entrypoint Script -# ============================================================================= -# This script handles container startup for different services: -# - orion: Start the Orion worker (build execution client) -# - buck2: Run Buck2 commands -# - bash/sh: Interactive shell -# - help: Show usage information -# -# Note: Scorpio (FUSE filesystem) has been moved to: -# https://github.com/web3infra-foundation/scorpiofs -# ============================================================================= - -set -e - -# ----------------------------------------------------------------------------- -# Configuration -# ----------------------------------------------------------------------------- -CONFIG_DIR="/app/config" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# ----------------------------------------------------------------------------- -# Helper Functions -# ----------------------------------------------------------------------------- - -log_info() { - echo -e "${GREEN}[INFO]${NC} $1" -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $1" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -log_header() { - echo -e "${BLUE}============================================================${NC}" - echo -e "${BLUE} $1${NC}" - echo -e "${BLUE}============================================================${NC}" -} - -show_help() { - cat << 'EOF' -Mega Dev Image - Unified Development Environment -================================================= - -Usage: docker run [OPTIONS] mega-dev: [ARGS...] - -Commands: - orion Start the Orion worker (build execution client) - orion-worker Alias for `orion` - buck2 [ARGS] Run Buck2 with specified arguments - bash, sh Start an interactive shell - help Show this help message - -Examples: - # Start Orion Worker (build execution client) - docker run -d --name orion-worker \ - -e SERVER_WS=ws://your-orion-server:8004/ws \ - mega-dev:latest orion - - # Run Buck2 build - docker run --rm -v $(pwd):/workspace mega-dev:latest buck2 build //... - - # Interactive shell - docker run --rm -it mega-dev:latest bash - -Environment Variables: - SERVER_WS - WebSocket URL for Orion server - BUCK_PROJECT_ROOT - Buck2 project root (default: /workspace) - BUILD_TMP - Temporary build directory - RUST_LOG - Logging level (default: info) - MEGA_BASE_URL - Mega server base URL (default: http://localhost:8000) - MEGA_LFS_URL - Mega LFS URL (default: same as MEGA_BASE_URL) - SCORPIO_CONFIG - Custom scorpio.toml path (optional, auto-generated if not set) - -Note: Scorpio (FUSE filesystem) has been moved to a separate repository: - https://github.com/web3infra-foundation/scorpiofs - -For more information, see the documentation in orion/ -EOF -} - - - -# Wait for a TCP service to be available -wait_for_service() { - local host=$1 - local port=$2 - local timeout=${3:-30} - local elapsed=0 - - log_info "Waiting for $host:$port (timeout: ${timeout}s)..." - - while ! nc -z "$host" "$port" 2>/dev/null; do - sleep 1 - elapsed=$((elapsed + 1)) - if [ $elapsed -ge $timeout ]; then - log_error "Timeout waiting for $host:$port" - return 1 - fi - done - - log_info "$host:$port is available" - return 0 -} - -setup_directories() { - log_info "Setting up directories..." - mkdir -p /workspace - - # Setup scorpio directories for Antares overlay filesystem - mkdir -p /var/lib/scorpio/{store,antares/{upper,cl,mnt}} - mkdir -p /etc/scorpio -} - -# Generate scorpio.toml from template or use user-provided config -setup_scorpio_config() { - local template="/app/config/scorpio.toml.template" - local config="/etc/scorpio/scorpio.toml" - - # Honour user-provided config first - if [ -n "${SCORPIO_CONFIG:-}" ] && [ -f "${SCORPIO_CONFIG}" ]; then - log_info "Using provided SCORPIO_CONFIG: ${SCORPIO_CONFIG}" - elif [ -f "$template" ]; then - log_info "Generating scorpio configuration from template..." - # Export variables for envsubst (child process needs exported vars) - export MEGA_BASE_URL MEGA_LFS_URL - envsubst < "$template" > "$config" - export SCORPIO_CONFIG="$config" - log_info "Scorpio config written to: $config" - else - log_warn "No scorpio config template or SCORPIO_CONFIG found" - fi -} - -# Print version information -print_versions() { - log_header "Mega Dev Image - Version Information" - echo "" - echo "Components:" - if command -v orion &>/dev/null; then - echo " Orion Worker: installed" - else - echo " Orion Worker: missing" - fi - echo " Buck2: $(buck2 --version 2>/dev/null || echo 'unknown')" - echo "" - echo "Image Labels:" - if [ -f /etc/mega-image-info ]; then - cat /etc/mega-image-info - fi - echo "" - echo "Note: Scorpio (FUSE) moved to https://github.com/web3infra-foundation/scorpiofs" - echo "" -} - -# ----------------------------------------------------------------------------- -# Service Start Functions -# ----------------------------------------------------------------------------- - -start_orion_worker() { - log_header "Starting Orion Worker" - - print_versions - - # Orion worker configuration - : "${SERVER_WS:=ws://127.0.0.1:8004/ws}" - : "${BUCK_PROJECT_ROOT:=/workspace}" - : "${BUILD_TMP:=/tmp/orion-builds}" - - # Scorpiofs config defaults - : "${MEGA_BASE_URL:=http://git.gitmega.com}" - : "${MEGA_LFS_URL:=${MEGA_BASE_URL}}" - - setup_directories - setup_scorpio_config - mkdir -p "${BUILD_TMP}" - - log_info "Configuration:" - echo " SERVER_WS: ${SERVER_WS}" - echo " ORION_WORKER_ID: ${ORION_WORKER_ID:-(auto)}" - echo " BUCK_PROJECT_ROOT: ${BUCK_PROJECT_ROOT}" - echo " BUILD_TMP: ${BUILD_TMP}" - echo " MEGA_BASE_URL: ${MEGA_BASE_URL}" - echo " MEGA_LFS_URL: ${MEGA_LFS_URL}" - echo " SCORPIO_CONFIG: ${SCORPIO_CONFIG:-not set}" - echo "" - - log_info "Starting orion worker..." - exec orion "$@" -} - -run_buck2() { - log_info "Running Buck2..." - exec buck2 "$@" -} - -# ----------------------------------------------------------------------------- -# Main Entry Point -# ----------------------------------------------------------------------------- - -main() { - local command="${1:-help}" - shift 1 2>/dev/null || true - - case "$command" in - orion|orion-worker) - start_orion_worker "$@" - ;; - buck2) - run_buck2 "$@" - ;; - bash|sh) - print_versions - exec /bin/bash "$@" - ;; - version|--version|-v) - print_versions - ;; - help|--help|-h) - show_help - ;; - *) - # If the command is an executable, run it directly - if command -v "$command" &>/dev/null; then - exec "$command" "$@" - else - log_error "Unknown command: $command" - echo "" - show_help - exit 1 - fi - ;; - esac -} - -# Run main function -main "$@" diff --git a/orion/runner-config/cleanup.sh b/orion/runner-config/cleanup.sh old mode 100644 new mode 100755 diff --git a/orion/runner-config/preflight.sh b/orion/runner-config/preflight.sh old mode 100644 new mode 100755 diff --git a/orion/runner-config/run.sh b/orion/runner-config/run.sh old mode 100644 new mode 100755 diff --git a/orion/scorpio.toml.template b/orion/scorpio.toml.template deleted file mode 100644 index d2219f5b9..000000000 --- a/orion/scorpio.toml.template +++ /dev/null @@ -1,31 +0,0 @@ -## Scorpio configuration template for Orion -## -## NOTE: -## - This file is rendered with `envsubst`, which only substitutes plain -## `$VAR` / `${VAR}` patterns (it does NOT understand Bash defaults like -## `${VAR:-default}`). -## - Defaults are provided by the entrypoint/workflow. - -# Mega server connection -base_url = "$MEGA_BASE_URL" -lfs_url = "$MEGA_LFS_URL" - -# Dicfuse data storage -store_path = "/var/lib/scorpio/store" - -# Antares overlay directories -antares_upper_root = "/var/lib/scorpio/antares/upper" -antares_cl_root = "/var/lib/scorpio/antares/cl" -antares_mount_root = "/var/lib/scorpio/antares/mnt" -antares_state_file = "/var/lib/scorpio/antares/state.toml" - -# Preload depth -antares_load_dir_depth = "3" - -# Dicfuse performance tuning -antares_dicfuse_stat_mode = "fast" -antares_dicfuse_open_buff_max_bytes = "67108864" -antares_dicfuse_open_buff_max_files = "1024" -antares_dicfuse_dir_sync_ttl_secs = "120" -antares_dicfuse_reply_ttl_secs = "60" - diff --git a/scripts/demo/README.md b/scripts/demo/README.md index 6c22c51b1..898c90fc7 100644 --- a/scripts/demo/README.md +++ b/scripts/demo/README.md @@ -7,7 +7,6 @@ A bash script for building Docker images for the demo environment on your local This script builds Docker images for the following components: - **mono-engine** - Mono engine service - **orion-server** - Orion server component -- **orion-client** - Orion client component - **mega-ui** - Mega UI web application Images are built using Docker Buildx and can be tagged with architecture-specific suffixes (e.g., `-arm64`, `-amd64`). @@ -154,8 +153,7 @@ These are hardcoded in the script and can be modified if needed. Images are built in the following order: 1. mono-engine 2. orion-server -3. orion-client -4. mega-ui +3. mega-ui This order ensures dependencies are built first. diff --git a/scripts/demo/build-demo-images-local.bat b/scripts/demo/build-demo-images-local.bat index b38f6af9b..ccc7fe26a 100644 --- a/scripts/demo/build-demo-images-local.bat +++ b/scripts/demo/build-demo-images-local.bat @@ -63,7 +63,7 @@ if exist "%SCRIPT_DIR%\..\..\Cargo.toml" ( rem Image Definitions rem Keep build order consistent with the bash script -set "IMAGE_ORDER=mono-engine orion-server orion-client mega-ui" +set "IMAGE_ORDER=mono-engine orion-server mega-ui" set "IMAGES_mono-engine_DOCKERFILE=mono/Dockerfile" set "IMAGES_mono-engine_CONTEXT=." @@ -77,10 +77,6 @@ set "IMAGES_orion-server_DOCKERFILE=orion-server/Dockerfile" set "IMAGES_orion-server_CONTEXT=." set "TAGS_orion-server=latest" -set "IMAGES_orion-client_DOCKERFILE=orion/Dockerfile" -set "IMAGES_orion-client_CONTEXT=." -set "TAGS_orion-client=latest" - rem Parse Arguments set "TARGET_IMAGE=" :parse_args diff --git a/scripts/demo/build-demo-images-local.sh b/scripts/demo/build-demo-images-local.sh index ccbf6aea0..7b8989f46 100644 --- a/scripts/demo/build-demo-images-local.sh +++ b/scripts/demo/build-demo-images-local.sh @@ -41,6 +41,8 @@ REPOSITORY="mega" REGISTRY="public.ecr.aws" SHOULD_PUSH=false # Default: only build, don't push SHOULD_PRUNE_BUILDKIT=false # Optional: prune buildkit cache after build +BUILDX_BUILDER_ARGS=() +BUILDX_PRUNE_ARGS=() # Auto-detect platform if not explicitly set # The script automatically detects your machine architecture and sets the appropriate platform @@ -97,7 +99,7 @@ get_image_tag() { is_valid_image() { case "$1" in - "mono-engine"|"mega-ui"|"orion-server"|"orion-client") return 0 ;; + "mono-engine"|"mega-ui"|"orion-server") return 0 ;; *) return 1 ;; esac } @@ -175,6 +177,40 @@ setup_buildx() { local builder_name="mega-builder" + if [ "$SHOULD_PUSH" != "true" ]; then + # Local-only builds prefer a docker-driver builder so the image is loaded + # directly into the local Docker engine (no extra --load tar export). + # Try common docker-driver builder names; fall back to the currently + # active builder if none of them are present. + local local_builder="" + local candidate + for candidate in default desktop-linux orbstack colima; do + if docker buildx inspect "${candidate}" >/dev/null 2>&1; then + # Only pick docker-driver builders here; container builders + # would re-introduce the --load round-trip we are trying to avoid. + if docker buildx inspect "${candidate}" 2>/dev/null \ + | grep -E '^Driver:[[:space:]]+docker$' >/dev/null; then + local_builder="${candidate}" + break + fi + fi + done + + if [ -n "${local_builder}" ]; then + log_info "Using local docker-driver buildx builder: ${local_builder}" + BUILDX_BUILDER_ARGS=(--builder "${local_builder}") + BUILDX_PRUNE_ARGS=(--builder "${local_builder}") + else + log_warn "No docker-driver buildx builder found; using currently active builder." + log_warn "Tip: enable Docker Desktop's default builder, or run 'docker buildx use default'." + BUILDX_BUILDER_ARGS=() + BUILDX_PRUNE_ARGS=() + fi + + log_info "Buildx setup complete ✓" + return 0 + fi + # Reuse existing builder if present; otherwise create it. # NOTE: `docker buildx ls | grep` is not reliable enough (can false-match), so prefer `inspect`. if docker buildx inspect "${builder_name}" >/dev/null 2>&1; then @@ -188,6 +224,9 @@ setup_buildx() { fi fi + BUILDX_BUILDER_ARGS=(--builder "${builder_name}") + BUILDX_PRUNE_ARGS=(--builder "${builder_name}") + # Ensure the builder is bootstrapped (idempotent) if ! docker buildx inspect "${builder_name}" --bootstrap >/dev/null 2>&1; then log_error "Failed to bootstrap buildx builder: ${builder_name}" @@ -222,37 +261,14 @@ setup_buildx() { log_info "Buildx setup complete ✓" } -rotate_local_cache_dir() { - local cache_dir=$1 - local cache_dir_new="${cache_dir}-new" - - # BuildKit local cache grows monotonically if we keep writing into the same directory. - # Rotate to a temporary directory and replace atomically so stale layers are dropped. - if [ -d "${cache_dir_new}" ]; then - rm -rf "${cache_dir_new}" - fi - mkdir -p "${cache_dir_new}" - echo "${cache_dir_new}" -} - -finalize_local_cache_dir() { - local cache_dir=$1 - local cache_dir_new="${cache_dir}-new" - - if [ -d "${cache_dir_new}" ]; then - rm -rf "${cache_dir}" - mv "${cache_dir_new}" "${cache_dir}" - fi -} - prune_buildkit_cache() { if [ "${SHOULD_PRUNE_BUILDKIT}" != "true" ]; then return 0 fi - log_info "Pruning BuildKit cache (builder: mega-builder, filter: unused for 24h)..." - if ! docker buildx prune --builder mega-builder --force --filter "until=24h" >/dev/null 2>&1; then - log_warn "BuildKit cache prune failed. You can retry manually: docker buildx prune --builder mega-builder --force" + log_info "Pruning BuildKit cache (filter: unused for 24h)..." + if ! docker buildx prune "${BUILDX_PRUNE_ARGS[@]}" --force --filter "until=24h" >/dev/null 2>&1; then + log_warn "BuildKit cache prune failed. You can retry manually: docker buildx prune --force" return 0 fi @@ -299,8 +315,6 @@ build_and_push() { fi local image_tag_with_arch="${image_tag}-${arch_suffix}" local image_repo="${REGISTRY}/${REGISTRY_ALIAS}/${REPOSITORY}/${image_name}" - local cache_dir="${REPO_ROOT}/.buildx-cache/${image_name}-${arch_suffix}" - local cache_dir_export="" # Verify paths exist (use absolute paths) local full_dockerfile="${REPO_ROOT}/${dockerfile_path}" @@ -377,17 +391,18 @@ build_and_push() { # Build command arguments local build_args=( - --builder mega-builder + "${BUILDX_BUILDER_ARGS[@]}" --platform "${TARGET_PLATFORMS}" --file "${dockerfile_path}" --tag "${image_repo}:${image_tag_with_arch}" - --progress=plain + --progress=auto --build-arg BUILDKIT_INLINE_CACHE=1 ) - # Always load the image into the local Docker engine first. - # This guarantees the pushed artifact is a single-architecture image manifest. - build_args+=(--load) + if [ "$SHOULD_PUSH" = "true" ]; then + # Load before docker push so the pushed artifact is a single-architecture image manifest. + build_args+=(--load) + fi if [ "$image_name" = "mega-ui" ]; then build_args+=(--build-arg APP_ENV=demo) @@ -398,26 +413,21 @@ build_and_push() { build_args+=("${cache_from_args[@]}") fi - # Persist BuildKit cache across builder recreation to speed up local rebuilds. - mkdir -p "${cache_dir}" - cache_dir_export=$(rotate_local_cache_dir "${cache_dir}") - build_args+=(--cache-from "type=local,src=${cache_dir}") - build_args+=(--cache-to "type=local,dest=${cache_dir_export},mode=max") - - # Add cache-to (inline cache is always useful) - build_args+=(--cache-to type=inline) + if [ "$SHOULD_PUSH" = "true" ]; then + # Inline cache helps subsequent remote builds without writing a local layer tar cache. + build_args+=(--cache-to type=inline) + fi # Add build context build_args+=("${build_context}") log_info "output build args: ${build_args[*]}" - # Build (load into local engine) + # Build the image. if ! docker buildx build "${build_args[@]}"; then log_error "Failed to build ${image_name}" return 1 fi - finalize_local_cache_dir "${cache_dir}" # Push if requested (ensures single-arch manifest is uploaded) if [ "$SHOULD_PUSH" = "true" ]; then