Mirror static toolchains to GitHub Release #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Mirror static toolchains to GitHub Release | |
| permissions: | |
| contents: write | |
| on: | |
| workflow_dispatch: | |
| workflow_call: | |
| jobs: | |
| prepare-release: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Configure Git Identity | |
| run: | | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git config user.name "github-actions[bot]" | |
| - name: Ensure Single Release | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| # Check if the release already exists | |
| TARGET_REPO="${{ github.repository }}" | |
| TAG_NAME="toolchain-cache" | |
| if ! git rev-parse "$TAG_NAME" >/dev/null 2>&1; then | |
| echo "Tag not found. Creating backdated tag and release..." | |
| GIT_COMMITTER_DATE="2015-01-01T12:00:00" git tag -a "$TAG_NAME" -m "Internal Toolchains Cache" | |
| git push origin "$TAG_NAME" | |
| echo "Release not found. Creating..." | |
| gh release create "$TAG_NAME" --title "Toolchains Mirror Cache" --notes "Deduplicated Toolchains for Kernel Build" --repo "$TARGET_REPO" | |
| else | |
| echo "Release 'toolchain-cache' already exists. Skipping creation." | |
| fi | |
| generate-mirror-matrix: | |
| runs-on: ubuntu-latest | |
| needs: prepare-release | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout Code (to access configs/) | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: devices | |
| sparse-checkout: | | |
| configs/ | |
| sparse-checkout-cone-mode: false | |
| - name: Fetch manifests from main repo | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| curl -sSfL https://github.com/WildKernels/OnePlus_KernelSU_SUSFS/archive/refs/heads/main.tar.gz \ | |
| | tar xz --wildcards '*/manifests/*' --strip-components=1 | |
| - name: Generate Mirror Matrix | |
| id: generate-config | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| echo "[" > all_configs.json | |
| mapfile -t files < <(find configs/ -name "*.json") | |
| for i in "${!files[@]}"; do | |
| cat "${files[$i]}" >> all_configs.json | |
| [[ $((i+1)) -lt ${#files[@]} ]] && echo "," >> all_configs.json | |
| done | |
| echo "]" >> all_configs.json | |
| - name: Resolve Unique Projects | |
| id: set-matrix | |
| shell: python | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| run: | | |
| import xml.etree.ElementTree as ET | |
| import json | |
| import os | |
| import subprocess | |
| import shutil | |
| with open('all_configs.json', 'r') as f: | |
| configs = json.load(f) | |
| unique_toolchains = {} | |
| TOOLCHAIN_MAP = { | |
| "clang/host/linux-x86": "clang", | |
| "prebuilts/rust": "rust", | |
| "prebuilts/clang-tools": "clang-tools", | |
| "prebuilts/build-tools": "build-tools", | |
| "AnyKernel3": "AnyKernel3" | |
| } | |
| print(f"Processing {len(configs)} configurations...") | |
| print("-" * 50) | |
| for cfg in configs: | |
| op_model = cfg.get('model', 'Unknown') | |
| op_branch = cfg.get('branch', '') | |
| op_manifest = cfg.get('manifest', '') | |
| op_os_version = cfg.get('os_version', '').lower() | |
| xml_file = "temp_manifest.xml" | |
| try: | |
| if op_manifest.startswith("https://"): | |
| subprocess.run(f"curl -LfsS {op_manifest} -o {xml_file}", shell=True, check=True) | |
| elif op_branch.startswith("wild/"): | |
| shutil.copy(f"manifests/{op_os_version}/{op_manifest}", xml_file) | |
| else: | |
| url = f"https://raw.githubusercontent.com/OnePlusOSS/kernel_manifest/refs/heads/{op_branch}/{op_manifest}" | |
| subprocess.run(f"curl -LfsS {url} -o {xml_file}", shell=True, check=True) | |
| root = ET.parse(xml_file).getroot() | |
| remotes = {r.get('name'): r.get('fetch').rstrip('/') for r in root.findall('remote')} | |
| default = root.find('default') | |
| def_remote = default.get('remote') | |
| def_rev = default.get('revision') | |
| for project in root.findall('project'): | |
| name = project.get('name') | |
| type_label = None | |
| for repo_key, label in TOOLCHAIN_MAP.items(): | |
| if repo_key in name: | |
| type_label = label | |
| break | |
| if type_label: | |
| remote_name = project.get('remote', def_remote) | |
| rev = project.get('revision', def_rev) | |
| base_url = remotes.get(remote_name, "").rstrip('/') | |
| cache_filename = f"{type_label}-{rev}.tar.gz" | |
| if cache_filename not in unique_toolchains: | |
| if "googlesource.com" in base_url: | |
| dl_url = f"{base_url}/{name}/+archive/{rev}.tar.gz" | |
| elif "github.com" in base_url: | |
| dl_url = f"{base_url}/{name}/archive/{rev}.tar.gz" | |
| elif "git.codelinaro.org" in base_url or "gitlab.com" in base_url: | |
| dl_url = f"{base_url}/{name}/-/archive/{rev}.tar.gz" | |
| else: | |
| error_msg = f"❌ [FATAL] Unknown toolchain provider for: {base_url}" | |
| print(f"::error::{error_msg}") | |
| raise ValueError(error_msg) | |
| unique_toolchains[cache_filename] = {"rev": rev, "url": dl_url, "name": name, "type_label": type_label, "cache_file": cache_filename} | |
| print(f"🆕 [{op_model}][{op_os_version}] -> New {type_label} found: {rev}") | |
| else: | |
| print(f"♻️ [{op_model}][{op_os_version}] -> Using existing {type_label}: {rev}") | |
| except Exception as e: | |
| print(f"⚠️ Failed to process {op_manifest}: {e}") | |
| matrix = {"include": list(unique_toolchains.values())} | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as f: | |
| f.write(f"matrix={json.dumps(matrix)}\n") | |
| print(f"✅ Found {len(unique_toolchains)} unique projects to mirror.") | |
| mirror-to-release: | |
| name: build (${{ matrix.type_label }}, ${{ matrix.rev }}) | |
| needs: [prepare-release, generate-mirror-matrix] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.generate-mirror-matrix.outputs.matrix) }} | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Sync & Upload | |
| run: | | |
| TARGET_REPO="${{ github.repository }}" | |
| REV="${{ matrix.rev }}" | |
| FILENAME="${{ matrix.cache_file }}" | |
| BASE_URL="https://github.com/${TARGET_REPO}/releases/download/toolchain-cache" | |
| echo "🔍 Checking if $FILENAME exists in mirror..." | |
| STATUS=$(curl -IsL -o /dev/null -w "%{http_code}" "$BASE_URL/$FILENAME") | |
| PART_STATUS=$(curl -IsL -o /dev/null -w "%{http_code}" "$BASE_URL/${FILENAME}.partaa") | |
| if [ "$STATUS" -eq 200 ] || [ "$PART_STATUS" -eq 200 ]; then | |
| echo "✅ ${{ matrix.type_label }} revision ${{ matrix.rev }} already cached." | |
| else | |
| echo "📥 Not found. Downloading ${{ matrix.type_label }} from source..." | |
| aria2c -x16 -s16 -k1M -j5 --file-allocation=none --console-log-level=error --summary-interval=0 --retry-wait=5 --max-tries=10 -o "$FILENAME" "${{ matrix.url }}" | |
| FILE_SIZE=$(stat -c%s "$FILENAME") | |
| MAX_SIZE=2000000000 # ~1.86GB | |
| if [ "$FILE_SIZE" -gt "$MAX_SIZE" ]; then | |
| echo "⚠️ File > 2GB. Splitting..." | |
| split -b 1500M -a 2 "$FILENAME" "${FILENAME}.part" | |
| for part in "${FILENAME}".part*; do | |
| echo "📤 Uploading $part..." | |
| gh release upload "toolchain-cache" "$part" --clobber --repo "$TARGET_REPO" | |
| done | |
| else | |
| echo "📤 Uploading single file..." | |
| gh release upload "toolchain-cache" "$FILENAME" --clobber --repo "$TARGET_REPO" | |
| fi | |
| fi |