🛠️ ToolChains Generator #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🛠️ ToolChains Generator | |
| permissions: | |
| contents: write | |
| on: | |
| schedule: | |
| - cron: '0 0 * * 0' # Every Sunday at 00:00 UTC | |
| workflow_dispatch: | |
| workflow_call: | |
| jobs: | |
| prepare-release: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Ensure Single Release | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| # Check if the release already exists | |
| TARGET_REPO="${{ github.repository }}" | |
| TAG_NAME="toolchain-cache" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| if ! git rev-parse "$TAG_NAME" >/dev/null 2>&1; then | |
| echo "Tag not found. Creating backdated tag and release..." | |
| GIT_COMMITTER_DATE="2015-01-01T12:00:00" git tag -a "$TAG_NAME" -m "Internal Toolchains Cache" | |
| git push origin "$TAG_NAME" | |
| echo "Release not found. Creating..." | |
| gh release create "$TAG_NAME" --title "Toolchains Mirror Cache" --notes "Deduplicated Toolchains for Kernel Build" --repo "$TARGET_REPO" | |
| else | |
| echo "Release 'toolchain-cache' already exists. Skipping creation." | |
| fi | |
| generate-mirror-matrix: | |
| runs-on: ubuntu-latest | |
| needs: prepare-release | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v6 | |
| - name: Generate Mirror Matrix | |
| id: generate-config | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| echo "[" > all_configs.json | |
| mapfile -t files < <(find configs/ -name "*.json") | |
| for i in "${!files[@]}"; do | |
| cat "${files[$i]}" >> all_configs.json | |
| [[ $((i+1)) -lt ${#files[@]} ]] && echo "," >> all_configs.json | |
| done | |
| echo "]" >> all_configs.json | |
| - name: Resolve Unique Projects | |
| id: set-matrix | |
| shell: python | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| run: | | |
| import xml.etree.ElementTree as ET | |
| import json | |
| import os | |
| import subprocess | |
| import shutil | |
| with open('all_configs.json', 'r') as f: | |
| configs = json.load(f) | |
| unique_toolchains = {} | |
| TOOLCHAIN_MAP = { | |
| "clang/host/linux-x86": "clang", | |
| "prebuilts/rust": "rust", | |
| "prebuilts/clang-tools": "clang-tools", | |
| "prebuilts/build-tools": "build-tools", | |
| "AnyKernel3": "AnyKernel3" | |
| } | |
| print(f"Processing {len(configs)} configurations...") | |
| print("-" * 50) | |
| for cfg in configs: | |
| op_model = cfg.get('model', 'Unknown') | |
| op_branch = cfg.get('branch', '') | |
| op_manifest = cfg.get('manifest', '') | |
| op_os_version = cfg.get('os_version', '').lower() | |
| xml_file = "temp_manifest.xml" | |
| try: | |
| if op_manifest.startswith("https://"): | |
| subprocess.run(f"curl -LfsS {op_manifest} -o {xml_file}", shell=True, check=True) | |
| elif op_branch.startswith("wild/"): | |
| shutil.copy(f"manifests/{op_os_version}/{op_manifest}", xml_file) | |
| else: | |
| url = f"https://raw.githubusercontent.com/OnePlusOSS/kernel_manifest/refs/heads/{op_branch}/{op_manifest}" | |
| subprocess.run(f"curl -LfsS {url} -o {xml_file}", shell=True, check=True) | |
| root = ET.parse(xml_file).getroot() | |
| remotes = {r.get('name'): r.get('fetch').rstrip('/') for r in root.findall('remote')} | |
| default = root.find('default') | |
| def_remote = default.get('remote') if default is not None else None | |
| def_rev = default.get('revision') if default is not None else None | |
| for project in root.findall('project'): | |
| name = project.get('name') | |
| type_label = None | |
| for repo_key, label in TOOLCHAIN_MAP.items(): | |
| if repo_key in name: | |
| type_label = label | |
| break | |
| if type_label: | |
| remote_name = project.get('remote', def_remote) | |
| rev = project.get('revision', def_rev) | |
| base_url = remotes.get(remote_name, "").rstrip('/') | |
| cache_filename = f"{type_label}-{rev}.tar.gz" | |
| if not remote_name or not rev: | |
| print(f"⚠️ Skipping toolchain project '{name}' because remote or revision cannot be resolved.") | |
| continue | |
| if cache_filename not in unique_toolchains: | |
| if "googlesource.com" in base_url: | |
| dl_url = f"{base_url}/{name}/+archive/{rev}.tar.gz" | |
| elif "github.com" in base_url: | |
| dl_url = f"{base_url}/{name}/archive/{rev}.tar.gz" | |
| elif "git.codelinaro.org" in base_url or "gitlab.com" in base_url: | |
| dl_url = f"{base_url}/{name}/-/archive/{rev}.tar.gz" | |
| else: | |
| error_msg = f"❌ [FATAL] Unknown toolchain provider for: {base_url}" | |
| print(f"::error::{error_msg}") | |
| raise ValueError(error_msg) | |
| unique_toolchains[cache_filename] = {"rev": rev, "url": dl_url, "name": name, "type_label": type_label, "cache_file": cache_filename} | |
| print(f"🆕 [{op_model}][{op_os_version}] -> New {type_label} found: {rev}") | |
| else: | |
| print(f"♻️ [{op_model}][{op_os_version}] -> Using existing {type_label}: {rev}") | |
| except Exception as e: | |
| print(f"⚠️ Failed to process {op_manifest}: {e}") | |
| matrix = {"include": list(unique_toolchains.values())} | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as f: | |
| f.write(f"matrix={json.dumps(matrix)}\n") | |
| print(f"✅ Found {len(unique_toolchains)} unique projects to mirror.") | |
| mirror-to-release: | |
| name: build (${{ matrix.type_label }}, ${{ matrix.rev }}) | |
| needs: [prepare-release, generate-mirror-matrix] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.generate-mirror-matrix.outputs.matrix) }} | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Sync & Upload | |
| run: | | |
| TARGET_REPO="${{ github.repository }}" | |
| REV="${{ matrix.rev }}" | |
| FILENAME="${{ matrix.cache_file }}" | |
| BASE_URL="https://github.com/${TARGET_REPO}/releases/download/toolchain-cache" | |
| echo "🔍 Checking if $FILENAME exists in mirror..." | |
| STATUS=$(curl -IsL -o /dev/null -w "%{http_code}" "$BASE_URL/$FILENAME") | |
| PART_STATUS=$(curl -IsL -o /dev/null -w "%{http_code}" "$BASE_URL/${FILENAME}.partaa") | |
| if [ "$STATUS" -eq 200 ] || [ "$PART_STATUS" -eq 200 ]; then | |
| echo "✅ ${{ matrix.type_label }} revision ${{ matrix.rev }} already cached." | |
| else | |
| echo "📥 Not found. Downloading ${{ matrix.type_label }} from source..." | |
| aria2c -x16 -s16 -k1M -j5 --file-allocation=none --console-log-level=error --summary-interval=0 --retry-wait=5 --max-tries=10 -o "$FILENAME" "${{ matrix.url }}" | |
| FILE_SIZE=$(stat -c%s "$FILENAME") | |
| MAX_SIZE=2000000000 # ~1.86GB | |
| if [ "$FILE_SIZE" -gt "$MAX_SIZE" ]; then | |
| echo "⚠️ File > 2GB. Splitting..." | |
| split -b 1500M -a 2 "$FILENAME" "${FILENAME}.part" | |
| for part in "${FILENAME}".part*; do | |
| echo "📤 Uploading $part..." | |
| gh release upload "toolchain-cache" "$part" --clobber --repo "$TARGET_REPO" | |
| done | |
| else | |
| echo "📤 Uploading single file..." | |
| gh release upload "toolchain-cache" "$FILENAME" --clobber --repo "$TARGET_REPO" | |
| fi | |
| fi |