mirror-toolchains.yml

name: 🛠️ ToolChains Generator

permissions:
  contents: write

on:
  schedule:
    - cron: '0 0 * * 0' # Every Sunday at 00:00 UTC
  workflow_dispatch:
  workflow_call:

jobs:
  prepare-release:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Ensure Single Release
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # Check if the release already exists
          TARGET_REPO="${{ github.repository }}"
          TAG_NAME="toolchain-cache"
          
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          
          if ! git rev-parse "$TAG_NAME" >/dev/null 2>&1; then
            echo "Tag not found. Creating backdated tag and release..."
            
            GIT_COMMITTER_DATE="2015-01-01T12:00:00" git tag -a "$TAG_NAME" -m "Internal Toolchains Cache"
            git push origin "$TAG_NAME"
            
            echo "Release not found. Creating..."
            gh release create "$TAG_NAME" --title "Toolchains Mirror Cache" --notes "Deduplicated Toolchains for Kernel Build" --repo "$TARGET_REPO"
          else
            echo "Release 'toolchain-cache' already exists. Skipping creation."
          fi

  generate-mirror-matrix:
    runs-on: ubuntu-latest
    needs: prepare-release
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout Code
        uses: actions/checkout@v6

      - name: Generate Mirror Matrix
        id: generate-config
        shell: bash
        run: |
          set -euo pipefail
          
          echo "[" > all_configs.json
          mapfile -t files < <(find configs/ -name "*.json")
          for i in "${!files[@]}"; do
            cat "${files[$i]}" >> all_configs.json
            [[ $((i+1)) -lt ${#files[@]} ]] && echo "," >> all_configs.json
          done
          echo "]" >> all_configs.json

      - name: Resolve Unique Projects
        id: set-matrix
        shell: python
        env:
          PYTHONUNBUFFERED: "1"
        run: |
          import xml.etree.ElementTree as ET
          import json
          import os
          import subprocess
          import shutil
          
          with open('all_configs.json', 'r') as f:
              configs = json.load(f)
          
          unique_toolchains = {}
          TOOLCHAIN_MAP = {
              "clang/host/linux-x86": "clang",
              "prebuilts/rust": "rust",
              "prebuilts/clang-tools": "clang-tools",
              "prebuilts/build-tools": "build-tools",
              "AnyKernel3": "AnyKernel3"
          }
          
          print(f"Processing {len(configs)} configurations...")
          print("-" * 50)
          
          for cfg in configs:
              op_model = cfg.get('model', 'Unknown')
              op_branch = cfg.get('branch', '')
              op_manifest = cfg.get('manifest', '')
              op_os_version = cfg.get('os_version', '').lower()
              
              xml_file = "temp_manifest.xml"
              
              try:
                  if op_manifest.startswith("https://"):
                      subprocess.run(f"curl -LfsS {op_manifest} -o {xml_file}", shell=True, check=True)
                  elif op_branch.startswith("wild/"):
                      shutil.copy(f"manifests/{op_os_version}/{op_manifest}", xml_file)
                  else:
                      url = f"https://raw.githubusercontent.com/OnePlusOSS/kernel_manifest/refs/heads/{op_branch}/{op_manifest}"
                      subprocess.run(f"curl -LfsS {url} -o {xml_file}", shell=True, check=True)
                  
                  root = ET.parse(xml_file).getroot()
                  remotes = {r.get('name'): r.get('fetch').rstrip('/') for r in root.findall('remote')}
                  default = root.find('default')
                  def_remote = default.get('remote') if default is not None else None
                  def_rev = default.get('revision') if default is not None else None
                  
                  for project in root.findall('project'):
                      name = project.get('name')
                      type_label = None
                      
                      for repo_key, label in TOOLCHAIN_MAP.items():
                        if repo_key in name:
                            type_label = label
                            break
                      
                      if type_label:
                        remote_name = project.get('remote', def_remote)
                        rev = project.get('revision', def_rev)
                        base_url = remotes.get(remote_name, "").rstrip('/')
                        cache_filename = f"{type_label}-{rev}.tar.gz"
                        
                        if not remote_name or not rev:
                          print(f"⚠️ Skipping toolchain project '{name}' because remote or revision cannot be resolved.")
                          continue
                        
                        if cache_filename not in unique_toolchains:
                            if "googlesource.com" in base_url:
                                dl_url = f"{base_url}/{name}/+archive/{rev}.tar.gz"
                            elif "github.com" in base_url:
                                dl_url = f"{base_url}/{name}/archive/{rev}.tar.gz"
                            elif "git.codelinaro.org" in base_url or "gitlab.com" in base_url:
                                dl_url = f"{base_url}/{name}/-/archive/{rev}.tar.gz"
                            else:
                                error_msg = f"❌ [FATAL] Unknown toolchain provider for: {base_url}"
                                print(f"::error::{error_msg}")
                                raise ValueError(error_msg)
                            unique_toolchains[cache_filename] = {"rev": rev, "url": dl_url, "name": name, "type_label": type_label, "cache_file": cache_filename}
                            print(f"🆕 [{op_model}][{op_os_version}] -> New {type_label} found: {rev}")
                        else:
                            print(f"♻️ [{op_model}][{op_os_version}] -> Using existing {type_label}: {rev}")
              except Exception as e:
                  print(f"⚠️ Failed to process {op_manifest}: {e}")
          matrix = {"include": list(unique_toolchains.values())}
          with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
              f.write(f"matrix={json.dumps(matrix)}\n")
          print(f"✅ Found {len(unique_toolchains)} unique projects to mirror.")

  mirror-to-release:
    name: build (${{ matrix.type_label }}, ${{ matrix.rev }})
    needs: [prepare-release, generate-mirror-matrix]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.generate-mirror-matrix.outputs.matrix) }}
    env:
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Sync & Upload
        run: |
          TARGET_REPO="${{ github.repository }}"
          REV="${{ matrix.rev }}"
          FILENAME="${{ matrix.cache_file }}"
          BASE_URL="https://github.com/${TARGET_REPO}/releases/download/toolchain-cache"
          
          echo "🔍 Checking if $FILENAME exists in mirror..."
          
          STATUS=$(curl -IsL -o /dev/null -w "%{http_code}" "$BASE_URL/$FILENAME")
          PART_STATUS=$(curl -IsL -o /dev/null -w "%{http_code}" "$BASE_URL/${FILENAME}.partaa")
          
          if [ "$STATUS" -eq 200 ] || [ "$PART_STATUS" -eq 200 ]; then
            echo "✅ ${{ matrix.type_label }} revision ${{ matrix.rev }} already cached."
          else
            echo "📥 Not found. Downloading ${{ matrix.type_label }} from source..."
            
            aria2c -x16 -s16 -k1M -j5 --file-allocation=none --console-log-level=error --summary-interval=0 --retry-wait=5 --max-tries=10 -o "$FILENAME" "${{ matrix.url }}"
            
            FILE_SIZE=$(stat -c%s "$FILENAME")
            MAX_SIZE=2000000000 # ~1.86GB
            
            if [ "$FILE_SIZE" -gt "$MAX_SIZE" ]; then
              echo "⚠️ File > 2GB. Splitting..."
              split -b 1500M -a 2 "$FILENAME" "${FILENAME}.part"
              
              for part in "${FILENAME}".part*; do
                echo "📤 Uploading $part..."
                gh release upload "toolchain-cache" "$part" --clobber --repo "$TARGET_REPO"
              done
            else
              echo "📤 Uploading single file..."
              gh release upload "toolchain-cache" "$FILENAME" --clobber --repo "$TARGET_REPO"
            fi
          fi