Add workflow to sync Kernel source code #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Download and configure Kernel Source code' | ||
|
Check failure on line 1 in .github/workflows/sync-kernel-source.yml
|
||
| inputs: | ||
| source_location: | ||
| description: 'Folder path to save Kernel source' | ||
| required: true | ||
| type: string | ||
| github_token: | ||
| description: 'GitHub Token' | ||
| required: true | ||
| debug: | ||
| description: 'Enable Logs' | ||
| required: false | ||
| type: boolean | ||
| default: false | ||
| runs: | ||
| using: 'composite' | ||
| steps: | ||
| - name: Download and Prepare Manifest | ||
| shell: bash | ||
| working-directory: ${{ inputs.source_location }} | ||
| run: | | ||
| # Download and Prepare Manifest | ||
| if [[ "$OP_MANIFEST" == https://* ]]; then | ||
| curl --fail --show-error --location --proto '=https' "$OP_MANIFEST" -o manifest.xml | ||
| elif [[ "$OP_BRANCH" == wild/* ]]; then | ||
| cp "../manifests/$(echo "$OP_OS_VERSION" | tr '[:upper:]' '[:lower:]')/$OP_MANIFEST" manifest.xml | ||
| else | ||
| curl --fail --show-error --location --proto '=https' "https://raw.githubusercontent.com/OnePlusOSS/kernel_manifest/refs/heads/$OP_BRANCH/$OP_MANIFEST" -o manifest.xml | ||
| fi | ||
| - name: Download Manifest Archives | ||
| shell: python | ||
| env: | ||
| PYTHONUNBUFFERED: "1" | ||
| GITHUB_TOKEN: ${{ inputs.github_token }} | ||
| DEBUG: ${{ inputs.debug }} | ||
| working-directory: ${{ inputs.source_location }} | ||
| run: | | ||
| # Download Manifest Archives | ||
| import xml.etree.ElementTree as ET | ||
| import subprocess | ||
| import os, shutil | ||
| import time | ||
| import glob | ||
| from concurrent.futures import ThreadPoolExecutor | ||
| import requests | ||
| MAX_WORKERS = (os.cpu_count() or 2) * 4 | ||
| NPROC = int(subprocess.check_output("nproc", shell=True).strip()) | ||
| TARGET_REPO = "${{ github.repository }}" | ||
| TOOLCHAIN_MAP = { | ||
| "clang/host/linux-x86": "clang", | ||
| "prebuilts/rust": "rust", | ||
| "prebuilts/clang-tools": "clang-tools", | ||
| "prebuilts/build-tools": "build-tools", | ||
| "AnyKernel3": "AnyKernel3" | ||
| } | ||
| DEBUG = os.environ.get("DEBUG", "false").lower() == "true" | ||
| aria_quiet_flags = "--quiet --summary-interval=0" if not DEBUG else "" | ||
| curl_headers = ( | ||
| "-H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' " | ||
| "-H 'Accept: */*' " | ||
| "-H 'Connection: keep-alive' " | ||
| "--tcp-fastopen" | ||
| ) | ||
| aria2_headers = ( | ||
| "--header='User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' " | ||
| "--header='Accept: */*' " | ||
| "--header='Connection: keep-alive' " | ||
| ) | ||
| print("::group::Download Manifest Archives") | ||
| def sync_project(task): | ||
| name, path, url, strip, rev = task | ||
| if path not in ["./", "."]: | ||
| os.makedirs(path, exist_ok=True) | ||
| start_time = time.time() | ||
| MAX_ATTEMPTS = 3 | ||
| for attempt in range(1, MAX_ATTEMPTS + 1): | ||
| shm_workspace = f"/dev/shm/sync_{hash(name) % 1000000}" | ||
| if os.path.exists(shm_workspace): | ||
| shutil.rmtree(shm_workspace, ignore_errors=True) | ||
| os.makedirs(shm_workspace, exist_ok=True) | ||
| print(f"Syncing: {name} -> {path}") | ||
| print(f" [PENDING] {name}") | ||
| try: | ||
| label = None | ||
| for repo_key, type_label in TOOLCHAIN_MAP.items(): | ||
| if repo_key in name: | ||
| label = type_label | ||
| break | ||
| found_in_cache = False | ||
| if label: | ||
| base_filename = f"{label}-{rev}.tar.gz" | ||
| BASE_URL = f"https://github.com/{TARGET_REPO}/releases/download/toolchain-cache" | ||
| check_single = f"curl -IsL {curl_headers} --retry 10 --connect-timeout 30 -o /dev/null -w '%{{http_code}}' {BASE_URL}/{base_filename}" | ||
| if subprocess.check_output(check_single, shell=True).decode().strip() == "200": | ||
| if DEBUG: print(f" [CACHE] Downloading single file: {base_filename}") | ||
| aria_cmd = f"aria2c -x16 -s16 -k1M -j16 --retry-wait=5 --max-tries=10 -d {shm_workspace} --file-allocation=none {aria_quiet_flags} {aria2_headers} -o {base_filename} {BASE_URL}/{base_filename}" | ||
| subprocess.run(aria_cmd, shell=True, check=True, capture_output=not DEBUG) | ||
| found_in_cache = True | ||
| else: | ||
| if DEBUG: print(f" [CACHE] Single file not found, checking parts for {base_filename}...") | ||
| parts_to_download = [] | ||
| for suffix in [f"a{c}" for c in "abcdefghijklmnopqrstuvwxyz"]: | ||
| part_name = f"{base_filename}.part{suffix}" | ||
| check_part = f"curl -IsL {curl_headers} --retry 10 --connect-timeout 30 -o /dev/null -w '%{{http_code}}' {BASE_URL}/{part_name}" | ||
| if subprocess.check_output(check_part, shell=True).decode().strip() == "200": | ||
| parts_to_download.append(f"{BASE_URL}/{part_name}") | ||
| else: | ||
| break | ||
| if parts_to_download: | ||
| if DEBUG: print(f" [CACHE] Found {len(parts_to_download)} parts. Downloading...") | ||
| input_file_path = f"{base_filename}.urls" | ||
| with open(input_file_path, "w") as f: | ||
| for part_url in parts_to_download: | ||
| p_suffix = part_url.split('.part')[-1] | ||
| p_name = f"{base_filename}.part{p_suffix}" | ||
| f.write(f"{part_url}\n out={p_name}\n") | ||
| aria_bulk_cmd = ( | ||
| f"aria2c -x16 -s16 -k1M -j16 --retry-wait=5 --max-tries=10 -d {shm_workspace} --file-allocation=none " | ||
| f"{aria_quiet_flags} {aria2_headers} " | ||
| f"--input-file={input_file_path}" | ||
| ) | ||
| subprocess.run(aria_bulk_cmd, shell=True, check=True, capture_output=not DEBUG) | ||
| os.remove(input_file_path) | ||
| found_in_cache = True | ||
| if not found_in_cache: | ||
| print(f"::error::[FATAL ERROR] Toolchain {base_filename} NOT FOUND in GitHub Release cache!") | ||
| print(f"::error::[FATAL ERROR] Please Run Mirror Toolchain Workflow or select sync toolchain during build!!!") | ||
| return False | ||
| if found_in_cache: | ||
| parts = sorted(glob.glob(os.path.join(shm_workspace, f"*.part*"))) | ||
| source = f"cat {' '.join(parts)} | " if parts else f"cat {os.path.join(shm_workspace, base_filename)} | " | ||
| extract_cmd = f"{source} tar -I 'pigz -p {NPROC} -b 256' -x --record-size=1M --no-same-owner -C {path} {strip}" | ||
| subprocess.run(extract_cmd, shell=True, check=True) | ||
| else: | ||
| direct_file = f"dynamic_file.tar.gz" | ||
| archive_path = os.path.join(shm_workspace, direct_file) | ||
| aria_cmd = f"aria2c -x16 -s16 -k1M -j16 --allow-overwrite=true --retry-wait=5 --max-tries=10 -d {shm_workspace} -o {direct_file} {aria_quiet_flags} {aria2_headers} '{url}'" | ||
| if DEBUG: print(f" [DIRECT] Fetching: {url}") | ||
| subprocess.run(aria_cmd, shell=True, check=True, capture_output=not DEBUG) | ||
| extract_cmd = f"tar -I 'pigz -p {NPROC} -b 256' -x --record-size=1M -f {archive_path} -C {path} {strip}" | ||
| subprocess.run(extract_cmd, shell=True, check=True) | ||
| duration = time.time() - start_time | ||
| print(f"Synced {name} successfully! ({duration:.2f}s)") | ||
| return True | ||
| except subprocess.CalledProcessError as e: | ||
| print(f" [RETRY {attempt}/{MAX_ATTEMPTS}] {name} failed with exit code {e.returncode}. Command: {e.cmd}") | ||
| print(f" [ERROR] Command failed for {name}: {e.cmd}") | ||
| print(f" Stderr: {e.stderr.decode() if e.stderr else 'No stderr'}") | ||
| if attempt < MAX_ATTEMPTS: | ||
| time.sleep(5) | ||
| except Exception as e: | ||
| print(f" [RETRY {attempt}/{MAX_ATTEMPTS}] {name} failed with exit code {e.returncode}. Command: {e.cmd}") | ||
| print(f" [ERROR] Failed to sync {name}") | ||
| if attempt < MAX_ATTEMPTS: | ||
| time.sleep(5) | ||
| finally: | ||
| if os.path.exists(shm_workspace): | ||
| shutil.rmtree(shm_workspace, ignore_errors=True) | ||
| return False | ||
| global_start = time.time() | ||
| with open('manifest.xml', 'r') as f: | ||
| manifest_content = f.read() | ||
| root = ET.fromstring(manifest_content) | ||
| top_dir = os.getcwd() | ||
| remotes = {r.get('name'): r.get('fetch').rstrip('/') for r in root.findall('remote')} | ||
| default = root.find('default') | ||
| def_remote = default.get('remote') if default is not None else None | ||
| def_rev = default.get('revision') if default is not None else None | ||
| sync_tasks = [] | ||
| post_process_data = [] | ||
| for project in root.findall('project'): | ||
| name = project.get('name') | ||
| path = project.get('path', name) | ||
| remote_name = project.get('remote', def_remote) | ||
| rev = project.get('revision', def_rev) | ||
| base_url = remotes.get(remote_name) | ||
| if not base_url: continue | ||
| if "github.com" in base_url: | ||
| url = f"{base_url}/{name}/archive/{rev}.tar.gz" | ||
| strip = "--strip-components=1" | ||
| elif "googlesource.com" in base_url: | ||
| url = f"{base_url}/{name}/+archive/{rev}.tar.gz" | ||
| strip = "" | ||
| elif "git.codelinaro.org" in base_url: | ||
| url = f"{base_url}/{name}/-/archive/{rev}.tar.gz" | ||
| strip = "--strip-components=1" | ||
| else: | ||
| continue | ||
| sync_tasks.append((name, path, url, strip, rev)) | ||
| for child in project: | ||
| if child.tag in ['linkfile', 'copyfile']: | ||
| post_process_data.append((path, child)) | ||
| if DEBUG: | ||
| print(f"Starting parallel sync of {len(sync_tasks)} projects...") | ||
| with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: | ||
| success_list = list(executor.map(sync_project, sync_tasks)) | ||
| if not all(success_list): | ||
| print("::error::One or more projects failed to sync!") | ||
| print("::endgroup::") | ||
| exit(1) | ||
| print("Processing linkfiles and copyfiles...") | ||
| for path, child in post_process_data: | ||
| src_rel = child.get('src') | ||
| dest_rel = child.get('dest') | ||
| if not src_rel or not dest_rel: continue | ||
| src_path = os.path.join(top_dir, path, src_rel) | ||
| dest_path = os.path.join(top_dir, dest_rel) | ||
| os.makedirs(os.path.dirname(dest_path), exist_ok=True) | ||
| if child.tag == 'linkfile': | ||
| if os.path.lexists(dest_path): os.remove(dest_path) | ||
| rel_target = os.path.relpath(src_path, os.path.dirname(dest_path)) | ||
| os.symlink(rel_target, dest_path) | ||
| print(f" [Link] {dest_rel} -> {src_rel}") | ||
| elif child.tag == 'copyfile': | ||
| shutil.copy2(src_path, dest_path) | ||
| print(f" [Copy] {dest_rel} from {src_rel}") | ||
| total_duration = time.time() - global_start | ||
| minutes = int(total_duration // 60) | ||
| seconds = total_duration % 60 | ||
| print(f"Kernel Sync completed in {minutes}m {seconds:.2f}s") | ||
| print("::endgroup::") | ||
