update_user_libs.py

#!/usr/bin/env python3
"""Update the copy of user libs (nethelper, picozero, espzero ...).

User libs are not yet packaged on PyPI; this script mirrors them into the
local repository to make them easily installable.

Usage:
    python update_user_libs.py nethelper.py
    python update_user_libs.py picozero.py
    python update_user_libs.py espzero
"""
import json
import base64
import os
import subprocess
from urllib.parse import urljoin
from urllib.request import build_opener
import sys

DEST = ''
NETHELPER_DEST = 'mu/resources/pygamezero/nethelper.py'
PICOZERO_DEST  = 'mu/resources/pico/picozero.py'
ESPZERO_DEST   = 'mu/resources/esp32'

NETHELPER_REPO_URL = 'https://api.github.com/repos/roboticsware/nethelper/'
PICOZERO_REPO_URL  = 'https://api.github.com/repos/roboticsware/picozero/'
ESPZERO_REPO_URL   = 'https://api.github.com/repos/roboticsware/espzero/'

HEADER = '''"""

This module is directly copied from

    https://github.com/roboticsware/nethelper

at revision {sha}
and used under CC0.

"""
# flake8: noqa: E501
'''

ESPZERO_HEADER = '''\
# This file is part of espzero, automatically synced from
# https://github.com/roboticsware/espzero at revision {sha}.
# Do not edit this file directly — run update_user_libs.py espzero instead.

'''


# Customise the opener here if you need to
opener = build_opener()


def read_json(url):
    """Download and decode a JSON resource from the given URL."""
    resp = opener.open(url)
    charset = resp.headers.get_content_charset()
    data = resp.read().decode(charset)
    return json.loads(data)


def get_tree(file):
    """Download the repository tree, returning a decoded JSON structure."""
    print('Downloading repository tree...')

    if file == 'nethelper.py':
        REPO_URL = NETHELPER_REPO_URL
    elif file == 'picozero.py':
        REPO_URL = PICOZERO_REPO_URL
    elif file == 'espzero':
        REPO_URL = ESPZERO_REPO_URL
    else:
        raise ValueError("Unknown library: {}".format(file))

    url = urljoin(REPO_URL, 'git/trees/HEAD?recursive=1')
    return read_json(url)


def get_file(file):
    """Download the tree state and named file.

    Return a tuple of the current repo version hash and the file's data.
    """
    tree = get_tree(file)
    for f in tree['tree']:
        if file in f['path']:
            break
    else:
        raise ValueError("Could not find the module to download.")

    url = f['url']
    print('Downloading', file, 'module...')
    blob = read_json(url)
    data = base64.b64decode(blob['content']).decode('utf8')
    return tree['sha'], data


def update_espzero():
    """Download all .py files from the espzero repo and write them to ESPZERO_DEST.

    Preserves the package directory structure (e.g. profiles/ subdir).
    Prepends ESPZERO_HEADER to every file so the source revision is traceable.
    """
    tree = get_tree('espzero')
    sha = tree['sha']
    header = ESPZERO_HEADER.format(sha=sha)

    py_blobs = [f for f in tree['tree']
                if f['type'] == 'blob' and f['path'].endswith('.py')]

    for entry in py_blobs:
        rel_path  = entry['path']           # e.g. '_hal.py' or 'profiles/auto.py'
        dest_path = os.path.join(ESPZERO_DEST, rel_path)

        # Ensure sub-directories exist (e.g. mu/resources/esp32/profiles/)
        dest_dir = os.path.dirname(dest_path)
        if dest_dir and not os.path.exists(dest_dir):
            os.makedirs(dest_dir)
            print('Created directory:', dest_dir)

        print('Downloading', rel_path, '...')
        blob = read_json(entry['url'])
        data = base64.b64decode(blob['content']).decode('utf8')

        with open(dest_path, 'w', encoding='utf8') as f:
            f.write(header + data)

        print('  ->', dest_path)

    print("\nespzero updated to revision", sha[:7],
          "({} files)".format(len(py_blobs)))


def update_local():
    """Download a new copy of the file and write it to DEST.

    Include a header based on the template HEADER.
    """
    global DEST
    FILE = sys.argv[1]

    if FILE == 'espzero':
        update_espzero()
        return

    if FILE == 'nethelper.py':
        DEST = NETHELPER_DEST
    elif FILE == 'picozero.py':
        DEST = PICOZERO_DEST
    else:
        raise ValueError("Unknown library: {}".format(FILE))

    sha, data = get_file(FILE)
    header = HEADER.format(sha=sha)
    with open(DEST, 'w', encoding='utf8') as f:
        f.write(header + data)
    print("Updated", FILE, "to revision", sha[:7])
    autopep8()


def autopep8():
    """Use autopep8 to fix formatting problems."""
    print("Running autopep8")
    subprocess.check_call(['autopep8', '-i', DEST])


if __name__ == '__main__':
    update_local()