diff --git a/CHANGELOG.md b/CHANGELOG.md index cc553efe5..9f07e3bee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,9 @@ # Future Release * System + * Upgrade from 32 bit raspbian Bookworm Desktop to 64 bit raspbian Trixie Lite * Upgraded volume calculations to preserve relative positions when hitting the min or max setting via source volume bar + * Added volume matching between AmpliPi and Spotify and vice-versa * Update our spotify provider `go-librespot` to `0.7.3` * Upgrade from Logitech Media Server 8.5.2 to Lyrion Music Server 9.0.3 * Added in-place preamp recovery when I2C writes fail persistently diff --git a/amplipi/app.py b/amplipi/app.py index 6ff3c5c07..b410baea5 100644 --- a/amplipi/app.py +++ b/amplipi/app.py @@ -972,7 +972,7 @@ async def get_response(self, path, scope): # Website -app.mount('/', StaticFiles(directory=WEB_DIR, html=True), name='web') +app.mount('/', CachelessFiles(directory=WEB_DIR, html=True), name='web') def create_app(mock_ctrl=None, mock_streams=None, config_file=None, delay_saves=None, settings: models.AppSettings = models.AppSettings()) -> FastAPI: diff --git a/amplipi/auth.py b/amplipi/auth.py index acd333b47..72ea608f9 100644 --- a/amplipi/auth.py +++ b/amplipi/auth.py @@ -51,7 +51,7 @@ # This is duplicated from amplipi.config intentionally. This file is pulled in # via the updater and we'd like to avoid as many external Amplipi dependencies in the updater # as possible. -USER_CONFIG_DIR = os.path.join(os.path.expanduser('~'), '.config', 'amplipi') +USER_CONFIG_DIR = os.path.join('/data', '.config', 'amplipi') USERS_FILE = os.path.join(USER_CONFIG_DIR, 'users.json') diff --git a/amplipi/ctrl.py b/amplipi/ctrl.py index 36e167f04..801a16feb 100644 --- a/amplipi/ctrl.py +++ b/amplipi/ctrl.py @@ -852,7 +852,7 @@ def set_vol(): vol, vol_f, vol_f_delta, vol_min, or vol_max. """ # Field precedence: vol (db) > vol_delta > vol (float) - # vol (db) is first in precedence yet last in the stack to cover the default case of no volume change + # vol (db) is first in precedence yet last in the stack to cover the default case of a None volume change, but when it does have a value it overrides the other options if update.vol_delta_f is not None and update.vol is None: true_vol_f = zone.vol_f + zone.vol_f_overflow expected_vol_total = update.vol_delta_f + true_vol_f @@ -860,7 +860,7 @@ def set_vol(): vol_db = utils.vol_float_to_db(vol_f_new, zone.vol_min, zone.vol_max) zone.vol_f_overflow = 0 if models.MIN_VOL_F < expected_vol_total and expected_vol_total < models.MAX_VOL_F \ - else utils.clamp((expected_vol_total - vol_f_new), models.MIN_VOL_F_OVERFLOW, models.MAX_VOL_F_OVERFLOW) + else utils.clamp((expected_vol_total - vol_f_new), models.MIN_VOL_F_OVERFLOW, models.MAX_VOL_F_OVERFLOW) # Clamp the remaining delta to be between -1 and 1 elif update.vol_f is not None and update.vol is None: diff --git a/amplipi/defaults.py b/amplipi/defaults.py index 76f69eb3a..c2a24894f 100644 --- a/amplipi/defaults.py +++ b/amplipi/defaults.py @@ -14,7 +14,7 @@ RCAs = [996, 997, 998, 999] LMS_DEFAULTS = [1000, 1001, 1002, 1003] -USER_CONFIG_DIR = os.path.join(os.path.expanduser('~'), '.config', 'amplipi') +USER_CONFIG_DIR = os.path.join('/data', '.config', 'amplipi') DEFAULT_CONFIG = { # This is the system state response that will come back from the amplipi box "version": 1, diff --git a/amplipi/display/check_pass b/amplipi/display/check_pass index f0d8aa072..c1419cf7b 100755 --- a/amplipi/display/check_pass +++ b/amplipi/display/check_pass @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/home/pi/.local/share/uv/python/cpython-3.8.20-linux-aarch64-gnu/bin/python3.8 # This is a separate script as it must be run with root privileges # Example: sudo python check_pass 'raspberry' @@ -9,5 +9,7 @@ import sys default_password = sys.argv[1] current_password = spwd.getspnam('pi').sp_pwd if crypt.crypt(default_password, current_password) == current_password: + print("Correct Password") sys.exit(0) +print("Incorrect Password") sys.exit(1) diff --git a/amplipi/display/common.py b/amplipi/display/common.py index 8a52e36db..46641e2bb 100644 --- a/amplipi/display/common.py +++ b/amplipi/display/common.py @@ -75,7 +75,7 @@ class DefaultPass: the stored default AmpliPi password.""" # Password config location - PASS_DIR = os.path.join(os.path.expanduser('~'), '.config', 'amplipi') + PASS_DIR = os.path.join('/data', '.config', 'amplipi') PASS_FILE = os.path.join(PASS_DIR, 'default_password.txt') DEFAULT_PI_PASSWORD = 'raspberry' @@ -115,7 +115,7 @@ def get_default_pw(self) -> str: def check_pw(pw: str) -> bool: """ Check if the given password is the pi user's current password. """ try: - subprocess.run(f'sudo python3 amplipi/display/check_pass {pw}', shell=True, check=True) + subprocess.run(f'sudo ./amplipi/display/check_pass {pw}', shell=True, check=True) return True except subprocess.CalledProcessError: return False @@ -177,9 +177,9 @@ def get_zone_status(zone, sources) -> STATUS: else: return STATUS.PLAYING elif source_for_zone.state == 'stopped': - return STATUS.STOPPED + return STATUS.STOPPED elif source_for_zone.state == 'paused': - return STATUS.PAUSED + return STATUS.PAUSED return STATUS.IGNORE return STATUS.IGNORE @@ -222,7 +222,7 @@ def get_emoji_status(url: str, max_length: int = 16) -> Union[str, int]: result = '' if status_counts[STATUS.PLAYING] > 0: - result += f'▶x{status_counts[STATUS.PLAYING]} ' + result += f'▶x{status_counts[STATUS.PLAYING]} ' if status_counts[STATUS.PAUSED] > 0: result += f'⏸x{status_counts[STATUS.PAUSED]} ' if status_counts[STATUS.STOPPED] > 0: @@ -248,7 +248,7 @@ def get_status(url: str, no_serial_ok: bool = False, emoji: bool = True, max_len result_status = st # Check if API is running - api_on = subprocess.run("systemctl --user is-active amplipi.service".split(), stdout=subprocess.DEVNULL) + api_on = subprocess.run("systemctl is-active amplipi.service".split(), stdout=subprocess.DEVNULL) if api_on.returncode != 0: return DisplayError.NO_AMPLIPI_SERVICE, None, 0 diff --git a/amplipi/rt.py b/amplipi/rt.py index c789b3ee3..92beeb7d6 100644 --- a/amplipi/rt.py +++ b/amplipi/rt.py @@ -283,6 +283,10 @@ def _recover_preamps(self) -> bool: try: self.reset_preamps() self.set_i2c_addr() + try: + self.bus.close() + except Exception: + pass self.bus = SMBus(1) for addr, regs in list(self.preamps.items()): for reg, val in enumerate(regs): @@ -319,6 +323,10 @@ def write_byte_data(self, preamp_addr, reg, data): # Fallback 1: reopen the bus handle and retry (transient bus glitch). try: time.sleep(0.001) + try: + self.bus.close() + except Exception: + pass self.bus = SMBus(1) self.bus.write_byte_data(preamp_addr, reg, data) except Exception: diff --git a/amplipi/updater/asgi.py b/amplipi/updater/asgi.py index 1cf6cdd96..ff9cd53e7 100644 --- a/amplipi/updater/asgi.py +++ b/amplipi/updater/asgi.py @@ -36,12 +36,16 @@ import shutil import asyncio +import hashlib +import lzma + import configparser # web framework import requests from fastapi import FastAPI, Request, File, UploadFile, Depends, APIRouter, Response from fastapi.staticfiles import StaticFiles +from fastapi.exceptions import HTTPException from sse_starlette.sse import EventSourceResponse from starlette.responses import FileResponse # web server @@ -49,8 +53,10 @@ # models # pylint: disable=no-name-in-module from pydantic import BaseModel +from typing import Optional, Callable +from enum import Enum -from ..auth import CookieOrParamAPIKey, router as auth_router, set_password_hash, unset_password_hash,\ +from ..auth import CookieOrParamAPIKey, router as auth_router, set_password_hash, unset_password_hash, \ NotAuthenticatedException, not_authenticated_exception_handler, create_access_key app = FastAPI() @@ -143,7 +149,7 @@ class ReleaseInfo(BaseModel): app.mount("/static", StaticFiles(directory=f"{dir_path}/static"), name="static") INSTALL_DIR = os.getenv('INSTALL_DIR', os.getcwd()) -USER_CONFIG_DIR = os.path.join(os.path.expanduser('~'), '.config', 'amplipi') +USER_CONFIG_DIR = os.path.join('/data', '.config', 'amplipi') # if we have a broken configuration, the updater should still function # as a failsafe. This structure & some code was copied from @@ -288,11 +294,14 @@ async def start_upload(file: UploadFile = File(...)): def download(url, file_name): """ Download a binary file from @url to @file_name """ + # (connect timeout, read timeout) - read timeout is the max gap between + # received chunks, not a cap on total download time + response = requests.get(url, stream=True, timeout=(10, 30)) + response.raise_for_status() with open(file_name, "wb") as file: - # get request - response = requests.get(url) - # write to file - file.write(response.content) + for chunk in response.iter_content(chunk_size=4 * 1024 * 1024): + if chunk: + file.write(chunk) # TODO: verify file has amplipi version @@ -498,6 +507,249 @@ def request_support(): return Response(content=f"failed to request tunnel: {e}", media_type="text/html") +class ImageMetadata(BaseModel): + """ Expected checksum/size of a single image file (root or boot) from the update manifest """ + filename: str + sha256: str + size: int + + +class UpdateManifest(BaseModel): + """ Schema for manifest.json, contains size and checksum info for the image(s). Root is required, boot is optional """ + version: str + boot: Optional[ImageMetadata] = None + root: ImageMetadata + + +class BootPair(BaseModel): + """ + Partition mappings for the boot and root partitions of OS A and B + Given that there's only two slots, this is just a schema for hardcoding the mappings for the BootSlot Enum + """ + boot: int + root: int + + +class BootSlot(Enum): + """ + RPi A:B tryboot has two boot slots: A and B + This enum contains the partition mappings for both slots + """ + A = BootPair(boot=2, root=5) + B = BootPair(boot=3, root=6) + + +class PartitionSize(Enum): + """ + The size of boot and root partitions, used to provide a progress bar for the reflashing step + Hardcoded using the byte size of the uncompressed images + """ + BOOT = 268435456 + ROOT = 11470372864 + + +def get_checksum(path: str, total_size: int, progress_cb: Optional[Callable] = None) -> str: + """ sha256 checksum of a file (path) in 4MB chunks, reporting progress via progress_cb(done, total) callback as it goes """ + h = hashlib.sha256() + hashed_size = 0 + with open(path, "rb") as f: + while chunk := f.read(4 * 1024 * 1024): + h.update(chunk) + hashed_size += len(chunk) + if progress_cb: + progress_cb(hashed_size, total_size) + return h.hexdigest() + + +@router.post('/update/flash') +async def flash_partition(): + """ + Validate the update package downloaded to /data/update and then flash the inactive boot slot + see the BootSlot enum for slot mapping details + """ + + def flash(image: str, partition: int, total: int, progress_cb: Optional[Callable] = None): + """ Decompress an image and stream it straight into /dev/mmcblk0p{partition} via dd, reporting + progress via the progress_cb(written, total) callback as it goes """ + written = 0 + dd = subprocess.Popen( + # conv=fsync forces dd to flush all writes to disk before it exits, so a successful return + # here means the image is actually durable on the card, not just sitting in a write cache + ['sudo', 'dd', f'of=/dev/mmcblk0p{partition}', 'bs=4M', 'conv=fsync'], + stdin=subprocess.PIPE, stderr=subprocess.PIPE + ) + # This is chunked both to prevent loading a massive (potentially too large) file into memory all at once and to provide chunk-by-chunk feedback to the user for how the update is going + with lzma.open(image, 'rb') as src: + while chunk := src.read(4 * 1024 * 1024): + dd.stdin.write(chunk) + written += len(chunk) + if progress_cb: + progress_cb(written, total) + dd.stdin.close() + dd.wait() + if dd.returncode != 0: + raise RuntimeError(f'dd failed: {dd.stderr.read().decode()}') + + manifest_dir = "/data/update/manifest.json" + root_img = "/data/update/root.img.xz" + boot_img = "/data/update/boot.img.xz" + # BOOT_SLOT is an env_var set by the active boot partition's commandline.txt + if os.environ.get("BOOT_SLOT") != "A" and os.environ.get("BOOT_SLOT") != "B": + raise Exception("Boot slot could not be read") + + active_slot = BootSlot.A if os.environ.get("BOOT_SLOT") == "A" else BootSlot.B + target_slot = BootSlot.B if os.environ.get("BOOT_SLOT") == "A" else BootSlot.A + + q: asyncio.Queue = asyncio.Queue() + loop = asyncio.get_running_loop() + + def progress(done, total, label): + loop.call_soon_threadsafe(q.put_nowait, {'type': 'info', 'message': f'{label}: {done/total:.0%}'}) + + def do_checks(): + """ + Load manifest.json and verify the downloaded root (and boot, if present) images actually + match the size/checksum it declares, before anything is allowed to touch a partition + """ + manifest = None + if os.path.exists(manifest_dir): + with open(manifest_dir, "r", encoding="UTF-8") as f: + manifest = UpdateManifest(**json.load(f)) + + if manifest is None: + raise RuntimeError("Manifest unable to load due to error") + if not os.path.exists(root_img): + raise RuntimeError("Root image not found") + + root_size = os.path.getsize(root_img) + if manifest.root.size != root_size: + raise RuntimeError("Root image size does not match expected value") + if manifest.root.sha256 != get_checksum(root_img, root_size, lambda done, total: progress(done, total, "Verifying root image")): + raise RuntimeError("Root image checksum does not match expected value") + + if manifest.boot is not None: + if not os.path.exists(boot_img): + raise RuntimeError("Boot image expected but not found") + boot_size = os.path.getsize(boot_img) + if manifest.boot.size != boot_size: + raise RuntimeError("Boot image size does not match expected value") + if manifest.boot.sha256 != get_checksum(boot_img, boot_size, lambda done, total: progress(done, total, "Verifying boot image")): + raise RuntimeError("Boot image checksum does not match expected value") + + return manifest + + async def stream(): + """ SSE generator driving the whole flash: checks, then flash root (+boot), then patch the + target slot so it's bootable, yielding progress/status messages to the client throughout. + + The actual work (do_checks/flash) is blocking IO, so it's run in a thread pool executor + (loop.run_in_executor) instead of directly in this coroutine, to avoid blocking the event + loop for the whole operation. A background thread can't `yield` into this async generator + directly, so `progress()` instead pushes messages onto `q` via call_soon_threadsafe, and this + generator polls the executor future's `.done()` state, draining `q` each time it wakes up. + This same drain-while-polling shape repeats below for the root and boot flash steps. """ + future = loop.run_in_executor(None, do_checks) + while not future.done(): + await asyncio.sleep(0.1) + while not q.empty(): + yield {'data': json.dumps(q.get_nowait())} + await asyncio.sleep(0) # let any final call_soon_threadsafe callbacks land before the last drain + while not q.empty(): + yield {'data': json.dumps(q.get_nowait())} + try: + manifest = future.result() + yield {'data': json.dumps({'type': 'success', 'message': 'All checks successful!'})} + except Exception as e: + yield {'data': json.dumps({'type': 'error', 'message': str(e)})} + return + + # Congrats, everything is in place, you've survived this far, time to actually do anything at all + try: + yield {'data': json.dumps({'type': 'info', 'message': f'Currently on slot {active_slot.name}, will flash slot {target_slot.name} (root p{target_slot.value.root}, boot p{target_slot.value.boot})'})} + + yield {'data': json.dumps({'type': 'info', 'message': 'Flashing root image...'})} + root_future = loop.run_in_executor(None, lambda: flash(root_img, target_slot.value.root, PartitionSize.ROOT.value, lambda done, total: progress(done, total, 'Flashing root'))) + while not root_future.done(): + await asyncio.sleep(0.1) + while not q.empty(): + yield {'data': json.dumps(q.get_nowait())} + await asyncio.sleep(0) + while not q.empty(): + yield {'data': json.dumps(q.get_nowait())} + root_future.result() + yield {'data': json.dumps({'type': 'success', 'message': 'Root image flashed'})} + + if manifest.boot is not None: + yield {'data': json.dumps({'type': 'info', 'message': 'Flashing boot image...'})} + boot_future = loop.run_in_executor(None, lambda: flash(boot_img, target_slot.value.boot, PartitionSize.BOOT.value, lambda done, total: progress(done, total, 'Flashing boot'))) + while not boot_future.done(): + await asyncio.sleep(0.1) + while not q.empty(): + yield {'data': json.dumps(q.get_nowait())} + await asyncio.sleep(0) + while not q.empty(): + yield {'data': json.dumps(q.get_nowait())} + boot_future.result() + yield {'data': json.dumps({'type': 'success', 'message': 'Boot image flashed'})} + except Exception as e: + yield {'data': json.dumps({'type': 'error', 'message': f'Update failed mid-flash: {e}'})} + return + + try: + # /data/tmpmnt is the mountpoint used for whichever partition is being operated on at the time, either the inactive boot or root + # Necessary for making sure individual files have the proper details such as making sure the boot points to the correct root partition + yield {'data': json.dumps({'type': 'info', 'message': 'Patching boot partition...'})} + if not os.path.exists("/data/tmpmnt"): + os.mkdir("/data/tmpmnt") + subprocess.run(["sudo", "umount", "/data/tmpmnt"]) # In case the user put something there + subprocess.run(["sudo", "mount", f"/dev/mmcblk0p{target_slot.value.boot}", "/data/tmpmnt"], check=True) + + # The section below used to be more pythonic by using with open(...) as f:, reading, and writing to the file + # That is no longer the case as all of these operations require higher privs to touch a boot partition that doesn't belong to the user doing the changes + if manifest.boot is not None: + yield {'data': json.dumps({'type': 'info', 'message': 'Patching cmdline.txt'})} + content = subprocess.run(['sudo', 'cat', '/data/tmpmnt/cmdline.txt'], capture_output=True, text=True, check=True).stdout + content = re.sub(rf'(root=PARTUUID=[0-9a-f]+-0){active_slot.value.root}\b', rf'\g<1>{target_slot.value.root}', content) + content = content.replace(f"BOOT_SLOT={active_slot.name}", f"BOOT_SLOT={target_slot.name}") + subprocess.run(['sudo', 'tee', '/data/tmpmnt/cmdline.txt'], input=content, text=True, check=True) + + yield {'data': json.dumps({'type': 'info', 'message': 'Patching root partition...'})} + # All systems originate from the same ancestor image. The following tools cleanse the root partition of identifiable info + # so that A and B don't have a case of mistaken identity by sharing these identifiers + fsck = subprocess.run(["sudo", "e2fsck", "-p", f"/dev/mmcblk0p{target_slot.value.root}"]) + if fsck.returncode not in (0, 1): + raise RuntimeError(f"e2fsck exited with code {fsck.returncode} on /dev/mmcblk0p{target_slot.value.root}") + subprocess.run(["sudo", "tune2fs", "-U", "random", f"/dev/mmcblk0p{target_slot.value.root}"], check=True) + + # Create the update-pending file that the update validation service will use to detect an update happened post-reboot + subprocess.run(['sudo', 'tee', '/data/tmpmnt/update-pending'], input=str(target_slot.value.boot), text=True, check=True) + subprocess.run(["sudo", "umount", "/data/tmpmnt"], check=True) + + yield {'data': json.dumps({'type': 'info', 'message': 'Patching root fstab...'})} + subprocess.run(["sudo", "mount", f"/dev/mmcblk0p{target_slot.value.root}", "/data/tmpmnt"], check=True) + + # Mark the root with the slot letter so you know which partition you're in by simply running `ls` + subprocess.run(["sudo", "touch", f"/data/tmpmnt/home/pi/SLOT_{target_slot.name}"], check=True) + subprocess.run(["sudo", "chown", "pi:pi", f"/data/tmpmnt/home/pi/SLOT_{target_slot.name}"], check=True) + fstab = subprocess.run(['sudo', 'cat', '/data/tmpmnt/etc/fstab'], capture_output=True, text=True, check=True).stdout + # The root image was captured from whichever slot was active on the machine that built it, + # so its baked-in fstab still has that slot's boot/root partition numbers. Since this image + # always lands on the slot opposite whatever's active on *this* device, remap both digits so + # /boot/firmware and / mount from the partitions this slot actually occupies here. + fstab = re.sub(rf'(PARTUUID=[0-9a-f]+-0){active_slot.value.boot}\b', rf'\g<1>{target_slot.value.boot}', fstab) + fstab = re.sub(rf'(PARTUUID=[0-9a-f]+-0){active_slot.value.root}\b', rf'\g<1>{target_slot.value.root}', fstab) + subprocess.run(['sudo', 'tee', '/data/tmpmnt/etc/fstab'], input=fstab, text=True, check=True) + subprocess.run(["sudo", "umount", "/data/tmpmnt"], check=True) + + yield {'data': json.dumps({'type': 'success', 'message': 'Imaging successful!'})} + + except Exception as e: + yield {'data': json.dumps({'type': 'error', 'message': f'Update failed post-flash: {e}'})} + return + + return EventSourceResponse(stream()) + + app.include_router(auth_router) app.include_router(router) diff --git a/amplipi/utils.py b/amplipi/utils.py index c4777673d..ef1b05642 100644 --- a/amplipi/utils.py +++ b/amplipi/utils.py @@ -256,9 +256,9 @@ def get_folder(relative_folder, mock=False): Abstracts the directory structure. TODO: This does not find the correct directory when testing. """ if relative_folder == "config": - folder = os.path.join(os.path.expanduser('~'), '.config', 'amplipi') + folder = os.path.join("/data", '.config', 'amplipi') elif relative_folder == "web": - folder = os.path.join(os.path.expanduser('~'), '.config', 'amplipi', 'web') + folder = os.path.join('/data', '.config', 'amplipi', 'web') else: folder = os.path.join(os.path.expanduser('~'), 'amplipi-dev', relative_folder) @@ -490,7 +490,7 @@ def clear_custom_configs(): logger.exception(f"failed to delete user {user.pw_name}: {e}") # Remove these paths whole-cloth. They do not need special permissions. - for path in ["/var/lib/support_tunnel/device.db", "/home/pi/.config/amplipi/users.json"]: + for path in ["/var/lib/support_tunnel/device.db", "/data/.config/amplipi/users.json"]: try: os.remove(path) except Exception as e: diff --git a/bin/arm64/go-librespot b/bin/arm64/go-librespot new file mode 100755 index 000000000..5f0a7949c --- /dev/null +++ b/bin/arm64/go-librespot @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c0534347a168c2df8d8e95401f5c82f4c47a58a409d2d4e769c78ea39e4aa06 +size 16145128 diff --git a/bin/arm64/pianobar b/bin/arm64/pianobar new file mode 100755 index 000000000..baa762465 --- /dev/null +++ b/bin/arm64/pianobar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cdae9c2ce435cf053766887092c1c891bb1d5943859c52552564ed907b20c85 +size 151296 diff --git a/bin/arm64/shairport-sync b/bin/arm64/shairport-sync new file mode 100755 index 000000000..c2807e7ff --- /dev/null +++ b/bin/arm64/shairport-sync @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6457caedf491fa8bd1d274fb86b1efc9f8640f7a3c68af76dab881d4f2ba4eab +size 1265072 diff --git a/bin/arm64/shairport-sync-ap2 b/bin/arm64/shairport-sync-ap2 new file mode 100755 index 000000000..fa37d6b70 --- /dev/null +++ b/bin/arm64/shairport-sync-ap2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0952e3bdb69a9bc145c6ad3fec24e133dd44a8a1148ed4d0587e3822b0c2d1c3 +size 1722896 diff --git a/bin/arm64/shairport-sync-metadata-reader b/bin/arm64/shairport-sync-metadata-reader new file mode 100755 index 000000000..eede43b29 --- /dev/null +++ b/bin/arm64/shairport-sync-metadata-reader @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d94ae9e70c5dfaa54ffa0c9295244bda3235d04a148bf583934a172ae0b76828 +size 82560 diff --git a/bin/generate_bins.bash b/bin/generate_bins.bash index 6e02d1bec..3125a80d0 100755 --- a/bin/generate_bins.bash +++ b/bin/generate_bins.bash @@ -18,7 +18,7 @@ This builds binaries on a Pi for now, TODO: cross compile everything! local_only=false while [[ "$#" -gt 0 ]]; do case "$1" in - --local_only) local_only=true ;; + --only-local) local_only=true ;; -h|--help) printf "$helptext"; exit 0 ;; *) printf "Unknown parameter passed: $1\n\n" printf "$helptext" @@ -120,7 +120,7 @@ if [[ ! -d '/home/pi' ]] ; then echo "failed to build binaries, try running this script directly on the pi to debug" exit -1 fi - scp $RPI_IP_ADDRESS:$x/* arm/ + scp $RPI_IP_ADDRESS:$x/* arm64/ fi # build files locally build diff --git a/bin/x64/shairport-sync b/bin/x64/shairport-sync index f352c0997..dc152803f 100755 --- a/bin/x64/shairport-sync +++ b/bin/x64/shairport-sync @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3ba8fbb12c4e81e9effd1e189058b4981e136708ab75278f9c7cbc4d8b955996 -size 1530128 +oid sha256:4168813eb328e031ca802d070f4fba9ebf7fabcaab24fbe6eb42a50c73516a04 +size 1289752 diff --git a/bin/x64/shairport-sync-ap2 b/bin/x64/shairport-sync-ap2 index 2154be6be..132311a91 100755 --- a/bin/x64/shairport-sync-ap2 +++ b/bin/x64/shairport-sync-ap2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bb165e9efbd832d5665de2053b6a1d27aa55e1fcf3dee14585dca1461b00abcd -size 2167048 +oid sha256:3fd6bb5501921adb9a1fa8608134ee8e5a427c06005a8cff59bec6b6acf17865 +size 1849496 diff --git a/bin/x64/shairport-sync-metadata-reader b/bin/x64/shairport-sync-metadata-reader new file mode 100755 index 000000000..d3cd84d0e --- /dev/null +++ b/bin/x64/shairport-sync-metadata-reader @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e92db116715953ea96b38433166709fdc6b900493aa5c3d27ed5ebacb4341a3c +size 31792 diff --git a/config/autoboot.txt b/config/autoboot.txt new file mode 100644 index 000000000..96cbb8746 --- /dev/null +++ b/config/autoboot.txt @@ -0,0 +1,13 @@ +[all] +tryboot_a_b=1 +boot_partition=2 + +[tryboot] +boot_partition=3 + +if exist mmc 0:1 /boot_partition.txt then + load mmc 0:1 ${loadaddr} /boot_partition.txt + env import -t ${loadaddr} ${filesize} +fi + +boot mmc 0:${boot_partition} diff --git a/docs/imaging_etcher.md b/docs/imaging_etcher.md index 2d9fae256..1894a8386 100644 --- a/docs/imaging_etcher.md +++ b/docs/imaging_etcher.md @@ -5,10 +5,6 @@ * A micro USB cable * Your AmpliPi -## Optional Step: Preserve your config - Assuming you're not reflashing due to a corrupted config, you can make sure to save your configuration file for later upload to save yourself the manual setup post-flash. You can get your configuration file by going to Settings -> Configuration -> Download Config. If you can't reach the app, you can also access it by SSHing in using the credentials on the screen on the front of your unit (username: pi@{IP} ; password: {4 word string on front display}) or through the USB connection that will be achieved in steps 1 , 2, and 4 and then mounting the connected drive. - Once you're in, the config file is at **/home/pi/.config/amplipi/house.json** - ## 1. Get the Latest AmpliPi Image Download the latest AmpliPi image from [here](https://storage.googleapis.com/amplipi-img/amplipi_0.4.8.img.xz) ([md5sum](https://storage.googleapis.com/amplipi-img/CHECKSUMS), [.sig](https://storage.googleapis.com/amplipi-img/CHECKSUMS.sig)) and save it to your computer. diff --git a/docs/manual/TROUBLESHOOTING.md b/docs/manual/TROUBLESHOOTING.md index 694976655..716622408 100644 --- a/docs/manual/TROUBLESHOOTING.md +++ b/docs/manual/TROUBLESHOOTING.md @@ -45,7 +45,7 @@ Manual backups are taken by navigating to ⚙ -> Config -> Download Config. This Automated backups are also taken nightly at 2AM and stored for 90 days, and also whenever you upgrade to a newer version of software. These are raw system backups, accessible only by an advanced user via the backend. These backups help either support, or a technical user, to restore prior system configuration from a particular point in time. If the below instructions do not make sense to you, you should feel free to email [support@micro-nova.com](mailto:support@micro-nova.com) and we can help restore a backup from your appliance. -These backups are dated tarballs stored at `/home/pi/backups`. This tarball contains the entire `/home/pi/.config/amplipi` directory. To restore this backup: +These backups are dated tarballs stored at `/home/pi/backups`. This tarball contains the entire `/data/.config/amplipi` directory. To restore this backup: 1. Stop the AmpliPi service (`systemctl stop --user amplipi` as the `pi` user). 1. Unpack a backup tarball and overwrite the contents of `.config/amplipi` (something like `tar --force-local -xvzf backups/config_2024-08-22T12:42:31-04:00_pre-fw-upgrade.tgz -C /`). Here we use `config_2024-08-22T12:42:31-04:00_pre-fw-upgrade.tgz` as an example backup file. 1. Start AmpliPi again (`systemctl start --user amplipi`).` diff --git a/scripts/amplipi-postflash.service b/scripts/amplipi-postflash.service new file mode 100644 index 000000000..4a3f8a2c0 --- /dev/null +++ b/scripts/amplipi-postflash.service @@ -0,0 +1,13 @@ +[Unit] +Description=AmpliPi post-flash setup (package install and custom scripts) +After=network-online.target +Before=amplipi.service amplipi-tasks.service +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/amplipi-postflash.sh +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/scripts/amplipi-postflash.sh b/scripts/amplipi-postflash.sh new file mode 100644 index 000000000..3566ecdbf --- /dev/null +++ b/scripts/amplipi-postflash.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +PENDING_FILE="/boot/firmware/update-pending" +PACKAGES_FILE="/data/packages.apt" +SCRIPTS_DIR="/data/update_scripts" +UPDATE_LOG="/data/update-log.txt" + +log() { + local msg="[$(date -Iseconds)] amplipi-postflash: $*" + echo "${msg}" + echo "${msg}" >> "${UPDATE_LOG}" 2>/dev/null || true +} + +if [ ! -f "${PENDING_FILE}" ]; then + exit 0 +fi + +log "Trial boot detected — running post-flash setup" + +# Refresh apt package index (cleared during image prep to reduce image size) +log "Refreshing apt package index..." +apt-get update -qq || log "Warning: apt-get update failed — package installs may not work" + +# Install user-customized packages that need to survive OTA updates. +# Add package names (one per line) to /data/packages.apt on p7. +if [ -f "${PACKAGES_FILE}" ]; then + log "Installing packages from ${PACKAGES_FILE}..." + DEBIAN_FRONTEND=noninteractive xargs apt-get install -y < "${PACKAGES_FILE}" \ + || log "Warning: some packages from ${PACKAGES_FILE} failed to install" +fi + +# Run device-specific customization scripts from p7. +# These survive OTA updates and are re-applied on each new slot's first boot. +if [ -d "${SCRIPTS_DIR}" ]; then + for script in "${SCRIPTS_DIR}"/*.sh; do + [ -f "${script}" ] || continue + log "Running ${script}..." + bash "${script}" || log "Warning: ${script} exited non-zero" + done +fi + +log "Post-flash setup complete" diff --git a/scripts/amplipi-tryboot-verify.service b/scripts/amplipi-tryboot-verify.service new file mode 100755 index 000000000..13a54c85b --- /dev/null +++ b/scripts/amplipi-tryboot-verify.service @@ -0,0 +1,12 @@ +[Unit] +Description=AmpliPi tryboot verification and commit +After=network-online.target amplipi.service amplipi-tasks.service redis-server.service +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/amplipi-tryboot-verify.sh +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/scripts/amplipi-tryboot-verify.sh b/scripts/amplipi-tryboot-verify.sh new file mode 100755 index 000000000..90fdf480b --- /dev/null +++ b/scripts/amplipi-tryboot-verify.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +BOOT_MOUNT="/boot/firmware" +AUTOBOOT_MOUNT="/boot/autoboot" +PENDING_FILE="${BOOT_MOUNT}/update-pending" +AUTOBOOT_FILE="${AUTOBOOT_MOUNT}/autoboot.txt" +UPDATE_LOG="/data/update-log.txt" + +log() { + local msg="[$(date -Iseconds)] amplipi-tryboot-verify: $*" + echo "${msg}" + echo "${msg}" >> "${UPDATE_LOG}" 2>/dev/null || true +} + +# Retry helper: retry N times with DELAY seconds between attempts +retry() { + local n=$1 delay=$2; shift 2 + for i in $(seq 1 "${n}"); do + if "$@" &>/dev/null; then return 0; fi + [ "${i}" -lt "${n}" ] && sleep "${delay}" + done + return 1 +} + +current_part=$( [[ "${BOOT_SLOT}" == "A" ]] && echo 2 || echo 3 ) + +commit() { # Swap which boot slot is considered primary and secondary p1's autoboot.txt + local old_part + if [ "${current_part}" = "2" ]; then old_part=3; else old_part=2; fi + + log "Committing: p${current_part} becomes default, p${old_part} becomes tryboot" + + mount -o remount,rw "${AUTOBOOT_MOUNT}" + python3 /usr/local/bin/update_autoboot.py "${AUTOBOOT_FILE}" "${current_part}" "${old_part}" + mount -o remount,ro "${AUTOBOOT_MOUNT}" + + mount -o remount,rw "${BOOT_MOUNT}" + rm -f "${PENDING_FILE}" + mount -o remount,ro "${BOOT_MOUNT}" + + log "Commit complete. Default: p${current_part} | Tryboot: p${old_part}" +} + +revert() { # If anything is unsuccessful, revert to the previous boot slot + local reason="$1" + log "Health check failed: ${reason} — rebooting to trigger auto-revert" + mount -o remount,rw "${BOOT_MOUNT}" 2>/dev/null || true + rm -f "${PENDING_FILE}" + mount -o remount,ro "${BOOT_MOUNT}" 2>/dev/null || true + systemctl reboot +} + +# ---- Main ---- + +if [ ! -f "${PENDING_FILE}" ]; then + log "No update pending — exiting" + exit 0 +fi + +expected_part=$(tr -d '[:space:]' < "${PENDING_FILE}") +log "Trial boot detected. Expected p${expected_part}, running on p${current_part}" + +if [ "${current_part}" != "${expected_part}" ]; then + revert "Booted on wrong partition (expected p${expected_part}, got p${current_part})" + exit 0 +fi + +log "Running health checks" + +failed_checks=() + +retry 12 5 systemctl is-active amplipi || failed_checks+=("amplipi service") +retry 12 5 systemctl is-active amplipi-tasks || failed_checks+=("amplipi-tasks service") +retry 6 5 systemctl is-active redis-server || failed_checks+=("redis-server service") +retry 12 5 curl -sf --max-time 5 http://localhost/api || failed_checks+=("API health check") + +if [ "${#failed_checks[@]}" -gt 0 ]; then + revert "Failed: ${failed_checks[*]}" + exit 0 +fi + +commit diff --git a/scripts/backup_config.sh b/scripts/backup_config.sh index 3676d5a65..1502d16c3 100755 --- a/scripts/backup_config.sh +++ b/scripts/backup_config.sh @@ -1,13 +1,13 @@ #!/bin/bash date="$(date --iso-8601='seconds')" -backup_dir="${HOME}/backups" +backup_dir="/data/backups" suffix="${1}" backup_filename="${backup_dir}/config_${date}${suffix}.tgz" echo "taking a config backup..." mkdir -p "${backup_dir}" -tar --one-file-system -czf "${backup_filename}" "${HOME}/.config/amplipi/" +tar --one-file-system -czf "${backup_filename}" "/data/.config/amplipi/" # remove all backups older than 90 days echo "removing old backups..." diff --git a/scripts/bootstrap_pi b/scripts/bootstrap_pi index 7ada53db9..106386d20 100755 --- a/scripts/bootstrap_pi +++ b/scripts/bootstrap_pi @@ -9,10 +9,10 @@ Bootstrap a Raspberry Pi Compute Module for first-time AmpliPi setup. --hostname NAME: The hostname to set on the pi, so once this script successfully completes connect via {hostname}.local. Default is amplipi - --cli-only: Use the lite version of Raspberry Pi OS which does not have + --desktop: Use the full version of Raspberry Pi OS which has a desktop environment --img FILE.img: Specify a .img file to write to the Pi. This option will not - modify the image contents, so ignores the --cli-only option + modify the image contents, so ignores the --desktop option --no-instr: Skip printing manual instructions. Pi must already be in bootloader mode before running this script. --debug: Prints all commands to be run and disables cleanup on exit @@ -28,7 +28,7 @@ write_speed_mb=6 # MB/s write speed to the Pi's EMMC over USB hostname='amplipi' # Hostname to set on the Pi debug=false # Enable/disable debugging -lite=false # Use non-desktop Pi image... +desktop=false # Use desktop containing Pi image... img_file='' # ...or use this image file explicitly instructions=true # Skip manual instruction step hostname_only=false # Configure only hostname in root partition @@ -53,7 +53,7 @@ while [[ "$#" -gt 0 ]]; do exit 1 fi ;; - --cli-only) lite=true ;; + --desktop) desktop=true ;; --no-instr) instructions=false ;; --debug) debug=true ;; -h|--help) printf "$helptext"; exit 0 ;; @@ -65,6 +65,12 @@ while [[ "$#" -gt 0 ]]; do shift done +echo " + Please note that the bootstrap_pi script is no longer able to make a fully functional AmpliPi from point zero and now requires some manual working + + If you'd prefer a less laborious solution, please see the imaging instructions at /docs/imaging_etcher.md +" + $debug && set -x current_dir="$(dirname "$(realpath ${BASH_SOURCE[0]})")" @@ -161,32 +167,29 @@ if ! $connected; then fi if [[ -z $img_file ]]; then - if $lite; then - release=2020-08-20-raspios-buster-armhf-lite - url=http://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2020-08-24/$release.zip + if $desktop; then + release=2025-12-04-raspios-trixie-armhf-full.img + url=https://downloads.raspberrypi.org/raspios_full_armhf/images/raspios_full_armhf-2025-12-04/$release.xz else - release=2020-08-20-raspios-buster-armhf - url=http://downloads.raspberrypi.org/raspios_armhf/images/raspios_armhf-2020-08-24/$release.zip + release=2025-12-04-raspios-trixie-arm64-lite.img + url=https://downloads.raspberrypi.org/raspios_lite_arm64/images/raspios_lite_arm64-2025-12-04/$release.xz fi - zip_file=$HOME/Downloads/$release.zip - if [[ ! -f $zip_file ]]; then + compressed_image=$HOME/Downloads/$release.xz + img_file=$HOME/Downloads/$release + if [[ ! -f $compressed_image ]]; then echo "Downloading Raspberry Pi OS" - wget $url -O $zip_file # save for later + wget $url -O $compressed_image # save for later fi - mkdir $release - pushd $release echo "Extracting Raspberry Pi OS" - unzip $zip_file - popd # $release - img_file=$release/$release.img -else - # Delay to ensure Pi disk device has been loaded - sleep 1 + unxz -fk $compressed_image # -f to overwrite existing, -k to keep the .xz for later reuse fi +echo "waiting for 5 seconds to load the pi's filesystem" +sleep 5 + # Check for Raspberry Pi device path disk_base_path=/dev/disk/by-id/usb-RPi-MSD -diskpath=$(ls $disk_base_path* 2>/dev/null | head -n1) +diskpath=$(ls $disk_base_path* | head -n1) if [ ! -z $diskpath ]; then echo "Raspberry Pi device found at $diskpath" else @@ -200,6 +203,7 @@ echo "Copying the image to the AmpliPi. This takes about $est_time minutes." echo "Go get a coffee or something :)" echo "Started at: $(date)" sudo dd if=$img_file of=$diskpath bs=4MiB status=progress +sudo sync # Validate that everything is flushed from memory and into disk before announcing success echo "Finished at: $(date)" # First arg is partition path to mount @@ -211,6 +215,11 @@ config_boot () { echo "Enabling SSH access" sudo touch ssh + # Adds default pi user with password raspberry as this is no longer a default in modern raspbian + # This would be seen as a security vulnerability if we didn't reset the password with our cleanup script + echo "Adding default pi user with password 'raspberry'" + echo pi:$6$p1c80DC32OHpYn0W$G26kjOgiFZoQ3N64X7/8dQgdOWUD.1hLE/dLEXkCpg0EUrifOUSy.23VE3sC3Jmdgz7r5GjlUPWEI2PqYigUe. > userconf + # extra config to make sure the USB serial tty is disabled # (we use this to communicate with the AmpliPi preamp) echo "Disabling USB serial tty" @@ -297,5 +306,32 @@ Bootstrapping successful. Please complete the following steps: 2. Unplug the AmpliPi unit. 3. Plug it back in to power it on. 4. Run the deploy script to install the latest AmpliPi version. + 5. (Optional) Run the cleanup script to make the system 'factory fresh' with a randomgen ssh password and empty logs +" + + +echo " +NOTE: a modern AmpliPi setup is more complicated than simple bash scripting will allow, to match what a normal Pi would contain please follow the above directions, and then: + 1. use `sudo dd if=/dev/sdx1 of=amplipi-boot.img status=progress bs=4MiB` to make an image of your boot directory, changing sdx into whatever sd your device mounted to + 2. use `sudo dd if=/dev/sdx2 of=amplipi-root.img status=progress bs=4MiB` to make an image of your root directory, changing sdx into whatever sd your device mounted to + 3. use parted or gparted to wipe all partitions and make the following partitions: + + p1: FAT32, 516 MB, flash with amplipi-boot.img + p2: FAT32, 516 MB, flash with amplipi-boot.img + p3: FAT32, 516 MB, flash with amplipi-boot.img + p4: Extended partition containing p5-7 containing remainder of disk + p5: EXT4, 10939 MB, flash with amplipi-root.img + p6: EXT4, 10939 MB, flash with amplipi-root.img + p7: EXT4, sized to fill the remaining space + + 4. Edit the contents of the partitions as so: + + p1: add autoboot.txt as seen in /config/autoboot.txt + p2: adjust cmdline.txt to point to partition 5 + p3: adjust cmdline.txt to point to partition 6 + + p5: Add p7 to fstab as /data + p6: Add p7 to fstab as /data + p7: create a .config folder " cleanup "success" diff --git a/scripts/check-online b/scripts/check-online index bc606ec0f..7f308ae3a 100755 --- a/scripts/check-online +++ b/scripts/check-online @@ -2,7 +2,7 @@ # Update the offline/online state set -e -USER_CONFIG_DIR="${HOME}/.config/amplipi" +USER_CONFIG_DIR="/data/.config/amplipi" ONLINE_FILE="${USER_CONFIG_DIR}/online" # add file if needed diff --git a/scripts/check-release b/scripts/check-release index 4986f2ed9..f0ee6399c 100755 --- a/scripts/check-release +++ b/scripts/check-release @@ -2,7 +2,7 @@ # Update the latest release set -e -USER_CONFIG_DIR="${HOME}/.config/amplipi" +USER_CONFIG_DIR="/data/.config/amplipi" LATEST_RELEASE_FILE="${USER_CONFIG_DIR}/latest_release" # add file if needed diff --git a/scripts/cleanup b/scripts/cleanup index 16c306b7d..b0b4852f5 100755 --- a/scripts/cleanup +++ b/scripts/cleanup @@ -63,9 +63,9 @@ if ! ssh -o PasswordAuthentication=no $user_host 'echo "AmpliPi has your SSH Key fi echo -e "reverting amplipi back to stock configuration" -ssh $user_host "systemctl --user stop amplipi" -ssh $user_host "find ~/.config/amplipi -maxdepth 1 -type f -delete" # intentionally avoids deleting folders used as mountpoints -ssh $user_host "systemctl --user start amplipi" +ssh $user_host "sudo systemctl stop amplipi" +ssh $user_host "find /data/.config/amplipi -maxdepth 1 -type f -delete" # intentionally avoids deleting folders used as mountpoints +ssh $user_host "sudo systemctl start amplipi" echo -e "removing deployments" ssh $user_host "rm -rf amplipi-*.tar.gz amplipi-dev/amplipi-*" @@ -83,6 +83,16 @@ ssh $user_host "rm -f ~/Desktop/*.desktop" echo -e "\ncleaning up amplipi logs:" ssh $user_host "sudo journalctl --rotate && sleep 1 && sudo journalctl --vacuum-time=1s" +echo -e "\nremoving A/B update artifacts" +# Remove update state from both boot partitions (p2=boot-A, p3=boot-B) +ssh $user_host "sudo mkdir -p /mnt/bootp && sudo mount /dev/mmcblk0p2 /mnt/bootp && sudo rm -f /mnt/bootp/update-pending && sudo umount /mnt/bootp && sudo mount /dev/mmcblk0p3 /mnt/bootp && sudo rm -f /mnt/bootp/update-pending && sudo umount /mnt/bootp && sudo rmdir /mnt/bootp" +# Remove update package and log from shared data partition +ssh $user_host "sudo rm -rf /data/update/ && sudo rm -f /data/update-log.txt" + +echo -e "\nregenerating SSH host keys (fresh identity per unit)" +# Wipe old keys then regenerate via symlinks into /data/ssh/ +ssh $user_host "sudo rm -f /data/ssh/* && sudo ssh-keygen -A" + echo -e "\nchanging hostname to amplipi" ssh $user_host "sudo sed -i 's/127\.0\.1\.1.*/127.0.1.1\tamplipi/' /etc/hosts" ssh $user_host "sudo sed -i 's/.*/amplipi/' /etc/hostname" @@ -93,7 +103,10 @@ ssh $user_host "amplipi-dev/scripts/set_pass" #echo -e "removing ssh key and erasing tracks" #ssh $user_host "rm -f ~/.ssh/*" # || true used to ignore the error ssh returns due to being cut off -echo -e "erasing tracks" +echo -e "erasing tracks (both slots)" +# Clear inactive slot's bash history +inactive_root=$(ssh $user_host "grep -q 'BOOT_SLOT=A' /proc/cmdline && echo /dev/mmcblk0p6 || echo /dev/mmcblk0p5") +ssh $user_host "sudo mkdir -p /mnt/inactive && sudo mount $inactive_root /mnt/inactive && sudo rm -f /mnt/inactive/home/pi/.bash_history && sudo umount /mnt/inactive && sudo rmdir /mnt/inactive" ssh $user_host "cat /dev/null > ~/.bash_history && history -c && sudo systemctl reboot" || true # verify amplipi.local get request is successful diff --git a/scripts/configure.py b/scripts/configure.py index 12d79bcf6..89fd52712 100755 --- a/scripts/configure.py +++ b/scripts/configure.py @@ -10,6 +10,7 @@ import pathlib import pwd # username import glob +import tempfile from typing import List, Union, Tuple, Dict, Any, Optional import time import re @@ -81,7 +82,7 @@ 'base': { 'apt': ['python3-pip', 'python3-venv', 'curl', 'authbind', 'python3-pil', 'libopenjp2-7', # Pillow dependencies - 'libatlas-base-dev', # numpy dependencies + 'libopenblas-dev', # numpy dependencies. was libatlas-base-dev, is no longer thanks to https://github.com/numpy/numpy/issues/29108#issuecomment-3371130468 'stm32flash', # Programming Preamp Board 'xkcdpass', # Random passphrase generation 'systemd-journal-remote', # Remote/web based log access @@ -91,6 +92,46 @@ 'libgirepository1.0-dev', 'libcairo2-dev', ], }, + 'updates': { + 'copy': [ + { + 'from': 'scripts/amplipi-tryboot-verify.sh', + 'to': '/usr/local/bin/amplipi-tryboot-verify.sh', + 'sudo': 'true', + }, + { + 'from': 'scripts/update_autoboot.py', + 'to': '/usr/local/bin/update_autoboot.py', + 'sudo': 'true', + }, + { + 'from': 'scripts/amplipi-tryboot-verify.service', + 'to': '/etc/systemd/system/amplipi-tryboot-verify.service', + 'sudo': 'true', + }, + { + 'from': 'scripts/amplipi-postflash.sh', + 'to': '/usr/local/bin/amplipi-postflash.sh', + 'sudo': 'true', + }, + { + 'from': 'scripts/amplipi-postflash.service', + 'to': '/etc/systemd/system/amplipi-postflash.service', + 'sudo': 'true', + }, + ], + 'script': [ + 'sudo chmod +x /usr/local/bin/amplipi-tryboot-verify.sh', + 'sudo chmod +x /usr/local/bin/update_autoboot.py', + 'sudo chmod +x /usr/local/bin/amplipi-postflash.sh', + + 'sudo chmod 444 /etc/systemd/system/amplipi-tryboot-verify.service', + 'sudo systemctl enable amplipi-tryboot-verify.service', + + 'sudo chmod 444 /etc/systemd/system/amplipi-postflash.service', + 'sudo systemctl enable amplipi-postflash.service', + ], + }, 'usb': { 'apt': [ 'udisks2', 'udiskie', # Required to mount filesystem without desktop installed @@ -207,6 +248,9 @@ 'popd', ] }, + 'poetry': { + 'script': ['curl -sSL https://install.python-poetry.org | python3 -'] + }, # streams # TODO: can stream dependencies be aggregated from the streams themselves? 'airplay': { @@ -252,11 +296,13 @@ 'apt': ['libcrypt-openssl-rsa-perl', 'libio-socket-ssl-perl', 'libopusfile0', 'squeezelite'], 'copy': [{'from': 'bin/ARCH/find_lms_server', 'to': 'streams/find_lms_server'}], 'script': [ + # uname -m returns aarch64 on 64-bit (Trixie) and armv7l on 32-bit (Buster), this helps sort 64 or 32 bit LMS + 'LMS_ARCH=$([ "$(uname -m)" = "aarch64" ] && echo arm64 || echo arm)', 'if [ ! $(dpkg-query --show --showformat=\'${Status}\' lyrionmusicserver | grep -q installed) ]; then ' - ' wget -nv https://downloads.lms-community.org/LyrionMusicServer_v9.0.3/lyrionmusicserver_9.0.3_arm.deb -O /tmp/lyrionmusicserver_9.0.3.deb', + ' wget -nv https://downloads.lms-community.org/LyrionMusicServer_v9.0.3/lyrionmusicserver_9.0.3_${LMS_ARCH}.deb -O /tmp/lyrionmusicserver_9.0.3.deb', ' sudo dpkg -i /tmp/lyrionmusicserver_9.0.3.deb', - ' if [ ! -e /home/pi/.config/amplipi/lms_mode ] ; then sudo systemctl disable lyrionmusicserver; fi', - ' if [ ! -e /home/pi/.config/amplipi/lms_mode ] ; then sudo systemctl stop lyrionmusicserver; fi', + ' if [ ! -e /data/.config/amplipi/lms_mode ] ; then sudo systemctl disable lyrionmusicserver; fi', + ' if [ ! -e /data/.config/amplipi/lms_mode ] ; then sudo systemctl stop lyrionmusicserver; fi', 'fi', 'sudo systemctl stop squeezelite', 'sudo systemctl disable squeezelite', @@ -299,21 +345,34 @@ }, 'pandora': { 'apt': [ - 'libao4', 'libavcodec58', 'libavfilter7', 'libavformat58', 'libavutil56', 'libc6', 'libcurl3-gnutls', 'libgcrypt20', 'libjson-c3' + 'libavfilter-dev', 'libcurl4-openssl-dev', 'libjson-c-dev', 'libao-dev' ], 'copy': [{'from': 'bin/ARCH/pianobar', 'to': 'streams/pianobar'}], }, 'bluetooth': { 'amplipi_only': True, - 'apt': ['libsndfile1', 'libsndfile1-dev', 'libbluetooth-dev', 'bluealsa', 'python-dbus', - 'libasound2-dev', 'git', 'autotools-dev', 'automake', 'libtool', 'm4'], + 'apt': ['libsndfile1', 'libsndfile1-dev', 'libbluetooth-dev', 'python3-dbus', + 'libasound2-dev', 'git', 'autotools-dev', 'automake', 'libtool', 'm4', + 'build-essential', 'pkg-config', 'python3-docutils', 'libdbus-1-dev', + 'libglib2.0-dev', 'libsbc-dev'], 'script': [ + + # Install bluealsa from git + 'echo installing bluealsa from source', + 'git clone https://github.com/arkq/bluez-alsa', + 'cd bluez-alsa', + 'autoreconf --install --force', + './configure --disable-aac --disable-ldac --disable-aptx --disable-opus', + 'sudo make -j$(nproc)', + 'sudo make install', + 'cd ..', + # referencing arm here is okay because bluetooth is marked as 'amplipi_only' 'sudo cp bin/arm/rtl8761b_fw /lib/firmware/rtl_bt/rtl8761b_fw.bin', 'sudo cp bin/arm/rtl8761b_config /lib/firmware/rtl_bt/rtl8761b_config.bin', 'sudo cp config/bluetooth/main.conf /etc/bluetooth/main.conf', - # TODO: investigate where to put these services - 'sudo cp config/bluetooth/bluealsa.service /lib/systemd/system/', + 'sudo cp config/bluetooth/bluealsa.service /etc/systemd/system/', + 'sudo rm -f /usr/lib/systemd/system/bluealsa.service', 'sudo cp streams/bluetooth_agent /usr/local/bin/', 'sudo cp config/bluetooth/bluetooth_agent.service /etc/systemd/system/', @@ -347,6 +406,15 @@ } +def filter_deps(dep: str, dep_filter: List[str]) -> bool: + ret = len(dep_filter) > 0 and dep not in dep_filter + if ret: + print(f"\n{dep} not in {dep_filter}, skipping...\n") + time.sleep(1) + + return ret + + def _check_and_update_streamer(env): """Check if this is a streamer (no preamp firmware)""" is_streamer_path = os.path.join(env['config_dir'], 'is_streamer') @@ -362,7 +430,7 @@ def _check_and_setup_platform(development, ci_mode): 'platform_supported': False, 'script_dir': script_dir, 'base_dir': script_dir.rsplit('/', 1)[0], - 'config_dir': os.path.join(os.path.expanduser('~'), '.config', 'amplipi'), + 'config_dir': os.path.join('/data', '.config', 'amplipi'), 'is_amplipi': False, 'is_streamer': False, 'arch': 'unknown', @@ -384,8 +452,12 @@ def _check_and_setup_platform(development, ci_mode): env['arch'] = 'x64' elif 'armv7l' in lplatform: env['arch'] = 'arm' + elif 'aarch64' in lplatform: + env['arch'] = 'arm64' env['is_amplipi'] = 'amplipi' in platform.node() # checks hostname + if env['is_amplipi']: + env['config_dir'] = '/data/.config/amplipi' if env['arch'] == 'x64' and env['has_apt']: # possibly a development machine running a debian-based distro @@ -395,6 +467,10 @@ def _check_and_setup_platform(development, ci_mode): # possibly a Rasperry Pi running Raspbian env['platform_supported'] = True + if env['arch'] == 'arm64': + # 64 bit raspbian OS + env['platform_supported'] = True + if development: # We're explicitly overriding any checks here; assume we're supported. env['platform_supported'] = True @@ -452,12 +528,20 @@ def _setup_loopbacks(base_dir) -> List[Task]: ]).run()] -def _install_os_deps(env, progress, deps=_os_deps.keys()) -> List[Task]: +def _install_os_deps(env, progress, with_alsa, deps=_os_deps.keys(), dep_filter: List[str] = [], development: bool = False) -> List[Task]: def print_progress(tasks): progress(tasks) return tasks tasks = [] + # The commit service leaves the boot partition ro for safety between boots, but apt + # operations below (dist-upgrade, package installs) can trigger kernel postinst scripts + # (e.g. update-initramfs) that write here, so remount rw for the duration of dependency + # install and restore ro at the end. + _boot_firmware = "/boot/firmware" + tasks += print_progress([Task(f"remount {_boot_firmware} rw", + ['sudo', 'mount', '-o', 'remount,rw', _boot_firmware]).run()]) + # Comment out deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi from /etc/apt/sources.list to avoid hitting up a now empty apt source tasks += print_progress([Task('Deactivate apt updates for outdated OS', args='file=/etc/apt/sources.list; ' @@ -476,14 +560,23 @@ def print_progress(tasks): # Upgrade current packages print_progress( [Task("upgrading debian packages, this will take 10+ minutes", success=True)]) - tasks += print_progress([Task('upgrade debian packages', - 'sudo apt-get dist-upgrade --assume-yes'.split()).run()]) + if development: + # Show verbose printout of what debian packages are being installed if developing + # this helps find out what step of the upgrade process you get hung up on (or if it's just a really long install) + tasks += print_progress([Task('upgrade debian packages', + 'sudo DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade --assume-yes'.split()).run()]) + else: + tasks += print_progress([Task('upgrade debian packages', + 'sudo apt-get dist-upgrade --assume-yes'.split()).run()]) # organize stuff to install packages = set() files = [] scripts: Dict[str, List[str]] = {} for dep in deps: + if filter_deps(dep, dep_filter): + continue + install_steps = _os_deps[dep] if install_steps.get('amplipi_only', False) and not env['is_amplipi']: continue @@ -508,7 +601,7 @@ def print_progress(tasks): if _sudo or not _parent_dir.exists(): tasks += print_progress([Task(f"creating parent dir(s) for {_from}", f"{_sudo}mkdir -p {_parent_dir}".split()).run()]) tasks += print_progress([Task(f"copy -f {_from} to {_to}", f"{_sudo}cp -f {_from} {_to}".split()).run()]) # shairport needs the -f if it is running - if env['is_amplipi'] or env['is_ci']: + if with_alsa or env['is_amplipi'] or env['is_ci']: # copy alsa configuration file _from = f"{env['base_dir']}/config/asound.conf" _to = "/etc/asound.conf" @@ -516,7 +609,7 @@ def print_progress(tasks): [Task(f"copy {_from} to {_to}", f"sudo cp {_from} {_to}".split()).run()]) # copy boot_config.txt RPi firmware configuration file _boot_config_from = f"{env['base_dir']}/config/boot_config.txt" - _boot_config_to = "/boot/config.txt" + _boot_config_to = f"{_boot_firmware}/config.txt" tasks += print_progress([Task(f"copy {_boot_config_from} to {_boot_config_to}", f"sudo cp {_boot_config_from} {_boot_config_to}".split()).run()]) # fix usb soundcard name @@ -558,6 +651,9 @@ def print_progress(tasks): # Run scripts for dep, script in scripts.items(): + print(f"\ndep: {dep} \nscript: {script}\n") + if filter_deps(dep, dep_filter): + continue sh_loc = f'{env["base_dir"]}/install_{dep}.sh' with open(sh_loc, 'a') as sh: for scrap in script: @@ -577,6 +673,9 @@ def print_progress(tasks): tasks += print_progress(_stop_service('shairport-sync', system=True)) tasks += print_progress(_disable_service('shairport-sync', system=True)) + tasks += print_progress([Task(f"remount {_boot_firmware} ro", + ['sudo', 'mount', '-o', 'remount,ro', _boot_firmware]).run()]) + return tasks @@ -591,6 +690,15 @@ def _install_python_deps(env: dict, deps: List[str]): return tasks +def _install_custom_deps(dep): + tasks = [] + _, extension = os.path.splitext(dep) + if os.path.isfile(f"/data/update_scripts/{dep}") and extension.lower() == ".sh": + tasks += [Task(f'install custom settings from {dep}', + f'bash /data/update_scripts/{dep}'.split()).run()] + return tasks + + def _add_desktop_icon(env, directory: pathlib.Path, name, command) -> Task: """ Add a desktop icon to the pi """ entry = f"""[Desktop Entry] @@ -631,80 +739,90 @@ def _setup_tmpfs(config_dir, env): return tasks -def _web_service(directory: str): +def _web_service(directory: str, user: str = 'pi'): return f"""\ [Unit] Description=Amplipi Home Audio System After=network.target [Service] +User={user} +Group={user} Type=simple WorkingDirectory={directory} ExecStart=/usr/bin/authbind --deep {directory}/venv/bin/python -m uvicorn --host 0.0.0.0 --port 80 amplipi.asgi:application Restart=always [Install] -WantedBy=default.target +WantedBy=multi-user.target """ -def _tasks_service(directory: str): +def _tasks_service(directory: str, user: str = 'pi'): return f"""\ [Unit] Description=AmpliPi Background Tasks After=redis-server.service [Service] +User={user} +Group={user} Type=simple WorkingDirectory={directory} ExecStart={directory}/venv/bin/python -m celery -A amplipi.tasks worker Restart=always [Install] -WantedBy=default.target +WantedBy=multi-user.target """ -def _update_service(directory: str, port: int = 5001): +def _update_service(directory: str, port: int = 5001, user: str = 'pi'): return f"""\ [Unit] Description=Amplipi Software Updater After=network.target [Service] +User={user} +Group={user} Type=simple WorkingDirectory={directory} ExecStart={directory}/venv/bin/python -m uvicorn amplipi.updater.asgi:app --host 0.0.0.0 --port {port} Restart=on-abort [Install] -WantedBy=default.target +WantedBy=multi-user.target """ -def _display_service(directory: str): +def _display_service(directory: str, user: str = 'pi'): return f"""\ [Unit] Description=Amplipi Front Panel Display [Service] +User={user} +Group={user} Type=simple WorkingDirectory={directory} ExecStart={directory}/venv/bin/python -m amplipi.display.display Restart=on-abort [Install] -WantedBy=default.target +WantedBy=multi-user.target """ -def _audiodetector_service(base_dir: str, config_dir: str): +def _audiodetector_service(base_dir: str, config_dir: str, user: str = 'pi'): return f"""\ [Unit] Description=Amplipi RCA Input Audio Detector -ConditionPathExists=!/home/pi/.config/amplipi/is_streamer +ConditionPathExists=!/data/.config/amplipi/is_streamer [Service] +User={user} +Group={user} Type=simple WorkingDirectory={config_dir}/srcs ExecStart={base_dir}/amplipi/audiodetector/audiodetector @@ -712,7 +830,7 @@ def _audiodetector_service(base_dir: str, config_dir: str): RestartSec=10 [Install] -WantedBy=default.target +WantedBy=multi-user.target """ @@ -724,7 +842,7 @@ def systemctl_cmd(system: bool) -> str: return 'systemctl --user' -def _service_status(service: str, system: bool = False) -> Tuple[List[Task], bool]: +def _service_status(service: str, system: bool = True) -> Tuple[List[Task], bool]: # Status can be: active, reloading, inactive, failed, activating, or deactivating cmd = f'{systemctl_cmd(system)} is-active {service}' tasks = [Task(f'Check {service} status', cmd.split()).run()] @@ -734,10 +852,8 @@ def _service_status(service: str, system: bool = False) -> Tuple[List[Task], boo active = 'active' in tasks[0].output and 'inactive' not in tasks[0].output return (tasks, active) -# Stop a systemd service. By default use the Session (user) session - -def _stop_service(name: str, system: bool = False) -> List[Task]: +def _stop_service(name: str, system: bool = True) -> List[Task]: service = f'{name}.service' tasks, running = _service_status(service, system) if running: @@ -746,50 +862,52 @@ def _stop_service(name: str, system: bool = False) -> List[Task]: return tasks -def _remove_service(name: str) -> List[Task]: +def _remove_service(name: str, system: bool = True) -> List[Task]: filename = f'{name}.service' - directory = pathlib.Path.home().joinpath('.config/systemd/user') tasks = [Task(f'Remove {filename}')] - try: - # Delete the service file - pathlib.Path(directory).joinpath(filename).unlink() - tasks[0].output = f'Removed {filename}' - tasks[0].success = True - except Exception as exc: - tasks[0].output = str(exc) - tasks[0].success = False + if system: + path = pathlib.Path('/etc/systemd/system') / filename + result = subprocess.run(['sudo', 'rm', '-f', str(path)], capture_output=True) + tasks[0].success = result.returncode == 0 + tasks[0].output = f'Removed {path}' if tasks[0].success else result.stderr.decode() + else: + path = pathlib.Path.home().joinpath('.config/systemd/user') / filename + try: + path.unlink() + tasks[0].output = f'Removed {path}' + tasks[0].success = True + except Exception as exc: + tasks[0].output = str(exc) + tasks[0].success = False return tasks -def _enable_service(name: str, system: bool = False) -> List[Task]: +def _enable_service(name: str, system: bool = True) -> List[Task]: service = f'{name}.service' cmd = f'{systemctl_cmd(system)} enable {service}' tasks = [Task(f'Enable {service}', cmd.split()).run()] return tasks -def _disable_service(name: str, system: bool = False) -> List[Task]: +def _disable_service(name: str, system: bool = True) -> List[Task]: service = f'{name}.service' cmd = f'{systemctl_cmd(system)} disable {service}' tasks = [Task(f'Disable {service}', cmd.split()).run()] return tasks -def _start_restart_service(name: str, restart: bool, test_url: Union[None, str] = None) -> List[Task]: +def _start_restart_service(name: str, restart: bool, test_url: Union[None, str] = None, system: bool = True) -> List[Task]: service = f'{name}.service' if restart: - tasks = [ - Task(f'Restart {service}', f'systemctl --user restart {service}'.split()).run()] + tasks = [Task(f'Restart {service}', f'{systemctl_cmd(system)} restart {service}'.split()).run()] else: - # just start - tasks = [ - Task(f'Start {service}', f'systemctl --user start {service}'.split()).run()] + tasks = [Task(f'Start {service}', f'{systemctl_cmd(system)} start {service}'.split()).run()] # wait a bit, so initial failures are detected before is-active is called if tasks[-1].success: # we need to check if the service is running for _ in range(50): # retry for 10 seconds, giving the service time to start - task_check, running = _service_status(service) + task_check, running = _service_status(service, system) if running: break time.sleep(0.2) @@ -805,20 +923,20 @@ def _start_restart_service(name: str, restart: bool, test_url: Union[None, str] elif name == 'amplipi': tasks[-1].output += "\ntry checking this service failure using 'scripts/run_debug_webserver' on the system" tasks.append(Task( - f'Check {service} Status', f'systemctl --user status {service}'.split()).run()) + f'Check {service} Status', f'{systemctl_cmd(system)} status {service}'.split()).run()) elif 'amplipi-updater' in name: tasks[-1].output += "\ntry debugging this service failure using 'scripts/run_debug_updater' on the system" tasks.append(Task( - f'Check {service} Status', f'systemctl --user status {service}'.split()).run()) + f'Check {service} Status', f'{systemctl_cmd(system)} status {service}'.split()).run()) return tasks -def _start_service(name: str, test_url: Union[None, str] = None) -> List[Task]: - return _start_restart_service(name, restart=False, test_url=test_url) +def _start_service(name: str, test_url: Union[None, str] = None, system: bool = True) -> List[Task]: + return _start_restart_service(name, restart=False, test_url=test_url, system=system) -def _restart_service(name: str, test_url: Union[None, str] = None) -> List[Task]: - return _start_restart_service(name, restart=True, test_url=test_url) +def _restart_service(name: str, test_url: Union[None, str] = None, system: bool = True) -> List[Task]: + return _start_restart_service(name, restart=True, test_url=test_url, system=system) def _create_dir(directory: str) -> List[Task]: @@ -839,25 +957,42 @@ def _create_dir(directory: str) -> List[Task]: def _create_service(name: str, config: str, env: dict) -> List[Task]: filename = f'{name}.service' - directory = pathlib.Path.home().joinpath('.config/systemd/user') tasks = [] - # create the systemd directory if it doesn't already exist - tasks += _create_dir(str(directory)) - - # create the service file, overwriting any existing one - tasks.append(Task(f'Create {filename}')) - try: - with directory.joinpath(filename).open('w+') as svc_file: - svc_file.write(config) - tasks[-1].success = True - tasks[-1].output = f'Created {filename}' - except: - tasks[-1].output = f'Failed to create {filename}' - - if not env['is_ci']: - # recreate systemd's dependency tree + if env.get('is_ci', False): + # CI: write to user-level directory without sudo + directory = pathlib.Path.home().joinpath('.config/systemd/user') + tasks += _create_dir(str(directory)) + tasks.append(Task(f'Create {filename}')) + try: + with directory.joinpath(filename).open('w+') as svc_file: + svc_file.write(config) + tasks[-1].success = True + tasks[-1].output = f'Created {filename}' + except Exception as exc: + tasks[-1].output = f'Failed to create {filename}: {exc}' tasks.append(Task('Reload systemd config', 'systemctl --user daemon-reload'.split()).run()) + else: + # Hardware: write to system-level directory using a temp file + sudo cp + dest = f'/etc/systemd/system/{filename}' + tasks.append(Task(f'Create {filename}')) + tmp_path = None + try: + with tempfile.NamedTemporaryFile(mode='w', suffix='.service', delete=False) as tmp: + tmp.write(config) + tmp_path = tmp.name + result = subprocess.run(['sudo', 'cp', tmp_path, dest], capture_output=True) + tasks[-1].success = result.returncode == 0 + tasks[-1].output = f'Created {dest}' if tasks[-1].success else result.stderr.decode() + except Exception as exc: + tasks[-1].output = f'Failed to create {filename}: {exc}' + finally: + if tmp_path: + try: + os.unlink(tmp_path) + except Exception: + pass + tasks.append(Task('Reload systemd config', 'sudo systemctl daemon-reload'.split()).run()) return tasks @@ -883,22 +1018,10 @@ def _configure_authbind() -> List[Task]: return tasks -# Enable linger so that user manager is started at boot -def _enable_linger(user: str, env) -> List[Task]: - if env['is_ci']: - # https://unix.stackexchange.com/a/721463 - margs = [ - 'sudo mkdir -p /var/lib/systemd/linger'.split(), - f'sudo touch /var/lib/systemd/linger/{user}'.split() - ] - else: - margs = [f'sudo loginctl enable-linger {user}'.split()] - return [Task(f'Enable linger for {user} user', multiargs=margs).run()] - def _api_key() -> Optional[str]: """ Get a singular API key for use with the updater """ - user_file_path = os.path.join(os.path.expanduser('~'), '.config', 'amplipi', 'users.json') + user_file_path = os.path.join('/data', '.config', 'amplipi', 'users.json') try: with open(user_file_path, encoding='utf-8') as user_file: users = json.load(user_file) @@ -985,6 +1108,7 @@ def print_progress(tasks): progress(tasks) return tasks + user = env.get('user', 'pi') tasks = [] if not env['is_ci']: @@ -998,7 +1122,7 @@ def print_progress(tasks): # bringup amplipi and updater separately tasks += print_progress(_configure_authbind()) - tasks += print_progress(_create_service('amplipi', _web_service(env['base_dir']), env)) + tasks += print_progress(_create_service('amplipi', _web_service(env['base_dir'], user), env)) tasks += print_progress(_enable_service('amplipi')) if not env['is_ci']: tasks += print_progress(_start_service('amplipi', test_url='http://0.0.0.0')) @@ -1006,7 +1130,7 @@ def print_progress(tasks): return tasks tasks += print_progress([_check_version('http://0.0.0.0/api')]) - tasks += print_progress(_create_service('amplipi-updater', _update_service(env['base_dir']), env)) + tasks += print_progress(_create_service('amplipi-updater', _update_service(env['base_dir'], user=user), env)) tasks += print_progress(_enable_service('amplipi-updater')) if not env['is_ci']: if restart_updater: @@ -1015,22 +1139,18 @@ def print_progress(tasks): else: # start a second updater service and check if it serves a url # this allow us to verify the update the updater probably works - tasks += print_progress(_create_service('amplipi-updater-test', _update_service(env['base_dir'], port=5002), env)) + tasks += print_progress(_create_service('amplipi-updater-test', _update_service(env['base_dir'], port=5002, user=user), env)) tasks += print_progress(_start_service('amplipi-updater-test', test_url='http://0.0.0.0:5002/update')) # stop and disable the service so it doesn't start up on a reboot tasks += print_progress(_stop_service('amplipi-updater-test')) tasks += print_progress(_remove_service('amplipi-updater-test')) # bring up amplipi-tasks - tasks += print_progress(_create_service('amplipi-tasks', _tasks_service(env['base_dir']), env)) + tasks += print_progress(_create_service('amplipi-tasks', _tasks_service(env['base_dir'], user), env)) tasks += print_progress(_enable_service('amplipi-tasks')) if not env['is_ci']: tasks += print_progress(_restart_service('amplipi-tasks')) - if env['is_amplipi'] or env['is_ci']: - # start the user manager at boot, instead of after first login - # this is needed so the user systemd services start at boot - tasks += print_progress(_enable_linger(env['user'], env)) return tasks @@ -1038,15 +1158,12 @@ def _update_display(env: dict, progress) -> List[Task]: def print_progress(tasks): progress(tasks) return tasks + user = env.get('user', 'pi') tasks = [] - tasks += print_progress(_create_service('amplipi-display', _display_service(env['base_dir']), env)) + tasks += print_progress(_create_service('amplipi-display', _display_service(env['base_dir'], user), env)) tasks += print_progress(_enable_service('amplipi-display')) if not env['is_ci']: tasks += print_progress(_restart_service('amplipi-display')) - if env['is_amplipi']: - # start the user manager at boot, instead of after first login - # this is needed so the user systemd services start at boot - tasks += print_progress(_enable_linger(env['user'], env)) return tasks @@ -1057,15 +1174,13 @@ def print_progress(tasks): return tasks if not env['is_amplipi'] and not env['is_ci']: return [Task(name='Update Audio Detector', output='Not on AmpliPi', success=False)] + user = env.get('user', 'pi') tasks = [] tasks += print_progress([Task('Build audiodetector', f'make -C {env["base_dir"]}/amplipi/audiodetector'.split()).run()]) - tasks += print_progress(_create_service('amplipi-audiodetector', _audiodetector_service(env['base_dir'], env['config_dir']), env)) + tasks += print_progress(_create_service('amplipi-audiodetector', _audiodetector_service(env['base_dir'], env['config_dir'], user), env)) tasks += print_progress(_enable_service('amplipi-audiodetector')) if not env['is_ci']: tasks += print_progress(_restart_service('amplipi-audiodetector')) - # start the user manager at boot, instead of after first login - # this is needed so the user systemd services start at boot - tasks += print_progress(_enable_linger(env['user'], env)) return tasks @@ -1194,9 +1309,10 @@ def add_tests(env, progress) -> List[Task]: return tasks -def install(os_deps=True, python_deps=True, web=True, restart_updater=False, +def install(os_deps=True, python_deps=True, custom_deps=True, web=True, restart_updater=False, display=True, audiodetector=True, firmware=True, password=True, - progress=print_task_results, development=False, ci_mode=False) -> bool: + progress=print_task_results, development=False, ci_mode=False, with_alsa=False, + dep_filter: List[str] = []) -> bool: """ Install and configure AmpliPi's dependencies """ # pylint: disable=too-many-return-statements tasks = [Task('setup')] @@ -1209,14 +1325,15 @@ def failed(): return True return False - # Find the version number line, break off the version= portion, then split on the decimals to separate major, middle, and minor revisions - version = re.search(r'version=(\d+\.\d+\.\d+)', str(_check_version('http://0.0.0.0/api').output)).group(1).split(".") - # Example output: - # using: http://0.0.0.0/api - # version=0.3.1 - if int(version[0]) == 0 and int(version[1]) < 4: # Is the version less than version 0.4.0? - print("Your version is too old to update automatically, please update manually using this guide: https://github.com/micro-nova/AmpliPi/blob/main/docs/imaging_etcher.md") - return False + if not development: + # Find the version number line, break off the version= portion, then split on the decimals to separate major, middle, and minor revisions + version = re.search(r'version=(\d+\.\d+\.\d+)', str(_check_version('http://0.0.0.0/api').output)).group(1).split(".") + # Example output: + # using: http://0.0.0.0/api + # version=0.3.1 + if int(version[0]) == 0 and int(version[1]) < 4: # Is the version less than version 0.4.0? + print("Your version is too old to update automatically, please update manually using this guide: https://github.com/micro-nova/AmpliPi/blob/main/docs/imaging_etcher.md") + return False env = _check_and_setup_platform(development, ci_mode) if not env['platform_supported'] and not development: @@ -1237,7 +1354,7 @@ def failed(): if failed(): return False if os_deps: - tasks += _install_os_deps(env, progress, _os_deps) + tasks += _install_os_deps(env, progress, with_alsa, _os_deps, dep_filter, development) if failed(): print('OS dependency install step failed, exiting...') return False @@ -1251,6 +1368,17 @@ def failed(): if failed(): print('Python dependency install step failed, exiting...') return False + if custom_deps: + custom_deps_dir = "/data/update_scripts" + os.makedirs(custom_deps_dir, exist_ok=True) + custom_deps = os.listdir(custom_deps_dir) + for dep in custom_deps: + if dep != "README.md": + custom_tasks = _install_custom_deps(dep) + progress(custom_tasks) + tasks += custom_tasks + if failed(): + print(f'Custom dependency {dep} failed to install') if web: tasks += _update_web(env, restart_updater, progress) if failed(): @@ -1304,13 +1432,17 @@ def failed(): help='Install python dependencies (using venv)') parser.add_argument('--os-deps', action='store_true', default=False, help='Install os dependencies using apt') + parser.add_argument('--dep-filter', action='append', default=[], + help='Define what OS deps to install (useful for testing new install flows efficiently)') + parser.add_argument('--custom-deps', action='store_true', default=False, + help='Install custom dependencies from /data/update_scripts') parser.add_argument('--web', '--webserver', action='store_true', default=False, help="Install and configure webserver") parser.add_argument('--restart-updater', '--reboot', action='store_true', default=False, help="""Restart AmpliPis OS, rebooting all of Ampli's services \ Only do this if you are running this from the command line. \ When this is set False system will need to be restarted to complete an update""") - # --restart-updater is needed by the web updater and hasn't been changed to --reboot to simplify updgrade/downgrade logic + # --restart-updater is needed by the web updater and hasn't been changed to --reboot to simplify upgrade/downgrade logic parser.add_argument('--display', action='store_true', default=False, help="Install and run the front-panel display service") parser.add_argument('--audiodetector', action='store_true', default=False, @@ -1323,6 +1455,8 @@ def failed(): help="Enable development mode.") parser.add_argument('--ci-mode', action='store_true', default=False, help="Enable CI mode, for automated builds. This mode doesn't attempt to start or check services.") + parser.add_argument('--with-alsa', action='store_true', default=False, + help="Configure alsa config. Automatically set to True if the device being configured has hostname 'amplipi'") flags = parser.parse_args() print('Configuring AmpliPi installation') has_args = flags.python_deps or flags.os_deps or flags.web or flags.restart_updater or flags.display or flags.firmware @@ -1330,10 +1464,10 @@ def failed(): print(' WARNING: expected some arguments, check --help for more information') if sys.version_info.major < 3 or sys.version_info.minor < 7: print(' WARNING: minimum python version is 3.7') - result = install(os_deps=flags.os_deps, python_deps=flags.python_deps, web=flags.web, - display=flags.display, audiodetector=flags.audiodetector, + result = install(os_deps=flags.os_deps, python_deps=flags.python_deps, custom_deps=flags.custom_deps, + web=flags.web, display=flags.display, audiodetector=flags.audiodetector, firmware=flags.firmware, password=flags.password, restart_updater=flags.restart_updater, development=flags.development, - ci_mode=flags.ci_mode) + ci_mode=flags.ci_mode, with_alsa=flags.with_alsa, dep_filter=flags.dep_filter) if not result: sys.exit(1) diff --git a/scripts/deploy b/scripts/deploy index 5d60fbefb..3834780ae 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -11,18 +11,36 @@ HELP="Install/Update AmpliPi software on a remote system defined by USER@HOST (d --fw: program the latest preamp firmware --pw: generate and set a new random password - --no-deps: skip installing os-deps and python-deps on AmpliPi" + --dev: skip a few version checks and increase verbosity + --alsa: configure alsa settings + --no-deps: skip installing os-deps, python-deps, and custom-deps on AmpliPi + --no-web: Skip compiling the web directory. May result in failures or odd behavior if you haven't compiled recently, but saves a lot of time if you have compiled recently." user_host='pi@amplipi.local' user_host_set=false fw=false pw=false +dev=false +alsa=false deps=true +dep_filter=() +web=true while [[ "$#" -gt 0 ]]; do case $1 in --fw) fw=true ;; --pw) pw=true ;; + --alsa) alsa=true ;; + --dev) dev=true ;; --no-deps) deps=false ;; + --dep-filter) + if [[ -z "$2" || "$2" =~ ^-- ]]; then + echo "Error: --dep-filter requires a value" + exit 1 + fi + dep_filter+=("$2") + shift + ;; + --no-web) web=false ;; -h|--help) echo -e "$HELP"; exit 0 ;; *) if ! $user_host_set; then user_host=$1 @@ -115,18 +133,37 @@ if ! which git-lfs; then echo "git-lfs needs to be installed to use this script" fi -# create a virtual environment and install pip dependencies -if [[ ! -d ../venv ]] || [[ ! -e ../venv/bin/activate ]] && [[ ! -e ../venv/Scripts/activate ]]; then - echo "" +if ! command uv --version >/dev/null 2>&1; then + echo "" + echo "uv not found, installing..." + + curl -LsSf https://astral.sh/uv/install.sh | sh + + export PATH="$HOME/.local/bin:$PATH" +fi + +if [[ ! -d ../venv ]] || [[ ! -e ../venv/bin/activate && ! -e ../venv/Scripts/activate ]]; then + echo "" echo "Setting up virtual environment" + mkdir -p ../venv - $python -m venv ../venv + + if command -v uv >/dev/null 2>&1; then + echo "Using uv to create Python 3.8 venv" + uv venv venv --python 3.8 + else + echo "Falling back to python venv" + $python -m venv ../venv + fi fi + +# activate venv (cross-platform) if [[ -e ../venv/Scripts/activate ]]; then source ../venv/Scripts/activate else source ../venv/bin/activate fi + $python -m pip install --upgrade pip $python -m pip install poetry echo -e "Finished checking dependencies\n" @@ -156,12 +193,14 @@ else fi poetry version ${git_info} -echo "Building web app" -pushd ../web # Change to web directory -npm install # Install nodejs dependencies -echo "VITE_BACKEND_VERSION=$(poetry version -s)" > .env # Collect the version number and bake it into the frontend -npm run build # Build the web app -popd +if $web; then + echo "Building web app" + pushd ../web # Change to web directory + npm install # Install nodejs dependencies + echo "VITE_BACKEND_VERSION=$(poetry version -s)" > .env # Collect the version number and bake it into the frontend + npm run build # Build the web app + popd +fi # pull in the latest binaries git lfs checkout @@ -202,7 +241,12 @@ ssh $user_host "chmod +x amplipi-dev/scripts/configure.py" opts="" $fw && opts="$opts --firmware" $pw && opts="$opts --password" -$deps && opts="$opts --os-deps --python-deps" +$alsa && opts="$opts --with-alsa" +$deps && opts="$opts --os-deps --python-deps --custom-deps" +$dev && opts="$opts --development" +if [[ ${#dep_filter[@]} -gt 0 ]]; then + opts="$opts ${dep_filter[@]/#/--dep-filter }" +fi ssh $user_host -t "python3 amplipi-dev/scripts/configure.py --web --restart-updater --display --audiodetector$opts" || echo "" printf "Waiting for AmpliPi to restart" diff --git a/scripts/install_python_deps.bash b/scripts/install_python_deps.bash index b249d5c24..6a95d043e 100755 --- a/scripts/install_python_deps.bash +++ b/scripts/install_python_deps.bash @@ -3,8 +3,21 @@ set -e cd "$( dirname "$0" )"/.. -python3 -m venv venv # TODO --clear removes all packages, make this a flag to this script -. venv/bin/activate -pip3 install --upgrade pip wheel # Avoid errors about using legacy 'setup.py install' -pip3 install -r requirements.txt -deactivate + +VENV=/home/pi/amplipi-dev/venv + +export PATH="$HOME/.local/bin:$PATH" + +if [[ ! -d $VENV ]] || [[ ! -e $VENV/bin/python ]]; then + echo "" + echo "Setting up virtual environment" + if ! command -v uv &>/dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh + fi + uv venv $VENV --python 3.8 +fi + +# uv pip bypasses PEP 668 (Trixie blocks system pip) and doesn't require pip in the venv +uv pip install --python $VENV/bin/python -r requirements.txt + +echo "install python deps complete!" diff --git a/scripts/set_pass b/scripts/set_pass index 31fccda68..98a5f9143 100755 --- a/scripts/set_pass +++ b/scripts/set_pass @@ -21,7 +21,7 @@ # https://www.raspberrypi.com/documentation/computers/configuration.html#using-key-based-authentication set -e -confdir="$HOME/.config/amplipi" +confdir="/data/.config/amplipi" # First verify the current user is 'pi' #if [[ $USER != pi ]]; then diff --git a/scripts/update_autoboot.py b/scripts/update_autoboot.py new file mode 100644 index 000000000..cad7cf028 --- /dev/null +++ b/scripts/update_autoboot.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""Patch autoboot.txt boot_partition values after a successful OTA commit.""" + +import sys +import re + +path, new_default, new_tryboot = sys.argv[1], sys.argv[2], sys.argv[3] +content = open(path).read() + +content = re.sub( + r'(\[all\].*?boot_partition=)\d+', + lambda m: m.group(1) + new_default, + content, flags=re.DOTALL +) +content = re.sub( + r'(\[tryboot\].*?boot_partition=)\d+', + lambda m: m.group(1) + new_tryboot, + content, flags=re.DOTALL +) + +open(path, 'w').write(content) diff --git a/streams/lms_metadata.py b/streams/lms_metadata.py index 2b65f0f0d..544d69b1e 100644 --- a/streams/lms_metadata.py +++ b/streams/lms_metadata.py @@ -56,7 +56,7 @@ class LMSMetadataReader: def __init__(self, name: str, vsrc: int, server: Optional[str] = None, port: Optional[int] = 9000, meta_ref: Optional[int] = 2): self.player_name = name self.folder = os.path.join( - os.path.expanduser('~'), '.config', 'amplipi', 'srcs', f'v{vsrc}' + '/data', '.config', 'amplipi', 'srcs', f'v{vsrc}' ) self.server = server self.port = port @@ -229,7 +229,10 @@ def connect(self): self.meta.track = track_data.get('title') or "" if song_data.get('artwork_url'): - self.meta.image_url = song_data.get('artwork_url') + artwork_url = song_data.get('artwork_url') + if artwork_url.startswith('/'): + artwork_url = f"http://{self.server}:{self.port}{artwork_url}" + self.meta.image_url = artwork_url elif song_data.get('coverid'): self.meta.image_url = f"http://{self.server}:{self.port}/music/{song_data['coverid']}/cover.jpg?id={song_data['coverid']}" else: diff --git a/streams/pianobar b/streams/pianobar index edac4069a..cc4502058 100755 Binary files a/streams/pianobar and b/streams/pianobar differ diff --git a/tests/meta_test.bash b/tests/meta_test.bash index c35190e8d..02272cf56 100755 --- a/tests/meta_test.bash +++ b/tests/meta_test.bash @@ -2,24 +2,24 @@ # Test different chunks of the sp_meta.py script. Be sure to include src arg echo -e "Running first test. If I get stuck on any test, terminate the test with ^C\n" -cat /home/pi/.config/amplipi/python/tests/sample_info.txt | /home/pi/python/sp_meta.py $1 +cat /data/.config/amplipi/python/tests/sample_info.txt | /home/pi/python/sp_meta.py $1 echo -e "First test complete. Showing file differences from known good -\n" -diff /home/pi/.config/amplipi/srcs/$1/currentSong /home/pi/python/tests/test_metadata_script/kg_currentSong --color=always +diff /data/.config/amplipi/srcs/$1/currentSong /home/pi/python/tests/test_metadata_script/kg_currentSong --color=always echo -e "Running second test...\n" -cat /home/pi/.config/amplipi/python/tests/sample_sinfo.txt | /home/pi/python/sp_meta.py $1 +cat /data/.config/amplipi/python/tests/sample_sinfo.txt | /home/pi/python/sp_meta.py $1 echo -e "Second test complete. Showing file differences from known good -\n" -diff /home/pi/.config/amplipi/srcs/$1/sourceInfo /home/pi/python/tests/test_metadata_script/kg_sourceInfo --color=always +diff /data/.config/amplipi/srcs/$1/sourceInfo /home/pi/python/tests/test_metadata_script/kg_sourceInfo --color=always echo -e "Running third test...\n" -cat /home/pi/.config/amplipi/python/tests/sample_info2.txt | /home/pi/python/sp_meta.py $1 +cat /data/.config/amplipi/python/tests/sample_info2.txt | /home/pi/python/sp_meta.py $1 echo -e "Third test complete. Showing file differences from known good -\n" -diff /home/pi/.config/amplipi/srcs/$1/currentSong /home/pi/python/tests/test_metadata_script/kg_currentSong2 --color=always +diff /data/.config/amplipi/srcs/$1/currentSong /home/pi/python/tests/test_metadata_script/kg_currentSong2 --color=always echo -e "Running fourth test...\n" -cat /home/pi/.config/amplipi/python/tests/sample_info3.txt | /home/pi/python/sp_meta.py $1 +cat /data/.config/amplipi/python/tests/sample_info3.txt | /home/pi/python/sp_meta.py $1 echo -e "Fourth test complete. Showing file differences from known good -\n" -diff /home/pi/.config/amplipi/srcs/$1/currentSong /home/pi/python/tests/test_metadata_script/kg_currentSong3 --color=always +diff /data/.config/amplipi/srcs/$1/currentSong /home/pi/python/tests/test_metadata_script/kg_currentSong3 --color=always echo All tests complete. diff --git a/tests/test_rest.py b/tests/test_rest.py index 2bfcaef3b..2c9a10463 100644 --- a/tests/test_rest.py +++ b/tests/test_rest.py @@ -607,8 +607,29 @@ def test_patch_zones_vol_delta(client): if z['id'] in range(6): assert z['vol_f'] - (zones[z['id']]['vol_f'] + 0.1) < 0.0001 and z["vol_f_overflow"] == 0 - # test overflowing deltas - rv = client.patch('/api/zones', json={'zones': [z['id'] for z in zones], 'update': {'vol_delta_f': -1.0}}) + +def test_patch_zones_vol_delta_overflow(client): + """ Try changing multiple zones volume using volume delta with an overflowing vol_f """ + zones = [z for z in base_config()['zones']] + # set each zone to a random initial volume + for z in zones: + z['vol_f'] = random.uniform(0.0, 0.9) # random initial volume + rv = client.patch('/api/zones/{}'.format(z['id']), json={'vol_f': z['vol_f']}) + assert rv.status_code == HTTPStatus.OK + # update each zones volume by 100% + rv = client.patch('/api/zones', json={'zones': [z['id'] for z in zones], 'update': {'vol_delta_f': 1.0}}) + assert rv.status_code == HTTPStatus.OK + jrv = rv.json() + assert len(jrv['zones']) >= 6 + # check that each update worked as expected + for z in jrv['zones']: + if z['id'] in range(6): + assert z['vol_f'] == amplipi.models.MIN_VOL_F + assert z["vol_f_overflow"] == zones[z['id']]['vol_f'] + 0.1 - 1 + + # test oversized deltas + rv = client.patch('/api/zones', json={'zones': [z['id'] for z in zones], 'update': {'vol_delta_f': -10.0}}) + assert rv.status_code == HTTPStatus.OK jrv = rv.json() assert len(jrv['zones']) >= 6