diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index ab962c9..370eb61 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -19,7 +19,7 @@ start_date_option, timeout_option, pass_client, - pass_session + pass_session, ) from ..downloader import Downloader as NewDownloader, Status from ..exceptions import ( @@ -27,7 +27,7 @@ DirectoryDoesNotExists, DownloadUrlExpired, NotDownloadableAsAAX, - VoucherNeedRefresh + VoucherNeedRefresh, ) from ..models import Library from ..utils import datetime_type, Downloader @@ -35,9 +35,7 @@ logger = logging.getLogger("audible_cli.cmds.cmd_download") -CLIENT_HEADERS = { - "User-Agent": "Audible/671 CFNetwork/1240.0.4 Darwin/20.6.0" -} +CLIENT_HEADERS = {"User-Agent": "Audible/671 CFNetwork/1240.0.4 Darwin/20.6.0"} QUEUE = None @@ -54,6 +52,7 @@ def __init__(self): self._voucher_saved: int = 0 self._aycl = 0 self._aycl_voucher = 0 + self._errors: int = 0 @property def aax(self): @@ -137,6 +136,14 @@ def count_voucher_saved(self): self._voucher_saved += 1 logger.debug(f"Currently saved voucher files: {self.voucher_saved}") + @property + def errors(self): + return self._errors + + def count_error(self): + self._errors += 1 + logger.debug(f"Currently encountered errors: {self.errors}") + def as_dict(self) -> dict: return { "aax": self.aax, @@ -148,7 +155,8 @@ def as_dict(self) -> dict: "voucher": self.voucher, "voucher_saved": self.voucher_saved, "aycl": self.aycl, - "aycl_voucher": self.aycl_voucher + "aycl_voucher": self.aycl_voucher, + "errors": self.errors, } def has_downloads(self): @@ -163,16 +171,15 @@ def has_downloads(self): async def download_cover( - client, output_dir, base_filename, item, res, overwrite_existing + client, output_dir, base_filename, item, res, overwrite_existing ): filename = f"{base_filename}_({str(res)}).jpg" filepath = output_dir / filename url = item.get_cover_url(res) if url is None: - logger.error( - f"No COVER with size {res} found for {item.full_title}" - ) + logger.error(f"No COVER with size {res} found for {item.full_title}") + counter.count_error() return dl = Downloader(url, filepath, client, overwrite_existing, "image/jpeg") @@ -182,9 +189,7 @@ async def download_cover( counter.count_cover() -async def download_pdf( - client, output_dir, base_filename, item, overwrite_existing -): +async def download_pdf(client, output_dir, base_filename, item, overwrite_existing): url = item.get_pdf_url() if url is None: logger.info(f"No PDF found for {item.full_title}") @@ -193,8 +198,11 @@ async def download_pdf( filename = base_filename + ".pdf" filepath = output_dir / filename dl = Downloader( - url, filepath, client, overwrite_existing, - ["application/octet-stream", "application/pdf"] + url, + filepath, + client, + overwrite_existing, + ["application/octet-stream", "application/pdf"], ) downloaded = await dl.run(stream=False, pb=False) @@ -203,7 +211,7 @@ async def download_pdf( async def download_chapters( - output_dir, base_filename, item, quality, overwrite_existing, chapter_type + output_dir, base_filename, item, quality, overwrite_existing, chapter_type ): if not output_dir.is_dir(): raise DirectoryDoesNotExists(output_dir) @@ -211,17 +219,13 @@ async def download_chapters( filename = base_filename + "-chapters.json" file = output_dir / filename if file.exists() and not overwrite_existing: - logger.info( - f"File {file} already exists. Skip saving chapters" - ) + logger.info(f"File {file} already exists. Skip saving chapters") return True try: metadata = await item.get_content_metadata(quality, chapter_type=chapter_type) except NotFoundError: - logger.info( - f"No chapters found for {item.full_title}." - ) + logger.info(f"No chapters found for {item.full_title}.") return metadata = json.dumps(metadata, indent=4) async with aiofiles.open(file, "w") as f: @@ -230,26 +234,20 @@ async def download_chapters( counter.count_chapter() -async def download_annotations( - output_dir, base_filename, item, overwrite_existing -): +async def download_annotations(output_dir, base_filename, item, overwrite_existing): if not output_dir.is_dir(): raise DirectoryDoesNotExists(output_dir) filename = base_filename + "-annotations.json" file = output_dir / filename if file.exists() and not overwrite_existing: - logger.info( - f"File {file} already exists. Skip saving annotations" - ) + logger.info(f"File {file} already exists. Skip saving annotations") return True try: annotation = await item.get_annotations() except NotFoundError: - logger.info( - f"No annotations found for {item.full_title}." - ) + logger.info(f"No annotations found for {item.full_title}.") return annotation = json.dumps(annotation, indent=4) async with aiofiles.open(file, "w") as f: @@ -273,8 +271,15 @@ async def _get_audioparts(item): async def _add_audioparts_to_queue( - client, output_dir, filename_mode, filename_length, item, quality, overwrite_existing, - aax_fallback, download_mode + client, + output_dir, + filename_mode, + filename_length, + item, + quality, + overwrite_existing, + aax_fallback, + download_mode, ): parts = await _get_audioparts(item) @@ -302,13 +307,20 @@ async def _add_audioparts_to_queue( cover_sizes=None, quality=quality, overwrite_existing=overwrite_existing, - aax_fallback=aax_fallback + aax_fallback=aax_fallback, ) async def download_aax( - client, output_dir, base_filename, item, quality, overwrite_existing, - aax_fallback, filename_mode, filename_length + client, + output_dir, + base_filename, + item, + quality, + overwrite_existing, + aax_fallback, + filename_mode, + filename_length, ): # url, codec = await item.get_aax_url(quality) try: @@ -324,7 +336,7 @@ async def download_aax( quality=quality, overwrite_existing=overwrite_existing, filename_mode=filename_mode, - filename_length=filename_length + filename_length=filename_length, ) raise @@ -334,9 +346,7 @@ async def download_aax( dl = NewDownloader( source=url, client=client, - expected_types=[ - "audio/aax", "audio/vnd.audible.aax", "audio/audible" - ] + expected_types=["audio/aax", "audio/vnd.audible.aax", "audio/audible"], ) downloaded = await dl.run(target=filepath, force_reload=overwrite_existing) @@ -410,8 +420,14 @@ async def _reuse_voucher(lr_file, item): async def download_aaxc( - client, output_dir, base_filename, item, quality, overwrite_existing, - filename_mode, filename_length + client, + output_dir, + base_filename, + item, + quality, + overwrite_existing, + filename_mode, + filename_length, ): lr, url, codec = None, None, None @@ -419,27 +435,26 @@ async def download_aaxc( if not overwrite_existing: codec, _ = item._get_codec(quality) if codec is not None: - filepath = pathlib.Path( - output_dir) / f"{base_filename}-{codec}.aaxc" + filepath = pathlib.Path(output_dir) / f"{base_filename}-{codec}.aaxc" lr_file = filepath.with_suffix(".voucher") if lr_file.is_file(): if filepath.is_file(): - logger.info( - f"File {lr_file} already exists. Skip download." - ) - logger.info( - f"File {filepath} already exists. Skip download." - ) + logger.info(f"File {lr_file} already exists. Skip download.") + logger.info(f"File {filepath} already exists. Skip download.") return try: lr, url, codec = await _reuse_voucher(lr_file, item) except DownloadUrlExpired: - logger.debug(f"Download url in {lr_file} is expired. Refreshing license.") + logger.debug( + f"Download url in {lr_file} is expired. Refreshing license." + ) overwrite_existing = True except VoucherNeedRefresh: - logger.debug(f"Refresh date for voucher {lr_file} reached. Refreshing license.") + logger.debug( + f"Refresh date for voucher {lr_file} reached. Refreshing license." + ) overwrite_existing = True is_aycl = item.benefit_id == "AYCL" @@ -455,14 +470,11 @@ async def download_aaxc( else: ext = "aaxc" - filepath = pathlib.Path( - output_dir) / f"{base_filename}-{codec}.{ext}" + filepath = pathlib.Path(output_dir) / f"{base_filename}-{codec}.{ext}" lr_file = filepath.with_suffix(".voucher") if lr_file.is_file() and not overwrite_existing: - logger.info( - f"File {lr_file} already exists. Skip download." - ) + logger.info(f"File {lr_file} already exists. Skip download.") else: lr = json.dumps(lr, indent=4) async with aiofiles.open(lr_file, "w") as f: @@ -474,8 +486,11 @@ async def download_aaxc( source=url, client=client, expected_types=[ - "audio/aax", "audio/vnd.audible.aax", "audio/mpeg", "audio/x-m4a", - "audio/audible" + "audio/aax", + "audio/vnd.audible.aax", + "audio/mpeg", + "audio/x-m4a", + "audio/audible", ], ) downloaded = await dl.run(target=filepath, force_reload=overwrite_existing) @@ -497,7 +512,7 @@ async def download_aaxc( quality=quality, overwrite_existing=overwrite_existing, aax_fallback=False, - download_mode="aaxc" + download_mode="aaxc", ) @@ -508,6 +523,7 @@ async def consume(ignore_errors): await cmd(**kwargs) except Exception as e: logger.error(e) + counter.count_error() if not ignore_errors: raise finally: @@ -515,22 +531,22 @@ async def consume(ignore_errors): def queue_job( - get_cover, - get_pdf, - get_annotation, - get_chapters, - get_aax, - get_aaxc, - client, - output_dir, - filename_mode, - filename_length, - item, - cover_sizes, - chapter_type, - quality, - overwrite_existing, - aax_fallback + get_cover, + get_pdf, + get_annotation, + get_chapters, + get_aax, + get_aaxc, + client, + output_dir, + filename_mode, + filename_length, + item, + cover_sizes, + chapter_type, + quality, + overwrite_existing, + aax_fallback, ): base_filename = item.create_base_filename(filename_mode, filename_length) @@ -543,7 +559,7 @@ def queue_job( "base_filename": base_filename, "item": item, "res": cover_size, - "overwrite_existing": overwrite_existing + "overwrite_existing": overwrite_existing, } QUEUE.put_nowait((cmd, kwargs)) @@ -554,7 +570,7 @@ def queue_job( "output_dir": output_dir, "base_filename": base_filename, "item": item, - "overwrite_existing": overwrite_existing + "overwrite_existing": overwrite_existing, } QUEUE.put_nowait((cmd, kwargs)) @@ -566,7 +582,7 @@ def queue_job( "item": item, "quality": quality, "overwrite_existing": overwrite_existing, - "chapter_type": chapter_type + "chapter_type": chapter_type, } QUEUE.put_nowait((cmd, kwargs)) @@ -576,7 +592,7 @@ def queue_job( "output_dir": output_dir, "base_filename": base_filename, "item": item, - "overwrite_existing": overwrite_existing + "overwrite_existing": overwrite_existing, } QUEUE.put_nowait((cmd, kwargs)) @@ -591,7 +607,7 @@ def queue_job( "overwrite_existing": overwrite_existing, "aax_fallback": aax_fallback, "filename_mode": filename_mode, - "filename_length": filename_length + "filename_length": filename_length, } QUEUE.put_nowait((cmd, kwargs)) @@ -605,7 +621,7 @@ def queue_job( "quality": quality, "overwrite_existing": overwrite_existing, "filename_mode": filename_mode, - "filename_length": filename_length + "filename_length": filename_length, } QUEUE.put_nowait((cmd, kwargs)) @@ -626,6 +642,9 @@ def display_counter(): if diff > 0: echo(f"Unsaved voucher: {diff}") continue + elif k == "errors": + # Errors are already logged, skip displaying in counter + continue echo(f"New {k} files: {v}") else: echo("No new files downloaded.") @@ -633,113 +652,88 @@ def display_counter(): @click.command("download") @click.option( - "--output-dir", "-o", + "--output-dir", + "-o", type=click.Path(exists=True, dir_okay=True), default=pathlib.Path().cwd(), - help="output dir, uses current working dir as default" + help="output dir, uses current working dir as default", ) @click.option( "--all", is_flag=True, - help="download all library items, overrides --asin and --title options" -) -@click.option( - "--asin", "-a", - multiple=True, - help="asin of the audiobook" -) -@click.option( - "--title", "-t", - multiple=True, - help="title of the audiobook (partial search)" + help="download all library items, overrides --asin and --title options", ) +@click.option("--asin", "-a", multiple=True, help="asin of the audiobook") @click.option( - "--aax", - is_flag=True, - help="Download book in aax format" + "--title", "-t", multiple=True, help="title of the audiobook (partial search)" ) +@click.option("--aax", is_flag=True, help="Download book in aax format") @click.option( - "--aaxc", - is_flag=True, - help="Download book in aaxc format incl. voucher file" + "--aaxc", is_flag=True, help="Download book in aaxc format incl. voucher file" ) @click.option( "--aax-fallback", is_flag=True, - help="Download book in aax format and fallback to aaxc, if former is not supported." + help="Download book in aax format and fallback to aaxc, if former is not supported.", ) @click.option( - "--quality", "-q", + "--quality", + "-q", default="best", show_default=True, type=click.Choice(["best", "high", "normal"]), - help="download quality" + help="download quality", ) @click.option( - "--pdf", - is_flag=True, - help="downloads the pdf in addition to the audiobook" + "--pdf", is_flag=True, help="downloads the pdf in addition to the audiobook" ) @click.option( - "--cover", - is_flag=True, - help="downloads the cover in addition to the audiobook" + "--cover", is_flag=True, help="downloads the cover in addition to the audiobook" ) @click.option( "--cover-size", - type=click.Choice(["252", "315", "360", "408", "500", "558", "570", "882", - "900", "1215"]), + type=click.Choice( + ["252", "315", "360", "408", "500", "558", "570", "882", "900", "1215"] + ), default=["500"], multiple=True, - help="The cover pixel size. This option can be provided multiple times." -) -@click.option( - "--chapter", - is_flag=True, - help="Saves chapter metadata as JSON file." + help="The cover pixel size. This option can be provided multiple times.", ) +@click.option("--chapter", is_flag=True, help="Saves chapter metadata as JSON file.") @click.option( "--chapter-type", default="config", type=click.Choice(["Flat", "Tree", "config"], case_sensitive=False), - help="The chapter type." + help="The chapter type.", ) @click.option( "--annotation", is_flag=True, - help="saves the annotations (e.g. bookmarks, notes) as JSON file" + help="saves the annotations (e.g. bookmarks, notes) as JSON file", ) @start_date_option @end_date_option +@click.option("--no-confirm", "-y", is_flag=True, help="start without confirm") +@click.option("--overwrite", is_flag=True, help="rename existing files") @click.option( - "--no-confirm", "-y", - is_flag=True, - help="start without confirm" + "--ignore-errors", is_flag=True, help="ignore errors and continue with the rest" ) @click.option( - "--overwrite", - is_flag=True, - help="rename existing files" -) -@click.option( - "--ignore-errors", - is_flag=True, - help="ignore errors and continue with the rest" -) -@click.option( - "--jobs", "-j", + "--jobs", + "-j", type=int, default=3, show_default=True, - help="number of simultaneous downloads" + help="number of simultaneous downloads", ) @click.option( - "--filename-mode", "-f", + "--filename-mode", + "-f", type=click.Choice( ["config", "ascii", "asin_ascii", "unicode", "asin_unicode", "asin_only"] ), default="config", - help="Filename mode to use. [default: config]" + help="Filename mode to use. [default: config]", ) @click.option( "--filename-length", @@ -752,12 +746,10 @@ def display_counter(): @click.option( "--resolve-podcasts", is_flag=True, - help="Resolve podcasts to download a single episode via asin or title" + help="Resolve podcasts to download a single episode via asin or title", ) @click.option( - "--ignore-podcasts", - is_flag=True, - help="Ignore a podcast if it has episodes" + "--ignore-podcasts", is_flag=True, help="Ignore a podcast if it has episodes" ) @bunch_size_option @pass_session @@ -773,8 +765,7 @@ async def cli(session, api_client, **params): titles = params.get("title") if get_all and (asins or titles): raise click.BadOptionUsage( - "--all", - "`--all` can not be used together with `--asin` or `--title`" + "--all", "`--all` can not be used together with `--asin` or `--title`" ) # what to download @@ -788,14 +779,14 @@ async def cli(session, api_client, **params): ) get_aax = True if get_aaxc: - logger.warning("--aaxc is redundant with --aax-fallback (fallback already uses aaxc).") + logger.warning( + "--aaxc is redundant with --aax-fallback (fallback already uses aaxc)." + ) get_annotation = params.get("annotation") get_chapters = params.get("chapter") get_cover = params.get("cover") get_pdf = params.get("pdf") - if not any( - [get_aax, get_aaxc, get_annotation, get_chapters, get_cover, get_pdf] - ): + if not any([get_aax, get_aaxc, get_annotation, get_chapters, get_cover, get_pdf]): raise click.BadOptionUsage( "", "No download format specified. You must select at least one of:\n\n" @@ -808,7 +799,7 @@ async def cli(session, api_client, **params): " --cover Download cover image\n" " --chapter Download chapter metadata\n" " --annotation Download annotations/bookmarks\n\n" - "Example: audible download --aaxc -a " + "Example: audible download --aaxc -a ", ) # additional options @@ -825,7 +816,7 @@ async def cli(session, api_client, **params): "", "Cannot use --ignore-podcasts with --resolve-podcasts. " "Choose one: either resolve podcasts to individual episodes, " - "or ignore them entirely." + "or ignore them entirely.", ) bunch_size = session.params.get("bunch_size") @@ -833,8 +824,7 @@ async def cli(session, api_client, **params): end_date = session.params.get("end_date") if all([start_date, end_date]) and start_date > end_date: raise click.BadOptionUsage( - "", - "Start date must be before or equal to the end date." + "", "Start date must be before or equal to the end date." ) if start_date is not None: @@ -842,19 +832,21 @@ async def cli(session, api_client, **params): f"Selected start date: {start_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ')}" ) if end_date is not None: - logger.info( - f"Selected end date: {end_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ')}" - ) + logger.info(f"Selected end date: {end_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ')}") chapter_type = params.get("chapter_type") if chapter_type == "config": - chapter_type = session.config.get_profile_option( - session.selected_profile, "chapter_type") or "Tree" + chapter_type = ( + session.config.get_profile_option(session.selected_profile, "chapter_type") + or "Tree" + ) filename_mode = params.get("filename_mode") if filename_mode == "config": - filename_mode = session.config.get_profile_option( - session.selected_profile, "filename_mode") or "ascii" + filename_mode = ( + session.config.get_profile_option(session.selected_profile, "filename_mode") + or "ascii" + ) filename_length = params.get("filename_length") @@ -892,9 +884,8 @@ async def cli(session, api_client, **params): if not ignore_errors: logger.error(f"Asin {asin} not found in library.") raise click.Abort() - logger.error( - f"Skip asin {asin}: Not found in library" - ) + logger.error(f"Skip asin {asin}: Not found in library") + counter.count_error() for title in titles: match = library.search_item_by_title(title) @@ -913,15 +904,14 @@ async def cli(session, api_client, **params): answer = await questionary.checkbox( f"Found the following matches for '{title}'. Which you want to download?", - choices=choices + choices=choices, ).unsafe_ask_async() if answer is not None: [jobs.append(i) for i in answer] else: - logger.error( - f"Skip title {title}: Not found in library" - ) + logger.error(f"Skip title {title}: Not found in library") + counter.count_error() # set queue global QUEUE @@ -938,9 +928,7 @@ async def cli(session, api_client, **params): items.remove(item) if item._children is None: - await item.get_child_items( - start_date=start_date, end_date=end_date - ) + await item.get_child_items(start_date=start_date, end_date=end_date) for i in item._children: if i.asin not in jobs: @@ -968,13 +956,11 @@ async def cli(session, api_client, **params): chapter_type=chapter_type, quality=quality, overwrite_existing=overwrite_existing, - aax_fallback=aax_fallback + aax_fallback=aax_fallback, ) # schedule the consumer - consumers = [ - asyncio.create_task(consume(ignore_errors)) for _ in range(sim_jobs) - ] + consumers = [asyncio.create_task(consume(ignore_errors)) for _ in range(sim_jobs)] try: # wait until the consumer has processed all items await QUEUE.join() @@ -985,3 +971,7 @@ async def cli(session, api_client, **params): await asyncio.gather(*consumers, return_exceptions=True) display_counter() + + # Return non-zero exit code if any errors occurred + if counter.errors > 0: + return 1