v2.1.7: 修复永久网域的重定向bug，重构api提高可扩展性 (#85)

Author: hect0x7

src/jmcomic/__init__.py

@@ -1,7 +1,9 @@
 # 模块依赖关系如下:
 # 被依赖方 <--- 使用方
-# config <--- entity <--- toolkit <--- client <--- option
+# config <--- entity <--- toolkit <--- client <--- option <--- downloader
 
-__version__ = '2.1.6'
+__version__ = '2.1.7'
 
+# noinspection PyUnresolvedReferences
+from common import *
 from .api import *

src/jmcomic/api.py

@@ -1,129 +1,56 @@
-from .jm_option import *
-
-
-def download_album(jm_album_id, option=None):
-    """
-    下载一个本子集，入口api
-    @param jm_album_id: 禁漫的本子的id，类型可以是str/int/iterable[str]。
-    如果是iterable[str]，则会调用批量下载方法 download_album_batch
-    @param option: 下载选项，为空默认是 JmOption.default()
-    """
-
-    if not isinstance(jm_album_id, (str, int)):
-        return download_album_batch(jm_album_id, option)
-
-    option, jm_client = build_client(option)
-    album: JmAlbumDetail = jm_client.get_album_detail(jm_album_id)
-
-    option.before_album(album)
-    execute_by_condition(
-        iter_objs=album,
-        apply=lambda photo: download_by_photo_detail(photo, option),
-        count_batch=option.decide_photo_batch_count(album)
-    )
-    option.after_album(album)
-
-
-def download_photo(jm_photo_id, option=None):
-    """
-    下载一个本子的一章，入口api
-    """
-    option, jm_client = build_client(option)
-    photo = jm_client.get_photo_detail(jm_photo_id)
-    download_by_photo_detail(photo, option)
-
-
-def download_by_photo_detail(photo: JmPhotoDetail, option=None):
-    """
-    下载一个本子的一章，根据 photo
-    @param photo: 本子章节信息
-    @param option: 选项
-    """
-    option, jm_client = build_client(option)
-
-    # 下载准备
-    use_cache = option.download_cache
-    decode_image = option.download_image_decode
-    jm_client.check_photo(photo)
-
-    # 下载每个图片的函数
-    def download_image(image: JmImageDetail):
-        img_save_path = option.decide_image_filepath(image)
-        image.is_exists = file_exists(img_save_path)
-
-        option.before_image(image, img_save_path)
-        if use_cache is True and image.is_exists:
-            return
-        jm_client.download_by_image_detail(
-            image,
-            img_save_path,
-            decode_image=decode_image,
-        )
-        option.after_image(image, img_save_path)
-
-    option.before_photo(photo)
-    execute_by_condition(
-        iter_objs=photo,
-        apply=download_image,
-        count_batch=option.decide_image_batch_count(photo)
-    )
-    option.before_photo(photo)
+from .jm_downloader import *
 
 
 def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
                          option=None,
-                         wait_finish=True,
-                         ) -> List[Thread]:
+                         ):
     """
-    批量下载album，每个album一个线程，使用的是同一个option。
+    批量下载album.
+    一个album，对应一个线程，对应一个option
 
-    @param jm_album_id_iter: album_id的可迭代对象
+    @param jm_album_id_iter: album_id的迭代器
     @param option: 下载选项，为空默认是 JmOption.default()
-    @param wait_finish: 是否要等待这些下载线程全部完成
-    @return 返回值是List[Thread]，里面是每个下载漫画的线程。
     """
-    if option is None:
-        option = JmOption.default()
+    from common import multi_thread_launcher
 
-    return thread_pool_executor(
-        iter_objs=set(JmcomicText.parse_to_album_id(album_id) for album_id in jm_album_id_iter),
-        apply_each_obj_func=lambda album_id: download_album(album_id, option),
-        wait_finish=wait_finish,
+    return multi_thread_launcher(
+        iter_objs=set(
+            JmcomicText.parse_to_album_id(album_id)
+            for album_id in jm_album_id_iter
+        ),
+        apply_each_obj_func=lambda aid: download_album(aid, option),
     )
 
 
-def execute_by_condition(iter_objs, apply: Callable, count_batch: int):
+def download_album(jm_album_id, option=None):
     """
-    章节/图片的下载调度逻辑
+    下载一个本子
+    @param jm_album_id: 禁漫的本子的id，类型可以是str/int/iterable[str]。
+    如果是iterable[str]，则会调用 download_album_batch
+    @param option: 下载选项，为空默认是 JmOption.default()
     """
-    count_real = len(iter_objs)
 
-    if count_batch >= count_real:
-        # 一个图/章节 对应 一个线程
-        multi_thread_launcher(
-            iter_objs=iter_objs,
-            apply_each_obj_func=apply,
-        )
-    else:
-        # 创建batch个线程的线程池
-        thread_pool_executor(
-            iter_objs=iter_objs,
-            apply_each_obj_func=apply,
-            max_workers=count_batch,
-        )
+    if not isinstance(jm_album_id, (str, int)):
+        return download_album_batch(jm_album_id, option)
 
+    with new_downloader(option) as dler:
+        dler.download_album(jm_album_id)
 
-def build_client(option: Optional[JmOption]) -> Tuple[JmOption, JmcomicClient]:
+
+def download_photo(jm_photo_id, option=None):
     """
-    处理option的判空，并且创建jm_client
+    下载一个章节
     """
+    with new_downloader(option) as dler:
+        dler.download_photo(jm_photo_id)
+
+
+def new_downloader(option=None):
     if option is None:
-        option = JmOption.default()
+        option = JmModuleConfig.option_class().default()
 
-    jm_client = option.build_jm_client()
-    return option, jm_client
+    return JmModuleConfig.downloader_class()(option)
 
 
-def create_option(filepath: str) -> JmOption:
-    option = JmOption.from_file(filepath)
-    return option
+def create_option(filepath):
+    return JmModuleConfig.option_class().from_file(filepath)

src/jmcomic/jm_client_impl.py

@@ -85,7 +85,7 @@ def request_with_retry(self,
 
     # noinspection PyMethodMayBeStatic, PyUnusedLocal
     def before_retry(self, e, kwargs, retry_count, url):
-        jm_debug('error', str(e))
+        jm_debug('retry', str(e))
 
     def enable_cache(self, debug=False):
         def wrap_func_cache(func_name, cache_dict_name):
@@ -131,8 +131,11 @@ def get_jmcomic_url(self, postman=None):
     def get_jmcomic_domain_all(self, postman=None):
         return JmModuleConfig.get_jmcomic_domain_all(postman or self.get_root_postman())
 
+    # noinspection PyUnusedLocal
     def fallback(self, request, url, domain_index, retry_count, **kwargs):
-        raise AssertionError(f"请求重试全部失败: [{url}], {self.domain_list}")
+        msg = f"请求重试全部失败: [{url}], {self.domain_list}"
+        jm_debug('fallback', "msg")
+        raise AssertionError(msg)
 
 
 # 基于网页实现的JmClient
@@ -406,3 +409,15 @@ def headers_key_ts(self):
             "user-agent": "okhttp/3.12.1",
             "accept-encoding": "gzip",
         }, key_ts
+
+
+class AsyncSaveImageClient(JmImageClient):
+
+    def __init__(self, workers=None) -> None:
+        from concurrent.futures import ThreadPoolExecutor, Future
+        self.executor = ThreadPoolExecutor(max_workers=workers)
+        self.future_list: List[Future] = []
+
+    def save_image_resp(self, *args, **kwargs):
+        future = self.executor.submit(lambda: super().save_image_resp(*args, **kwargs))
+        self.future_list.append(future)

src/jmcomic/jm_client_interface.py

@@ -236,8 +236,10 @@ def download_image(self,
 
         resp.require_success()
 
-        # gif图无需加解密，需要最先判断
+        return self.save_image_resp(decode_image, img_save_path, img_url, resp, scramble_id)
 
+    def save_image_resp(self, decode_image, img_save_path, img_url, resp, scramble_id):
+        # gif图无需加解密，需要最先判断
         if self.img_is_not_need_to_decode(img_url, resp):
             JmImageSupport.save_resp_img(resp, img_save_path, False)
         else:
@@ -248,7 +250,7 @@ def download_by_image_detail(self,
                                  img_save_path,
                                  decode_image=True,
                                  ):
-        self.download_image(
+        return self.download_image(
             image.download_url,
             img_save_path,
             image.scramble_id,

src/jmcomic/jm_config.py

@@ -57,6 +57,24 @@ class JmModuleConfig:
     enable_jm_debug = True
     debug_executor = default_jm_debug
     postman_constructor = default_postman_constructor
+    DOWNLOADER_CLASS = None
+    OPTION_CLASS = None
+
+    @classmethod
+    def downloader_class(cls):
+        if cls.DOWNLOADER_CLASS is not None:
+            return cls.DOWNLOADER_CLASS
+
+        from .jm_downloader import JmDownloader
+        return JmDownloader
+
+    @classmethod
+    def option_class(cls):
+        if cls.OPTION_CLASS is not None:
+            return cls.OPTION_CLASS
+
+        from .jm_option import JmOption
+        return JmOption
 
     @classmethod
     @field_cache("DOMAIN")
@@ -115,8 +133,7 @@ def get_jmcomic_url(cls, postman=None):
         """
         postman = postman or cls.new_postman(session=True)
 
-        resp = postman.get(cls.JM_REDIRECT_URL)
-        url = resp.url
+        url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
         cls.jm_debug('获取禁漫地址', f'[{cls.JM_REDIRECT_URL}] → [{url}]')
         return url