From e42d081b7a3150cd7037c8724f0910b1e5625053 Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Thu, 25 Jun 2026 00:24:17 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20=E4=BF=AE=E5=A4=8D=20async=20downlo?= =?UTF-8?q?ader=20=E9=87=8D=E8=AF=95=E9=85=8D=E7=BD=AE=E4=B8=94=E6=94=B9?= =?UTF-8?q?=E9=80=A0=20batch=20download=20=E8=BF=94=E5=9B=9E=E5=80=BC?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/jmcomic/__init__.py | 2 +- src/jmcomic/api.py | 45 ++++++++++++++++++++-------------- src/jmcomic/jm_async_client.py | 3 ++- src/jmcomic/jm_downloader.py | 34 +++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 21 deletions(-) diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index 0c32f00c..4d0b420b 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,7 +2,7 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.7.0' +__version__ = '2.7.1' from .api import * from .jm_plugin import * diff --git a/src/jmcomic/api.py b/src/jmcomic/api.py index d28dea7d..223150c7 100644 --- a/src/jmcomic/api.py +++ b/src/jmcomic/api.py @@ -2,7 +2,7 @@ from .jm_downloader import * -__DOWNLOAD_API_RET = Tuple[JmAlbumDetail, JmDownloader] +__DOWNLOAD_API_RET = DownloadResult def download_batch(download_api, @@ -10,11 +10,13 @@ def download_batch(download_api, option=None, downloader=None, **kwargs, - ) -> Set[__DOWNLOAD_API_RET]: + ) -> BatchResult: """ 批量下载 album / photo - 一个album/photo，对应一个线程，对应一个option + 一个album/photo，对应一个线程，对应一个option。 + 返回 BatchResult(set)，支持 for album, dler in result 遍历。 + 失败项收集在 result.failed 中，不会静默丢失。 :param download_api: 下载api :param jm_id_iter: jmid (album_id, photo_id) 的迭代器 @@ -26,22 +28,25 @@ def download_batch(download_api, if option is None: option = JmModuleConfig.option_class().default() - result = set() + result = BatchResult() def callback(*ret): result.add(ret) + def _safe_download(aid): + """batch 内部的单任务包装：确保异常被收集而非静默丢失""" + try: + download_api(aid, option, downloader, callback=callback, **kwargs) + except Exception as e: + jm_log('batch.failed', f'批量下载失败: [{aid}], 异常: [{e}]', e) + result.failed[str(aid)] = e + multi_thread_launcher( iter_objs=set( JmcomicText.parse_to_jm_id(jmid) for jmid in jm_id_iter ), - apply_each_obj_func=lambda aid: download_api(aid, - option, - downloader, - callback=callback, - **kwargs, - ), + apply_each_obj_func=_safe_download, wait_finish=True ) @@ -81,7 +86,7 @@ def download_album(jm_album_id, callback(album, dler) if check_exception: dler.raise_if_has_exception() - return album, dler + return DownloadResult(album, dler) def download_photo(jm_photo_id, @@ -106,7 +111,7 @@ def download_photo(jm_photo_id, callback(photo, dler) if check_exception: dler.raise_if_has_exception() - return photo, dler + return DownloadResult(photo, dler) def new_downloader(option=None, downloader=None) -> JmDownloader: @@ -183,7 +188,7 @@ async def download_album_async(jm_album_id, if check_exception: dler.raise_if_has_exception() - return album, dler + return DownloadResult(album, dler) async def download_photo_async(jm_photo_id, @@ -213,7 +218,7 @@ async def download_photo_async(jm_photo_id, if check_exception: dler.raise_if_has_exception() - return photo, dler + return DownloadResult(photo, dler) async def download_batch_async(download_api, @@ -221,10 +226,11 @@ async def download_batch_async(download_api, option=None, downloader=None, **kwargs, - ): + ) -> BatchResult: """ 异步批量下载 album / photo。 - 容错机制：单个 album/photo 失败不会中止整批，也不会丢失其它已完成结果。 + - 返回 BatchResult(set)，失败项收集在 result.failed 中。 """ if option is None: option = JmModuleConfig.option_class().default() @@ -236,12 +242,13 @@ async def download_batch_async(download_api, return_exceptions=True, ) - # 失败不抛出，但要记录，便于排查 - result_set = set() + # 失败不抛出，但要记录到 result.failed，便于调用者排查 + result = BatchResult() for jmid, r in zip(jm_ids, results): if isinstance(r, BaseException): jm_log('async.batch.failed', f'批量下载失败: [{jmid}], 异常: [{r}]', r) + result.failed[str(jmid)] = r else: - result_set.add(r) + result.add(r) - return result_set + return result diff --git a/src/jmcomic/jm_async_client.py b/src/jmcomic/jm_async_client.py index 4d9236f1..e7bee243 100644 --- a/src/jmcomic/jm_async_client.py +++ b/src/jmcomic/jm_async_client.py @@ -57,7 +57,8 @@ class AsyncJmApiClient(AsyncJmcomicClient): def __init__(self, option: JmOption, max_clients=None, **kwargs): self.option = option self._domain_list = self._resolve_domain_list() - self._retry_times = option.client.get('retry_times', 5) or 5 + retry_times = option.client.get('retry_times') + self._retry_times = retry_times if retry_times is not None else 5 self._timeout = option.client.get('timeout', 30) or 30 # AsyncSession 句柄池大小：优先用调用方（下载器）传入的实际图片并发， # 否则回退到 option 配置；避免因默认限制导致真实并发被隐式压低。 diff --git a/src/jmcomic/jm_downloader.py b/src/jmcomic/jm_downloader.py index 3608ed92..986196a7 100644 --- a/src/jmcomic/jm_downloader.py +++ b/src/jmcomic/jm_downloader.py @@ -1,3 +1,5 @@ +from typing import NamedTuple + from .jm_option import * @@ -240,6 +242,38 @@ def raise_if_has_exception(self): ) +class DownloadResult(NamedTuple): + """单个下载结果。 + + 支持解包：album, dler = download_album(id) + 也支持属性访问：result.detail, result.downloader + NamedTuple 是 tuple 子类，isinstance(result, tuple) 为 True。 + """ + detail: DetailEntity + downloader: BaseDownloader + + +class BatchResult(set): + """批量下载结果集。 + + 继承 set，完全兼容旧写法 for album, dler in result。 + 新增 .failed 属性，记录失败的 jm_id 和对应异常。 + """ + + def __init__(self): + super().__init__() + self.failed: Dict[str, BaseException] = {} + + @property + def all_succeeded(self) -> bool: + return len(self.failed) == 0 + + @property + def total(self) -> int: + """预期总数（成功 + 失败）""" + return len(self) + len(self.failed) + + class JmDownloader(BaseDownloader): """ JmDownloader = BaseDownloader + 同步 I/O 调度逻辑 From 8313abfe0c875c2c267dbcef5fe9c4f22785e6d2 Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:35:59 +0800 Subject: [PATCH 2/4] Update jm_entity.py --- src/jmcomic/jm_entity.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/jmcomic/jm_entity.py b/src/jmcomic/jm_entity.py index 1de21c94..bfb8d5b7 100644 --- a/src/jmcomic/jm_entity.py +++ b/src/jmcomic/jm_entity.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from functools import lru_cache from common import * @@ -37,7 +38,7 @@ def is_page(cls): return False -class IndexedEntity: +class IndexedEntity(Sequence): def getindex(self, index: int): raise NotImplementedError @@ -46,12 +47,14 @@ def __len__(self): def __getitem__(self, item) -> Any: if isinstance(item, slice): - start = item.start or 0 - stop = item.stop or len(self) - step = item.step or 1 + start, stop, step = item.indices(len(self)) return [self.getindex(index) for index in range(start, stop, step)] elif isinstance(item, int): + if item < 0: + item += len(self) + if item < 0 or item >= len(self): + raise IndexError(f"index {item} out of range") return self.getindex(item) else: From aa74af52e301700dc1d3bdbf8b3e712f1cbfbba9 Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:38:44 +0800 Subject: [PATCH 3/4] Update test_jm_client.py --- tests/test_jmcomic/test_jm_client.py | 42 ++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/test_jmcomic/test_jm_client.py b/tests/test_jmcomic/test_jm_client.py index e7a484a8..7abc93be 100644 --- a/tests/test_jmcomic/test_jm_client.py +++ b/tests/test_jmcomic/test_jm_client.py @@ -357,3 +357,45 @@ def test_ranking(self): This test retrieves the page 1 ranking data from the configured client and writes it to standard output. """ print(self.client.month_ranking(1)) + + def test_indexed_entity_slice(self): + from collections.abc import Sequence + + # 获取真实的 JmSearchPage 对象 + page = self.client.search_site("中文", page=1) + empty_page = self.client.search_site("测试很长的不知道什么鬼123", page=1) + + self.assertTrue(isinstance(page, Sequence)) + + # 切片越界应该正常截断，不报错 + items = page[:1000] + self.assertEqual(len(items), len(page)) + self.assertEqual(page[1000:2000], []) + + self.assertEqual(empty_page[:10], []) + + # 负数切片 + if len(page) >= 2: + rev = page[::-1] + self.assertEqual(rev[0], page[-1]) + self.assertEqual(rev[-1], page[0]) + + if len(page) >= 2: + self.assertEqual(page[-1], page[len(page) - 1]) + self.assertEqual(page[-2], page[len(page) - 2]) + + with self.assertRaises(IndexError): + _ = page[len(page) + 10] + with self.assertRaises(IndexError): + _ = page[-(len(page) + 10)] + + res = [] + for i in page: + res.append(i) + self.assertEqual(len(res), len(page)) + + if len(page) > 0: + # JmSearchPage overrides __iter__ to yield different elements than __getitem__ + # So we take an item from iter() to test __contains__ + first_item = next(iter(page)) + self.assertTrue(first_item in page) From 75a4cb1b5b5db3703e661936bdd7aca62fa0ad6a Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Sat, 4 Jul 2026 18:42:14 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20download=5Fbatc?= =?UTF-8?q?h=20=E6=88=90=E5=8A=9F=E6=A0=87=E8=AE=B0=E6=97=B6=E5=BA=8F?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=8C=E9=81=BF=E5=85=8D=E9=83=A8=E5=88=86?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E9=A1=B9=E5=90=8C=E6=97=B6=E5=87=BA=E7=8E=B0?= =?UTF-8?q?=E5=9C=A8=20success=20=E5=92=8C=20failed=20=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/jmcomic/api.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/jmcomic/api.py b/src/jmcomic/api.py index 223150c7..13aea801 100644 --- a/src/jmcomic/api.py +++ b/src/jmcomic/api.py @@ -30,13 +30,11 @@ def download_batch(download_api, result = BatchResult() - def callback(*ret): - result.add(ret) - def _safe_download(aid): """batch 内部的单任务包装：确保异常被收集而非静默丢失""" try: - download_api(aid, option, downloader, callback=callback, **kwargs) + ret = download_api(aid, option, downloader, **kwargs) + result.add(ret) except Exception as e: jm_log('batch.failed', f'批量下载失败: [{aid}], 异常: [{e}]', e) result.failed[str(aid)] = e