Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/jmcomic/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader

__version__ = '2.7.0'
__version__ = '2.7.1'

from .api import *
from .jm_plugin import *
Expand Down
47 changes: 26 additions & 21 deletions src/jmcomic/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,21 @@

from .jm_downloader import *

__DOWNLOAD_API_RET = Tuple[JmAlbumDetail, JmDownloader]
__DOWNLOAD_API_RET = DownloadResult


def download_batch(download_api,
jm_id_iter: Union[Iterable, Generator],
option=None,
downloader=None,
**kwargs,
) -> Set[__DOWNLOAD_API_RET]:
) -> BatchResult:
"""
批量下载 album / photo

一个album/photo,对应一个线程,对应一个option
一个album/photo,对应一个线程,对应一个option。
返回 BatchResult(set),支持 for album, dler in result 遍历。
失败项收集在 result.failed 中,不会静默丢失。

:param download_api: 下载api
:param jm_id_iter: jmid (album_id, photo_id) 的迭代器
Expand All @@ -26,22 +28,23 @@ def download_batch(download_api,
if option is None:
option = JmModuleConfig.option_class().default()

result = set()
result = BatchResult()

def callback(*ret):
result.add(ret)
def _safe_download(aid):
"""batch 内部的单任务包装:确保异常被收集而非静默丢失"""
try:
ret = download_api(aid, option, downloader, **kwargs)
result.add(ret)
except Exception as e:
jm_log('batch.failed', f'批量下载失败: [{aid}], 异常: [{e}]', e)
result.failed[str(aid)] = e

multi_thread_launcher(
iter_objs=set(
JmcomicText.parse_to_jm_id(jmid)
for jmid in jm_id_iter
),
apply_each_obj_func=lambda aid: download_api(aid,
option,
downloader,
callback=callback,
**kwargs,
),
apply_each_obj_func=_safe_download,
wait_finish=True
)

Expand Down Expand Up @@ -81,7 +84,7 @@ def download_album(jm_album_id,
callback(album, dler)
if check_exception:
dler.raise_if_has_exception()
return album, dler
return DownloadResult(album, dler)


def download_photo(jm_photo_id,
Expand All @@ -106,7 +109,7 @@ def download_photo(jm_photo_id,
callback(photo, dler)
if check_exception:
dler.raise_if_has_exception()
return photo, dler
return DownloadResult(photo, dler)


def new_downloader(option=None, downloader=None) -> JmDownloader:
Expand Down Expand Up @@ -183,7 +186,7 @@ async def download_album_async(jm_album_id,
if check_exception:
dler.raise_if_has_exception()

return album, dler
return DownloadResult(album, dler)


async def download_photo_async(jm_photo_id,
Expand Down Expand Up @@ -213,18 +216,19 @@ async def download_photo_async(jm_photo_id,
if check_exception:
dler.raise_if_has_exception()

return photo, dler
return DownloadResult(photo, dler)


async def download_batch_async(download_api,
jm_id_iter,
option=None,
downloader=None,
**kwargs,
):
) -> BatchResult:
"""
异步批量下载 album / photo。
- 容错机制:单个 album/photo 失败不会中止整批,也不会丢失其它已完成结果。
- 返回 BatchResult(set),失败项收集在 result.failed 中。
"""
if option is None:
option = JmModuleConfig.option_class().default()
Expand All @@ -236,12 +240,13 @@ async def download_batch_async(download_api,
return_exceptions=True,
)

# 失败不抛出,但要记录,便于排查
result_set = set()
# 失败不抛出,但要记录到 result.failed,便于调用者排查
result = BatchResult()
for jmid, r in zip(jm_ids, results):
if isinstance(r, BaseException):
jm_log('async.batch.failed', f'批量下载失败: [{jmid}], 异常: [{r}]', r)
result.failed[str(jmid)] = r
else:
result_set.add(r)
result.add(r)

return result_set
return result
3 changes: 2 additions & 1 deletion src/jmcomic/jm_async_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ class AsyncJmApiClient(AsyncJmcomicClient):
def __init__(self, option: JmOption, max_clients=None, **kwargs):
self.option = option
self._domain_list = self._resolve_domain_list()
self._retry_times = option.client.get('retry_times', 5) or 5
retry_times = option.client.get('retry_times')
self._retry_times = retry_times if retry_times is not None else 5
self._timeout = option.client.get('timeout', 30) or 30
# AsyncSession 句柄池大小:优先用调用方(下载器)传入的实际图片并发,
# 否则回退到 option 配置;避免因默认限制导致真实并发被隐式压低。
Expand Down
34 changes: 34 additions & 0 deletions src/jmcomic/jm_downloader.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from typing import NamedTuple

from .jm_option import *


Expand All @@ -8,7 +10,7 @@
def wrapper(self, *args, **kwargs):
self: JmDownloader
try:
return func(self, *args, **kwargs)

Check failure on line 13 in src/jmcomic/jm_downloader.py

View workflow job for this annotation

GitHub Actions / test (3.9, ubuntu-latest)

test_partial_exception

Check failure on line 13 in src/jmcomic/jm_downloader.py

View workflow job for this annotation

GitHub Actions / test (3.10, ubuntu-latest)

test_partial_exception
except Exception as e:
detail: JmBaseEntity = args[0]
if detail.is_image():
Expand Down Expand Up @@ -240,6 +242,38 @@
)


class DownloadResult(NamedTuple):
"""单个下载结果。

支持解包:album, dler = download_album(id)
也支持属性访问:result.detail, result.downloader
NamedTuple 是 tuple 子类,isinstance(result, tuple) 为 True。
"""
detail: DetailEntity
downloader: BaseDownloader


class BatchResult(set):
"""批量下载结果集。

继承 set,完全兼容旧写法 for album, dler in result。
新增 .failed 属性,记录失败的 jm_id 和对应异常。
"""

def __init__(self):
super().__init__()
self.failed: Dict[str, BaseException] = {}

@property
def all_succeeded(self) -> bool:
return len(self.failed) == 0

@property
def total(self) -> int:
"""预期总数(成功 + 失败)"""
return len(self) + len(self.failed)


class JmDownloader(BaseDownloader):
"""
JmDownloader = BaseDownloader + 同步 I/O 调度逻辑
Expand Down
11 changes: 7 additions & 4 deletions src/jmcomic/jm_entity.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from collections.abc import Sequence
from functools import lru_cache

from common import *
Expand Down Expand Up @@ -37,7 +38,7 @@ def is_page(cls):
return False


class IndexedEntity:
class IndexedEntity(Sequence):
def getindex(self, index: int):
raise NotImplementedError

Expand All @@ -46,12 +47,14 @@ def __len__(self):

def __getitem__(self, item) -> Any:
if isinstance(item, slice):
start = item.start or 0
stop = item.stop or len(self)
step = item.step or 1
start, stop, step = item.indices(len(self))
return [self.getindex(index) for index in range(start, stop, step)]

elif isinstance(item, int):
if item < 0:
item += len(self)
if item < 0 or item >= len(self):
raise IndexError(f"index {item} out of range")
return self.getindex(item)

else:
Expand Down
42 changes: 42 additions & 0 deletions tests/test_jmcomic/test_jm_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -357,3 +357,45 @@ def test_ranking(self):
This test retrieves the page 1 ranking data from the configured client and writes it to standard output.
"""
print(self.client.month_ranking(1))

def test_indexed_entity_slice(self):
from collections.abc import Sequence

# 获取真实的 JmSearchPage 对象
page = self.client.search_site("中文", page=1)
empty_page = self.client.search_site("测试很长的不知道什么鬼123", page=1)

self.assertTrue(isinstance(page, Sequence))

# 切片越界应该正常截断,不报错
items = page[:1000]
self.assertEqual(len(items), len(page))
self.assertEqual(page[1000:2000], [])

self.assertEqual(empty_page[:10], [])

# 负数切片
if len(page) >= 2:
rev = page[::-1]
self.assertEqual(rev[0], page[-1])
self.assertEqual(rev[-1], page[0])

if len(page) >= 2:
self.assertEqual(page[-1], page[len(page) - 1])
self.assertEqual(page[-2], page[len(page) - 2])

with self.assertRaises(IndexError):
_ = page[len(page) + 10]
with self.assertRaises(IndexError):
_ = page[-(len(page) + 10)]

res = []
for i in page:
res.append(i)
self.assertEqual(len(res), len(page))

if len(page) > 0:
# JmSearchPage overrides __iter__ to yield different elements than __getitem__
# So we take an item from iter() to test __contains__
first_item = next(iter(page))
self.assertTrue(first_item in page)
Loading