From 6cf0d74fb5c49aa8b019bed656914d4c319abeab Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Sat, 6 Sep 2025 17:32:47 +0800 Subject: [PATCH 1/8] =?UTF-8?q?=E5=A2=9E=E5=BC=BA=E5=9F=9F=E5=90=8D?= =?UTF-8?q?=E9=87=8D=E8=AF=95=E6=9C=BA=E5=88=B6=20(#472)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/jmcomic/jm_option.py | 2 +- src/jmcomic/jm_plugin.py | 75 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py index 32b4b36e..f71b6c8f 100644 --- a/src/jmcomic/jm_option.py +++ b/src/jmcomic/jm_option.py @@ -419,7 +419,7 @@ def decide_domain_list(): if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient): raise NotImplementedError(clazz) - client: AbstractJmClient = clazz( + client: JmcomicClient = clazz( postman=postman, domain_list=decide_domain_list(), retry_times=retry_times, diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py index 27a7b8f6..b1243de6 100644 --- a/src/jmcomic/jm_plugin.py +++ b/src/jmcomic/jm_plugin.py @@ -1221,23 +1221,86 @@ def new_decide_dir(photo, ensure_exists=True) -> str: class AdvancedRetryPlugin(JmOptionPlugin): plugin_key = 'advanced-retry' + def __init__(self, option: JmOption): + super().__init__(option) + self.retry_config = None + def invoke(self, retry_config, **kwargs): + self.require_param(isinstance(retry_config, dict), '必须配置retry_config为dict') + self.retry_config = retry_config + new_jm_client: Callable = self.option.new_jm_client def hook_new_jm_client(*args, **kwargs): - client: AbstractJmClient = new_jm_client(*args, **kwargs) + client: JmcomicClient = new_jm_client(*args, **kwargs) client.domain_retry_strategy = self.request_with_retry + client.domain_req_failed_counter = {} + from threading import Lock + client.domain_counter_lock = Lock() return client self.option.new_jm_client = hook_new_jm_client def request_with_retry(self, - client, - request, - url, - is_image, + client: AbstractJmClient, + request: Callable, + url: str, + is_image: bool, **kwargs, ): - pass + """ + 实现如下域名重试机制： + - 对域名列表轮询请求，配置：retry_rounds + - 限制单个域名最大失败次数，配置：retry_domain_max_times + - 轮询域名列表前，根据历史失败次数对域名列表排序，失败多的后置 + """ + + def do_request(domain): + url_to_use = url + if url_to_use.startswith('/'): + # path → url + url_to_use = client.of_api_url(url, domain) + client.update_request_with_specify_domain(kwargs, domain, is_image) + jm_log(client.log_topic(), client.decode(url_to_use)) + elif is_image: + # 图片url + client.update_request_with_specify_domain(kwargs, None, is_image) + + resp = request(url_to_use, **kwargs) + resp = client.raise_if_resp_should_retry(resp, is_image) + return resp + + retry_domain_max_times: int = self.retry_config['retry_domain_max_times'] + retry_rounds: int = self.retry_config['retry_rounds'] + for rindex in range(retry_rounds): + domain_list = self.get_sorted_domain(client, retry_domain_max_times) + for i, domain in enumerate(domain_list): + if self.failed_count(client, domain) >= retry_domain_max_times: + continue + + try: + return do_request(domain) + except Exception as e: + from common import traceback_print_exec + traceback_print_exec() + jm_log('req.error', str(e)) + self.update_failed_count(client, domain) + + def get_sorted_domain(self, client: JmcomicClient, times): + domain_list = client.get_domain_list() + return sorted( + filter(lambda d: self.failed_count(client, d) < times, domain_list), + key=lambda d: self.failed_count(client, d) + ) + + # noinspection PyUnresolvedReferences + def update_failed_count(self, client: AbstractJmClient, domain: str): + with client.domain_counter_lock: + client.domain_req_failed_counter[domain] = self.failed_count(client, domain) + 1 + + @staticmethod + def failed_count(client: JmcomicClient, domain: str) -> int: + # noinspection PyUnresolvedReferences + return client.domain_req_failed_counter.get(domain, 0) From 32059dd70389893b1dc111a4cb0fd5283c9a6b02 Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Sat, 6 Sep 2025 17:37:13 +0800 Subject: [PATCH 2/8] =?UTF-8?q?=E5=A2=9E=E5=BC=BA=E5=9F=9F=E5=90=8D?= =?UTF-8?q?=E9=87=8D=E8=AF=95=E6=9C=BA=E5=88=B6=20(#472)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- assets/option/option_test_api.yml | 8 +++++++- src/jmcomic/jm_plugin.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/assets/option/option_test_api.yml b/assets/option/option_test_api.yml index 29efdaa6..f4c0b7b4 100644 --- a/assets/option/option_test_api.yml +++ b/assets/option/option_test_api.yml @@ -25,4 +25,10 @@ plugins: - plugin: client_proxy kwargs: proxy_client_key: photo_concurrent_fetcher_proxy - whitelist: [ api, ] \ No newline at end of file + whitelist: [ api, ] + + - plugin: advanced-retry + kwargs: + retry_config: + retry_rounds: 3 # 一共对域名列表重试3轮 + retry_domain_max_times: 5 # 当一个域名重试次数超过5次，忽略该域名，不再重试 \ No newline at end of file diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py index b1243de6..37a3ac42 100644 --- a/src/jmcomic/jm_plugin.py +++ b/src/jmcomic/jm_plugin.py @@ -1288,6 +1288,8 @@ def do_request(domain): jm_log('req.error', str(e)) self.update_failed_count(client, domain) + return client.fallback(request, url, 0, 0, is_image, **kwargs) + def get_sorted_domain(self, client: JmcomicClient, times): domain_list = client.get_domain_list() return sorted( From a1b9597e7e0bca7319a81fa544a30056e276f827 Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:47:36 +0800 Subject: [PATCH 3/8] Update README.md --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 07afce1a..ebea8062 100644 --- a/README.md +++ b/README.md @@ -192,3 +192,11 @@ jmcomic 123 Repo Card + + +## Contributors ✨ + +Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): + + + From 60929f9074b73c0aa28d8d4d72ebc8e4d867fa5c Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Tue, 9 Sep 2025 11:49:29 +0800 Subject: [PATCH 4/8] Revert "Update README.md" This reverts commit a1b9597e7e0bca7319a81fa544a30056e276f827. --- README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/README.md b/README.md index ebea8062..07afce1a 100644 --- a/README.md +++ b/README.md @@ -192,11 +192,3 @@ jmcomic 123 Repo Card - - -## Contributors ✨ - -Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): - - - From 3355c3368fb8cff0d8b3e3ff26e99cef414ae7f8 Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Tue, 9 Sep 2025 12:00:24 +0800 Subject: [PATCH 5/8] add release Changelog --- .github/release.yml | 21 +++++++++++++++++++++ .github/workflows/release_auto.yml | 3 ++- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 .github/release.yml diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 00000000..d2e47199 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,21 @@ +# .github/release.yml + +changelog: + exclude: + labels: + - ignore-for-release + authors: + - octocat + categories: + - title: 🏕 Features + labels: + - '*' + exclude: + labels: + - dependencies + - title: 👒 Dependencies + labels: + - dependencies + - title: Other Changes + labels: + - "*" diff --git a/.github/workflows/release_auto.yml b/.github/workflows/release_auto.yml index e6a5b324..3b85c38c 100644 --- a/.github/workflows/release_auto.yml +++ b/.github/workflows/release_auto.yml @@ -29,12 +29,13 @@ jobs: python .github/release.py "$commit_message" - name: Create Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ steps.tb.outputs.tag }} body_path: release_body.txt + generate_release_notes: true - name: Build run: | From d39cc8bcbcf6a990392d8ffaf936f920088f5f81 Mon Sep 17 00:00:00 2001 From: RSLN-creator <3316399314@qq.com> Date: Tue, 9 Sep 2025 12:36:06 +0800 Subject: [PATCH 6/8] Update jm_client_interface.py (#478) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 重写json()方法，专门处理API响应的清理 --- src/jmcomic/jm_client_interface.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/jmcomic/jm_client_interface.py b/src/jmcomic/jm_client_interface.py index f5226948..11c0b43c 100644 --- a/src/jmcomic/jm_client_interface.py +++ b/src/jmcomic/jm_client_interface.py @@ -101,6 +101,17 @@ def __init__(self, resp, ts: str): super().__init__(resp) self.ts = ts + # 重写json()方法，专门处理API响应的清理 + @field_cache() + def json(self) -> Dict: + from .jm_toolkit import safe_parse_json # 导入清理函数 + try: + # 仅对API响应进行清理（保留原始JmJsonResp的逻辑不变） + clean_text = safe_parse_json(self.resp.text) + return clean_text + except Exception as e: + ExceptionTool.raises_resp(f'API json解析失败: {e}', self, JsonResolveFailException) + @property def is_success(self) -> bool: return super().is_success and self.json()['code'] == 200 From e695d72f0598b8eb8a64921b9a19c56fead33802 Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Tue, 9 Sep 2025 13:07:47 +0800 Subject: [PATCH 7/8] try_parse_json_object --- src/jmcomic/jm_client_interface.py | 7 +++++++ src/jmcomic/jm_toolkit.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/jmcomic/jm_client_interface.py b/src/jmcomic/jm_client_interface.py index 11c0b43c..712eee3c 100644 --- a/src/jmcomic/jm_client_interface.py +++ b/src/jmcomic/jm_client_interface.py @@ -116,6 +116,13 @@ def json(self) -> Dict: def is_success(self) -> bool: return super().is_success and self.json()['code'] == 200 + def json(self) -> Dict: + try: + text = self.resp.text + return JmcomicText.try_parse_json_object(text) + except Exception as e: + ExceptionTool.raises_resp(f'json解析失败: {e}', self, JsonResolveFailException) + @property @field_cache() def decoded_data(self) -> str: diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py index e5eada1f..b0c6cb40 100644 --- a/src/jmcomic/jm_toolkit.py +++ b/src/jmcomic/jm_toolkit.py @@ -61,6 +61,8 @@ class JmcomicText: # 提取接口返回值信息 pattern_ajax_favorite_msg = compile(r'(.*?)') + # 提取api接口返回值里的json，防止返回值里有无关日志导致json解析报错 + pattern_api_response_json_object = re.compile(r'\{.*?}') @classmethod def parse_to_jm_domain(cls, text: str): @@ -344,6 +346,18 @@ def try_mkdir(cls, save_dir: str): raise e return save_dir + # noinspection PyTypeChecker + @classmethod + def try_parse_json_object(cls, text: str) -> dict: + import json + text = text.strip() + if text.startswith('{') and text.endswith('}'): + # fast case + return json.loads(text) + + for match in cls.pattern_api_response_json_object.finditer(text): + return json.loads(match.group(0)) + # 支持dsl: #{???} -> os.getenv(???) JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env) From 650d271eee9a0cd42387553bf089d4fab1f9d1fd Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Tue, 9 Sep 2025 16:15:40 +0800 Subject: [PATCH 8/8] polish code --- src/jmcomic/__init__.py | 2 +- src/jmcomic/jm_client_impl.py | 4 ++-- src/jmcomic/jm_client_interface.py | 18 ++++-------------- src/jmcomic/jm_toolkit.py | 28 ++++++++++++++++------------ 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index a4e9c979..5aff6914 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,7 +2,7 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.6.6' +__version__ = '2.6.7' from .api import * from .jm_plugin import * diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index be29b9a8..0a801f35 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -895,7 +895,7 @@ def require_resp_status_ok(self, resp: JmApiResp): 检查返回数据中的status字段是否为ok """ data = resp.model_data - if data.status == 'ok': + if data.status != 'ok': ExceptionTool.raises_resp(data.msg, resp) def req_api(self, url, get=True, require_success=True, **kwargs) -> JmApiResp: @@ -995,7 +995,7 @@ def raise_if_resp_should_retry(self, resp, is_image): # 找到第一个有效字符 ExceptionTool.require_true( char == '{', - f'请求不是json格式，强制重试！响应文本: [{resp.text}]' + f'请求不是json格式，强制重试！响应文本: [{JmcomicText.limit_text(text, 200)}]' ) return resp diff --git a/src/jmcomic/jm_client_interface.py b/src/jmcomic/jm_client_interface.py index 712eee3c..59802827 100644 --- a/src/jmcomic/jm_client_interface.py +++ b/src/jmcomic/jm_client_interface.py @@ -101,28 +101,18 @@ def __init__(self, resp, ts: str): super().__init__(resp) self.ts = ts - # 重写json()方法，专门处理API响应的清理 + # 重写json()方法，可以忽略一些非json格式的脏数据 @field_cache() def json(self) -> Dict: - from .jm_toolkit import safe_parse_json # 导入清理函数 try: - # 仅对API响应进行清理（保留原始JmJsonResp的逻辑不变） - clean_text = safe_parse_json(self.resp.text) - return clean_text + return JmcomicText.try_parse_json_object(self.resp.text) except Exception as e: - ExceptionTool.raises_resp(f'API json解析失败: {e}', self, JsonResolveFailException) - + ExceptionTool.raises_resp(f'json解析失败: {e}', self, JsonResolveFailException) + @property def is_success(self) -> bool: return super().is_success and self.json()['code'] == 200 - def json(self) -> Dict: - try: - text = self.resp.text - return JmcomicText.try_parse_json_object(text) - except Exception as e: - ExceptionTool.raises_resp(f'json解析失败: {e}', self, JsonResolveFailException) - @property @field_cache() def decoded_data(self) -> str: diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py index b0c6cb40..7ec84d5c 100644 --- a/src/jmcomic/jm_toolkit.py +++ b/src/jmcomic/jm_toolkit.py @@ -62,7 +62,7 @@ class JmcomicText: # 提取接口返回值信息 pattern_ajax_favorite_msg = compile(r'(.*?)') # 提取api接口返回值里的json，防止返回值里有无关日志导致json解析报错 - pattern_api_response_json_object = re.compile(r'\{.*?}') + pattern_api_response_json_object = compile(r'\{[\s\S]*?}') @classmethod def parse_to_jm_domain(cls, text: str): @@ -348,15 +348,25 @@ def try_mkdir(cls, save_dir: str): # noinspection PyTypeChecker @classmethod - def try_parse_json_object(cls, text: str) -> dict: + def try_parse_json_object(cls, resp_text: str) -> dict: import json - text = text.strip() + text = resp_text.strip() if text.startswith('{') and text.endswith('}'): # fast case return json.loads(text) for match in cls.pattern_api_response_json_object.finditer(text): - return json.loads(match.group(0)) + try: + return json.loads(match.group(0)) + except Exception as e: + jm_log('parse_json_object.error', e) + + raise AssertionError(f'未解析出json数据: {cls.limit_text(resp_text, 200)}') + + @classmethod + def limit_text(cls, text: str, limit: int) -> str: + length = len(text) + return text if length <= limit else (text[:limit] + f'...({length - limit}') # 支持dsl: #{???} -> os.getenv(???) @@ -464,10 +474,7 @@ def parse_html_to_search_page(cls, html: str) -> JmSearchPage: # 这里不作解析，因为没什么用... tags = cls.pattern_html_search_tags.findall(tag_text) content.append(( - album_id, { - 'name': title, # 改成name是为了兼容 parse_api_resp_to_page - 'tags': tags - } + album_id, dict(name=title, tags=tags) # 改成name是为了兼容 parse_api_resp_to_page )) return JmSearchPage(content, total) @@ -482,10 +489,7 @@ def parse_html_to_category_page(cls, html: str) -> JmSearchPage: for (album_id, title, tag_text) in album_info_list: tags = cls.pattern_html_search_tags.findall(tag_text) content.append(( - album_id, { - 'name': title, # 改成name是为了兼容 parse_api_resp_to_page - 'tags': tags - } + album_id, dict(name=title, tags=tags) # 改成name是为了兼容 parse_api_resp_to_page )) return JmSearchPage(content, total)