增强域名重试机制 (#472)

.github/release.yml

@@ -0,0 +1,21 @@
+# .github/release.yml
+
+changelog:
+  exclude:
+    labels:
+      - ignore-for-release
+    authors:
+      - octocat
+  categories:
+    - title: 🏕 Features
+      labels:
+        - '*'
+      exclude:
+        labels:
+          - dependencies
+    - title: 👒 Dependencies
+      labels:
+        - dependencies
+    - title: Other Changes
+      labels:
+        - "*"

.github/workflows/release_auto.yml

@@ -29,12 +29,13 @@ jobs:
           python .github/release.py "$commit_message"
 
       - name: Create Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           tag_name: ${{ steps.tb.outputs.tag }}
           body_path: release_body.txt
+          generate_release_notes: true
 
       - name: Build
         run: |

assets/option/option_test_api.yml

@@ -25,4 +25,10 @@ plugins:
     - plugin: client_proxy
       kwargs:
         proxy_client_key: photo_concurrent_fetcher_proxy
-        whitelist: [ api, ]
+        whitelist: [ api, ]
+
+    - plugin: advanced-retry
+      kwargs:
+        retry_config:
+          retry_rounds: 3 # 一共对域名列表重试3轮
+          retry_domain_max_times: 5 # 当一个域名重试次数超过5次，忽略该域名，不再重试

src/jmcomic/__init__.py

@@ -2,7 +2,7 @@
 # 被依赖方 <--- 使用方
 # config <--- entity <--- toolkit <--- client <--- option <--- downloader
 
-__version__ = '2.6.6'
+__version__ = '2.6.7'
 
 from .api import *
 from .jm_plugin import *

src/jmcomic/jm_client_impl.py

@@ -895,7 +895,7 @@ def require_resp_status_ok(self, resp: JmApiResp):
         检查返回数据中的status字段是否为ok
         """
         data = resp.model_data
-        if data.status == 'ok':
+        if data.status != 'ok':
             ExceptionTool.raises_resp(data.msg, resp)
 
     def req_api(self, url, get=True, require_success=True, **kwargs) -> JmApiResp:
@@ -995,7 +995,7 @@ def raise_if_resp_should_retry(self, resp, is_image):
                 # 找到第一个有效字符
                 ExceptionTool.require_true(
                     char == '{',
-                    f'请求不是json格式，强制重试！响应文本: [{resp.text}]'
+                    f'请求不是json格式，强制重试！响应文本: [{JmcomicText.limit_text(text, 200)}]'
                 )
                 return resp
 

src/jmcomic/jm_client_interface.py

@@ -101,6 +101,14 @@ def __init__(self, resp, ts: str):
         super().__init__(resp)
         self.ts = ts
 
+    # 重写json()方法，可以忽略一些非json格式的脏数据
+    @field_cache()
+    def json(self) -> Dict:
+        try:
+            return JmcomicText.try_parse_json_object(self.resp.text)
+        except Exception as e:
+            ExceptionTool.raises_resp(f'json解析失败: {e}', self, JsonResolveFailException)
+
     @property
     def is_success(self) -> bool:
         return super().is_success and self.json()['code'] == 200

src/jmcomic/jm_option.py

@@ -419,7 +419,7 @@ def decide_domain_list():
         if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient):
             raise NotImplementedError(clazz)
 
-        client: AbstractJmClient = clazz(
+        client: JmcomicClient = clazz(
             postman=postman,
             domain_list=decide_domain_list(),
             retry_times=retry_times,

src/jmcomic/jm_plugin.py

@@ -1221,23 +1221,88 @@ def new_decide_dir(photo, ensure_exists=True) -> str:
 class AdvancedRetryPlugin(JmOptionPlugin):
     plugin_key = 'advanced-retry'
 
+    def __init__(self, option: JmOption):
+        super().__init__(option)
+        self.retry_config = None
+
     def invoke(self,
                retry_config,
                **kwargs):
+        self.require_param(isinstance(retry_config, dict), '必须配置retry_config为dict')
+        self.retry_config = retry_config
+
         new_jm_client: Callable = self.option.new_jm_client
 
         def hook_new_jm_client(*args, **kwargs):
-            client: AbstractJmClient = new_jm_client(*args, **kwargs)
+            client: JmcomicClient = new_jm_client(*args, **kwargs)
             client.domain_retry_strategy = self.request_with_retry
+            client.domain_req_failed_counter = {}
+            from threading import Lock
+            client.domain_counter_lock = Lock()
             return client
 
         self.option.new_jm_client = hook_new_jm_client
 
     def request_with_retry(self,
-                           client,
-                           request,
-                           url,
-                           is_image,
+                           client: AbstractJmClient,
+                           request: Callable,
+                           url: str,
+                           is_image: bool,
                            **kwargs,
                            ):
-        pass
+        """
+        实现如下域名重试机制：
+        - 对域名列表轮询请求，配置：retry_rounds
+        - 限制单个域名最大失败次数，配置：retry_domain_max_times
+        - 轮询域名列表前，根据历史失败次数对域名列表排序，失败多的后置
+        """
+
+        def do_request(domain):
+            url_to_use = url
+            if url_to_use.startswith('/'):
+                # path → url
+                url_to_use = client.of_api_url(url, domain)
+                client.update_request_with_specify_domain(kwargs, domain, is_image)
+                jm_log(client.log_topic(), client.decode(url_to_use))
+            elif is_image:
+                # 图片url
+                client.update_request_with_specify_domain(kwargs, None, is_image)
+
+            resp = request(url_to_use, **kwargs)
+            resp = client.raise_if_resp_should_retry(resp, is_image)
+            return resp
+
+        retry_domain_max_times: int = self.retry_config['retry_domain_max_times']
+        retry_rounds: int = self.retry_config['retry_rounds']
+        for rindex in range(retry_rounds):
+            domain_list = self.get_sorted_domain(client, retry_domain_max_times)
+            for i, domain in enumerate(domain_list):
+                if self.failed_count(client, domain) >= retry_domain_max_times:
+                    continue
+
+                try:
+                    return do_request(domain)
+                except Exception as e:
+                    from common import traceback_print_exec
+                    traceback_print_exec()
+                    jm_log('req.error', str(e))
+                    self.update_failed_count(client, domain)
+
+        return client.fallback(request, url, 0, 0, is_image, **kwargs)
+
+    def get_sorted_domain(self, client: JmcomicClient, times):
+        domain_list = client.get_domain_list()
+        return sorted(
+            filter(lambda d: self.failed_count(client, d) < times, domain_list),
+            key=lambda d: self.failed_count(client, d)
+        )
+
+    # noinspection PyUnresolvedReferences
+    def update_failed_count(self, client: AbstractJmClient, domain: str):
+        with client.domain_counter_lock:
+            client.domain_req_failed_counter[domain] = self.failed_count(client, domain) + 1
+
+    @staticmethod
+    def failed_count(client: JmcomicClient, domain: str) -> int:
+        # noinspection PyUnresolvedReferences
+        return client.domain_req_failed_counter.get(domain, 0)

src/jmcomic/jm_toolkit.py

@@ -61,6 +61,8 @@ class JmcomicText:
 
     # 提取接口返回值信息
     pattern_ajax_favorite_msg = compile(r'</button>(.*?)</div>')
+    # 提取api接口返回值里的json，防止返回值里有无关日志导致json解析报错
+    pattern_api_response_json_object = compile(r'\{[\s\S]*?}')
 
     @classmethod
     def parse_to_jm_domain(cls, text: str):
@@ -344,6 +346,28 @@ def try_mkdir(cls, save_dir: str):
             raise e
         return save_dir
 
+    # noinspection PyTypeChecker
+    @classmethod
+    def try_parse_json_object(cls, resp_text: str) -> dict:
+        import json
+        text = resp_text.strip()
+        if text.startswith('{') and text.endswith('}'):
+            # fast case
+            return json.loads(text)
+
+        for match in cls.pattern_api_response_json_object.finditer(text):
+            try:
+                return json.loads(match.group(0))
+            except Exception as e:
+                jm_log('parse_json_object.error', e)
+
+        raise AssertionError(f'未解析出json数据: {cls.limit_text(resp_text, 200)}')
+
+    @classmethod
+    def limit_text(cls, text: str, limit: int) -> str:
+        length = len(text)
+        return text if length <= limit else (text[:limit] + f'...({length - limit}')
+
 
 # 支持dsl: #{???} -> os.getenv(???)
 JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env)
@@ -450,10 +474,7 @@ def parse_html_to_search_page(cls, html: str) -> JmSearchPage:
             # 这里不作解析，因为没什么用...
             tags = cls.pattern_html_search_tags.findall(tag_text)
             content.append((
-                album_id, {
-                    'name': title,  # 改成name是为了兼容 parse_api_resp_to_page
-                    'tags': tags
-                }
+                album_id, dict(name=title, tags=tags)  # 改成name是为了兼容 parse_api_resp_to_page
             ))
 
         return JmSearchPage(content, total)
@@ -468,10 +489,7 @@ def parse_html_to_category_page(cls, html: str) -> JmSearchPage:
         for (album_id, title, tag_text) in album_info_list:
             tags = cls.pattern_html_search_tags.findall(tag_text)
             content.append((
-                album_id, {
-                    'name': title,  # 改成name是为了兼容 parse_api_resp_to_page
-                    'tags': tags
-                }
+                album_id, dict(name=title, tags=tags)  # 改成name是为了兼容 parse_api_resp_to_page
             ))
 
         return JmSearchPage(content, total)