fix(site): keep redirect failures from aborting site requests (#7227)

2026-10-31 20:34:48 +08:00 · 2026-10-28 18:43:19 +08:00
parent be65e047b0
commit 252b702875
7 changed files with 213 additions and 18 deletions
--- a/app/chain/site.py
+++ b/app/chain/site.py
@@ -745,7 +745,7 @@ class SiteChain(InteractionChainMixin, ChainBase):
            if res:
                html_text = res.text
            else:
-                logger.error(f"获取站点页面失败：{url}")
+                logger.error(f"获取站点页面失败：{url_tools.url_origin_for_log(url)}")
                return favicon_url, None
        html = etree.HTML(html_text)
        try:
@@ -760,7 +760,7 @@ class SiteChain(InteractionChainMixin, ChainBase):
            ) as res:
                if res:
                    return favicon_url, base64.b64encode(res.content).decode()
-                logger.error(f"获取站点图标失败：{favicon_url}")
+                logger.error(f"获取站点图标失败：{url_tools.url_origin_for_log(favicon_url)}")
        finally:
            if html is not None:
                del html
--- a/app/foundation/url.py
+++ b/app/foundation/url.py
@@ -283,6 +283,24 @@ def mask_url_credentials(url: str) -> str:
    )


+def url_origin_for_log(url: str) -> str:
+    """
+    只保留协议、主机与端口，用于记录来自站点页面或重定向的地址。
+
+    路径和查询参数可能携带签名、passkey 或会话令牌，且参数名无法穷举，因此一律省略；
+    userinfo 同样不保留。
+
+    :param url: 需要记录的地址
+    :return: 来源地址；地址无法解析时返回固定占位文本
+    """
+    try:
+        parts = urlparse(url)
+        port = f":{parts.port}" if parts.port else ""
+    except ValueError:
+        return "（地址无法解析）"
+    return f"{parts.scheme}://{parts.hostname or ''}{port}"
+
+
 def parse_address(
    address: str,
    include_scheme: bool = True,
--- a/app/startup/composition/site.py
+++ b/app/startup/composition/site.py
@@ -2,7 +2,7 @@
 站点访问技术端口的宿主组合根。
 """

-from contextlib import contextmanager
+from contextlib import ExitStack, contextmanager
 from typing import Any, Callable, Iterator, Mapping, Optional, cast
 from urllib.parse import urljoin, urlsplit

@@ -43,7 +43,29 @@ from app.chain.site import (
    configure_site_ports,
    reset_site_ports,
 )
-from app.foundation.url import UrlUtils
+from app.foundation.url import UrlUtils, url_origin_for_log
+from app.runtime.log import logger
+
+
+def _redirect_destination(url: str, response: Any) -> tuple[Optional[str], Optional[str]]:
+    """
+    解析一跳重定向的目标地址，只接受不带 URL 凭据的 HTTP(S) 目标。
+
+    :param url: 当前跳的请求地址
+    :param response: 当前跳的重定向响应
+    :return: 目标地址与失败原因，二者恰有一个非空
+    """
+    location = response.headers.get("Location")
+    if not location:
+        return None, "站点请求重定向缺少目标地址"
+    try:
+        destination = urljoin(url, location)
+        target = urlsplit(destination)
+    except ValueError:
+        return None, "站点请求重定向目标无效"
+    if target.scheme not in ("http", "https") or not target.hostname or target.username or target.password:
+        return None, "站点请求重定向目标无效"
+    return destination, None


 class _RssHttpAdapter:
@@ -214,6 +236,10 @@ class _SiteHttpAdapter:
        """
        打开响应并在所有返回或异常分支后关闭底层连接。

+        GET 请求逐跳跟随重定向，跨域跳转清除 Cookie 与认证头。重定向缺少目标、目标无效、出现环路
+        或超过跳数上限时只记录来源地址的告警并提供 None，与请求失败的语义一致，
+        由调用方按无法打开站点处理，不能以异常中断 CookieCloud 同步等批量流程。
+
        :param method: HTTP 请求方法
        :param url: 完整请求地址
        :param headers: 额外请求头
@@ -226,9 +252,11 @@ class _SiteHttpAdapter:
        """
        request_headers = dict(headers or {})
        seen = set()
+        failure = "站点请求重定向次数超过上限"
        for _hop in range(9):
            if url in seen:
-                raise ValueError("站点请求重定向出现环路")
+                failure = "站点请求重定向出现环路"
+                break
            seen.add(url)
            request = RequestUtils(
                headers=request_headers, cookies=cookie or "", ua=ua or "",
@@ -237,17 +265,22 @@ class _SiteHttpAdapter:
            request_options: dict[str, Any] = {} if json_body is None else {"json": dict(json_body)}
            if method.upper() == "GET":
                request_options["allow_redirects"] = False
-            with request.response_manager(method=method, url=url, **request_options) as response:
+            with ExitStack() as stack:
+                try:
+                    response = stack.enter_context(
+                        request.response_manager(method=method, url=url, **request_options)
+                    )
+                except ValueError:
+                    # 请求地址或响应 Location 畸形时 requests 会在打开响应时抛出，禁用自动跟随也会预解析 Location
+                    failure = "站点请求地址或重定向目标无效"
+                    break
                if method.upper() != "GET" or response is None or response.status_code not in (301, 302, 303, 307, 308):
                    yield cast(Optional[SiteResponsePort], response)
                    return
-                location = response.headers.get("Location")
-                if not location:
-                    raise ValueError("站点请求重定向缺少目标地址")
-                destination = urljoin(url, location)
-                target = urlsplit(destination)
-                if target.scheme not in ("http", "https") or not target.hostname or target.username or target.password:
-                    raise ValueError("站点请求重定向目标无效")
+                destination, invalid_reason = _redirect_destination(url, response)
+                if destination is None:
+                    failure = invalid_reason or failure
+                    break
                if not UrlUtils.same_origin(url, destination):
                    cookie = None
                    request_headers = {
@@ -255,7 +288,8 @@ class _SiteHttpAdapter:
                        if key.lower() not in ("cookie", "authorization", "proxy-authorization", "referer")
                    }
                url = destination
-        raise ValueError("站点请求重定向次数超过上限")
+        logger.warning(f"{failure}：{url_origin_for_log(url)}")
+        yield None


 class _SiteBrowserAdapter:
--- a/docs/architecture-overview.md
+++ b/docs/architecture-overview.md
@@ -817,7 +817,7 @@ flowchart LR
 | 指标 | 当前值 |
 |---|---:|
 | Python 模块 | 1142 |
-| 内部导入边 | 9,850 |
+| 内部导入边 | 9,852 |
 | 非平凡 SCC | 1（29 个模块的 TMDB vendor 精确 containment） |
 | Application / Chain 具体 Adapter 直连 | 0 / 0 |
 | Direct egress | 51（精确 policy 准入，Application / Chain 均为 0） |
--- a/tests/fixtures/architecture/dependency-baseline.json
+++ b/tests/fixtures/architecture/dependency-baseline.json
@@ -959,8 +959,8 @@
      "runtime_only": true
    }
  },
-  "edge_count": 9850,
-  "edge_sha256": "4ee56369dee810a01662c3c2f63eb016d57c308836d1e6ca9254ca31f434dc6f",
+  "edge_count": 9852,
+  "edge_sha256": "281f9cb108557ce8bb78b1c4a7249fcc983d0b0b53b9691f0b6f72de79e7d61c",
  "edges": [
    "app -> app.foundation",
    "app -> app.foundation.environment",
@@ -10232,6 +10232,8 @@
    "app.startup.composition.site -> app.chain.site",
    "app.startup.composition.site -> app.foundation",
    "app.startup.composition.site -> app.foundation.url",
+    "app.startup.composition.site -> app.runtime",
+    "app.startup.composition.site -> app.runtime.log",
    "app.startup.composition.subscription -> app.adapters",
    "app.startup.composition.subscription -> app.adapters.external",
    "app.startup.composition.subscription -> app.adapters.external.server",
--- a/tests/test_site_chain_ports.py
+++ b/tests/test_site_chain_ports.py
@@ -437,3 +437,130 @@ def test_site_chain_has_no_concrete_adapter_imports() -> None:
    ]

    assert violations == []
+
+
+def _redirect(location: Optional[str]) -> SimpleNamespace:
+    """
+    构造带可选目标地址的重定向响应。
+    """
+    return SimpleNamespace(status_code=302, headers={"Location": location} if location else {})
+
+
+def _scripted_request(monkeypatch, responses: list) -> list[dict]:
+    """
+    用按序回放的 RequestUtils 替身替换组合根传输，并记录每一跳的构造参数和请求参数。
+    """
+    hops: list[dict] = []
+    queue = list(responses)
+
+    class ScriptedRequest:
+        """
+        记录每一跳携带的 Cookie 与请求地址。
+        """
+
+        def __init__(self, **kwargs) -> None:
+            """
+            保存 adapter 传入的 Cookie 等构造参数。
+            """
+            self.options = kwargs
+
+        @contextmanager
+        def response_manager(self, **kwargs):
+            """
+            依次返回预设响应。
+            """
+            hops.append({**self.options, **kwargs})
+            item = queue.pop(0)
+            if isinstance(item, Exception):
+                # 模拟 requests 在打开响应时即抛出的异常，例如预解析畸形 Location
+                raise item
+            yield item
+
+    monkeypatch.setattr(site_composition, "RequestUtils", ScriptedRequest)
+    return hops
+
+
+@pytest.mark.parametrize("responses,reason", [
+    ([_redirect("/dl/1/auth?X-Amz-Signature=sig")] * 2, "站点请求重定向出现环路"),
+    ([_redirect(None)], "站点请求重定向缺少目标地址"),
+    ([_redirect("ftp://site.test/file")], "站点请求重定向目标无效"),
+    ([_redirect("https://user:pass@site.test/")], "站点请求重定向目标无效"),
+    ([_redirect("https://[bad/path")], "站点请求重定向目标无效"),
+    ([_redirect(f"/page{index}") for index in range(9)], "站点请求重定向次数超过上限"),
+    ([ValueError("Invalid IPv6 URL")], "站点请求地址或重定向目标无效"),
+    ([_redirect("/next"), ValueError("Invalid IPv6 URL")], "站点请求地址或重定向目标无效"),
+], ids=[
+    "loop", "missing-location", "invalid-scheme", "userinfo", "malformed", "too-many-hops",
+    "transport-rejects-location", "transport-rejects-next-hop",
+])
+def test_site_adapter_redirect_failures_yield_none(monkeypatch, responses, reason) -> None:
+    """
+    重定向失败按端口约定提供 None 并告警，不抛出异常打断 CookieCloud 同步等批量流程；
+    告警只含来源地址，路径与查询参数中的签名、passkey 等凭据都不写入日志。
+    """
+    hops = _scripted_request(monkeypatch, responses)
+    warnings = []
+    monkeypatch.setattr(site_composition.logger, "warning", warnings.append)
+
+    with site_composition._SiteHttpAdapter().open(
+        method="GET", url="https://site.test/index.php?passkey=secret", cookie="uid=1",
+    ) as response:
+        assert response is None
+
+    assert len(hops) == len(responses)
+    assert warnings == [f"{reason}：https://site.test"]
+
+
+def test_cookiecloud_sync_continues_after_redirect_loop(monkeypatch) -> None:
+    """
+    单个新站点的重定向环路只记为该站点添加失败，同步继续处理后续站点。
+    """
+    hops = _scripted_request(monkeypatch, [_redirect("/")])
+    monkeypatch.setattr(site_composition.logger, "warning", lambda _message: None)
+    _configure(site_composition._SiteHttpAdapter())
+    chain = _chain()
+    chain.runtime_config.cookiecloud_blacklist = ""
+
+    result = chain._sync_cookiecloud_domain(
+        domain="loop.test",
+        cookie="uid=1",
+        indexer={"name": "环路站点", "domain": "https://loop.test/"},
+        site_info=None,
+        repository=SimpleNamespace(),
+        rsshelper=SimpleNamespace(),
+    )
+
+    # 跳回同一地址在第二跳发出前即判定为环路
+    assert result == (0, 0, 1, False)
+    assert [hop["url"] for hop in hops] == ["https://loop.test/"]
+
+
+def test_site_adapter_does_not_swallow_caller_errors(monkeypatch) -> None:
+    """
+    调用方在响应上下文内抛出的异常必须原样传播，不能被当作重定向失败吞掉。
+    """
+    _scripted_request(monkeypatch, [_FakeResponse(200, text="ok")])
+    warnings = []
+    monkeypatch.setattr(site_composition.logger, "warning", warnings.append)
+
+    with pytest.raises(ValueError, match="调用方错误"):
+        with site_composition._SiteHttpAdapter().open(method="GET", url="https://site.test/") as response:
+            assert response is not None
+            raise ValueError("调用方错误")
+
+    assert warnings == []
+
+
+def test_favicon_failure_logs_only_site_origin(monkeypatch) -> None:
+    """
+    图标或页面读取失败的日志只记录来源地址，页面提供的图标 URL 中的令牌不能写入日志。
+    """
+    _configure(_FakeHttpPort(_FakeResponse(200, text='<link rel="icon" href="/icon.png?token=secret">'), None))
+    errors = []
+    monkeypatch.setattr(site_module.logger, "error", errors.append)
+
+    favicon_url, content = SiteChain._SiteChain__parse_favicon("https://site.test/?passkey=secret", "uid=1", "UA")
+
+    assert favicon_url == "https://site.test/icon.png?token=secret"
+    assert content is None
+    assert errors == ["获取站点图标失败：https://site.test"]
--- a/tests/test_url_utils.py
+++ b/tests/test_url_utils.py
@@ -4,7 +4,7 @@

 import pytest

-from app.foundation.url import UrlUtils, mask_url_credentials, url_matches_trusted_hosts
+from app.foundation.url import UrlUtils, mask_url_credentials, url_matches_trusted_hosts, url_origin_for_log


@pytest.mark.parametrize(
@@ -128,3 +128,17 @@ def test_mask_url_credentials_applies_to_free_text_with_url():
    assert "SECRET" not in masked
    assert "passkey=<REDACTED>" in masked
    assert masked.startswith("Max retries exceeded with url: /torrentrss.php?")
+
+
+@pytest.mark.parametrize("url,expected", [
+    ("https://site.test/dl/1/auth?X-Amz-Signature=sig", "https://site.test"),
+    ("https://user:pass@site.test:8443/path?passkey=secret", "https://site.test:8443"),
+    ("http://site.test", "http://site.test"),
+    ("https://[bad/path", "（地址无法解析）"),
+    ("https://site.test:99999/", "（地址无法解析）"),
+])
+def test_url_origin_for_log_drops_path_query_and_userinfo(url, expected):
+    """
+    日志地址只保留协议、主机与端口，路径、查询参数和 userinfo 中的凭据都不保留。
+    """
+    assert url_origin_for_log(url) == expected