From 252b702875cba0fa2d4428376ef2829cc4c997b9 Mon Sep 17 00:00:00 2001
From: Cat <86354789+MiCat-S@users.noreply.github.com>
Date: Wed, 28 Oct 2026 18:43:19 +0800
Subject: [PATCH] fix(site): keep redirect failures from aborting site requests
(#7227)
---
app/chain/site.py | 4 +-
app/foundation/url.py | 18 +++
app/startup/composition/site.py | 58 ++++++--
docs/architecture-overview.md | 2 +-
.../architecture/dependency-baseline.json | 6 +-
tests/test_site_chain_ports.py | 127 ++++++++++++++++++
tests/test_url_utils.py | 16 ++-
7 files changed, 213 insertions(+), 18 deletions(-)
diff --git a/app/chain/site.py b/app/chain/site.py
index 0fa10ba6c..39df46073 100644
--- a/app/chain/site.py
+++ b/app/chain/site.py
@@ -745,7 +745,7 @@ class SiteChain(InteractionChainMixin, ChainBase):
if res:
html_text = res.text
else:
- logger.error(f"获取站点页面失败：{url}")
+ logger.error(f"获取站点页面失败：{url_tools.url_origin_for_log(url)}")
return favicon_url, None
html = etree.HTML(html_text)
try:
@@ -760,7 +760,7 @@ class SiteChain(InteractionChainMixin, ChainBase):
) as res:
if res:
return favicon_url, base64.b64encode(res.content).decode()
- logger.error(f"获取站点图标失败：{favicon_url}")
+ logger.error(f"获取站点图标失败：{url_tools.url_origin_for_log(favicon_url)}")
finally:
if html is not None:
del html
diff --git a/app/foundation/url.py b/app/foundation/url.py
index 53b64886b..c279de32e 100644
--- a/app/foundation/url.py
+++ b/app/foundation/url.py
@@ -283,6 +283,24 @@ def mask_url_credentials(url: str) -> str:
)
+def url_origin_for_log(url: str) -> str:
+ """
+ 只保留协议、主机与端口，用于记录来自站点页面或重定向的地址。
+
+ 路径和查询参数可能携带签名、passkey 或会话令牌，且参数名无法穷举，因此一律省略；
+ userinfo 同样不保留。
+
+ :param url: 需要记录的地址
+ :return: 来源地址；地址无法解析时返回固定占位文本
+ """
+ try:
+ parts = urlparse(url)
+ port = f":{parts.port}" if parts.port else ""
+ except ValueError:
+ return "（地址无法解析）"
+ return f"{parts.scheme}://{parts.hostname or ''}{port}"
+
+
def parse_address(
address: str,
include_scheme: bool = True,
diff --git a/app/startup/composition/site.py b/app/startup/composition/site.py
index 87847edbd..ce2d9e2be 100644
--- a/app/startup/composition/site.py
+++ b/app/startup/composition/site.py
@@ -2,7 +2,7 @@
站点访问技术端口的宿主组合根。
"""
-from contextlib import contextmanager
+from contextlib import ExitStack, contextmanager
from typing import Any, Callable, Iterator, Mapping, Optional, cast
from urllib.parse import urljoin, urlsplit
@@ -43,7 +43,29 @@ from app.chain.site import (
configure_site_ports,
reset_site_ports,
)
-from app.foundation.url import UrlUtils
+from app.foundation.url import UrlUtils, url_origin_for_log
+from app.runtime.log import logger
+
+
+def _redirect_destination(url: str, response: Any) -> tuple[Optional[str], Optional[str]]:
+ """
+ 解析一跳重定向的目标地址，只接受不带 URL 凭据的 HTTP(S) 目标。
+
+ :param url: 当前跳的请求地址
+ :param response: 当前跳的重定向响应
+ :return: 目标地址与失败原因，二者恰有一个非空
+ """
+ location = response.headers.get("Location")
+ if not location:
+ return None, "站点请求重定向缺少目标地址"
+ try:
+ destination = urljoin(url, location)
+ target = urlsplit(destination)
+ except ValueError:
+ return None, "站点请求重定向目标无效"
+ if target.scheme not in ("http", "https") or not target.hostname or target.username or target.password:
+ return None, "站点请求重定向目标无效"
+ return destination, None
class _RssHttpAdapter:
@@ -214,6 +236,10 @@ class _SiteHttpAdapter:
"""
打开响应并在所有返回或异常分支后关闭底层连接。
+ GET 请求逐跳跟随重定向，跨域跳转清除 Cookie 与认证头。重定向缺少目标、目标无效、出现环路
+ 或超过跳数上限时只记录来源地址的告警并提供 None，与请求失败的语义一致，
+ 由调用方按无法打开站点处理，不能以异常中断 CookieCloud 同步等批量流程。
+
:param method: HTTP 请求方法
:param url: 完整请求地址
:param headers: 额外请求头
@@ -226,9 +252,11 @@ class _SiteHttpAdapter:
"""
request_headers = dict(headers or {})
seen = set()
+ failure = "站点请求重定向次数超过上限"
for _hop in range(9):
if url in seen:
- raise ValueError("站点请求重定向出现环路")
+ failure = "站点请求重定向出现环路"
+ break
seen.add(url)
request = RequestUtils(
headers=request_headers, cookies=cookie or "", ua=ua or "",
@@ -237,17 +265,22 @@ class _SiteHttpAdapter:
request_options: dict[str, Any] = {} if json_body is None else {"json": dict(json_body)}
if method.upper() == "GET":
request_options["allow_redirects"] = False
- with request.response_manager(method=method, url=url, **request_options) as response:
+ with ExitStack() as stack:
+ try:
+ response = stack.enter_context(
+ request.response_manager(method=method, url=url, **request_options)
+ )
+ except ValueError:
+ # 请求地址或响应 Location 畸形时 requests 会在打开响应时抛出，禁用自动跟随也会预解析 Location
+ failure = "站点请求地址或重定向目标无效"
+ break
if method.upper() != "GET" or response is None or response.status_code not in (301, 302, 303, 307, 308):
yield cast(Optional[SiteResponsePort], response)
return
- location = response.headers.get("Location")
- if not location:
- raise ValueError("站点请求重定向缺少目标地址")
- destination = urljoin(url, location)
- target = urlsplit(destination)
- if target.scheme not in ("http", "https") or not target.hostname or target.username or target.password:
- raise ValueError("站点请求重定向目标无效")
+ destination, invalid_reason = _redirect_destination(url, response)
+ if destination is None:
+ failure = invalid_reason or failure
+ break
if not UrlUtils.same_origin(url, destination):
cookie = None
request_headers = {
@@ -255,7 +288,8 @@ class _SiteHttpAdapter:
if key.lower() not in ("cookie", "authorization", "proxy-authorization", "referer")
}
url = destination
- raise ValueError("站点请求重定向次数超过上限")
+ logger.warning(f"{failure}：{url_origin_for_log(url)}")
+ yield None
class _SiteBrowserAdapter:
diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md
index b1ce2ac08..9ef854226 100644
--- a/docs/architecture-overview.md
+++ b/docs/architecture-overview.md
@@ -817,7 +817,7 @@ flowchart LR
| 指标 | 当前值 |
|---|---:|
| Python 模块 | 1142 |
-| 内部导入边 | 9,850 |
+| 内部导入边 | 9,852 |
| 非平凡 SCC | 1（29 个模块的 TMDB vendor 精确 containment） |
| Application / Chain 具体 Adapter 直连 | 0 / 0 |
| Direct egress | 51（精确 policy 准入，Application / Chain 均为 0） |
diff --git a/tests/fixtures/architecture/dependency-baseline.json b/tests/fixtures/architecture/dependency-baseline.json
index 5bba03cc2..79292e700 100644
--- a/tests/fixtures/architecture/dependency-baseline.json
+++ b/tests/fixtures/architecture/dependency-baseline.json
@@ -959,8 +959,8 @@
"runtime_only": true
}
},
- "edge_count": 9850,
- "edge_sha256": "4ee56369dee810a01662c3c2f63eb016d57c308836d1e6ca9254ca31f434dc6f",
+ "edge_count": 9852,
+ "edge_sha256": "281f9cb108557ce8bb78b1c4a7249fcc983d0b0b53b9691f0b6f72de79e7d61c",
"edges": [
"app -> app.foundation",
"app -> app.foundation.environment",
@@ -10232,6 +10232,8 @@
"app.startup.composition.site -> app.chain.site",
"app.startup.composition.site -> app.foundation",
"app.startup.composition.site -> app.foundation.url",
+ "app.startup.composition.site -> app.runtime",
+ "app.startup.composition.site -> app.runtime.log",
"app.startup.composition.subscription -> app.adapters",
"app.startup.composition.subscription -> app.adapters.external",
"app.startup.composition.subscription -> app.adapters.external.server",
diff --git a/tests/test_site_chain_ports.py b/tests/test_site_chain_ports.py
index eab900890..1b8e138dd 100644
--- a/tests/test_site_chain_ports.py
+++ b/tests/test_site_chain_ports.py
@@ -437,3 +437,130 @@ def test_site_chain_has_no_concrete_adapter_imports() -> None:
]
assert violations == []
+
+
+def _redirect(location: Optional[str]) -> SimpleNamespace:
+ """
+ 构造带可选目标地址的重定向响应。
+ """
+ return SimpleNamespace(status_code=302, headers={"Location": location} if location else {})
+
+
+def _scripted_request(monkeypatch, responses: list) -> list[dict]:
+ """
+ 用按序回放的 RequestUtils 替身替换组合根传输，并记录每一跳的构造参数和请求参数。
+ """
+ hops: list[dict] = []
+ queue = list(responses)
+
+ class ScriptedRequest:
+ """
+ 记录每一跳携带的 Cookie 与请求地址。
+ """
+
+ def __init__(self, **kwargs) -> None:
+ """
+ 保存 adapter 传入的 Cookie 等构造参数。
+ """
+ self.options = kwargs
+
+ @contextmanager
+ def response_manager(self, **kwargs):
+ """
+ 依次返回预设响应。
+ """
+ hops.append({**self.options, **kwargs})
+ item = queue.pop(0)
+ if isinstance(item, Exception):
+ # 模拟 requests 在打开响应时即抛出的异常，例如预解析畸形 Location
+ raise item
+ yield item
+
+ monkeypatch.setattr(site_composition, "RequestUtils", ScriptedRequest)
+ return hops
+
+
+@pytest.mark.parametrize("responses,reason", [
+ ([_redirect("/dl/1/auth?X-Amz-Signature=sig")] * 2, "站点请求重定向出现环路"),
+ ([_redirect(None)], "站点请求重定向缺少目标地址"),
+ ([_redirect("ftp://site.test/file")], "站点请求重定向目标无效"),
+ ([_redirect("https://user:pass@site.test/")], "站点请求重定向目标无效"),
+ ([_redirect("https://[bad/path")], "站点请求重定向目标无效"),
+ ([_redirect(f"/page{index}") for index in range(9)], "站点请求重定向次数超过上限"),
+ ([ValueError("Invalid IPv6 URL")], "站点请求地址或重定向目标无效"),
+ ([_redirect("/next"), ValueError("Invalid IPv6 URL")], "站点请求地址或重定向目标无效"),
+], ids=[
+ "loop", "missing-location", "invalid-scheme", "userinfo", "malformed", "too-many-hops",
+ "transport-rejects-location", "transport-rejects-next-hop",
+])
+def test_site_adapter_redirect_failures_yield_none(monkeypatch, responses, reason) -> None:
+ """
+ 重定向失败按端口约定提供 None 并告警，不抛出异常打断 CookieCloud 同步等批量流程；
+ 告警只含来源地址，路径与查询参数中的签名、passkey 等凭据都不写入日志。
+ """
+ hops = _scripted_request(monkeypatch, responses)
+ warnings = []
+ monkeypatch.setattr(site_composition.logger, "warning", warnings.append)
+
+ with site_composition._SiteHttpAdapter().open(
+ method="GET", url="https://site.test/index.php?passkey=secret", cookie="uid=1",
+ ) as response:
+ assert response is None
+
+ assert len(hops) == len(responses)
+ assert warnings == [f"{reason}：https://site.test"]
+
+
+def test_cookiecloud_sync_continues_after_redirect_loop(monkeypatch) -> None:
+ """
+ 单个新站点的重定向环路只记为该站点添加失败，同步继续处理后续站点。
+ """
+ hops = _scripted_request(monkeypatch, [_redirect("/")])
+ monkeypatch.setattr(site_composition.logger, "warning", lambda _message: None)
+ _configure(site_composition._SiteHttpAdapter())
+ chain = _chain()
+ chain.runtime_config.cookiecloud_blacklist = ""
+
+ result = chain._sync_cookiecloud_domain(
+ domain="loop.test",
+ cookie="uid=1",
+ indexer={"name": "环路站点", "domain": "https://loop.test/"},
+ site_info=None,
+ repository=SimpleNamespace(),
+ rsshelper=SimpleNamespace(),
+ )
+
+ # 跳回同一地址在第二跳发出前即判定为环路
+ assert result == (0, 0, 1, False)
+ assert [hop["url"] for hop in hops] == ["https://loop.test/"]
+
+
+def test_site_adapter_does_not_swallow_caller_errors(monkeypatch) -> None:
+ """
+ 调用方在响应上下文内抛出的异常必须原样传播，不能被当作重定向失败吞掉。
+ """
+ _scripted_request(monkeypatch, [_FakeResponse(200, text="ok")])
+ warnings = []
+ monkeypatch.setattr(site_composition.logger, "warning", warnings.append)
+
+ with pytest.raises(ValueError, match="调用方错误"):
+ with site_composition._SiteHttpAdapter().open(method="GET", url="https://site.test/") as response:
+ assert response is not None
+ raise ValueError("调用方错误")
+
+ assert warnings == []
+
+
+def test_favicon_failure_logs_only_site_origin(monkeypatch) -> None:
+ """
+ 图标或页面读取失败的日志只记录来源地址，页面提供的图标 URL 中的令牌不能写入日志。
+ """
+ _configure(_FakeHttpPort(_FakeResponse(200, text=''), None))
+ errors = []
+ monkeypatch.setattr(site_module.logger, "error", errors.append)
+
+ favicon_url, content = SiteChain._SiteChain__parse_favicon("https://site.test/?passkey=secret", "uid=1", "UA")
+
+ assert favicon_url == "https://site.test/icon.png?token=secret"
+ assert content is None
+ assert errors == ["获取站点图标失败：https://site.test"]
diff --git a/tests/test_url_utils.py b/tests/test_url_utils.py
index c7acd7ba8..a80f343df 100644
--- a/tests/test_url_utils.py
+++ b/tests/test_url_utils.py
@@ -4,7 +4,7 @@
import pytest
-from app.foundation.url import UrlUtils, mask_url_credentials, url_matches_trusted_hosts
+from app.foundation.url import UrlUtils, mask_url_credentials, url_matches_trusted_hosts, url_origin_for_log
@pytest.mark.parametrize(
@@ -128,3 +128,17 @@ def test_mask_url_credentials_applies_to_free_text_with_url():
assert "SECRET" not in masked
assert "passkey=" in masked
assert masked.startswith("Max retries exceeded with url: /torrentrss.php?")
+
+
+@pytest.mark.parametrize("url,expected", [
+ ("https://site.test/dl/1/auth?X-Amz-Signature=sig", "https://site.test"),
+ ("https://user:pass@site.test:8443/path?passkey=secret", "https://site.test:8443"),
+ ("http://site.test", "http://site.test"),
+ ("https://[bad/path", "（地址无法解析）"),
+ ("https://site.test:99999/", "（地址无法解析）"),
+])
+def test_url_origin_for_log_drops_path_query_and_userinfo(url, expected):
+ """
+ 日志地址只保留协议、主机与端口，路径、查询参数和 userinfo 中的凭据都不保留。
+ """
+ assert url_origin_for_log(url) == expected