From 8c156d4f47e6b3bbdfce9bf9b59a59d137ba24c5 Mon Sep 17 00:00:00 2001 From: JontyLee <31982854+JontyLee@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:43:30 +0800 Subject: [PATCH] =?UTF-8?q?fix(rss):=20=E9=81=AE=E7=9B=96=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E6=97=A5=E5=BF=97=E4=B8=8E=E8=AF=8A=E6=96=AD=E6=8A=A5?= =?UTF-8?q?=E5=91=8A=E4=B8=AD=E7=9A=84=20URL=20=E5=87=AD=E6=8D=AE=20(#7157?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 站点 RSS 地址把 passkey、sign 等凭据放在查询串里，请求失败时这些原文会随 错误日志落盘，并被 Doctor 摘录进可外发的诊断报告；连接串形式的 userinfo 口令 同样存在。新增 app.foundation.url.mask_url_credentials 纯函数与两条共享正则， RSS 失败日志改用遮盖后的地址，Doctor 的日志遮盖规则复用同一模式，避免两处实现分叉。 --- app/application/rss.py | 8 +- app/doctor/checks.py | 4 + app/foundation/url.py | 35 +++++++++ docs/architecture-overview.md | 4 +- .../architecture/dependency-baseline.json | 6 +- tests/test_doctor.py | 29 +++++++- tests/test_rss_url_validation.py | 54 +++++++++++++- tests/test_url_utils.py | 73 ++++++++++++++++++- 8 files changed, 201 insertions(+), 12 deletions(-) diff --git a/app/application/rss.py b/app/application/rss.py index f6487f363..584736d6d 100644 --- a/app/application/rss.py +++ b/app/application/rss.py @@ -11,7 +11,7 @@ import dateutil.parser from lxml import etree from app.application.configuration import get_chain_runtime_config_snapshot -from app.foundation.url import UrlUtils +from app.foundation.url import UrlUtils, mask_url_credentials from app.runtime.log import logger @@ -435,15 +435,15 @@ class RssHelper: headers=headers, ) if not ret: - logger.error(f"获取RSS失败：请求返回空值，URL: {url}") + logger.error(f"获取RSS失败：请求返回空值，URL: {mask_url_credentials(url)}") return False except Exception as err: - logger.error(f"获取RSS失败：{str(err)} - {traceback.format_exc()}") + logger.error(mask_url_credentials(f"获取RSS失败：{str(err)} - {traceback.format_exc()}")) return False if ret: if ret.status_code != 200: - logger.error(f"RSS请求失败，状态码: {ret.status_code}, URL: {url}") + logger.error(f"RSS请求失败，状态码: {ret.status_code}, URL: {mask_url_credentials(url)}") return False ret_xml = None root = None diff --git a/app/doctor/checks.py b/app/doctor/checks.py index 17fdf342a..c7c0bfa17 100644 --- a/app/doctor/checks.py +++ b/app/doctor/checks.py @@ -26,6 +26,7 @@ from app.adapters.system.backup.database import verify_database_backup from app.adapters.system.backup.files import BackupFiles from app.adapters.system.host import SystemUtils from app.doctor.models import DoctorFinding, DoctorFindingStatus, DoctorReport, DoctorSeverity +from app.foundation.url import URL_QUERY_CREDENTIAL_PATTERN, URL_USERINFO_CREDENTIAL_PATTERN from app.runtime.settings import get_runtime_setting, update_runtime_setting from app.runtime.topology import process_topology_issue @@ -69,6 +70,9 @@ SENSITIVE_PATTERNS = ( re.compile(r"(?i)(api[_-]?token|token|password|secret|cookie)(\s*[:=]\s*)[^\s&]+"), re.compile(r"\bghp_[A-Za-z0-9]{20,}\b"), re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}\b"), + # 站点 RSS 与接口地址的凭据常以 passkey、sign 等查询参数或连接串口令出现，参数名本身需要保留供定位。 + URL_QUERY_CREDENTIAL_PATTERN, + URL_USERINFO_CREDENTIAL_PATTERN, ) diff --git a/app/foundation/url.py b/app/foundation/url.py index e584613c3..53b64886b 100644 --- a/app/foundation/url.py +++ b/app/foundation/url.py @@ -9,6 +9,19 @@ from typing import Iterable, Mapping, Optional, Tuple, Union from urllib import parse from urllib.parse import parse_qs, urlencode, urljoin, urlparse, urlunparse +REDACTED_URL_CREDENTIAL = "" +# 站点 RSS 与开放接口常把凭据放在查询串里，参数名保留用于定位，只遮盖参数值。 +URL_QUERY_CREDENTIAL_PATTERN = re.compile( + r"(?i)([?&](?:[a-z0-9]+[-_.])*(?:" + r"passkey|rsskey|authkey|auth_token|access_token|access_key|refresh_token|id_token" + r"|api_key|apikey|secret_key|secret|password|passwd|token|cookie|sign|pwd|auth|uid|pid|key" + r"))(\s*=\s*)[^\s&#]*" +) +# 主机端口与 userinfo 口令都由冒号引导，因此只在 authority 内以最后一个 @ 为边界遮盖口令。 +URL_USERINFO_CREDENTIAL_PATTERN = re.compile( + r"(?i)([a-z][a-z0-9+.\-]*://)([^\s/?#@:]*?:)(?=[^\s/?#]*@)[^\s/?#]*(?=@)" +) + class UrlUtils: """ @@ -248,6 +261,28 @@ def base_url(url: str) -> str: return f"{scheme}://{netloc}" +def mask_url_credentials(url: str) -> str: + """ + 遮盖文本中 URL 形态的凭据，保留其余内容用于定位。 + + 只按查询串凭据参数和 userinfo 口令两种 URL 语法做遮盖，不解析、不重写地址； + 主机、端口、路径、分页等非凭据参数保持原样，重复调用结果不变。 + + :param url: 需要脱敏的 URL，或包含 URL 的日志与异常文本 + :return: 凭据值替换为占位符后的文本；输入为空时原样返回 + """ + if not url: + return url + masked = URL_QUERY_CREDENTIAL_PATTERN.sub( + rf"\1\2{REDACTED_URL_CREDENTIAL}", + url, + ) + return URL_USERINFO_CREDENTIAL_PATTERN.sub( + rf"\1\2{REDACTED_URL_CREDENTIAL}", + masked, + ) + + def parse_address( address: str, include_scheme: bool = True, diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index d74e1c354..d4c989fba 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -811,13 +811,13 @@ flowchart LR ### 10.2 当前架构快照 -以下是 2026-10-21 核对的 `tests/fixtures/architecture/` 数据；这是源码合同快照， +以下是 2026-10-23 核对的 `tests/fixtures/architecture/` 数据；这是源码合同快照， 不代表某个已发布镜像或正在运行的实例已经验证。 | 指标 | 当前值 | |---|---:| | Python 模块 | 1129 | -| 内部导入边 | 9,711 | +| 内部导入边 | 9,713 | | 非平凡 SCC | 1（29 个模块的 TMDB vendor 精确 containment） | | Application / Chain 具体 Adapter 直连 | 0 / 0 | | Direct egress | 51（精确 policy 准入，Application / Chain 均为 0） | diff --git a/tests/fixtures/architecture/dependency-baseline.json b/tests/fixtures/architecture/dependency-baseline.json index 8b65a7fb0..a8b53632d 100644 --- a/tests/fixtures/architecture/dependency-baseline.json +++ b/tests/fixtures/architecture/dependency-baseline.json @@ -959,8 +959,8 @@ "runtime_only": true } }, - "edge_count": 9711, - "edge_sha256": "ab6b9c2a3353c8520468b3f7752ed1ed1c2c208a093afb50a7a5e6895d6f8ea0", + "edge_count": 9713, + "edge_sha256": "80a64865791510e96da79d2c47098559cae77c5401e1e6a3c0950c18044c38b5", "edges": [ "app -> app.foundation", "app -> app.foundation.environment", @@ -6824,6 +6824,8 @@ "app.doctor.checks -> app.adapters.system.host", "app.doctor.checks -> app.doctor", "app.doctor.checks -> app.doctor.models", + "app.doctor.checks -> app.foundation", + "app.doctor.checks -> app.foundation.url", "app.doctor.checks -> app.runtime", "app.doctor.checks -> app.runtime.settings", "app.doctor.checks -> app.runtime.topology", diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 928853f64..be9d872ba 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -1,5 +1,5 @@ """ -验证本地诊断报告格式、数据库损坏与备份可用性判定。 +验证本地诊断报告格式、数据库损坏与备份可用性判定、日志凭据遮盖。 """ from __future__ import annotations @@ -401,3 +401,30 @@ def test_doctor_ignores_errors_outside_log_window(tmp_path, monkeypatch): assert runner.report.find("logs.moviepilot.recent_errors") is None assert runner.report.find("logs.recent") is not None assert runner.report.status.value == "healthy" + + +def test_doctor_masks_url_credentials_in_recent_log_findings(tmp_path, monkeypatch): + """ + 诊断报告引用的日志行应遮盖 URL 查询凭据与连接串口令，保留主机和参数名。 + """ + monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path)) + app_log = settings.LOG_PATH / "moviepilot.log" + app_log.parent.mkdir(parents=True, exist_ok=True) + app_log.write_text( + f"【ERROR】{_current_log_timestamp()} - rss.py - 获取RSS失败：请求返回空值，" + "URL: https://example.com/torrentrss.php?id=1&passkey=SUPERSECRET&isdownload=0\n" + f"【ERROR】{_current_log_timestamp()} - database.py - 连接数据库失败 " + "postgres://moviepilot:DBPASSWORD@127.0.0.1:5432/moviepilot\n", + encoding="utf-8", + ) + + runner = DoctorRunner() + checks._check_logs(runner) + + finding = runner.report.find("logs.moviepilot.recent_errors") + assert finding is not None + assert "SUPERSECRET" not in finding.detail + assert "DBPASSWORD" not in finding.detail + assert "passkey=" in finding.detail + assert "id=1" in finding.detail + assert "postgres://moviepilot:@127.0.0.1:5432/moviepilot" in finding.detail diff --git a/tests/test_rss_url_validation.py b/tests/test_rss_url_validation.py index 08730b6be..850ac3399 100644 --- a/tests/test_rss_url_validation.py +++ b/tests/test_rss_url_validation.py @@ -1,5 +1,5 @@ """ -验证RSS 地址校验及访问安全边界。 +验证RSS 地址校验、访问安全边界与失败日志的凭据遮盖。 """ from types import SimpleNamespace @@ -8,6 +8,7 @@ from unittest.mock import Mock import pytest import app.chain.torrents as torrents_module +from app.application import rss as rss_module from app.application.rss import RssHelper, configure_rss_ports, reset_rss_ports from app.chain.torrents import TorrentsChain @@ -113,3 +114,54 @@ def test_torrents_rss_warns_once_and_skips_invalid_site_url(monkeypatch, rss_por assert warnings == ["站点 example.com RSS地址无效，跳过获取"] assert "passkey=secret" not in warnings[0] renew.assert_not_called() + + +@pytest.fixture +def rss_error_logs(monkeypatch): + """ + 收集 RSS 失败日志，避免用例输出真实错误日志。 + + :return: 收集到的 error 日志文本列表 + """ + errors: list[str] = [] + monkeypatch.setattr(rss_module.logger, "error", errors.append) + return errors + + +@pytest.mark.parametrize( + ("response", "expected_fragment"), + [ + (None, "获取RSS失败：请求返回空值"), + (SimpleNamespace(status_code=503, content=b"", text="", reason="Service Unavailable"), + "RSS请求失败，状态码: 503"), + ], +) +def test_rss_helper_error_logs_mask_query_passkey(rss_ports, rss_error_logs, response, expected_fragment): + """ + RSS 请求空响应与非 200 响应时，错误日志保留站点定位信息并遮盖 passkey。 + """ + rss_ports.get.return_value = response + + assert RssHelper().parse( + "https://example.com/torrentrss.php?id=1&linktype=1&passkey=SUPERSECRET" + ) is False + assert len(rss_error_logs) == 1 + message = rss_error_logs[0] + assert expected_fragment in message + assert "SUPERSECRET" not in message + assert "passkey=" in message + assert "id=1" in message + + +def test_rss_helper_exception_log_masks_url_credentials(rss_ports, rss_error_logs): + """ + 请求异常文本自带 RSS 相对地址时，错误日志同样不输出 passkey 原文。 + """ + rss_ports.get.side_effect = RuntimeError( + "Max retries exceeded with url: /torrentrss.php?passkey=SUPERSECRET" + ) + + assert RssHelper().parse("https://example.com/torrentrss.php?passkey=SUPERSECRET") is False + assert len(rss_error_logs) == 1 + assert "SUPERSECRET" not in rss_error_logs[0] + assert "passkey=" in rss_error_logs[0] diff --git a/tests/test_url_utils.py b/tests/test_url_utils.py index def8fed8b..c7acd7ba8 100644 --- a/tests/test_url_utils.py +++ b/tests/test_url_utils.py @@ -1,10 +1,10 @@ """ -验证URL 解析、拼接和规范化规则。 +验证URL 解析、拼接、规范化和凭据脱敏规则。 """ import pytest -from app.foundation.url import UrlUtils, url_matches_trusted_hosts +from app.foundation.url import UrlUtils, mask_url_credentials, url_matches_trusted_hosts @pytest.mark.parametrize( @@ -59,3 +59,72 @@ def test_url_matches_trusted_hosts_requires_exact_http_host(url, trusted_hosts, 受信主机匹配只放行 HTTP(S) 且端口或主机名精确命中的地址。 """ assert url_matches_trusted_hosts(url, trusted_hosts) is expected + + +@pytest.mark.parametrize( + ("url", "expected"), + [ + ("https://example.com/torrentrss.php?id=1&passkey=SECRET&isdownload=0", + "https://example.com/torrentrss.php?id=1&passkey=&isdownload=0"), + ("https://example.com/rss?access_token=SECRET", + "https://example.com/rss?access_token="), + ("https://example.com/rss?X-Plex-Token=SECRET&sort=title", + "https://example.com/rss?X-Plex-Token=&sort=title"), + ("https://example.com/rss?key=a%2Fb&uid=42&sign=deadbeef", + "https://example.com/rss?key=&uid=&sign="), + ("https://example.com/rss#item?passkey=SECRET", + "https://example.com/rss#item?passkey="), + ], +) +def test_mask_url_credentials_redacts_query_credentials(url, expected): + """ + 查询串凭据只遮盖参数值，保留主机、路径、参数名和非凭据参数。 + """ + assert mask_url_credentials(url) == expected + + +@pytest.mark.parametrize( + ("url", "expected"), + [ + ("http://admin:MyPass@192.168.1.5:5000/RPC2", + "http://admin:@192.168.1.5:5000/RPC2"), + ("redis://:p@ss@10.0.0.1:6379/0", "redis://:@10.0.0.1:6379/0"), + ("postgres://mp:TopSecret@127.0.0.1:5432/moviepilot", + "postgres://mp:@127.0.0.1:5432/moviepilot"), + ], +) +def test_mask_url_credentials_redacts_userinfo_password(url, expected): + """ + 连接串 userinfo 口令被遮盖，用户名、主机和端口保持可读。 + """ + assert mask_url_credentials(url) == expected + + +@pytest.mark.parametrize( + "url", + [ + "", + "http://192.168.1.5:32400/library", + "http://user@host:8080/p", + "https://example.com/a?b=1&monkey=2&turkey=3", + "https://example.com/t/p/w500/x.jpg", + "已遮盖 https://example.com/rss?passkey=", + ], +) +def test_mask_url_credentials_keeps_non_credential_text(url): + """ + 端口、无口令 userinfo 和非凭据参数都不被改写，重复遮盖保持幂等。 + """ + assert mask_url_credentials(url) == url + assert mask_url_credentials(mask_url_credentials(url)) == mask_url_credentials(url) + + +def test_mask_url_credentials_applies_to_free_text_with_url(): + """ + 异常文本中的相对路径查询串同样被遮盖，便于直接用于日志脱敏。 + """ + text = "Max retries exceeded with url: /torrentrss.php?passkey=SECRET (Caused by NewConnectionError)" + masked = mask_url_credentials(text) + assert "SECRET" not in masked + assert "passkey=" in masked + assert masked.startswith("Max retries exceeded with url: /torrentrss.php?")