nas/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-11-04 06:15:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(rss): 遮盖失败日志与诊断报告中的 URL 凭据 (#7157)

Browse Source 
站点 RSS 地址把 passkey、sign 等凭据放在查询串里，请求失败时这些原文会随
错误日志落盘，并被 Doctor 摘录进可外发的诊断报告；连接串形式的 userinfo 口令
同样存在。新增 app.foundation.url.mask_url_credentials 纯函数与两条共享正则，
RSS 失败日志改用遮盖后的地址，Doctor 的日志遮盖规则复用同一模式，避免两处实现分叉。
This commit is contained in:
JontyLee
2026-10-23 17:43:30 +08:00
committed by GitHub
parent 0797d548b3
commit 8c156d4f47
8 changed files with 201 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/application/rss.py
View File

@@ -11,7 +11,7 @@ import dateutil.parser
from lxml import etree
from app.application.configuration import get_chain_runtime_config_snapshot
from app.foundation.url import UrlUtils
from app.foundation.url import UrlUtils, mask_url_credentials
from app.runtime.log import logger
@@ -435,15 +435,15 @@ class RssHelper:
headers=headers,
)
if not ret:
logger.error(f"获取RSS失败：请求返回空值，URL: {url}")
logger.error(f"获取RSS失败：请求返回空值，URL: {mask_url_credentials(url)}")
return False
except Exception as err:
logger.error(f"获取RSS失败：{str(err)} - {traceback.format_exc()}")
logger.error(mask_url_credentials(f"获取RSS失败：{str(err)} - {traceback.format_exc()}"))
return False
if ret:
if ret.status_code != 200:
logger.error(f"RSS请求失败，状态码: {ret.status_code}, URL: {url}")
logger.error(f"RSS请求失败，状态码: {ret.status_code}, URL: {mask_url_credentials(url)}")
return False
ret_xml = None
root = None

4
app/doctor/checks.py
View File

@@ -26,6 +26,7 @@ from app.adapters.system.backup.database import verify_database_backup
from app.adapters.system.backup.files import BackupFiles
from app.adapters.system.host import SystemUtils
from app.doctor.models import DoctorFinding, DoctorFindingStatus, DoctorReport, DoctorSeverity
from app.foundation.url import URL_QUERY_CREDENTIAL_PATTERN, URL_USERINFO_CREDENTIAL_PATTERN
from app.runtime.settings import get_runtime_setting, update_runtime_setting
from app.runtime.topology import process_topology_issue
@@ -69,6 +70,9 @@ SENSITIVE_PATTERNS = (
re.compile(r"(?i)(api[_-]?token|token|password|secret|cookie)(\s*[:=]\s*)[^\s&]+"),
re.compile(r"\bghp_[A-Za-z0-9]{20,}\b"),
re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}\b"),
# 站点 RSS 与接口地址的凭据常以 passkey、sign 等查询参数或连接串口令出现，参数名本身需要保留供定位。
URL_QUERY_CREDENTIAL_PATTERN,
URL_USERINFO_CREDENTIAL_PATTERN,
)

35
app/foundation/url.py
View File

@@ -9,6 +9,19 @@ from typing import Iterable, Mapping, Optional, Tuple, Union
from urllib import parse
from urllib.parse import parse_qs, urlencode, urljoin, urlparse, urlunparse
REDACTED_URL_CREDENTIAL = "<REDACTED>"
# 站点 RSS 与开放接口常把凭据放在查询串里，参数名保留用于定位，只遮盖参数值。
URL_QUERY_CREDENTIAL_PATTERN = re.compile(
r"(?i)([?&](?:[a-z0-9]+[-_.])*(?:"
r"passkey|rsskey|authkey|auth_token|access_token|access_key|refresh_token|id_token"
r"|api_key|apikey|secret_key|secret|password|passwd|token|cookie|sign|pwd|auth|uid|pid|key"
r"))(\s*=\s*)[^\s&#]*"
)
# 主机端口与 userinfo 口令都由冒号引导，因此只在 authority 内以最后一个 @ 为边界遮盖口令。
URL_USERINFO_CREDENTIAL_PATTERN = re.compile(
r"(?i)([a-z][a-z0-9+.\-]*://)([^\s/?#@:]*?:)(?=[^\s/?#]*@)[^\s/?#]*(?=@)"
)
class UrlUtils:
"""
@@ -248,6 +261,28 @@ def base_url(url: str) -> str:
return f"{scheme}://{netloc}"
def mask_url_credentials(url: str) -> str:
"""
遮盖文本中 URL 形态的凭据，保留其余内容用于定位。
只按查询串凭据参数和 userinfo 口令两种 URL 语法做遮盖，不解析、不重写地址；
主机、端口、路径、分页等非凭据参数保持原样，重复调用结果不变。
:param url: 需要脱敏的 URL，或包含 URL 的日志与异常文本
:return: 凭据值替换为占位符后的文本；输入为空时原样返回
"""
if not url:
return url
masked = URL_QUERY_CREDENTIAL_PATTERN.sub(
rf"\1\2{REDACTED_URL_CREDENTIAL}",
url,
)
return URL_USERINFO_CREDENTIAL_PATTERN.sub(
rf"\1\2{REDACTED_URL_CREDENTIAL}",
masked,
)
def parse_address(
address: str,
include_scheme: bool = True,

4
docs/architecture-overview.md
View File

@@ -811,13 +811,13 @@ flowchart LR
### 10.2 当前架构快照
以下是 2026-10-21 核对的 `tests/fixtures/architecture/` 数据；这是源码合同快照，
以下是 2026-10-23 核对的 `tests/fixtures/architecture/` 数据；这是源码合同快照，
不代表某个已发布镜像或正在运行的实例已经验证。
| 指标 | 当前值 |
|---|---:|
| Python 模块 | 1129 |
| 内部导入边 | 9,711 |
| 内部导入边 | 9,713 |
| 非平凡 SCC | 1（29 个模块的 TMDB vendor 精确 containment） |
| Application / Chain 具体 Adapter 直连 | 0 / 0 |
| Direct egress | 51（精确 policy 准入，Application / Chain 均为 0） |

6
tests/fixtures/architecture/dependency-baseline.json vendored
View File

@@ -959,8 +959,8 @@
"runtime_only": true
}
},
"edge_count": 9711,
"edge_sha256": "ab6b9c2a3353c8520468b3f7752ed1ed1c2c208a093afb50a7a5e6895d6f8ea0",
"edge_count": 9713,
"edge_sha256": "80a64865791510e96da79d2c47098559cae77c5401e1e6a3c0950c18044c38b5",
"edges": [
"app -> app.foundation",
"app -> app.foundation.environment",
@@ -6824,6 +6824,8 @@
"app.doctor.checks -> app.adapters.system.host",
"app.doctor.checks -> app.doctor",
"app.doctor.checks -> app.doctor.models",
"app.doctor.checks -> app.foundation",
"app.doctor.checks -> app.foundation.url",
"app.doctor.checks -> app.runtime",
"app.doctor.checks -> app.runtime.settings",
"app.doctor.checks -> app.runtime.topology",

29
tests/test_doctor.py
View File

@@ -1,5 +1,5 @@
"""
验证本地诊断报告格式、数据库损坏与备份可用性判定。
验证本地诊断报告格式、数据库损坏与备份可用性判定、日志凭据遮盖。
"""
from __future__ import annotations
@@ -401,3 +401,30 @@ def test_doctor_ignores_errors_outside_log_window(tmp_path, monkeypatch):
assert runner.report.find("logs.moviepilot.recent_errors") is None
assert runner.report.find("logs.recent") is not None
assert runner.report.status.value == "healthy"
def test_doctor_masks_url_credentials_in_recent_log_findings(tmp_path, monkeypatch):
"""
诊断报告引用的日志行应遮盖 URL 查询凭据与连接串口令，保留主机和参数名。
"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
app_log = settings.LOG_PATH / "moviepilot.log"
app_log.parent.mkdir(parents=True, exist_ok=True)
app_log.write_text(
f"【ERROR】{_current_log_timestamp()} - rss.py - 获取RSS失败：请求返回空值，"
"URL: https://example.com/torrentrss.php?id=1&passkey=SUPERSECRET&isdownload=0\n"
f"【ERROR】{_current_log_timestamp()} - database.py - 连接数据库失败 "
"postgres://moviepilot:DBPASSWORD@127.0.0.1:5432/moviepilot\n",
encoding="utf-8",
)
runner = DoctorRunner()
checks._check_logs(runner)
finding = runner.report.find("logs.moviepilot.recent_errors")
assert finding is not None
assert "SUPERSECRET" not in finding.detail
assert "DBPASSWORD" not in finding.detail
assert "passkey=<REDACTED>" in finding.detail
assert "id=1" in finding.detail
assert "postgres://moviepilot:<REDACTED>@127.0.0.1:5432/moviepilot" in finding.detail

54
tests/test_rss_url_validation.py
View File

@@ -1,5 +1,5 @@
"""
验证RSS 地址校验及访问安全边界。
验证RSS 地址校验、访问安全边界与失败日志的凭据遮盖。
"""
from types import SimpleNamespace
@@ -8,6 +8,7 @@ from unittest.mock import Mock
import pytest
import app.chain.torrents as torrents_module
from app.application import rss as rss_module
from app.application.rss import RssHelper, configure_rss_ports, reset_rss_ports
from app.chain.torrents import TorrentsChain
@@ -113,3 +114,54 @@ def test_torrents_rss_warns_once_and_skips_invalid_site_url(monkeypatch, rss_por
assert warnings == ["站点 example.com RSS地址无效，跳过获取"]
assert "passkey=secret" not in warnings[0]
renew.assert_not_called()
@pytest.fixture
def rss_error_logs(monkeypatch):
"""
收集 RSS 失败日志，避免用例输出真实错误日志。
:return: 收集到的 error 日志文本列表
"""
errors: list[str] = []
monkeypatch.setattr(rss_module.logger, "error", errors.append)
return errors
@pytest.mark.parametrize(
("response", "expected_fragment"),
[
(None, "获取RSS失败：请求返回空值"),
(SimpleNamespace(status_code=503, content=b"", text="", reason="Service Unavailable"),
"RSS请求失败，状态码: 503"),
],
)
def test_rss_helper_error_logs_mask_query_passkey(rss_ports, rss_error_logs, response, expected_fragment):
"""
RSS 请求空响应与非 200 响应时，错误日志保留站点定位信息并遮盖 passkey。
"""
rss_ports.get.return_value = response
assert RssHelper().parse(
"https://example.com/torrentrss.php?id=1&linktype=1&passkey=SUPERSECRET"
) is False
assert len(rss_error_logs) == 1
message = rss_error_logs[0]
assert expected_fragment in message
assert "SUPERSECRET" not in message
assert "passkey=<REDACTED>" in message
assert "id=1" in message
def test_rss_helper_exception_log_masks_url_credentials(rss_ports, rss_error_logs):
"""
请求异常文本自带 RSS 相对地址时，错误日志同样不输出 passkey 原文。
"""
rss_ports.get.side_effect = RuntimeError(
"Max retries exceeded with url: /torrentrss.php?passkey=SUPERSECRET"
)
assert RssHelper().parse("https://example.com/torrentrss.php?passkey=SUPERSECRET") is False
assert len(rss_error_logs) == 1
assert "SUPERSECRET" not in rss_error_logs[0]
assert "passkey=<REDACTED>" in rss_error_logs[0]

73
tests/test_url_utils.py
View File

@@ -1,10 +1,10 @@
"""
验证URL 解析、拼接和规范化规则。
验证URL 解析、拼接、规范化和凭据脱敏规则。
"""
import pytest
from app.foundation.url import UrlUtils, url_matches_trusted_hosts
from app.foundation.url import UrlUtils, mask_url_credentials, url_matches_trusted_hosts
@pytest.mark.parametrize(
@@ -59,3 +59,72 @@ def test_url_matches_trusted_hosts_requires_exact_http_host(url, trusted_hosts,
受信主机匹配只放行 HTTP(S) 且端口或主机名精确命中的地址。
"""
assert url_matches_trusted_hosts(url, trusted_hosts) is expected
@pytest.mark.parametrize(
("url", "expected"),
[
("https://example.com/torrentrss.php?id=1&passkey=SECRET&isdownload=0",
"https://example.com/torrentrss.php?id=1&passkey=<REDACTED>&isdownload=0"),
("https://example.com/rss?access_token=SECRET",
"https://example.com/rss?access_token=<REDACTED>"),
("https://example.com/rss?X-Plex-Token=SECRET&sort=title",
"https://example.com/rss?X-Plex-Token=<REDACTED>&sort=title"),
("https://example.com/rss?key=a%2Fb&uid=42&sign=deadbeef",
"https://example.com/rss?key=<REDACTED>&uid=<REDACTED>&sign=<REDACTED>"),
("https://example.com/rss#item?passkey=SECRET",
"https://example.com/rss#item?passkey=<REDACTED>"),
],
)
def test_mask_url_credentials_redacts_query_credentials(url, expected):
"""
查询串凭据只遮盖参数值，保留主机、路径、参数名和非凭据参数。
"""
assert mask_url_credentials(url) == expected
@pytest.mark.parametrize(
("url", "expected"),
[
("http://admin:MyPass@192.168.1.5:5000/RPC2",
"http://admin:<REDACTED>@192.168.1.5:5000/RPC2"),
("redis://:p@ss@10.0.0.1:6379/0", "redis://:<REDACTED>@10.0.0.1:6379/0"),
("postgres://mp:TopSecret@127.0.0.1:5432/moviepilot",
"postgres://mp:<REDACTED>@127.0.0.1:5432/moviepilot"),
],
)
def test_mask_url_credentials_redacts_userinfo_password(url, expected):
"""
连接串 userinfo 口令被遮盖，用户名、主机和端口保持可读。
"""
assert mask_url_credentials(url) == expected
@pytest.mark.parametrize(
"url",
[
"",
"http://192.168.1.5:32400/library",
"http://user@host:8080/p",
"https://example.com/a?b=1&monkey=2&turkey=3",
"https://example.com/t/p/w500/x.jpg",
"已遮盖 https://example.com/rss?passkey=<REDACTED>",
],
)
def test_mask_url_credentials_keeps_non_credential_text(url):
"""
端口、无口令 userinfo 和非凭据参数都不被改写，重复遮盖保持幂等。
"""
assert mask_url_credentials(url) == url
assert mask_url_credentials(mask_url_credentials(url)) == mask_url_credentials(url)
def test_mask_url_credentials_applies_to_free_text_with_url():
"""
异常文本中的相对路径查询串同样被遮盖，便于直接用于日志脱敏。
"""
text = "Max retries exceeded with url: /torrentrss.php?passkey=SECRET (Caused by NewConnectionError)"
masked = mask_url_credentials(text)
assert "SECRET" not in masked
assert "passkey=<REDACTED>" in masked
assert masked.startswith("Max retries exceeded with url: /torrentrss.php?")