nas/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-11-03 13:55:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(cache): Redis 故障日志遮盖缓存键中携带的请求凭据 (#7220)

Browse Source
This commit is contained in:
JontyLee
2026-10-27 20:54:42 +08:00
committed by GitHub
parent 2c0ea1d0c6
commit 6e405a67b3
4 changed files with 154 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
app/adapters/cache/redis.py vendored
View File

@@ -15,6 +15,7 @@ from redis.asyncio import BlockingConnectionPool as AsyncBlockingConnectionPool
from redis.asyncio import Redis
from app.foundation.singleton import Singleton
from app.foundation.url import mask_url_credentials
from app.runtime.cache import DEFAULT_CACHE_REGION, AsyncCacheBackend, AtomicCacheBackend
from app.runtime.log import logger
from app.runtime.reload import ConfigReloadMixin
@@ -32,6 +33,24 @@ _socket_connect_timeout = 5
_health_check_interval = 60
def _mask_cache_key(key: Union[str, bytes], *, encoded: bool = False) -> str:
"""
生成只用于日志的缓存键文本，遮盖键中 URL 形态的凭据。
第三方请求封装会把带 api_key 的完整请求地址当作缓存键，Redis 抖动时原样记录就等于
把凭据写进日志。缓存读写仍然使用原始键，只有日志经过遮盖，主机、路径与非凭据参数
保持原样以便定位。
:param key: 区域内的原始缓存键，允许是尚未解码的字节串
:param encoded: 输入是否为 ``__make_redis_key`` 生成的 URL 编码键；为 True 时先解码再遮盖
:return: 凭据值替换为占位符后的键文本
"""
text = key.decode("utf-8", errors="replace") if isinstance(key, bytes) else str(key)
if encoded:
text = unquote(text)
return mask_url_credentials(text)
def serialize(value: Any) -> bytes:
"""
为缓存值添加格式标记，并序列化为字节。
@@ -206,7 +225,7 @@ class RedisHelper(ConfigReloadMixin, metaclass=Singleton):
parts = redis_key.split(":key:")
return unquote(parts[-1])
except Exception as e:
logger.warn("解析 Redis 缓存键失败：%r，错误：%s", redis_key, e)
logger.warn("解析 Redis 缓存键失败：%s，错误：%s", _mask_cache_key(redis_key, encoded=True), e)
return redis_key.decode("utf-8", errors="replace") if isinstance(redis_key, bytes) else redis_key
def set(self, key: str, value: Any, ttl: Optional[int] = None,
@@ -223,7 +242,7 @@ class RedisHelper(ConfigReloadMixin, metaclass=Singleton):
try:
self.store(key=key, value=value, ttl=ttl, region=region, **kwargs)
except Exception as e:
logger.error(f"Failed to set key: {key} in region: {region}, error: {e}")
logger.error(f"Failed to set key: {_mask_cache_key(key)} in region: {region}, error: {e}")
def store(self, key: str, value: Any, ttl: Optional[int] = None,
region: Optional[str] = "DEFAULT", **kwargs: Any) -> None:
@@ -258,7 +277,7 @@ class RedisHelper(ConfigReloadMixin, metaclass=Singleton):
redis_key = self.__make_redis_key(region, key)
return bool(client.exists(redis_key) == 1)
except Exception as e:
logger.error(f"Failed to exists key: {key} region: {region}, error: {e}")
logger.error(f"Failed to exists key: {_mask_cache_key(key)} region: {region}, error: {e}")
return False
def get(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]:
@@ -277,7 +296,7 @@ class RedisHelper(ConfigReloadMixin, metaclass=Singleton):
return deserialize(value)
return None
except Exception as e:
logger.error(f"Failed to get key: {key} in region: {region}, error: {e}")
logger.error(f"Failed to get key: {_mask_cache_key(key)} in region: {region}, error: {e}")
return None
def pop(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]:
@@ -323,7 +342,7 @@ class RedisHelper(ConfigReloadMixin, metaclass=Singleton):
redis_key = self.__make_redis_key(region, key)
client.delete(redis_key)
except Exception as e:
logger.error(f"Failed to delete key: {key} in region: {region}, error: {e}")
logger.error(f"Failed to delete key: {_mask_cache_key(key)} in region: {region}, error: {e}")
def clear(self, region: Optional[str] = None) -> None:
"""
@@ -597,7 +616,7 @@ class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton):
parts = redis_key.split(":key:")
return unquote(parts[-1])
except Exception as e:
logger.warn("解析 Redis 缓存键失败：%r，错误：%s", redis_key, e)
logger.warn("解析 Redis 缓存键失败：%s，错误：%s", _mask_cache_key(redis_key, encoded=True), e)
return redis_key.decode("utf-8", errors="replace") if isinstance(redis_key, bytes) else redis_key
async def set(self, key: str, value: Any, ttl: Optional[int] = None,
@@ -619,7 +638,7 @@ class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton):
kwargs.pop("maxsize", None)
await client.set(redis_key, serialized_value, ex=ttl, **kwargs)
except Exception as e:
logger.error(f"Failed to set key (async): {key} in region: {region}, error: {e}")
logger.error(f"Failed to set key (async): {_mask_cache_key(key)} in region: {region}, error: {e}")
async def exists(self, key: str, region: Optional[str] = "DEFAULT") -> bool:
"""
@@ -636,7 +655,7 @@ class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton):
result = await client.exists(redis_key)
return bool(result == 1)
except Exception as e:
logger.error(f"Failed to exists key (async): {key} region: {region}, error: {e}")
logger.error(f"Failed to exists key (async): {_mask_cache_key(key)} region: {region}, error: {e}")
return False
async def get(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]:
@@ -656,7 +675,7 @@ class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton):
return deserialize(value)
return None
except Exception as e:
logger.error(f"Failed to get key (async): {key} in region: {region}, error: {e}")
logger.error(f"Failed to get key (async): {_mask_cache_key(key)} in region: {region}, error: {e}")
return None
async def delete(self, key: str, region: Optional[str] = "DEFAULT") -> None:
@@ -672,7 +691,7 @@ class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton):
redis_key = self.__make_redis_key(region, key)
await client.delete(redis_key)
except Exception as e:
logger.error(f"Failed to delete key (async): {key} in region: {region}, error: {e}")
logger.error(f"Failed to delete key (async): {_mask_cache_key(key)} in region: {region}, error: {e}")
async def clear(self, region: Optional[str] = None) -> None:
"""

4
docs/architecture-overview.md
View File

@@ -811,13 +811,13 @@ flowchart LR
### 10.2 当前架构快照
以下是 2026-10-26 核对的 `tests/fixtures/architecture/` 数据；这是源码合同快照，
以下是 2026-10-27 核对的 `tests/fixtures/architecture/` 数据；这是源码合同快照，
不代表某个已发布镜像或正在运行的实例已经验证。
| 指标 | 当前值 |
|---|---:|
| Python 模块 | 1142 |
| 内部导入边 | 9,845 |
| 内部导入边 | 9,846 |
| 非平凡 SCC | 1（29 个模块的 TMDB vendor 精确 containment） |
| Application / Chain 具体 Adapter 直连 | 0 / 0 |
| Direct egress | 51（精确 policy 准入，Application / Chain 均为 0） |

5
tests/fixtures/architecture/dependency-baseline.json vendored
View File

@@ -959,8 +959,8 @@
"runtime_only": true
}
},
"edge_count": 9845,
"edge_sha256": "99374b39d7fc2d92d7bb86130dcaf3acced6fc093286c272cb7b3b2139e00d55",
"edge_count": 9846,
"edge_sha256": "07b850a0ae496213cf1d1dbc9355b48e9cda9948f3b6af9cb80d8504dae33445",
"edges": [
"app -> app.foundation",
"app -> app.foundation.environment",
@@ -972,6 +972,7 @@
"app.adapters.cache.backends -> app.runtime.settings",
"app.adapters.cache.redis -> app.foundation",
"app.adapters.cache.redis -> app.foundation.singleton",
"app.adapters.cache.redis -> app.foundation.url",
"app.adapters.cache.redis -> app.runtime",
"app.adapters.cache.redis -> app.runtime.cache",
"app.adapters.cache.redis -> app.runtime.log",

120
tests/test_cache_system.py
View File

@@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from app.adapters.cache import redis as redis_module
from app.adapters.cache.backends import AsyncFileBackend, FileBackend
from app.adapters.cache.redis import (
AsyncRedisBackend,
@@ -1148,3 +1149,122 @@ def test_file_backend_items_skip_directories(tmp_path):
(tmp_path / "region" / "empty_dir").mkdir()
assert list(cache.items(region="region")) == [("nested/value", b"value")]
# 第三方请求封装把带凭据的完整请求地址当作缓存键，这里用假值代表真实键形态
CREDENTIAL_CACHE_KEY = (
"request_('GET', 'https://api.example.com/3/person/25673"
"?api_key=SUPERSECRET&append_to_response=videos&language=zh', None, None)"
)
@pytest.fixture
def redis_failure_logs(monkeypatch):
"""
收集 Redis 后端故障日志，避免用例把错误打到真实日志。
:return: 收集到的错误与告警日志文本列表
"""
logs: list[str] = []
monkeypatch.setattr(redis_module.logger, "error", logs.append)
monkeypatch.setattr(redis_module.logger, "warn", logs.append)
return logs
def _failing_sync_client():
"""
构造各命令都抛出连接错误的同步 Redis 客户端替身。
:return: set/exists/get/delete 均失败的 MagicMock 客户端
"""
client = MagicMock()
error = ConnectionError("(104, 'Connection reset by peer')")
for command in ("set", "exists", "get", "delete"):
getattr(client, command).side_effect = error
return client
class _FailingAsyncClient:
"""
所有命令都抛出连接错误的异步 Redis 客户端替身。
"""
async def _fail(self, *_args, **_kwargs):
"""
模拟 Redis 命令在读写阶段失败。
"""
raise ConnectionError("(104, 'Connection reset by peer')")
set = _fail
exists = _fail
get = _fail
delete = _fail
def test_redis_helper_failure_logs_mask_cache_key_credentials(monkeypatch, redis_failure_logs):
"""
缓存键携带请求凭据时，Redis 故障日志不得输出凭据原文，并保留可定位的键结构。
"""
helper = RedisHelper()
monkeypatch.setattr(helper, "client", None)
monkeypatch.setattr(helper, "_connect", lambda: _failing_sync_client())
helper.set(CREDENTIAL_CACHE_KEY, {"result": 1}, region="TMDb:request")
assert helper.get(CREDENTIAL_CACHE_KEY, region="TMDb:request") is None
assert helper.exists(CREDENTIAL_CACHE_KEY, region="TMDb:request") is False
helper.delete(CREDENTIAL_CACHE_KEY, region="TMDb:request")
assert len(redis_failure_logs) == 4
for message in redis_failure_logs:
assert "SUPERSECRET" not in message
assert "api_key=<REDACTED>" in message
assert "3/person/25673" in message
assert "language=zh" in message
def test_async_redis_helper_failure_logs_mask_cache_key_credentials(monkeypatch, redis_failure_logs):
"""
异步后端与同步后端必须采用同样的日志脱敏语义。
"""
helper = AsyncRedisHelper()
async def _fake_connect():
"""
返回命令必然失败的异步客户端替身。
"""
return _FailingAsyncClient()
monkeypatch.setattr(helper, "client", None)
monkeypatch.setattr(helper, "_connect", _fake_connect)
async def exercise():
"""
依次触发异步后端的写入、读取、存在性判断与删除故障。
"""
await helper.set(CREDENTIAL_CACHE_KEY, {"result": 1}, region="TMDb:request")
assert await helper.get(CREDENTIAL_CACHE_KEY, region="TMDb:request") is None
assert await helper.exists(CREDENTIAL_CACHE_KEY, region="TMDb:request") is False
await helper.delete(CREDENTIAL_CACHE_KEY, region="TMDb:request")
asyncio.run(exercise())
assert len(redis_failure_logs) == 4
for message in redis_failure_logs:
assert "SUPERSECRET" not in message
assert "api_key=<REDACTED>" in message
assert "language=zh" in message
def test_mask_cache_key_masks_encoded_redis_keys_and_bytes():
"""
日志脱敏对 URL 编码后的 Redis 键与字节键同样生效。
"""
redis_key = f"region:TMDb:request:key:{redis_module.quote(CREDENTIAL_CACHE_KEY)}"
masked = redis_module._mask_cache_key(redis_key, encoded=True)
assert "SUPERSECRET" not in masked
assert "api_key=<REDACTED>" in masked
assert redis_module._mask_cache_key(b"region:poster:key:nested/value") == (
"region:poster:key:nested/value"
)