diff --git a/app/adapters/cache/redis.py b/app/adapters/cache/redis.py index 7d887c92b..b988c2b2d 100644 --- a/app/adapters/cache/redis.py +++ b/app/adapters/cache/redis.py @@ -15,6 +15,7 @@ from redis.asyncio import BlockingConnectionPool as AsyncBlockingConnectionPool from redis.asyncio import Redis from app.foundation.singleton import Singleton +from app.foundation.url import mask_url_credentials from app.runtime.cache import DEFAULT_CACHE_REGION, AsyncCacheBackend, AtomicCacheBackend from app.runtime.log import logger from app.runtime.reload import ConfigReloadMixin @@ -32,6 +33,24 @@ _socket_connect_timeout = 5 _health_check_interval = 60 +def _mask_cache_key(key: Union[str, bytes], *, encoded: bool = False) -> str: + """ + 生成只用于日志的缓存键文本，遮盖键中 URL 形态的凭据。 + + 第三方请求封装会把带 api_key 的完整请求地址当作缓存键，Redis 抖动时原样记录就等于 + 把凭据写进日志。缓存读写仍然使用原始键，只有日志经过遮盖，主机、路径与非凭据参数 + 保持原样以便定位。 + + :param key: 区域内的原始缓存键，允许是尚未解码的字节串 + :param encoded: 输入是否为 ``__make_redis_key`` 生成的 URL 编码键；为 True 时先解码再遮盖 + :return: 凭据值替换为占位符后的键文本 + """ + text = key.decode("utf-8", errors="replace") if isinstance(key, bytes) else str(key) + if encoded: + text = unquote(text) + return mask_url_credentials(text) + + def serialize(value: Any) -> bytes: """ 为缓存值添加格式标记，并序列化为字节。 @@ -206,7 +225,7 @@ class RedisHelper(ConfigReloadMixin, metaclass=Singleton): parts = redis_key.split(":key:") return unquote(parts[-1]) except Exception as e: - logger.warn("解析 Redis 缓存键失败：%r，错误：%s", redis_key, e) + logger.warn("解析 Redis 缓存键失败：%s，错误：%s", _mask_cache_key(redis_key, encoded=True), e) return redis_key.decode("utf-8", errors="replace") if isinstance(redis_key, bytes) else redis_key def set(self, key: str, value: Any, ttl: Optional[int] = None, @@ -223,7 +242,7 @@ class RedisHelper(ConfigReloadMixin, metaclass=Singleton): try: self.store(key=key, value=value, ttl=ttl, region=region, **kwargs) except Exception as e: - logger.error(f"Failed to set key: {key} in region: {region}, error: {e}") + logger.error(f"Failed to set key: {_mask_cache_key(key)} in region: {region}, error: {e}") def store(self, key: str, value: Any, ttl: Optional[int] = None, region: Optional[str] = "DEFAULT", **kwargs: Any) -> None: @@ -258,7 +277,7 @@ class RedisHelper(ConfigReloadMixin, metaclass=Singleton): redis_key = self.__make_redis_key(region, key) return bool(client.exists(redis_key) == 1) except Exception as e: - logger.error(f"Failed to exists key: {key} region: {region}, error: {e}") + logger.error(f"Failed to exists key: {_mask_cache_key(key)} region: {region}, error: {e}") return False def get(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]: @@ -277,7 +296,7 @@ class RedisHelper(ConfigReloadMixin, metaclass=Singleton): return deserialize(value) return None except Exception as e: - logger.error(f"Failed to get key: {key} in region: {region}, error: {e}") + logger.error(f"Failed to get key: {_mask_cache_key(key)} in region: {region}, error: {e}") return None def pop(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]: @@ -323,7 +342,7 @@ class RedisHelper(ConfigReloadMixin, metaclass=Singleton): redis_key = self.__make_redis_key(region, key) client.delete(redis_key) except Exception as e: - logger.error(f"Failed to delete key: {key} in region: {region}, error: {e}") + logger.error(f"Failed to delete key: {_mask_cache_key(key)} in region: {region}, error: {e}") def clear(self, region: Optional[str] = None) -> None: """ @@ -597,7 +616,7 @@ class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton): parts = redis_key.split(":key:") return unquote(parts[-1]) except Exception as e: - logger.warn("解析 Redis 缓存键失败：%r，错误：%s", redis_key, e) + logger.warn("解析 Redis 缓存键失败：%s，错误：%s", _mask_cache_key(redis_key, encoded=True), e) return redis_key.decode("utf-8", errors="replace") if isinstance(redis_key, bytes) else redis_key async def set(self, key: str, value: Any, ttl: Optional[int] = None, @@ -619,7 +638,7 @@ class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton): kwargs.pop("maxsize", None) await client.set(redis_key, serialized_value, ex=ttl, **kwargs) except Exception as e: - logger.error(f"Failed to set key (async): {key} in region: {region}, error: {e}") + logger.error(f"Failed to set key (async): {_mask_cache_key(key)} in region: {region}, error: {e}") async def exists(self, key: str, region: Optional[str] = "DEFAULT") -> bool: """ @@ -636,7 +655,7 @@ class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton): result = await client.exists(redis_key) return bool(result == 1) except Exception as e: - logger.error(f"Failed to exists key (async): {key} region: {region}, error: {e}") + logger.error(f"Failed to exists key (async): {_mask_cache_key(key)} region: {region}, error: {e}") return False async def get(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]: @@ -656,7 +675,7 @@ class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton): return deserialize(value) return None except Exception as e: - logger.error(f"Failed to get key (async): {key} in region: {region}, error: {e}") + logger.error(f"Failed to get key (async): {_mask_cache_key(key)} in region: {region}, error: {e}") return None async def delete(self, key: str, region: Optional[str] = "DEFAULT") -> None: @@ -672,7 +691,7 @@ class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton): redis_key = self.__make_redis_key(region, key) await client.delete(redis_key) except Exception as e: - logger.error(f"Failed to delete key (async): {key} in region: {region}, error: {e}") + logger.error(f"Failed to delete key (async): {_mask_cache_key(key)} in region: {region}, error: {e}") async def clear(self, region: Optional[str] = None) -> None: """ diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index 9dad795a3..df2271599 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -811,13 +811,13 @@ flowchart LR ### 10.2 当前架构快照 -以下是 2026-10-26 核对的 `tests/fixtures/architecture/` 数据；这是源码合同快照， +以下是 2026-10-27 核对的 `tests/fixtures/architecture/` 数据；这是源码合同快照， 不代表某个已发布镜像或正在运行的实例已经验证。 | 指标 | 当前值 | |---|---:| | Python 模块 | 1142 | -| 内部导入边 | 9,845 | +| 内部导入边 | 9,846 | | 非平凡 SCC | 1（29 个模块的 TMDB vendor 精确 containment） | | Application / Chain 具体 Adapter 直连 | 0 / 0 | | Direct egress | 51（精确 policy 准入，Application / Chain 均为 0） | diff --git a/tests/fixtures/architecture/dependency-baseline.json b/tests/fixtures/architecture/dependency-baseline.json index c8f8038bf..63fcd0877 100644 --- a/tests/fixtures/architecture/dependency-baseline.json +++ b/tests/fixtures/architecture/dependency-baseline.json @@ -959,8 +959,8 @@ "runtime_only": true } }, - "edge_count": 9845, - "edge_sha256": "99374b39d7fc2d92d7bb86130dcaf3acced6fc093286c272cb7b3b2139e00d55", + "edge_count": 9846, + "edge_sha256": "07b850a0ae496213cf1d1dbc9355b48e9cda9948f3b6af9cb80d8504dae33445", "edges": [ "app -> app.foundation", "app -> app.foundation.environment", @@ -972,6 +972,7 @@ "app.adapters.cache.backends -> app.runtime.settings", "app.adapters.cache.redis -> app.foundation", "app.adapters.cache.redis -> app.foundation.singleton", + "app.adapters.cache.redis -> app.foundation.url", "app.adapters.cache.redis -> app.runtime", "app.adapters.cache.redis -> app.runtime.cache", "app.adapters.cache.redis -> app.runtime.log", diff --git a/tests/test_cache_system.py b/tests/test_cache_system.py index 7911a4bd0..0dcfc8cf8 100644 --- a/tests/test_cache_system.py +++ b/tests/test_cache_system.py @@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest +from app.adapters.cache import redis as redis_module from app.adapters.cache.backends import AsyncFileBackend, FileBackend from app.adapters.cache.redis import ( AsyncRedisBackend, @@ -1148,3 +1149,122 @@ def test_file_backend_items_skip_directories(tmp_path): (tmp_path / "region" / "empty_dir").mkdir() assert list(cache.items(region="region")) == [("nested/value", b"value")] + + +# 第三方请求封装把带凭据的完整请求地址当作缓存键，这里用假值代表真实键形态 +CREDENTIAL_CACHE_KEY = ( + "request_('GET', 'https://api.example.com/3/person/25673" + "?api_key=SUPERSECRET&append_to_response=videos&language=zh', None, None)" +) + + +@pytest.fixture +def redis_failure_logs(monkeypatch): + """ + 收集 Redis 后端故障日志，避免用例把错误打到真实日志。 + + :return: 收集到的错误与告警日志文本列表 + """ + logs: list[str] = [] + monkeypatch.setattr(redis_module.logger, "error", logs.append) + monkeypatch.setattr(redis_module.logger, "warn", logs.append) + return logs + + +def _failing_sync_client(): + """ + 构造各命令都抛出连接错误的同步 Redis 客户端替身。 + + :return: set/exists/get/delete 均失败的 MagicMock 客户端 + """ + client = MagicMock() + error = ConnectionError("(104, 'Connection reset by peer')") + for command in ("set", "exists", "get", "delete"): + getattr(client, command).side_effect = error + return client + + +class _FailingAsyncClient: + """ + 所有命令都抛出连接错误的异步 Redis 客户端替身。 + """ + + async def _fail(self, *_args, **_kwargs): + """ + 模拟 Redis 命令在读写阶段失败。 + """ + raise ConnectionError("(104, 'Connection reset by peer')") + + set = _fail + exists = _fail + get = _fail + delete = _fail + + +def test_redis_helper_failure_logs_mask_cache_key_credentials(monkeypatch, redis_failure_logs): + """ + 缓存键携带请求凭据时，Redis 故障日志不得输出凭据原文，并保留可定位的键结构。 + """ + helper = RedisHelper() + monkeypatch.setattr(helper, "client", None) + monkeypatch.setattr(helper, "_connect", lambda: _failing_sync_client()) + + helper.set(CREDENTIAL_CACHE_KEY, {"result": 1}, region="TMDb:request") + assert helper.get(CREDENTIAL_CACHE_KEY, region="TMDb:request") is None + assert helper.exists(CREDENTIAL_CACHE_KEY, region="TMDb:request") is False + helper.delete(CREDENTIAL_CACHE_KEY, region="TMDb:request") + + assert len(redis_failure_logs) == 4 + for message in redis_failure_logs: + assert "SUPERSECRET" not in message + assert "api_key=" in message + assert "3/person/25673" in message + assert "language=zh" in message + + +def test_async_redis_helper_failure_logs_mask_cache_key_credentials(monkeypatch, redis_failure_logs): + """ + 异步后端与同步后端必须采用同样的日志脱敏语义。 + """ + helper = AsyncRedisHelper() + + async def _fake_connect(): + """ + 返回命令必然失败的异步客户端替身。 + """ + return _FailingAsyncClient() + + monkeypatch.setattr(helper, "client", None) + monkeypatch.setattr(helper, "_connect", _fake_connect) + + async def exercise(): + """ + 依次触发异步后端的写入、读取、存在性判断与删除故障。 + """ + await helper.set(CREDENTIAL_CACHE_KEY, {"result": 1}, region="TMDb:request") + assert await helper.get(CREDENTIAL_CACHE_KEY, region="TMDb:request") is None + assert await helper.exists(CREDENTIAL_CACHE_KEY, region="TMDb:request") is False + await helper.delete(CREDENTIAL_CACHE_KEY, region="TMDb:request") + + asyncio.run(exercise()) + + assert len(redis_failure_logs) == 4 + for message in redis_failure_logs: + assert "SUPERSECRET" not in message + assert "api_key=" in message + assert "language=zh" in message + + +def test_mask_cache_key_masks_encoded_redis_keys_and_bytes(): + """ + 日志脱敏对 URL 编码后的 Redis 键与字节键同样生效。 + """ + redis_key = f"region:TMDb:request:key:{redis_module.quote(CREDENTIAL_CACHE_KEY)}" + + masked = redis_module._mask_cache_key(redis_key, encoded=True) + + assert "SUPERSECRET" not in masked + assert "api_key=" in masked + assert redis_module._mask_cache_key(b"region:poster:key:nested/value") == ( + "region:poster:key:nested/value" + )