From 765b286fd7dbfa4e6332a5a87d188267ce9bf98e Mon Sep 17 00:00:00 2001 From: jxxghp Date: Fri, 12 Jun 2026 08:21:26 +0800 Subject: [PATCH] fix: improve cache locking mechanism and enhance key handling in file and redis backends --- app/core/cache.py | 32 ++++++++----- app/helper/redis.py | 6 +-- tests/test_cache_system.py | 92 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 15 deletions(-) create mode 100644 tests/test_cache_system.py diff --git a/app/core/cache.py b/app/core/cache.py index ca57377c..470646c8 100644 --- a/app/core/cache.py +++ b/app/core/cache.py @@ -421,7 +421,8 @@ class MemoryBackend(CacheBackend): region_cache = self.__get_region_cache(region) if region_cache is None: return False - return key in region_cache + with self._lock: + return key in region_cache def get(self, key: str, region: Optional[str] = DEFAULT_CACHE_REGION) -> Any: """ @@ -434,7 +435,8 @@ class MemoryBackend(CacheBackend): region_cache = self.__get_region_cache(region) if region_cache is None: return None - return region_cache.get(key) + with self._lock: + return region_cache.get(key) def delete(self, key: str, region: Optional[str] = DEFAULT_CACHE_REGION): """ @@ -447,7 +449,8 @@ class MemoryBackend(CacheBackend): if region_cache is None: return with self._lock: - del region_cache[key] + if key in region_cache: + del region_cache[key] def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None: """ @@ -803,8 +806,10 @@ class FileBackend(CacheBackend): :param region: 缓存的区 """ cache_path = self.base / region / key - if cache_path.exists(): + if cache_path.is_file(): cache_path.unlink() + elif cache_path.exists(): + shutil.rmtree(cache_path, ignore_errors=True) def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None: """ @@ -840,10 +845,11 @@ class FileBackend(CacheBackend): if not cache_path.exists(): yield from () return - for item in cache_path.iterdir(): + for item in sorted(cache_path.rglob("*")): if item.is_file(): - with open(item, 'r') as f: - yield item.as_posix(), f.read() + key = item.relative_to(cache_path).as_posix() + with open(item, 'rb') as f: + yield key, f.read() def close(self) -> None: """ @@ -916,8 +922,10 @@ class AsyncFileBackend(AsyncCacheBackend): :param region: 缓存的区 """ cache_path = AsyncPath(self.base) / region / key - if await cache_path.exists(): + if await cache_path.is_file(): await cache_path.unlink() + elif await cache_path.exists(): + await aioshutil.rmtree(cache_path, ignore_errors=True) async def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None: """ @@ -951,12 +959,12 @@ class AsyncFileBackend(AsyncCacheBackend): """ cache_path = AsyncPath(self.base) / region if not await cache_path.exists(): - yield "", None return - async for item in cache_path.iterdir(): + async for item in cache_path.rglob("*"): if await item.is_file(): - async with aiofiles.open(item, 'r') as f: - yield item.as_posix(), await f.read() + key = Path(str(item)).relative_to(Path(str(cache_path))).as_posix() + async with aiofiles.open(item, 'rb') as f: + yield key, await f.read() async def close(self) -> None: """ diff --git a/app/helper/redis.py b/app/helper/redis.py index 6b104be8..23191735 100644 --- a/app/helper/redis.py +++ b/app/helper/redis.py @@ -2,7 +2,7 @@ import asyncio import json import pickle from typing import Any, Optional, Generator, Tuple, AsyncGenerator, Union -from urllib.parse import quote +from urllib.parse import quote, unquote import redis from redis.asyncio import Redis @@ -160,7 +160,7 @@ class RedisHelper(ConfigReloadMixin, metaclass=Singleton): if isinstance(redis_key, bytes): redis_key = redis_key.decode("utf-8") parts = redis_key.split(":key:") - return parts[-1] + return unquote(parts[-1]) except Exception as e: logger.warn(f"Failed to parse redis key: {redis_key}, error: {e}") return redis_key @@ -410,7 +410,7 @@ class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton): if isinstance(redis_key, bytes): redis_key = redis_key.decode("utf-8") parts = redis_key.split(":key:") - return parts[-1] + return unquote(parts[-1]) except Exception as e: logger.warn(f"Failed to parse redis key: {redis_key}, error: {e}") return redis_key diff --git a/tests/test_cache_system.py b/tests/test_cache_system.py new file mode 100644 index 00000000..4cb9c4c9 --- /dev/null +++ b/tests/test_cache_system.py @@ -0,0 +1,92 @@ +import asyncio + +from app.core.cache import AsyncFileBackend, FileBackend, MemoryBackend +from app.helper.redis import RedisHelper + + +def test_file_backend_items_keep_relative_keys_and_bytes(tmp_path): + """ + 文件缓存遍历应返回可继续删除的相对 key，并保持二进制内容不变。 + """ + cache = FileBackend(base=tmp_path) + cache.set("nested/poster.jpg", b"\xff\xd8image", region="images") + + items = list(cache.items(region="images")) + + assert items == [("nested/poster.jpg", b"\xff\xd8image")] + assert cache.popitem(region="images") == ("nested/poster.jpg", b"\xff\xd8image") + assert not cache.exists("nested/poster.jpg", region="images") + + +def test_file_backend_delete_missing_key_is_noop(tmp_path): + """ + 删除不存在的文件缓存 key 应保持幂等，不向调用方抛出文件系统异常。 + """ + cache = FileBackend(base=tmp_path) + + cache.delete("missing", region="default") + + assert not cache.exists("missing", region="default") + + +def test_memory_backend_delete_missing_key_is_noop(): + """ + 内存缓存后端 delete 与其他后端保持一致，不存在时直接返回。 + """ + cache = MemoryBackend() + + cache.delete("missing", region="missing_delete") + + assert not cache.exists("missing", region="missing_delete") + + +def test_redis_original_key_decodes_quoted_key(): + """ + Redis items 返回的 key 应还原为原始缓存 key，确保带特殊字符的 key 可继续删除。 + """ + redis_key = b"region:DEFAULT:key:nested/poster%20one.jpg" + + assert RedisHelper._RedisHelper__get_original_key(redis_key) == "nested/poster one.jpg" + + +def test_async_file_backend_missing_region_has_no_items(tmp_path): + """ + 异步文件缓存缺失区域时应返回空迭代，而不是伪造空 key。 + """ + + async def collect_items(): + cache = AsyncFileBackend(base=tmp_path) + return [item async for item in cache.items(region="missing")] + + assert asyncio.run(collect_items()) == [] + + +def test_async_file_backend_items_keep_relative_keys_and_bytes(tmp_path): + """ + 异步文件缓存遍历应与同步文件缓存保持相同 key 和二进制语义。 + """ + + async def collect_items(): + cache = AsyncFileBackend(base=tmp_path) + await cache.set("nested/poster.jpg", b"\xff\xd8image", region="images") + items = [item async for item in cache.items(region="images")] + popped = await cache.popitem(region="images") + exists = await cache.exists("nested/poster.jpg", region="images") + return items, popped, exists + + items, popped, exists = asyncio.run(collect_items()) + + assert items == [("nested/poster.jpg", b"\xff\xd8image")] + assert popped == ("nested/poster.jpg", b"\xff\xd8image") + assert not exists + + +def test_file_backend_items_skip_directories(tmp_path): + """ + 文件缓存遍历应递归读取有效缓存文件，不把目录当成缓存项。 + """ + cache = FileBackend(base=tmp_path) + cache.set("nested/value", b"value", region="region") + (tmp_path / "region" / "empty_dir").mkdir() + + assert list(cache.items(region="region")) == [("nested/value", b"value")]