perf(memory): return idle allocator pages to the OS

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/runtime/gc.py

@@ -2,11 +2,13 @@
 内存回收装饰器模块
 提供装饰器用于在函数执行后立即回收内存
 """
+import ctypes
 import gc
 import functools
 import logging
 import psutil
 import os
+import sys
 from typing import Callable, Any, Optional
 
 logger = logging.getLogger(__name__)
@@ -83,6 +85,45 @@ def get_memory_usage() -> float:
         return 0.0
 
 
+# jemalloc 的 MALLCTL_ARENAS_ALL：对全部 arena 执行 purge
+_JEMALLOC_ARENAS_ALL_PURGE = b"arena.4096.purge"
+
+
+def release_allocator_memory() -> Optional[str]:
+    """
+    把 C 分配器中已释放但尚未归还的空闲页交还操作系统。
+
+    ``gc.collect()`` 只把 Python 对象交还分配器，jemalloc 默认没有后台回收线程，空闲 arena
+    的脏页会一直计入 RSS，glibc 的堆顶碎片同理。官方镜像通过 ``LD_PRELOAD`` 加载 jemalloc 时
+    调用 ``mallctl`` purge；源码部署等使用 glibc 的环境调用 ``malloc_trim(0)``。两者都只归还
+    空闲页、不影响存活对象，其他平台或符号不存在时不做任何事。
+
+    Returns:
+        实际使用的机制（``jemalloc`` 或 ``glibc``），未执行时返回 None
+    """
+    if not sys.platform.startswith("linux"):
+        return None
+    try:
+        # CDLL(None) 取当前进程的全局符号表，包含 LD_PRELOAD 进来的 jemalloc
+        libc = ctypes.CDLL(None)
+    except OSError:
+        return None
+    mallctl = getattr(libc, "mallctl", None)  # 动态符号：仅 jemalloc 提供
+    if mallctl is not None:
+        mallctl.argtypes = [ctypes.c_char_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t]
+        mallctl.restype = ctypes.c_int
+        if mallctl(_JEMALLOC_ARENAS_ALL_PURGE, None, None, None, 0) == 0:
+            return "jemalloc"
+        return None
+    malloc_trim = getattr(libc, "malloc_trim", None)  # 动态符号：仅 glibc 提供
+    if malloc_trim is not None:
+        malloc_trim.argtypes = [ctypes.c_size_t]
+        malloc_trim.restype = ctypes.c_int
+        malloc_trim(0)
+        return "glibc"
+    return None
+
+
 def memory_monitor(threshold_mb: Optional[float] = None) -> Callable:
     """
     内存监控装饰器，当内存使用超过阈值时自动触发垃圾回收

app/scheduler/maintenance.py

@@ -4,7 +4,7 @@ import gc
 
 from app.application.backup import BackupArtifact
 from app.application.database import get_database_governance
-from app.runtime.gc import get_memory_usage
+from app.runtime.gc import get_memory_usage, release_allocator_memory
 from app.runtime.log import logger
 from app.scheduler.contract import _SchedulerOwnerBase
 
@@ -30,9 +30,14 @@ class SchedulerMaintenanceOwner(_SchedulerOwnerBase):
         """
         memory_before = get_memory_usage()
         collected = gc.collect()
+        # gc 只把对象还给分配器，还需让分配器把空闲页交还系统，RSS 才会真正下降
+        allocator = release_allocator_memory()
         memory_after = get_memory_usage()
         memory_freed = memory_before - memory_after
-        logger.info(f"主动内存回收完成，回收对象数: {collected}，释放内存: {memory_freed:.2f} MB")
+        logger.info(
+            f"主动内存回收完成，回收对象数: {collected}，分配器归还: {allocator or '未执行'}，"
+            f"释放内存: {memory_freed:.2f} MB"
+        )
 
     @staticmethod
     async def agent_heartbeat() -> None:

docker/Dockerfile

@@ -285,7 +285,11 @@ RUN mkdir -p \
 # final 阶段: 安装运行时依赖和配置最终镜像
 FROM prepare_package AS final
 
+# jemalloc 默认按 CPU 数×4 建 arena 且没有后台回收线程，空闲 arena 的脏页会长期计入 RSS。
+# 后台线程按 5 秒衰减归还脏页（muzzy 直接归还，避免 MADV_FREE 页继续计入 RSS）；
+# arena 限制为 8 个，带 GIL 的 Python 大块分配竞争很小。用户可在 compose 中覆盖 MALLOC_CONF。
 ENV LD_PRELOAD="/usr/local/lib/libjemalloc.so" \
+    MALLOC_CONF="background_thread:true,max_background_threads:1,narenas:8,dirty_decay_ms:5000,muzzy_decay_ms:0" \
     MOVIEPILOT_DOCKER_SUPERVISOR="true"
 
 # 引入支持 amr 编码的静态 ffmpeg

tests/test_runtime_allocator_release.py

@@ -0,0 +1,105 @@
+"""分配器空闲页归还：按进程实际加载的分配器选择 jemalloc purge 或 glibc malloc_trim。"""
+
+from __future__ import annotations
+
+import ctypes
+
+import pytest
+
+from app.runtime import gc as runtime_gc
+from app.scheduler.maintenance import SchedulerMaintenanceOwner
+
+
+class _FakeSymbol:
+    """模拟 ctypes 导出函数，记录调用参数与签名设置。"""
+
+    def __init__(self, result: int = 0):
+        self.result = result
+        self.calls: list[tuple] = []
+        self.argtypes = None
+        self.restype = None
+
+    def __call__(self, *args):
+        self.calls.append(args)
+        return self.result
+
+
+class _FakeLib:
+    """只暴露给定符号的伪 libc，缺失符号与真实 ctypes 一样返回 AttributeError。"""
+
+    def __init__(self, **symbols):
+        self._symbols = symbols
+
+    def __getattr__(self, name):
+        try:
+            return self._symbols[name]
+        except KeyError:
+            raise AttributeError(name) from None
+
+
+@pytest.fixture
+def linux(monkeypatch):
+    monkeypatch.setattr(runtime_gc.sys, "platform", "linux")
+
+
+def _use_lib(monkeypatch, lib) -> None:
+    monkeypatch.setattr(runtime_gc.ctypes, "CDLL", lambda _name: lib)
+
+
+def test_jemalloc_purges_all_arenas(linux, monkeypatch):
+    mallctl = _FakeSymbol()
+    trim = _FakeSymbol()
+    _use_lib(monkeypatch, _FakeLib(mallctl=mallctl, malloc_trim=trim))
+
+    assert runtime_gc.release_allocator_memory() == "jemalloc"
+    assert mallctl.calls == [(b"arena.4096.purge", None, None, None, 0)]
+    assert mallctl.restype is ctypes.c_int
+    # jemalloc 生效时不再调用 glibc 接口
+    assert trim.calls == []
+
+
+def test_jemalloc_failure_does_not_fall_back_to_glibc(linux, monkeypatch):
+    mallctl = _FakeSymbol(result=22)
+    trim = _FakeSymbol()
+    _use_lib(monkeypatch, _FakeLib(mallctl=mallctl, malloc_trim=trim))
+
+    assert runtime_gc.release_allocator_memory() is None
+    assert trim.calls == []
+
+
+def test_glibc_trims_when_jemalloc_absent(linux, monkeypatch):
+    trim = _FakeSymbol(result=1)
+    _use_lib(monkeypatch, _FakeLib(malloc_trim=trim))
+
+    assert runtime_gc.release_allocator_memory() == "glibc"
+    assert trim.calls == [(0,)]
+
+
+def test_unknown_allocator_is_a_no_op(linux, monkeypatch):
+    _use_lib(monkeypatch, _FakeLib())
+
+    assert runtime_gc.release_allocator_memory() is None
+
+
+def test_non_linux_platform_skips_native_calls(monkeypatch):
+    monkeypatch.setattr(runtime_gc.sys, "platform", "darwin")
+
+    def unexpected(_name):
+        raise AssertionError("非 Linux 平台不应加载 C 库")
+
+    monkeypatch.setattr(runtime_gc.ctypes, "CDLL", unexpected)
+
+    assert runtime_gc.release_allocator_memory() is None
+
+
+def test_full_gc_releases_allocator_after_collect(monkeypatch):
+    order: list[str] = []
+    monkeypatch.setattr("app.scheduler.maintenance.gc.collect", lambda: order.append("collect") or 3)
+    monkeypatch.setattr(
+        "app.scheduler.maintenance.release_allocator_memory", lambda: order.append("release") or "jemalloc"
+    )
+    monkeypatch.setattr("app.scheduler.maintenance.get_memory_usage", lambda: 100.0)
+
+    SchedulerMaintenanceOwner.full_gc()
+
+    assert order == ["collect", "release"]