From 9af903e6772365cf35d71cc463a5874cf41f1cae Mon Sep 17 00:00:00 2001 From: InfinityPacer Date: Mon, 28 Sep 2026 02:24:17 +0800 Subject: [PATCH] perf(memory): return idle allocator pages to the OS Co-Authored-By: Claude Opus 5.5 --- app/runtime/gc.py | 41 +++++++++ app/scheduler/maintenance.py | 9 +- docker/Dockerfile | 4 + tests/test_runtime_allocator_release.py | 105 ++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 tests/test_runtime_allocator_release.py diff --git a/app/runtime/gc.py b/app/runtime/gc.py index 543f56e15..eaf70dfa6 100644 --- a/app/runtime/gc.py +++ b/app/runtime/gc.py @@ -2,11 +2,13 @@ 内存回收装饰器模块 提供装饰器用于在函数执行后立即回收内存 """ +import ctypes import gc import functools import logging import psutil import os +import sys from typing import Callable, Any, Optional logger = logging.getLogger(__name__) @@ -83,6 +85,45 @@ def get_memory_usage() -> float: return 0.0 +# jemalloc 的 MALLCTL_ARENAS_ALL：对全部 arena 执行 purge +_JEMALLOC_ARENAS_ALL_PURGE = b"arena.4096.purge" + + +def release_allocator_memory() -> Optional[str]: + """ + 把 C 分配器中已释放但尚未归还的空闲页交还操作系统。 + + ``gc.collect()`` 只把 Python 对象交还分配器，jemalloc 默认没有后台回收线程，空闲 arena + 的脏页会一直计入 RSS，glibc 的堆顶碎片同理。官方镜像通过 ``LD_PRELOAD`` 加载 jemalloc 时 + 调用 ``mallctl`` purge；源码部署等使用 glibc 的环境调用 ``malloc_trim(0)``。两者都只归还 + 空闲页、不影响存活对象，其他平台或符号不存在时不做任何事。 + + Returns: + 实际使用的机制（``jemalloc`` 或 ``glibc``），未执行时返回 None + """ + if not sys.platform.startswith("linux"): + return None + try: + # CDLL(None) 取当前进程的全局符号表，包含 LD_PRELOAD 进来的 jemalloc + libc = ctypes.CDLL(None) + except OSError: + return None + mallctl = getattr(libc, "mallctl", None) # 动态符号：仅 jemalloc 提供 + if mallctl is not None: + mallctl.argtypes = [ctypes.c_char_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t] + mallctl.restype = ctypes.c_int + if mallctl(_JEMALLOC_ARENAS_ALL_PURGE, None, None, None, 0) == 0: + return "jemalloc" + return None + malloc_trim = getattr(libc, "malloc_trim", None) # 动态符号：仅 glibc 提供 + if malloc_trim is not None: + malloc_trim.argtypes = [ctypes.c_size_t] + malloc_trim.restype = ctypes.c_int + malloc_trim(0) + return "glibc" + return None + + def memory_monitor(threshold_mb: Optional[float] = None) -> Callable: """ 内存监控装饰器，当内存使用超过阈值时自动触发垃圾回收 diff --git a/app/scheduler/maintenance.py b/app/scheduler/maintenance.py index f75edfe90..f0a80b3e6 100644 --- a/app/scheduler/maintenance.py +++ b/app/scheduler/maintenance.py @@ -4,7 +4,7 @@ import gc from app.application.backup import BackupArtifact from app.application.database import get_database_governance -from app.runtime.gc import get_memory_usage +from app.runtime.gc import get_memory_usage, release_allocator_memory from app.runtime.log import logger from app.scheduler.contract import _SchedulerOwnerBase @@ -30,9 +30,14 @@ class SchedulerMaintenanceOwner(_SchedulerOwnerBase): """ memory_before = get_memory_usage() collected = gc.collect() + # gc 只把对象还给分配器，还需让分配器把空闲页交还系统，RSS 才会真正下降 + allocator = release_allocator_memory() memory_after = get_memory_usage() memory_freed = memory_before - memory_after - logger.info(f"主动内存回收完成，回收对象数: {collected}，释放内存: {memory_freed:.2f} MB") + logger.info( + f"主动内存回收完成，回收对象数: {collected}，分配器归还: {allocator or '未执行'}，" + f"释放内存: {memory_freed:.2f} MB" + ) @staticmethod async def agent_heartbeat() -> None: diff --git a/docker/Dockerfile b/docker/Dockerfile index 0bc083b7c..fe57c222a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -285,7 +285,11 @@ RUN mkdir -p \ # final 阶段: 安装运行时依赖和配置最终镜像 FROM prepare_package AS final +# jemalloc 默认按 CPU 数×4 建 arena 且没有后台回收线程，空闲 arena 的脏页会长期计入 RSS。 +# 后台线程按 5 秒衰减归还脏页（muzzy 直接归还，避免 MADV_FREE 页继续计入 RSS）； +# arena 限制为 8 个，带 GIL 的 Python 大块分配竞争很小。用户可在 compose 中覆盖 MALLOC_CONF。 ENV LD_PRELOAD="/usr/local/lib/libjemalloc.so" \ + MALLOC_CONF="background_thread:true,max_background_threads:1,narenas:8,dirty_decay_ms:5000,muzzy_decay_ms:0" \ MOVIEPILOT_DOCKER_SUPERVISOR="true" # 引入支持 amr 编码的静态 ffmpeg diff --git a/tests/test_runtime_allocator_release.py b/tests/test_runtime_allocator_release.py new file mode 100644 index 000000000..ff5402a68 --- /dev/null +++ b/tests/test_runtime_allocator_release.py @@ -0,0 +1,105 @@ +"""分配器空闲页归还：按进程实际加载的分配器选择 jemalloc purge 或 glibc malloc_trim。""" + +from __future__ import annotations + +import ctypes + +import pytest + +from app.runtime import gc as runtime_gc +from app.scheduler.maintenance import SchedulerMaintenanceOwner + + +class _FakeSymbol: + """模拟 ctypes 导出函数，记录调用参数与签名设置。""" + + def __init__(self, result: int = 0): + self.result = result + self.calls: list[tuple] = [] + self.argtypes = None + self.restype = None + + def __call__(self, *args): + self.calls.append(args) + return self.result + + +class _FakeLib: + """只暴露给定符号的伪 libc，缺失符号与真实 ctypes 一样返回 AttributeError。""" + + def __init__(self, **symbols): + self._symbols = symbols + + def __getattr__(self, name): + try: + return self._symbols[name] + except KeyError: + raise AttributeError(name) from None + + +@pytest.fixture +def linux(monkeypatch): + monkeypatch.setattr(runtime_gc.sys, "platform", "linux") + + +def _use_lib(monkeypatch, lib) -> None: + monkeypatch.setattr(runtime_gc.ctypes, "CDLL", lambda _name: lib) + + +def test_jemalloc_purges_all_arenas(linux, monkeypatch): + mallctl = _FakeSymbol() + trim = _FakeSymbol() + _use_lib(monkeypatch, _FakeLib(mallctl=mallctl, malloc_trim=trim)) + + assert runtime_gc.release_allocator_memory() == "jemalloc" + assert mallctl.calls == [(b"arena.4096.purge", None, None, None, 0)] + assert mallctl.restype is ctypes.c_int + # jemalloc 生效时不再调用 glibc 接口 + assert trim.calls == [] + + +def test_jemalloc_failure_does_not_fall_back_to_glibc(linux, monkeypatch): + mallctl = _FakeSymbol(result=22) + trim = _FakeSymbol() + _use_lib(monkeypatch, _FakeLib(mallctl=mallctl, malloc_trim=trim)) + + assert runtime_gc.release_allocator_memory() is None + assert trim.calls == [] + + +def test_glibc_trims_when_jemalloc_absent(linux, monkeypatch): + trim = _FakeSymbol(result=1) + _use_lib(monkeypatch, _FakeLib(malloc_trim=trim)) + + assert runtime_gc.release_allocator_memory() == "glibc" + assert trim.calls == [(0,)] + + +def test_unknown_allocator_is_a_no_op(linux, monkeypatch): + _use_lib(monkeypatch, _FakeLib()) + + assert runtime_gc.release_allocator_memory() is None + + +def test_non_linux_platform_skips_native_calls(monkeypatch): + monkeypatch.setattr(runtime_gc.sys, "platform", "darwin") + + def unexpected(_name): + raise AssertionError("非 Linux 平台不应加载 C 库") + + monkeypatch.setattr(runtime_gc.ctypes, "CDLL", unexpected) + + assert runtime_gc.release_allocator_memory() is None + + +def test_full_gc_releases_allocator_after_collect(monkeypatch): + order: list[str] = [] + monkeypatch.setattr("app.scheduler.maintenance.gc.collect", lambda: order.append("collect") or 3) + monkeypatch.setattr( + "app.scheduler.maintenance.release_allocator_memory", lambda: order.append("release") or "jemalloc" + ) + monkeypatch.setattr("app.scheduler.maintenance.get_memory_usage", lambda: 100.0) + + SchedulerMaintenanceOwner.full_gc() + + assert order == ["collect", "release"]