feat(indexer): 适配 Milkie 站点搜索与连通测试

- 新增 MilkieSpider:基于 milkie.cc JSON API(x-milkie-auth 认证)的
  种子搜索与浏览,支持关键字分类收敛、服务端翻页、免费因子与
  媒体类型映射,下载链接按 API Key URL 编码拼接
- 注册 SPIDER_PARSER_CLASSES 与专用搜索参数投影
- SiteChain 新增 milkie.cc 连通性测试,401/403 判定 API Key 失效
- 补充 MilkieSpider 单元测试与同步异步参数一致性用例,同步依赖基线
This commit is contained in:
WenGuoyin
2026-10-08 17:18:10 +08:00
parent 656e2852f7
commit e19076f482
6 changed files with 433 additions and 3 deletions

View File

@@ -203,6 +203,7 @@ class SiteChain(InteractionChainMixin, ChainBase):
"hddolby.com": self.__hddolby_test,
"rousi.pro": self.__rousi_test,
"sunnypt.top": self.__sunnypt_test,
"milkie.cc": self.__milkie_test,
}
def refresh_userdata(self, site: dict = None) -> Optional[SiteUserData]:
@@ -403,6 +404,33 @@ class SiteChain(InteractionChainMixin, ChainBase):
return False, user_info.get("message", "鉴权已过期或无效")
return False, f"错误:{res.status_code} {res.reason}"
def __milkie_test(self, site: SiteSnapshot) -> Tuple[bool, str]:
"""
通过种子搜索接口测试 Milkie API Key
:param site: Milkie 站点配置
:return: 是否可用及状态信息
"""
domain = site.domain or site_rules.extract_domain(site.url)
with _open_site_response(
method="GET",
url=f"https://{domain}/api/v1/torrents?ps=1",
headers={
"Accept": "application/json",
"User-Agent": site.ua or self.runtime_config.user_agent,
"x-milkie-auth": site.apikey,
},
proxies=(self.runtime_config.proxy or {}) if site.proxy else {},
timeout=site.timeout or 15,
) as res:
if res is None:
return False, "无法打开网站!"
if res.status_code == 200:
return True, "连接成功"
if res.status_code in (401, 403):
return False, "API Key 无效或已过期"
return False, f"错误:{res.status_code} {res.reason}"
def __sunnypt_test(self, site: SiteSnapshot) -> Tuple[bool, str]:
"""
通过 profile 接口测试 SunnyPT API Key 和下载权限

View File

@@ -18,6 +18,7 @@ from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.modules.indexer.spider import SiteSpider
from app.modules.indexer.spider.haidan import HaiDanSpider
from app.modules.indexer.spider.hddolby import HddolbySpider
from app.modules.indexer.spider.milkie import MilkieSpider
from app.modules.indexer.spider.mtorrent import MTorrentSpider
from app.modules.indexer.spider.rousi import RousiSpider
from app.modules.indexer.spider.sunnypt import SunnyPTSpider
@@ -39,6 +40,7 @@ SPIDER_PARSER_CLASSES = {
"HDDolby": HddolbySpider,
"RousiPro": RousiSpider,
"SunnyPT": SunnyPTSpider,
"Milkie": MilkieSpider,
}
_SPECIALIZED_SEARCH_ARGUMENTS = {
@@ -50,6 +52,7 @@ _SPECIALIZED_SEARCH_ARGUMENTS = {
"Haidan": ("keyword", "mtype"),
"HDDolby": ("keyword", "mtype", "page"),
"RousiPro": ("keyword", "mtype", "cat", "page"),
"Milkie": ("keyword", "mtype", "page"),
}
_SPECIALIZED_SUBTITLE_SEARCH_ARGUMENTS = {

View File

@@ -0,0 +1,178 @@
from typing import Any, List, Optional, Tuple
from urllib.parse import quote
from app.adapters.network.http import AsyncRequestUtils, RequestUtils
from app.foundation import temporal as time_tools
from app.runtime.log import logger
from app.runtime.settings import get_runtime_setting
from app.schemas.types import MediaType
class MilkieSpider:
"""Milkie JSON API 种子搜索适配器。"""
_size = 100
_timeout = 15
_proxy: Optional[dict[str, Any]] = None
_searchurl = "%sapi/v1/torrents"
_downloadurl = "%sapi/v1/torrents/%s/torrent?key=%s"
_pageurl = "%sbrowse/%s"
# 电影分类
_movie_category = "1"
# 电视剧分类
_tv_category = "2"
# 音乐分类
_music_category = "3"
# 站点分类到媒体类型的映射,未覆盖的分类不输出媒体类型
_category_mapping = {
1: MediaType.MOVIE,
2: MediaType.TV,
3: MediaType.MUSIC,
}
def __init__(self, indexer: dict[str, Any]):
"""使用站点配置初始化 Milkie 请求上下文。"""
if indexer:
self._indexerid = indexer.get('id')
self._domain = indexer.get('domain')
self._searchurl = self._searchurl % self._domain
self._name = indexer.get('name')
if indexer.get('proxy'):
self._proxy = get_runtime_setting('PROXY')
self._apikey = indexer.get('apikey')
self._ua = indexer.get('ua')
self._timeout = indexer.get('timeout') or 15
@classmethod
def get_search_page_size(cls, keyword: Optional[str] = None) -> Optional[int]:
"""
获取搜索接口单页容量。
"""
return cls._size
def __get_params(self, keyword: Optional[str] = None, mtype: Optional[MediaType] = None,
page: Optional[int] = 0) -> dict[str, Any]:
"""
获取搜索参数
"""
params = {
"pi": int(page or 0),
"ps": self._size,
"oby": "created_at",
"odir": "desc"
}
if keyword:
params["query"] = keyword
# 按媒体类型收敛分类,提高关键字搜索的准确度
if mtype == MediaType.MOVIE:
params["categories"] = self._movie_category
elif mtype == MediaType.TV:
params["categories"] = self._tv_category
elif mtype == MediaType.MUSIC:
params["categories"] = self._music_category
return params
def __parse_created_at(self, created_at: str) -> str:
"""将站点 ISO8601 发布时间转换为本地日期时间文本。"""
parsed = time_tools.parse_datetime(created_at)
if not parsed:
return created_at or ""
return parsed.strftime("%Y-%m-%d %H:%M:%S")
def __parse_result(self, results: List[dict[str, Any]]) -> List[dict[str, Any]]:
"""
解析搜索结果
"""
torrents: List[dict[str, Any]] = []
if not results:
return torrents
for result in results:
category = self._category_mapping.get(result.get('category') or 0)
torrent = {
'title': result.get('releaseName'),
'description': result.get('id'),
# API Key 含 URL 保留字符,必须编码后拼入下载链接
'enclosure': self._downloadurl % (
self._domain, result.get('id'), quote(self._apikey or "", safe="")
),
'pubdate': self.__parse_created_at(result.get('createdAt') or ""),
'size': result.get('size'),
'seeders': result.get('seeders'),
'peers': result.get('leechers'),
'grabs': result.get('downloaded'),
# 站点无分享率考核,等效全站免费
'downloadvolumefactor': 0,
'uploadvolumefactor': 1,
'page_url': self._pageurl % (self._domain, result.get('id')),
'category': category.value if category else None
}
externals = result.get('externals') or {}
if externals.get('imdb'):
torrent['imdbid'] = externals.get('imdb')
torrents.append(torrent)
return torrents
def __process_response(self, res: Any) -> Tuple[bool, List[dict[str, Any]]]:
"""统一判定搜索响应状态并投影 Milkie 种子结果。"""
if res and res.status_code == 200:
results = res.json().get("torrents") or []
return False, self.__parse_result(results)
if res is not None:
if res.status_code in (401, 403):
logger.warn(f"{self._name} 搜索失败,API Key 无效或已过期")
else:
logger.warn(f"{self._name} 搜索失败,错误码:{res.status_code}")
return True, []
logger.warn(f"{self._name} 搜索失败,无法连接 {self._domain}")
return True, []
def __get_headers(self) -> dict[str, str]:
"""构造带 API Key 认证的请求头。"""
return {
"x-milkie-auth": self._apikey or "",
"User-Agent": self._ua or ""
}
def search(self, keyword: str, mtype: Optional[MediaType] = None,
page: Optional[int] = 0) -> Tuple[bool, List[dict[str, Any]]]:
"""
搜索
"""
if not self._apikey:
logger.warn(f"{self._name} 未配置 API Key,无法搜索")
return True, []
# 获取请求参数
params = self.__get_params(keyword, mtype, page)
# 发送请求
res = RequestUtils(
headers=self.__get_headers(),
proxies=self._proxy,
timeout=self._timeout
).get_res(url=self._searchurl, params=params)
return self.__process_response(res)
async def async_search(self, keyword: str, mtype: Optional[MediaType] = None,
page: Optional[int] = 0) -> Tuple[bool, List[dict[str, Any]]]:
"""
异步搜索
"""
if not self._apikey:
logger.warn(f"{self._name} 未配置 API Key,无法搜索")
return True, []
# 获取请求参数
params = self.__get_params(keyword, mtype, page)
# 发送请求
res = await AsyncRequestUtils(
headers=self.__get_headers(),
proxies=self._proxy,
timeout=self._timeout
).get_res(url=self._searchurl, params=params)
return self.__process_response(res)

View File

@@ -941,8 +941,8 @@
"runtime_only": true
}
},
"edge_count": 9401,
"edge_sha256": "82e88feed7d0c88ac1617017a301ed2979be7da87e995463b5e22b61c77e9620",
"edge_count": 9412,
"edge_sha256": "12ffbe3a91e7314ded4ce7996bffeb7497ba0e2434bb57e58e428da20dfd0ed6",
"edges": [
"app -> app.foundation",
"app -> app.foundation.environment",
@@ -7405,6 +7405,7 @@
"app.modules.indexer -> app.modules.indexer.spider",
"app.modules.indexer -> app.modules.indexer.spider.haidan",
"app.modules.indexer -> app.modules.indexer.spider.hddolby",
"app.modules.indexer -> app.modules.indexer.spider.milkie",
"app.modules.indexer -> app.modules.indexer.spider.mtorrent",
"app.modules.indexer -> app.modules.indexer.spider.rousi",
"app.modules.indexer -> app.modules.indexer.spider.sunnypt",
@@ -7628,6 +7629,16 @@
"app.modules.indexer.spider.hddolby -> app.runtime.settings",
"app.modules.indexer.spider.hddolby -> app.schemas",
"app.modules.indexer.spider.hddolby -> app.schemas.types",
"app.modules.indexer.spider.milkie -> app.adapters",
"app.modules.indexer.spider.milkie -> app.adapters.network",
"app.modules.indexer.spider.milkie -> app.adapters.network.http",
"app.modules.indexer.spider.milkie -> app.foundation",
"app.modules.indexer.spider.milkie -> app.foundation.temporal",
"app.modules.indexer.spider.milkie -> app.runtime",
"app.modules.indexer.spider.milkie -> app.runtime.log",
"app.modules.indexer.spider.milkie -> app.runtime.settings",
"app.modules.indexer.spider.milkie -> app.schemas",
"app.modules.indexer.spider.milkie -> app.schemas.types",
"app.modules.indexer.spider.mtorrent -> app.adapters",
"app.modules.indexer.spider.mtorrent -> app.adapters.network",
"app.modules.indexer.spider.mtorrent -> app.adapters.network.http",
@@ -10346,7 +10357,7 @@
"app.workflow.actions.transfer_file -> app.workflow",
"app.workflow.actions.transfer_file -> app.workflow.actions"
],
"module_count": 1112,
"module_count": 1113,
"modules": [
"app",
"app.adapters",
@@ -11121,6 +11132,7 @@
"app.modules.indexer.spider",
"app.modules.indexer.spider.haidan",
"app.modules.indexer.spider.hddolby",
"app.modules.indexer.spider.milkie",
"app.modules.indexer.spider.mtorrent",
"app.modules.indexer.spider.rousi",
"app.modules.indexer.spider.sunnypt",

View File

@@ -89,6 +89,10 @@ def _install_search_observers(monkeypatch, events: list[tuple]) -> None:
"Yema",
{"keyword": "clean keyword", "mtype": MediaType.MOVIE, "page": 3},
),
(
"Milkie",
{"keyword": "clean keyword", "mtype": MediaType.MOVIE, "page": 3},
),
(
"Haidan",
{"keyword": "clean keyword", "mtype": MediaType.MOVIE},

205
tests/test_milkie_spider.py Normal file
View File

@@ -0,0 +1,205 @@
# -*- coding: utf-8 -*-
# pylint: disable=no-name-in-module
from unittest.mock import AsyncMock, MagicMock
import pytest
from app.modules.indexer.spider import milkie as milkie_module
from app.modules.indexer.spider.milkie import MilkieSpider
from app.schemas.types import MediaType
def _build_indexer() -> dict:
"""构造 Milkie API Spider 所需的最小站点配置。"""
return {
"id": "milkie",
"name": "Milkie",
"domain": "https://milkie.cc/",
"apikey": "milkie+secret",
"ua": "MoviePilot-Test",
"proxy": False,
}
def _sample_torrent() -> dict:
"""构造 Milkie 搜索接口的一条真实形态结果。"""
return {
"id": "wrJoUrlySnVm",
"releaseName": "Dune.Part.Two.2024.2160p.UHD.BluRay.H265-GAZPROM",
"category": 1,
"createdAt": "2026-10-08T08:46:25+02:00",
"size": 1076476044,
"downloaded": 12,
"seeders": 11,
"leechers": 0,
"externals": {"imdb": "tt15239678", "tmdb": None, "tvmaze": None},
}
@pytest.fixture()
def milkie_spider():
"""构造不依赖真实站点配置的 MilkieSpider。"""
return MilkieSpider(_build_indexer())
def test_search_page_size_matches_api_limit(milkie_spider):
"""单页容量应与接口上限一致,供分页器正确切片。"""
assert MilkieSpider.get_search_page_size("dune") == 100
def test_keyword_search_maps_media_categories(milkie_spider):
"""带关键字的搜索应按媒体类型收敛分类。"""
params = milkie_spider._MilkieSpider__get_params(
"Dune", MediaType.MOVIE, 0
)
assert params["query"] == "Dune"
assert params["categories"] == "1"
assert params["pi"] == 0
assert params["ps"] == 100
def test_tv_and_music_category_mapping(milkie_spider):
"""剧集与音乐分别映射到站点 2、3 分类。"""
tv_params = milkie_spider._MilkieSpider__get_params("Dune", MediaType.TV, 1)
music_params = milkie_spider._MilkieSpider__get_params("Dune", MediaType.MUSIC, 0)
assert tv_params["categories"] == "2"
assert tv_params["pi"] == 1
assert music_params["categories"] == "3"
def test_keyword_search_without_media_type_omits_categories(milkie_spider):
"""未指定媒体类型时不提交分类,避免站点接口过滤失效。"""
params = milkie_spider._MilkieSpider__get_params("Dune", None, 0)
assert "categories" not in params
def test_browse_mode_supports_pagination(milkie_spider):
"""无关键字的浏览模式同样支持服务端翻页。"""
first_page = milkie_spider._MilkieSpider__get_params(None, None, 0)
second_page = milkie_spider._MilkieSpider__get_params(None, None, 1)
assert "query" not in first_page
assert first_page["pi"] == 0
assert second_page["pi"] == 1
assert first_page["oby"] == "created_at"
def test_parse_result_projects_torrent_fields(milkie_spider):
"""结果字段应完整映射到站点搜索契约。"""
torrents = milkie_spider._MilkieSpider__parse_result([_sample_torrent()])
assert len(torrents) == 1
torrent = torrents[0]
assert torrent["title"] == "Dune.Part.Two.2024.2160p.UHD.BluRay.H265-GAZPROM"
assert torrent["enclosure"] == (
"https://milkie.cc/api/v1/torrents/wrJoUrlySnVm"
"/torrent?key=milkie%2Bsecret"
)
assert torrent["page_url"] == "https://milkie.cc/browse/wrJoUrlySnVm"
assert torrent["size"] == 1076476044
assert torrent["seeders"] == 11
assert torrent["peers"] == 0
assert torrent["grabs"] == 12
assert torrent["pubdate"] == "2026-10-08 08:46:25"
assert torrent["imdbid"] == "tt15239678"
assert torrent["category"] == MediaType.MOVIE.value
def test_parse_result_marks_site_as_freeleech(milkie_spider):
"""站点无分享率考核,所有种子应视为免费。"""
torrents = milkie_spider._MilkieSpider__parse_result([_sample_torrent()])
assert torrents[0]["downloadvolumefactor"] == 0
assert torrents[0]["uploadvolumefactor"] == 1
def test_parse_result_category_fallback(milkie_spider):
"""游戏、软件等无对应媒体类型的分类不应输出媒体类型。"""
raw = _sample_torrent()
raw["category"] = 4
torrents = milkie_spider._MilkieSpider__parse_result([raw])
assert torrents[0]["category"] is None
def test_search_requires_api_key(milkie_spider, monkeypatch):
"""缺少 API Key 时应直接失败且不发起请求。"""
request_utils = MagicMock()
monkeypatch.setattr(milkie_module, "RequestUtils", request_utils)
milkie_spider._apikey = None
error, results = milkie_spider.search("Dune")
assert error is True
assert results == []
request_utils.assert_not_called()
def _install_json_response(monkeypatch, status_code: int, payload: dict) -> MagicMock:
"""在模块边界替换同步 HTTP 客户端并返回调用记录。"""
response = MagicMock()
response.status_code = status_code
response.json.return_value = payload
request_utils = MagicMock()
request_utils.return_value.get_res.return_value = response
monkeypatch.setattr(milkie_module, "RequestUtils", request_utils)
return request_utils
def test_search_sends_auth_header_and_parses_response(milkie_spider, monkeypatch):
"""搜索应携带 x-milkie-auth 请求头并解析 torrents 数组。"""
request_utils = _install_json_response(
monkeypatch, 200, {"hits": 1, "torrents": [_sample_torrent()]}
)
error, results = milkie_spider.search("Dune", MediaType.MOVIE)
assert error is False
assert len(results) == 1
headers = request_utils.call_args.kwargs["headers"]
assert headers["x-milkie-auth"] == "milkie+secret"
def test_search_reports_invalid_api_key(milkie_spider, monkeypatch):
"""401 响应应判定为 API Key 失效。"""
_install_json_response(monkeypatch, 401, {"message": "Unauthorized"})
error, results = milkie_spider.search("Dune")
assert error is True
assert results == []
def test_search_reports_unreachable_site(milkie_spider, monkeypatch):
"""网络不可达时应返回错误标志。"""
request_utils = MagicMock()
request_utils.return_value.get_res.return_value = None
monkeypatch.setattr(milkie_module, "RequestUtils", request_utils)
error, results = milkie_spider.search("Dune")
assert error is True
assert results == []
@pytest.mark.asyncio
async def test_async_search_matches_sync_behavior(milkie_spider, monkeypatch):
"""异步搜索应与同步搜索使用同样的认证与解析路径。"""
response = MagicMock()
response.status_code = 200
response.json.return_value = {"hits": 1, "torrents": [_sample_torrent()]}
async_request_utils = MagicMock()
async_request_utils.return_value.get_res = AsyncMock(return_value=response)
monkeypatch.setattr(milkie_module, "AsyncRequestUtils", async_request_utils)
error, results = await milkie_spider.async_search("Dune", MediaType.MOVIE)
assert error is False
assert results[0]["title"] == "Dune.Part.Two.2024.2160p.UHD.BluRay.H265-GAZPROM"
headers = async_request_utils.call_args.kwargs["headers"]
assert headers["x-milkie-auth"] == "milkie+secret"