mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-10-31 20:34:48 +08:00
fix(music): preserve track identity and normalize resource metadata
This commit is contained in:
@@ -1467,8 +1467,8 @@ class ScrapingChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
merged.album_artist = album.album_artist or album.artist or local_meta.album_artist
|
||||
merged.year = album.year or local_meta.year
|
||||
merged.total_tracks = album.total_tracks or local_meta.total_tracks
|
||||
merged.media_source = album.media_source or local_meta.media_source
|
||||
merged.media_id = album.media_id or local_meta.media_id
|
||||
# MetaMusic 的身份代表音轨;保留已有单曲 ID,不能用专辑 ID 补位,
|
||||
# 否则会误写 musicbrainz_trackid,使下次拿 release-group 查询 recording。
|
||||
return merged
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -280,6 +280,14 @@ _MUSIC_PAREN_SPEC_RE = re.compile(
|
||||
_MUSIC_EMPTY_BRACKET_RE = re.compile(r"[\((\[]\s*(?:[/+,\-]\s*)*[\))\]]")
|
||||
# 尾部花括号通常是唱片目录号或发布标记,仅在末尾剔除,保护正文中的花括号文本。
|
||||
_MUSIC_TRAILING_CATALOG_RE = re.compile(r"\s*\{[A-Za-z0-9][^{}]{0,40}\}\s*$")
|
||||
# 标准抓轨发布尾链有介质、音频格式和厂牌目录三重证据;不删除正文中的 CD 或花括号。
|
||||
_MUSIC_RESOURCE_CATALOG_TAIL_RE = re.compile(
|
||||
rf"(?<=[))])\s+-\s+(?:CD|SACD|WEB)\s+-\s+\[[^\]\r\n]*"
|
||||
rf"(?:{_MUSIC_FORMAT_TOKEN_ALT})[^\]\r\n]*\]"
|
||||
r"(?:\s+-\s+\d+(?:\.\d+)?\s*(?:bits?|kHz|Hz))*"
|
||||
r"\s+-\s+\{[^{}\r\n]+\}(?:\s+-\s+[A-Za-z0-9][A-Za-z0-9@._-]*)?\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
# 年份及完整发行日期括号提供候选消歧线索,规格或目录号不作为日期。
|
||||
_MUSIC_YEAR_RE = re.compile(
|
||||
r"[\(\[(【]((?:19|20)\d{2})"
|
||||
@@ -742,7 +750,10 @@ class MetaMusic(MetaBase):
|
||||
标题解析仍复用 Python/Rust 公共入口;副标题只补缺失字段,保留标题中
|
||||
已有的署名。专辑、曲序、曲名的明确多段格式在资源层统一补充。
|
||||
"""
|
||||
meta = cls.parse_query(title)
|
||||
meta = cls.parse_query(_MUSIC_RESOURCE_CATALOG_TAIL_RE.sub("", title))
|
||||
meta.org_string = title
|
||||
meta.apply_audio_quality(title)
|
||||
subtitle = (subtitle or "").replace("丨", "|")
|
||||
if not meta.title:
|
||||
# 整个作品名位于中文展示括号内时,旧解析器可能把它误当发布标签删除。
|
||||
bracket = re.match(r"^\s*[【《「]([^】》」]+)[】》」]", title)
|
||||
|
||||
@@ -33,6 +33,14 @@ _BARE_VERSION_SUFFIX = re.compile(
|
||||
r"现场|現場|混音|伴奏|不插电|不插電).*$", re.IGNORECASE,
|
||||
)
|
||||
_TITLE_LABEL = re.compile(r"^(?:专辑(?:名|名称)?|專輯(?:名|名稱)?|曲名|歌曲|album|title)\s*[::]\s*", re.I)
|
||||
_TITLE_CREDIT = re.compile(
|
||||
r"\s*[((]\s*(?:feat(?:uring)?\.?\s+[^()()]+|"
|
||||
r"(?:电视剧|電視劇|电影|電影|影视剧|影視劇|动画|動畫)?\s*《[^《》]+》"
|
||||
r"\s*(?:电视剧|電視劇|电影|電影|影视剧|影視劇|动画|動畫)?"
|
||||
r"\s*(?:[“「][^”」]+[”」])?\s*(?:主题曲|主題曲|片头曲|片頭曲|片尾曲|插曲))\s*[))]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_FEATURED_CREDIT = re.compile(r"[((]\s*feat(?:uring)?\.?\s+([^()()]+)[))]", re.IGNORECASE)
|
||||
_COLLECTIVE_ARTISTS = ("Various Artists", "Various", "VA", "群星", "众艺人", "眾藝人")
|
||||
_VERSION_YEAR = re.compile(r"(?<!\d)(?:19|20)\d{2}(?!\d)")
|
||||
_VERSION_DATE = re.compile(r"(?<!\d)((?:19|20)\d{2})(?:[-./]|年)\s*(\d{1,2})(?:[-./]|月)\s*(\d{1,2})日?(?!\d)")
|
||||
@@ -101,8 +109,8 @@ def music_artist_matches(music: MusicInfo, parsed_artists: Iterable[str]) -> boo
|
||||
|
||||
|
||||
def music_base_title(value: Optional[str], *, preserve_editions: bool = False) -> str:
|
||||
"""剥离已知版本后缀;数据源可保留发行版标签,不改变未知括号中的作品名。"""
|
||||
text = str(value or "")
|
||||
"""剥离明确署名、影视用途和版本注释,保留未知括号中的作品名。"""
|
||||
text = _TITLE_CREDIT.sub("", str(value or ""))
|
||||
if not preserve_editions:
|
||||
text = _EDITION.sub("", text)
|
||||
|
||||
@@ -219,8 +227,26 @@ def _version_dates(title: Optional[str], version: Optional[str]) -> tuple[set[in
|
||||
return years, dates
|
||||
|
||||
|
||||
def _resource_credits_match(music: MusicInfo, meta: MetaMusic) -> bool:
|
||||
"""资源显式客串署名必须得到目标艺术家或同一署名注释确认,不能把合作版匹配为独唱。"""
|
||||
credits = {
|
||||
music_text_key(artist)
|
||||
for value in _FEATURED_CREDIT.findall(meta.title or "")
|
||||
for artist in MetaMusic._split_artists(value)
|
||||
}
|
||||
confirmed = {music_text_key(artist) for artist in music_artists(music)}
|
||||
confirmed.update(
|
||||
music_text_key(artist)
|
||||
for value in _FEATURED_CREDIT.findall(music.title or "")
|
||||
for artist in MetaMusic._split_artists(value)
|
||||
)
|
||||
return credits <= confirmed
|
||||
|
||||
|
||||
def music_version_matches(music: MusicInfo, meta: MetaMusic) -> bool:
|
||||
"""资源匹配与候选确认共用录音版本约束,不从艺术家字段推断版本。"""
|
||||
if not _resource_credits_match(music, meta):
|
||||
return False
|
||||
target_title = music.album or music.title if music.music_type == MUSIC_ENTITY_ALBUM else music.title
|
||||
# 专辑类型描述整专版本,但单曲的所属专辑类型不能代替该录音自身的版本。
|
||||
album_versions = " ".join(music.secondary_types or []) if music.music_type == MUSIC_ENTITY_ALBUM else ""
|
||||
|
||||
87
docs/music-lifecycle-audit.md
Normal file
87
docs/music-lifecycle-audit.md
Normal file
@@ -0,0 +1,87 @@
|
||||
# 音乐全流程审计(2026-09-09)
|
||||
|
||||
## 结论
|
||||
|
||||
普通分轨专辑和独立单曲已接入通用搜索、订阅、下载、整理、刮削及媒体库刷新流程。
|
||||
这不等于任意音乐资源都可以无人值守处理:整轨音频加 CUE、SACD ISO、多专辑全集、
|
||||
演唱会视频,以及上游没有收录或缺少可靠署名的资源,仍有明确边界。
|
||||
|
||||
本轮基于后端 `ca01cceed13ff0060de22b733eb0fef08003e840` 和前端 `e5de69f4` 审查。
|
||||
没有向真实下载器添加任务,也没有修改生产数据库或媒体库。实站请求只获取音乐列表;
|
||||
识别请求使用公共 MusicBrainz 接口,单元测试全部使用隔离数据。
|
||||
|
||||
## 链路与边界
|
||||
|
||||
| 阶段 | 当前行为 | 需要保留的约束 |
|
||||
| --- | --- | --- |
|
||||
| 元数据搜索 | 前端专辑、单曲入口和 `useMediaSubscribe` 传递统一来源、ID、实体类型 | Recording 和 Release Group 不能互换;艺术家不能直接订阅 |
|
||||
| 站点搜索 | `SearchMusicOwner` 生成标题、艺人及可信别名关键词;`match_music_resource` 核对名称、署名、分类和版本 | 缺艺术家、不同版本、关联专辑只能作为候选;单曲在专辑中的存在不是独立单曲资源 |
|
||||
| 订阅 | 主动搜索与 RSS 均保留目标身份、暂停/取消状态和音质规则 | 专辑需有总曲目数;远端暂不可用时恢复已保存的订阅快照 |
|
||||
| 下载 | 下载前解析种子文件清单,按盘号/曲序或曲名去重;专辑达到预期数量后才标记覆盖 | 这是文件清单层的覆盖确认,不是逐曲声纹验证;没有文件清单或曲目数未知不能确认整专 |
|
||||
| 完成判定 | 专辑订阅消费下载层的 `confirmed_full_coverage`,单曲按成功添加下载任务处理 | “订阅完成”代表已提交符合条件的下载,不代表下载完成、整理成功和媒体库刷新完成 |
|
||||
| 整理 | 恢复下载历史中的音乐上下文,保留逐曲标签/文件名;本地目录可按曲序、曲名、时长匹配发行版 | 路径不可读时不能假定读到了标签;不能把整张专辑的标题和单曲 ID 复制给每个文件 |
|
||||
| 文件识别 | 指纹、标签、文件名分层识别,目录级匹配作为补充 | 需要可读取的音频、可用的指纹工具/服务及上游条目;失败时的本地展示对象不是远端识别成功 |
|
||||
| 刮削 | 各文件独立处理标签、封面和歌词,沿用统一存储与事件流程 | 单曲 ID 不适用于整目录;歌词缺失或请求预算耗尽不代表音频整理失败 |
|
||||
|
||||
整轨+CUE 不会被自动拆成多首独立音轨;ISO 不属于普通可写标签音频。
|
||||
多专辑合集无法仅凭种子主标题绑定成一个正确专辑,曲目数量也不能证明每首音频内容正确。
|
||||
经典音乐的作曲家与演奏者署名、非官方译名、拼音全集命名需要额外可靠证据,不能靠放宽
|
||||
艺术家/版本匹配来制造命中。
|
||||
|
||||
## 本轮修复
|
||||
|
||||
1. 专辑刮削合并曾将 Release Group ID 放入 `MetaMusic.media_id`,标签写入随后把它当作
|
||||
`musicbrainz_trackid`。现在只保留文件自身的单曲身份,缺失时不以专辑 ID 补位。
|
||||
验证覆盖“原本没有单曲 ID”和“已有有效单曲 ID”,并用真实 FLAC 完成写入、刮削、重读。
|
||||
2. 明确的客串署名和影视主题曲/插曲注释不再阻断同一作品匹配。客串必须由候选署名确认;
|
||||
Live、Remix、未知括号副标题和错误艺术家仍然被拒绝。
|
||||
3. 标准 `CD - [WAV整轨+CUE] - 位深 - 采样率 - {厂牌目录} - 发布者` 尾链不再污染
|
||||
专辑名;保留原始资源标题、年份和音质。副标题中的 `丨` 与普通竖线统一处理,避免把厂牌误识别为艺人。
|
||||
4. 更新批量工具:保留主副标题,复用生产模块入口,移除已失效的私有参数;没有远端 ID 的
|
||||
展示兜底不计命中。凭据按生产库注册域规则查询,避免 `pt.` 子域导致假报缺少 Cookie。
|
||||
采样与识别可单独执行,运行配置隔离,分页失败保留已采数据。
|
||||
|
||||
## 可复用采样方式
|
||||
|
||||
在仓库虚拟环境中执行。数据库通过 SQLite 只读 URI 打开,Cookie 不写入报告。
|
||||
默认数据库是本机 `~/Documents/moviepilot/user.db`,站点规则来自同工作区的
|
||||
`MoviePilot-Build/sites/private`。可通过环境变量 `PROXY_HOST` 提供必要的访问代理。
|
||||
|
||||
```bash
|
||||
.venv/bin/python scripts/music_recognize_batch_test.py --fetch-only --pages 1
|
||||
.venv/bin/python scripts/music_recognize_batch_test.py --limit 30 --music-type album
|
||||
.venv/bin/python scripts/music_recognize_batch_test.py --limit 30 --music-type recording
|
||||
```
|
||||
|
||||
TSV 样本和 CSV 报告保存在 `config/temp/`,兼容旧版纯标题和两列 TSV。
|
||||
复用资源列表只免去再次访问站点,识别仍会请求 MusicBrainz;这不是离线测试命令。
|
||||
不指定实体时的“命中”只是模块返回远端候选,需额外检查实体、艺人和版本,不能作为准确率。
|
||||
|
||||
本次本地采样、公共响应录制、前后回放及完整检查日志保存在工作区
|
||||
`.artifacts/music-audit-20260909/`。这些原始实站数据和运行状态不作为产品源码提交。
|
||||
|
||||
## 已确认的实站结果
|
||||
|
||||
只读采样得到 7 个站点的 206 条资源,保留主副标题。对全部样本运行修改前后的
|
||||
离线解析:修正 `Jay` 的抓轨尾链,以及 `Duo Improbabile` 资源中被误作艺人的厂牌字段;
|
||||
其余 204 条的名称、艺人、专辑、年份和格式字段未改变。这不是 206 条全部识别正确的声明。
|
||||
|
||||
`李佳薇 - 碧苍 (电视剧《与凤行》“碧海苍穹”主题曲)` 的真实 MusicBrainz 响应已保存为
|
||||
`tests/fixtures/music_lifecycle_recording.json`。相同响应在基线的单曲模式无命中,修复后同步和
|
||||
异步入口都命中 Recording `b64cad55-4c77-49cc-964a-fc3aaee5fa15`,无需回退到同名专辑。
|
||||
该回归严格重放已录制请求,不访问外部服务。
|
||||
|
||||
元数据对照选择 29 条跨站点音频资源。MusicBrainz 连续返回 503 后停止新增请求,按同一份
|
||||
公共响应记录离线回放,只有 16 条具备修改前后所需的完整响应。两版均有 10/16 条返回
|
||||
远端候选,**本轮没有证明总体命中率上升**;已证明的是一条单曲身份纠正及两条解析字段修正。
|
||||
其余 13 条包含网络失败、中断后尚未查询和修复后需要新查询的样本,统一标为不可评估,
|
||||
没有计作识别失败或成功。复杂古典署名、英文译名括号、无分隔拼音全集等仍有未解决样本。
|
||||
|
||||
检查包括后端全量测试、音乐专项回归、前端 5 个音乐相关测试文件(87 项)、架构策略和
|
||||
宿主快照、类型检查、复杂度、并发/异步/任务所有权/服务定位器/Ruff/Mypy 门禁,以及启动性能。
|
||||
复杂度检查曾因刮削类注释净增一行触及既有上限而失败,已调整注释后通过,没有修改任何基线。
|
||||
两个生成的真实 FLAC 文件已完成标签写入、专辑刮削和重新读取验证。
|
||||
|
||||
后端全量检查为 **9012 通过、9 跳过**;追加实站录制与双引擎回归后的音乐专项检查为
|
||||
**822 通过**,其中本轮新增回归 20 项。前端为 **87 通过**,架构策略为 **165 通过**,
|
||||
变更 Python 文件 Pylint 为 **10/10**,其余适用门禁通过。
|
||||
@@ -3,8 +3,8 @@
|
||||
|
||||
用途:
|
||||
1. fetch - 用生产数据库中的站点 Cookie 抓取多个站点音乐分区种子标题
|
||||
2. test - 对标题批量执行音乐识别链路(与识别测试页相同),输出 CSV 报告与命中率汇总
|
||||
3. 抓取结果落盘后可离线重跑 test,用于优化识别程序前后对比
|
||||
2. test - 复用生产 MusicBrainz 识别入口,输出 CSV 报告与候选命中率汇总
|
||||
3. 抓取结果落盘后可复用资源数据重测;识别仍会访问 MusicBrainz
|
||||
|
||||
约束:
|
||||
- 生产库仅以只读 URI 模式打开,不做任何写入
|
||||
@@ -19,18 +19,21 @@
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import yaml
|
||||
|
||||
# 保证从仓库根目录外执行时也能导入 app 包
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
# 生产数据库(site 表中的站点 Cookie)与站点索引配置源文件目录
|
||||
DB_PATH = Path.home() / "Documents" / "MoviePilot" / "user.db"
|
||||
DB_PATH = Path.home() / "Documents" / "moviepilot" / "user.db"
|
||||
INDEXER_DIR = Path.home() / "MPProjects" / "MoviePilot-Build" / "sites" / "private"
|
||||
|
||||
TITLES_FILE = ROOT / "config" / "temp" / "music_batch_titles.txt"
|
||||
@@ -62,6 +65,8 @@ SITES = [
|
||||
|
||||
def load_site_credentials(domains: list[str]) -> dict[str, dict]:
|
||||
"""只读打开生产库,按域名批量取出站点 Cookie 与 UA。"""
|
||||
from app.domain.site import extract_domain
|
||||
|
||||
if not DB_PATH.exists():
|
||||
sys.exit(f"生产数据库不存在:{DB_PATH}")
|
||||
credentials: dict[str, dict] = {}
|
||||
@@ -69,10 +74,10 @@ def load_site_credentials(domains: list[str]) -> dict[str, dict]:
|
||||
try:
|
||||
for domain in domains:
|
||||
row = con.execute(
|
||||
"SELECT cookie, ua FROM site WHERE domain = ?", (domain,)
|
||||
"SELECT cookie, ua, proxy FROM site WHERE domain = ?", (extract_domain(domain),)
|
||||
).fetchone()
|
||||
if row and row[0]:
|
||||
credentials[domain] = {"cookie": row[0], "ua": row[1] or None}
|
||||
credentials[domain] = {"cookie": row[0], "ua": row[1] or None, "proxy": bool(row[2])}
|
||||
finally:
|
||||
con.close()
|
||||
return credentials
|
||||
@@ -87,25 +92,26 @@ def build_indexer(site: dict, credential: dict) -> dict:
|
||||
with open(INDEXER_DIR / site["yml"], "r", encoding="utf-8") as f:
|
||||
indexer = yaml.safe_load(f)
|
||||
indexer["cookie"] = credential["cookie"]
|
||||
indexer["proxy"] = credential["proxy"]
|
||||
if credential["ua"]:
|
||||
indexer["ua"] = credential["ua"]
|
||||
indexer["browse"] = {"path": site["browse"]}
|
||||
return indexer
|
||||
|
||||
|
||||
def fetch_site_titles(site: dict, indexer: dict, max_pages: int) -> list[str]:
|
||||
def fetch_site_titles(site: dict, indexer: dict, max_pages: int) -> list[tuple[str, str]]:
|
||||
"""翻页抓取单个站点音乐分区种子标题,按标题去重并保留出现顺序。"""
|
||||
from app.modules.indexer.spider import SiteSpider
|
||||
from app.schemas.types import MediaType
|
||||
|
||||
titles: list[str] = []
|
||||
titles: list[tuple[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for page in range(max_pages):
|
||||
spider = SiteSpider(indexer=indexer, mtype=MediaType.MUSIC, page=page)
|
||||
torrents = spider.get_torrents()
|
||||
if spider.is_error:
|
||||
print(f" [{site['name']}] 第 {page} 页请求失败(Cookie 可能失效或站点不可达),跳过该站点")
|
||||
return []
|
||||
print(f" [{site['name']}] 第 {page} 页请求失败,保留已有样本并停止翻页")
|
||||
break
|
||||
if not torrents:
|
||||
break
|
||||
for torrent in torrents:
|
||||
@@ -113,15 +119,15 @@ def fetch_site_titles(site: dict, indexer: dict, max_pages: int) -> list[str]:
|
||||
if not title or title in seen:
|
||||
continue
|
||||
seen.add(title)
|
||||
titles.append(title)
|
||||
titles.append((title, torrent.get("description") or ""))
|
||||
if len(titles) >= site["limit"]:
|
||||
break
|
||||
print(f" [{site['name']}] 获取 {len(titles)} 条")
|
||||
return titles[:site["limit"]]
|
||||
|
||||
|
||||
def fetch_titles(site_keys: list[str], max_pages: int) -> list[tuple[str, str]]:
|
||||
"""按配置抓取多个站点音乐分区标题,返回 (站点名, 标题) 列表。"""
|
||||
def fetch_titles(site_keys: list[str], max_pages: int) -> list[tuple[str, str, str]]:
|
||||
"""按配置抓取多个站点音乐分区标题,返回 (站点名, 标题, 副标题) 列表。"""
|
||||
sites = [site for site in SITES if not site_keys or site["key"] in site_keys]
|
||||
unknown = set(site_keys) - {site["key"] for site in sites}
|
||||
if unknown:
|
||||
@@ -129,84 +135,66 @@ def fetch_titles(site_keys: list[str], max_pages: int) -> list[tuple[str, str]]:
|
||||
domains = []
|
||||
for site in sites:
|
||||
with open(INDEXER_DIR / site["yml"], "r", encoding="utf-8") as f:
|
||||
domain = (yaml.safe_load(f).get("domain") or "").replace("https://", "").replace("http://", "").rstrip("/")
|
||||
domain = urlsplit(yaml.safe_load(f).get("domain") or "").hostname or ""
|
||||
site["domain"] = domain
|
||||
domains.append(domain)
|
||||
credentials = load_site_credentials(domains)
|
||||
|
||||
results: list[tuple[str, str]] = []
|
||||
results: list[tuple[str, str, str]] = []
|
||||
for site in sites:
|
||||
credential = credentials.get(site["domain"])
|
||||
if not credential:
|
||||
print(f" [{site['name']}] 未配置 Cookie,跳过")
|
||||
continue
|
||||
indexer = build_indexer(site, credential)
|
||||
for title in fetch_site_titles(site, indexer, max_pages):
|
||||
results.append((site["name"], title))
|
||||
for title, description in fetch_site_titles(site, indexer, max_pages):
|
||||
results.append((site["name"], title, description))
|
||||
return results
|
||||
|
||||
|
||||
def recognize_one(module, title: str) -> dict:
|
||||
"""对单条标题执行与识别测试页相同的解析+识别链路,并给出失败归因。"""
|
||||
def recognize_one(module, title: str, description: str = "", music_type: str | None = None) -> dict:
|
||||
"""复用生产模块的完整识别入口;没有远端 ID 的展示兜底不能计为命中。"""
|
||||
from app.domain.meta.metamusic import MetaMusic
|
||||
from app.schemas.types import MediaSource, MediaType
|
||||
|
||||
row = {"title": title}
|
||||
row = {"title": title, "description": description, "requested_music_type": music_type or ""}
|
||||
try:
|
||||
meta = MetaMusic.parse_query(title)
|
||||
meta = MetaMusic.parse_resource(title, description)
|
||||
row.update({
|
||||
"parsed_title": meta.title,
|
||||
"parsed_artists": " / ".join(meta.artists or []),
|
||||
"parsed_album": meta.album or "",
|
||||
"parsed_format": meta.audio_format or "",
|
||||
})
|
||||
# 拆开检索与候选挑选两步,便于区分零命中与比对失配
|
||||
candidates = module._search_recordings(meta, limit=10)
|
||||
matched = module._select_candidate(meta, candidates, source="musicbrainz")
|
||||
hit_label = "命中"
|
||||
albums: list = []
|
||||
# 与正式链路一致:专辑挑选要求艺术家命中,无艺术家线索时跳过专辑回退检索
|
||||
if not matched and meta.artists:
|
||||
albums = module._search_albums(meta, limit=10)
|
||||
matched = module._select_album_candidate(meta, albums)
|
||||
hit_label = "命中(专辑)"
|
||||
if matched:
|
||||
matched = module.recognize_media(
|
||||
meta=meta, mtype=MediaType.MUSIC,
|
||||
media_source=MediaSource.MusicBrainz, music_type=music_type, cache=False,
|
||||
)
|
||||
if matched and matched.media_id:
|
||||
row.update({
|
||||
"status": hit_label,
|
||||
"status": "命中",
|
||||
"matched_title": matched.title,
|
||||
"matched_artists": " / ".join(matched.artists or []),
|
||||
"matched_album": matched.album or "",
|
||||
"matched_year": matched.year or "",
|
||||
"matched_music_type": matched.music_type,
|
||||
"media_id": matched.media_id,
|
||||
})
|
||||
elif candidates:
|
||||
# 有候选但全部比对失配,列出最接近的候选方便归因
|
||||
top = candidates[0]
|
||||
row.update({
|
||||
"status": "候选比对失配",
|
||||
"matched_title": f"{top.title} | {' / '.join(top.artists or [])}",
|
||||
})
|
||||
elif albums:
|
||||
row.update({
|
||||
"status": "专辑候选失配",
|
||||
"matched_title": f"{albums[0].title} | {' / '.join(albums[0].artists or [])}",
|
||||
})
|
||||
elif not meta.title:
|
||||
row.update({"status": "解析失败"})
|
||||
else:
|
||||
row.update({"status": "检索零命中"})
|
||||
except Exception as err: # pylint: disable=broad-except
|
||||
row.update({"status": "异常", "matched_title": str(err)[:200]})
|
||||
row["status"] = "未命中" if meta.title else "解析失败"
|
||||
except Exception as err: # pylint: disable=broad-exception-caught
|
||||
row.update({"status": "异常", "matched_title": type(err).__name__})
|
||||
return row
|
||||
|
||||
|
||||
def run_batch(entries: list[tuple[str, str]]) -> list[dict]:
|
||||
def run_batch(entries: list[tuple[str, str, str]], music_type: str | None = None) -> list[dict]:
|
||||
"""批量执行识别并写出 CSV 报告,打印命中率汇总。"""
|
||||
from app.modules.musicbrainz import MusicBrainzModule
|
||||
|
||||
module = MusicBrainzModule()
|
||||
module.init_module()
|
||||
rows = []
|
||||
for index, (site_name, title) in enumerate(entries, 1):
|
||||
row = recognize_one(module, title)
|
||||
for index, (site_name, title, description) in enumerate(entries, 1):
|
||||
row = recognize_one(module, title, description, music_type)
|
||||
row["site"] = site_name
|
||||
rows.append(row)
|
||||
if index % 20 == 0 or index == len(entries):
|
||||
@@ -214,8 +202,9 @@ def run_batch(entries: list[tuple[str, str]]) -> list[dict]:
|
||||
|
||||
REPORT_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
fieldnames = [
|
||||
"site", "title", "status", "parsed_title", "parsed_artists", "parsed_format",
|
||||
"matched_title", "matched_artists", "matched_album", "matched_year", "media_id",
|
||||
"site", "title", "description", "requested_music_type", "status",
|
||||
"parsed_title", "parsed_artists", "parsed_album", "parsed_format",
|
||||
"matched_title", "matched_artists", "matched_album", "matched_year", "matched_music_type", "media_id",
|
||||
]
|
||||
with open(REPORT_FILE, "w", newline="", encoding="utf-8-sig") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
|
||||
@@ -239,44 +228,49 @@ def run_batch(entries: list[tuple[str, str]]) -> list[dict]:
|
||||
return rows
|
||||
|
||||
|
||||
def read_titles_file() -> list[tuple[str, str]]:
|
||||
"""读取落盘标题,兼容旧版纯标题格式(无站点前缀记为「憨憨」)。"""
|
||||
entries: list[tuple[str, str]] = []
|
||||
for line in TITLES_FILE.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if "\t" in line:
|
||||
site_name, title = line.split("\t", 1)
|
||||
else:
|
||||
site_name, title = "憨憨", line
|
||||
entries.append((site_name, title))
|
||||
def read_titles_file() -> list[tuple[str, str, str]]:
|
||||
"""读取 TSV 主副标题,兼容历史二列 TSV 和纯标题文件。"""
|
||||
entries: list[tuple[str, str, str]] = []
|
||||
with TITLES_FILE.open(encoding="utf-8", newline="") as stream:
|
||||
for fields in csv.reader(stream, delimiter="\t"):
|
||||
if not fields or not any(fields):
|
||||
continue
|
||||
if len(fields) == 1:
|
||||
entries.append(("憨憨", fields[0], ""))
|
||||
else:
|
||||
entries.append((fields[0], fields[1], fields[2] if len(fields) > 2 else ""))
|
||||
return entries
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""在独立配置目录执行采样或重测,禁止识别缓存写入生产配置。"""
|
||||
parser = argparse.ArgumentParser(description="多站点音乐种子批量识别测试")
|
||||
parser.add_argument("--fetch", action="store_true", help="重新抓取种子标题(默认复用已保存列表)")
|
||||
parser.add_argument("--sites", default="", help="指定站点 key 逗号分隔,缺省抓取全部配置站点")
|
||||
parser.add_argument("--pages", type=int, default=3, help="每站最大翻页数")
|
||||
parser.add_argument("--fetch-only", action="store_true", help="只采样,不请求音乐元数据")
|
||||
parser.add_argument("--limit", type=int, default=0, help="最多识别多少条,0 表示全部")
|
||||
parser.add_argument("--music-type", choices=("recording", "album"), help="限定单曲或专辑,避免混合命中掩盖实体错误")
|
||||
args = parser.parse_args()
|
||||
if args.pages < 1 or args.limit < 0:
|
||||
parser.error("pages 必须大于 0,limit 不能小于 0")
|
||||
os.environ["CONFIG_DIR"] = str(ROOT / "config" / "temp" / "music-batch-runtime")
|
||||
|
||||
if args.fetch or not TITLES_FILE.exists():
|
||||
if args.fetch or args.fetch_only or not TITLES_FILE.exists():
|
||||
site_keys = [key.strip() for key in args.sites.split(",") if key.strip()]
|
||||
entries = fetch_titles(site_keys, max_pages=args.pages)
|
||||
if not entries:
|
||||
sys.exit("未抓取到任何种子标题")
|
||||
TITLES_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
TITLES_FILE.write_text(
|
||||
"\n".join(f"{site_name}\t{title}" for site_name, title in entries),
|
||||
encoding="utf-8",
|
||||
)
|
||||
with TITLES_FILE.open("w", encoding="utf-8", newline="") as stream:
|
||||
csv.writer(stream, delimiter="\t").writerows(entries)
|
||||
print(f"已保存 {len(entries)} 条标题到 {TITLES_FILE}")
|
||||
else:
|
||||
entries = read_titles_file()
|
||||
print(f"复用已保存的 {len(entries)} 条标题")
|
||||
|
||||
run_batch(entries)
|
||||
if not args.fetch_only:
|
||||
run_batch(entries[:args.limit] if args.limit else entries, args.music_type)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
1812
tests/fixtures/music_lifecycle_recording.json
vendored
Normal file
1812
tests/fixtures/music_lifecycle_recording.json
vendored
Normal file
File diff suppressed because it is too large
Load Diff
170
tests/test_music_lifecycle_audit.py
Normal file
170
tests/test_music_lifecycle_audit.py
Normal file
@@ -0,0 +1,170 @@
|
||||
"""音乐实站审计发现的名称匹配和刮削身份回归。"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.adapters.system import rust
|
||||
from app.application.audio import AudioMetadataHelper
|
||||
from app.chain.scraping import ScrapingChain
|
||||
from app.domain.context import MusicInfo
|
||||
from app.domain.meta.metamusic import MetaMusic
|
||||
from app.domain.meta import runtime
|
||||
from app.domain.metainfo import MetaInfo
|
||||
from app.domain.music import match_music_resource
|
||||
from app.modules.musicbrainz import MusicBrainzModule
|
||||
from app.schemas.types import MediaSource, MediaType
|
||||
from scripts import music_recognize_batch_test as batch
|
||||
|
||||
|
||||
@pytest.mark.parametrize("title,artist,annotation", [
|
||||
("一起飙高音", "黄明志", "(feat. 李佳薇)"),
|
||||
("黎明所愿", "李佳薇", "(电视剧《暗夜与黎明》插曲)"),
|
||||
("穿越星海", "李佳薇", "(《勿扰飞升》影视剧主题曲)"),
|
||||
("我要闭上眼睛", "李佳薇", "(《仙剑四》影视剧片尾曲)"),
|
||||
])
|
||||
def test_credited_recording_matches_without_marketing_annotation(title, artist, annotation):
|
||||
"""明确署名或影视用途注释不改变录音名称,检索与订阅匹配保持一致。"""
|
||||
resource = f"{artist} - {title} {annotation} (2024) - WEB-DL - 24bit ALAC-HHWEB"
|
||||
meta = MetaMusic.parse_resource(resource)
|
||||
artists = [artist, "李佳薇"] if "feat." in annotation else [artist]
|
||||
candidate = MusicInfo(media_source="musicbrainz", media_id="recording", title=title, artists=artists)
|
||||
|
||||
assert MusicBrainzModule._select_candidate(meta, [candidate], MediaSource.MusicBrainz) is candidate
|
||||
assert match_music_resource(candidate, resource).status == "exact"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("annotation", ["(Part Two)", "(Live)", "(Remix)", "(feature presentation)"])
|
||||
def test_music_annotation_cleanup_preserves_distinct_works(annotation):
|
||||
"""未知副标题和录音版本仍须拒绝,不能靠删除所有括号提升命中率。"""
|
||||
candidate = MusicInfo(media_source="musicbrainz", media_id="recording", title="Song", artists=["Artist"])
|
||||
meta = MetaMusic.parse_resource(f"Artist - Song {annotation} FLAC")
|
||||
|
||||
assert MusicBrainzModule._select_candidate(meta, [candidate], MediaSource.MusicBrainz) is None
|
||||
assert match_music_resource(candidate, meta.org_string).status != "exact"
|
||||
|
||||
|
||||
def test_featured_recording_does_not_match_solo_candidate():
|
||||
"""客串署名缺少来源佐证时,去注释不能把合作录音误配成独唱。"""
|
||||
candidate = MusicInfo(media_source="musicbrainz", media_id="recording", title="Song", artists=["Artist"])
|
||||
meta = MetaMusic.parse_resource("Artist - Song (feat. Guest) FLAC")
|
||||
|
||||
assert MusicBrainzModule._select_candidate(meta, [candidate], MediaSource.MusicBrainz) is None
|
||||
assert match_music_resource(candidate, meta.org_string).status != "exact"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("recording_id", [None, "f4f1b7b2-22b0-4ce0-9ac6-31c336c52812"])
|
||||
def test_album_scrape_never_writes_release_group_as_recording(recording_id):
|
||||
"""专辑合并一直验证到标签投影,避免 release-group ID 污染下一次单曲识别。"""
|
||||
local = MetaMusic(title="晴天", artists=["周杰伦"], track_number=3,
|
||||
media_source=MediaSource.MusicBrainz if recording_id else None, media_id=recording_id)
|
||||
album = MusicInfo(media_source="musicbrainz", media_id="album-id", music_type="album",
|
||||
title="叶惠美", album="叶惠美", artists=["周杰伦"], total_tracks=11)
|
||||
|
||||
merged = ScrapingChain._merge_music_album_metadata(local, album)
|
||||
tags = AudioMetadataHelper._tag_values(merged)
|
||||
|
||||
assert tags["musicbrainz_trackid"] == recording_id
|
||||
assert tags["title"] == "晴天"
|
||||
assert tags["album"] == "叶惠美"
|
||||
assert local.album is None
|
||||
|
||||
|
||||
@pytest.fixture(params=["python", "rust"])
|
||||
def music_parser(request, monkeypatch):
|
||||
"""使用真实两种引擎入口验证共享资源预处理,避免只修到 Python 回退。"""
|
||||
if request.param == "rust":
|
||||
if not rust.is_available():
|
||||
pytest.skip("moviepilot_rust 扩展未安装")
|
||||
monkeypatch.setattr(rust, "is_enabled", lambda: True)
|
||||
monkeypatch.setattr(runtime, "_metainfo_accelerator", rust if request.param == "rust" else None)
|
||||
return MetaInfo
|
||||
|
||||
|
||||
def test_resource_release_catalog_does_not_pollute_album_name(music_parser):
|
||||
"""标准 CD 抓轨尾链只提供发行和音质信息,不应进入专辑名称。"""
|
||||
title = ("周杰伦 - Jay (2000) - CD - [WAV整轨+CUE] - 16bit - 44.1kHz - "
|
||||
"{TW - Alfa Music 74321838942} - CFQG@HDFans")
|
||||
meta = music_parser(title, mtype=MediaType.MUSIC)
|
||||
|
||||
assert (meta.title, meta.artists, meta.year) == ("Jay", ["周杰伦"], 2000)
|
||||
assert meta.audio_format == "WAV"
|
||||
assert meta.org_string == title
|
||||
|
||||
|
||||
def test_vertical_separator_does_not_turn_record_label_into_artist(music_parser):
|
||||
"""站点竖线分隔的厂牌和编号不能被拼接为 artist-title 署名。"""
|
||||
meta = music_parser(
|
||||
"Duo.Improbabile.Improbabile.Genesi.2016.DSF.FLAC",
|
||||
"From NativeDSD丨VDM Records - VDM038009丨DSD256",
|
||||
mtype=MediaType.MUSIC,
|
||||
)
|
||||
|
||||
assert meta.artists == []
|
||||
|
||||
|
||||
def test_batch_uses_subtitle_and_does_not_count_display_fallback():
|
||||
"""评测必须复用正式入口并区分本地展示对象和远端身份命中。"""
|
||||
module = Mock()
|
||||
module.recognize_media.return_value = MusicInfo(title="晴天")
|
||||
|
||||
result = batch.recognize_one(module, "晴天 FLAC", "演唱:周杰伦;专辑:叶惠美")
|
||||
|
||||
assert result["status"] == "未命中"
|
||||
assert result["parsed_artists"] == "周杰伦"
|
||||
assert result["parsed_album"] == "叶惠美"
|
||||
assert module.recognize_media.call_args.kwargs["cache"] is False
|
||||
|
||||
|
||||
def test_batch_reads_multiline_subtitles_and_legacy_rows(tmp_path, monkeypatch):
|
||||
"""TSV 正确还原带换行的副标题,同时支持旧版只有主标题的采样。"""
|
||||
path = tmp_path / "titles.tsv"
|
||||
path.write_text('站点\t晴天\t"歌手:周杰伦\n专辑:叶惠美"\n旧站\t单曲\n旧标题\n', encoding="utf-8")
|
||||
monkeypatch.setattr(batch, "TITLES_FILE", path)
|
||||
|
||||
assert batch.read_titles_file() == [
|
||||
("站点", "晴天", "歌手:周杰伦\n专辑:叶惠美"),
|
||||
("旧站", "单曲", ""), ("憨憨", "旧标题", ""),
|
||||
]
|
||||
|
||||
|
||||
def test_batch_matches_registered_site_domain(tmp_path, monkeypatch):
|
||||
"""索引中的 pt 子域须按生产库注册域查找,不能把已有 Cookie 误报成缺失。"""
|
||||
path = tmp_path / "user.db"
|
||||
with sqlite3.connect(path) as connection:
|
||||
connection.execute("CREATE TABLE site (domain TEXT, cookie TEXT, ua TEXT, proxy INTEGER)")
|
||||
connection.execute("INSERT INTO site VALUES (?, ?, ?, ?)", ("example.org", "fixture", "test", 0))
|
||||
monkeypatch.setattr(batch, "DB_PATH", path)
|
||||
|
||||
result = batch.load_site_credentials(["pt.example.org"])
|
||||
|
||||
assert result["pt.example.org"]["cookie"] == "fixture"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
def test_real_recording_response_matches_tie_in_title(asynchronous, monkeypatch):
|
||||
"""实站名称和公共响应离线重放,确认修复能识别单曲而非回退到同名专辑。"""
|
||||
fixture = json.loads((Path(__file__).parent / "fixtures/music_lifecycle_recording.json").read_text(encoding="utf-8"))
|
||||
module = MusicBrainzModule()
|
||||
|
||||
def request(path, params=None):
|
||||
"""只接受已录制请求,意外新增出站路径直接使回归失败。"""
|
||||
return next(item["payload"] for item in fixture["requests"] if item["path"] == path and item["params"] == params)
|
||||
|
||||
async def async_request(path, params=None):
|
||||
"""异步入口消费相同公共响应。"""
|
||||
return request(path, params)
|
||||
|
||||
monkeypatch.setattr(module, "_request_json", request)
|
||||
monkeypatch.setattr(module, "_async_request_json", async_request)
|
||||
kwargs = dict(meta=MetaMusic.parse_resource(fixture["title"], fixture["description"]),
|
||||
media_source=MediaSource.MusicBrainz, mtype=MediaType.MUSIC, music_type="recording", cache=False)
|
||||
result = asyncio.run(module.async_recognize_media(**kwargs)) if asynchronous else module.recognize_media(**kwargs)
|
||||
|
||||
assert result.media_id == fixture["expected_id"]
|
||||
assert result.music_type == "recording"
|
||||
assert result.artists == ["李佳薇"]
|
||||
@@ -31,7 +31,7 @@ def _album_info() -> MusicInfo:
|
||||
)
|
||||
|
||||
|
||||
def test_album_scrape_merge_preserves_track_fields_and_applies_album_identity() -> None:
|
||||
def test_album_scrape_merge_preserves_track_fields_and_applies_album_metadata() -> None:
|
||||
"""专辑批量刮削应保留每首歌自己的曲名和曲序,只统一专辑级字段。"""
|
||||
local = MetaMusic(
|
||||
title="晴天",
|
||||
@@ -52,8 +52,8 @@ def test_album_scrape_merge_preserves_track_fields_and_applies_album_identity()
|
||||
assert merged.album_artist == "周杰伦"
|
||||
assert merged.year == 2003
|
||||
assert merged.total_tracks == 11
|
||||
assert merged.media_source == "musicbrainz"
|
||||
assert merged.media_id == "release-group-1"
|
||||
assert merged.media_source is None
|
||||
assert merged.media_id is None
|
||||
|
||||
|
||||
def test_album_directory_scrape_processes_each_track_and_reuses_cover() -> None:
|
||||
|
||||
Reference in New Issue
Block a user