diff --git a/backend/src/module/checker/checker.py b/backend/src/module/checker/checker.py index 43196586..97ccddb2 100644 --- a/backend/src/module/checker/checker.py +++ b/backend/src/module/checker/checker.py @@ -36,7 +36,7 @@ class Checker: return False @staticmethod - def check_version() -> bool: + def check_version() -> tuple[bool, int | None]: return version_check() @staticmethod diff --git a/backend/src/module/conf/config.py b/backend/src/module/conf/config.py index 1a9cf1f4..bc78fa27 100644 --- a/backend/src/module/conf/config.py +++ b/backend/src/module/conf/config.py @@ -38,10 +38,35 @@ class Settings(Config): def load(self): with open(CONFIG_PATH, "r", encoding="utf-8") as f: config = json.load(f) + config = self._migrate_old_config(config) config_obj = Config.parse_obj(config) self.__dict__.update(config_obj.__dict__) logger.info("Config loaded") + @staticmethod + def _migrate_old_config(config: dict) -> dict: + """Migrate old config field names (3.1.x) to current format (3.2.x).""" + program = config.get("program", {}) + # Rename sleep_time -> rss_time + if "sleep_time" in program and "rss_time" not in program: + program["rss_time"] = program.pop("sleep_time") + elif "sleep_time" in program: + program.pop("sleep_time") + # Rename times -> rename_time + if "times" in program and "rename_time" not in program: + program["rename_time"] = program.pop("times") + elif "times" in program: + program.pop("times") + # Remove deprecated data_version field + program.pop("data_version", None) + + # Remove deprecated rss_parser fields + rss_parser = config.get("rss_parser", {}) + for key in ("type", "custom_url", "token", "enable_tmdb"): + rss_parser.pop(key, None) + + return config + def save(self, config_dict: dict | None = None): if not config_dict: config_dict = self.dict() diff --git a/backend/src/module/conf/const.py b/backend/src/module/conf/const.py index c48409d7..606dfdfb 100644 --- a/backend/src/module/conf/const.py +++ b/backend/src/module/conf/const.py @@ -1,16 +1,13 @@ # -*- encoding: utf-8 -*- -from urllib.parse import parse_qs, urlparse - DEFAULT_SETTINGS = { "program": { - "sleep_time": 7200, - "times": 20, + "rss_time": 900, + "rename_time": 60, "webui_port": 7892, - "data_version": 4.0, }, "downloader": { "type": "qbittorrent", - "host": "127.0.0.1:8080", + "host": "172.17.0.1:8080", "username": "admin", "password": "adminadmin", "path": "/downloads/Bangumi", @@ -18,10 +15,6 @@ DEFAULT_SETTINGS = { }, "rss_parser": { "enable": True, - "type": "mikan", - "custom_url": "mikanani.me", - "token": "", - "enable_tmdb": False, "filter": ["720", "\\d+-\\d+"], "language": "zh", }, @@ -39,18 +32,27 @@ DEFAULT_SETTINGS = { "enable": False, "type": "http", "host": "", - "port": 1080, + "port": 0, "username": "", "password": "", }, "notification": {"enable": False, "type": "telegram", "token": "", "chat_id": ""}, + "experimental_openai": { + "enable": False, + "api_key": "", + "api_base": "https://api.openai.com/v1", + "api_type": "openai", + "api_version": "2023-05-15", + "model": "gpt-3.5-turbo", + "deployment_id": "", + }, } ENV_TO_ATTR = { "program": { - "AB_INTERVAL_TIME": ("sleep_time", lambda e: int(e)), - "AB_RENAME_FREQ": ("times", lambda e: int(e)), + "AB_INTERVAL_TIME": ("rss_time", lambda e: int(e)), + "AB_RENAME_FREQ": ("rename_time", lambda e: int(e)), "AB_WEBUI_PORT": ("webui_port", lambda e: int(e)), }, "downloader": { @@ -61,13 +63,8 @@ ENV_TO_ATTR = { }, "rss_parser": { "AB_RSS_COLLECTOR": ("enable", lambda e: e.lower() in ("true", "1", "t")), - "AB_RSS": [ - ("token", lambda e: parse_qs(urlparse(e).query).get("token", [None])[0]), - ("custom_url", lambda e: urlparse(e).netloc), - ], "AB_NOT_CONTAIN": ("filter", lambda e: e.split("|")), "AB_LANGUAGE": "language", - "AB_ENABLE_TMDB": ("enable_tmdb", lambda e: e.lower() in ("true", "1", "t")), }, "bangumi_manage": { "AB_RENAME": ("enable", lambda e: e.lower() in ("true", "1", "t")), diff --git a/backend/src/module/core/program.py b/backend/src/module/core/program.py index 887f19e0..81c138ca 100644 --- a/backend/src/module/core/program.py +++ b/backend/src/module/core/program.py @@ -8,6 +8,7 @@ from module.update import ( data_migration, first_run, from_30_to_31, + from_31_to_32, start_up, ) @@ -49,10 +50,14 @@ class Program(RenameThread, RSSThread): "[Core] Legacy data detected, starting data migration, please wait patiently." ) data_migration() - elif self.version_update: - # Update database - await from_30_to_31() - logger.info("[Core] Database updated.") + else: + need_update, last_minor = self.version_update + if need_update: + if last_minor is not None and last_minor == 0: + await from_30_to_31() + logger.info("[Core] Database migrated from 3.0 to 3.1.") + await from_31_to_32() + logger.info("[Core] Database updated.") if not self.img_cache: logger.info("[Core] No image cache exists, create image cache.") await cache_image() @@ -107,7 +112,8 @@ class Program(RenameThread, RSSThread): ) def update_database(self): - if not self.version_update: + need_update, _ = self.version_update + if not need_update: return {"status": "No update found."} else: start_up() diff --git a/backend/src/module/core/status.py b/backend/src/module/core/status.py index 86cc8672..c632ca18 100644 --- a/backend/src/module/core/status.py +++ b/backend/src/module/core/status.py @@ -50,8 +50,9 @@ class ProgramStatus(Checker): return LEGACY_DATA_PATH.exists() @property - def version_update(self): - return not self.check_version() + def version_update(self) -> tuple[bool, int | None]: + is_same, last_minor = self.check_version() + return not is_same, last_minor @property def database(self): diff --git a/backend/src/module/update/__init__.py b/backend/src/module/update/__init__.py index b53c09f7..db6d261f 100644 --- a/backend/src/module/update/__init__.py +++ b/backend/src/module/update/__init__.py @@ -1,4 +1,4 @@ -from .cross_version import from_30_to_31, cache_image +from .cross_version import cache_image, from_30_to_31, from_31_to_32 from .data_migration import data_migration from .startup import first_run, start_up from .version_check import version_check diff --git a/backend/src/module/update/cross_version.py b/backend/src/module/update/cross_version.py index afe74d13..fcd24790 100644 --- a/backend/src/module/update/cross_version.py +++ b/backend/src/module/update/cross_version.py @@ -1,3 +1,4 @@ +import logging import re from urllib3.util import parse_url @@ -6,6 +7,8 @@ from module.network import RequestContent from module.rss import RSSEngine from module.utils import save_image +logger = logging.getLogger(__name__) + async def from_30_to_31(): with RSSEngine() as db: @@ -32,6 +35,13 @@ async def from_30_to_31(): await db.add_rss(rss_link=rss, aggregate=aggregate) +async def from_31_to_32(): + """Migrate database schema from 3.1.x to 3.2.x.""" + with RSSEngine() as db: + db.create_table() # Handles adding new columns (e.g., air_weekday) + logger.info("[Migration] 3.1 -> 3.2 migration completed.") + + async def cache_image(): with RSSEngine() as db: bangumis = db.bangumi.search_all() diff --git a/backend/src/module/update/version_check.py b/backend/src/module/update/version_check.py index e0bc6fe8..4d484ae4 100644 --- a/backend/src/module/update/version_check.py +++ b/backend/src/module/update/version_check.py @@ -3,27 +3,33 @@ import semver from module.conf import VERSION, VERSION_PATH -def version_check() -> bool: +def version_check() -> tuple[bool, int | None]: + """Check if version has changed. + + Returns: + A tuple of (is_same_version, last_minor_version). + last_minor_version is None if no upgrade is needed. + """ if VERSION == "DEV_VERSION": - return True + return True, None if VERSION == "local": - return True + return True, None if not VERSION_PATH.exists(): with open(VERSION_PATH, "w") as f: f.write(VERSION + "\n") - return False + return False, None else: with open(VERSION_PATH, "r+") as f: # Read last version versions = f.readlines() - last_version = versions[-1] + last_version = versions[-1].strip() last_ver = semver.VersionInfo.parse(last_version) now_ver = semver.VersionInfo.parse(VERSION) if now_ver.minor == last_ver.minor: - return True + return True, None else: if now_ver.minor > last_ver.minor: f.write(VERSION + "\n") - return False + return False, last_ver.minor else: - return True + return True, None diff --git a/backend/src/test/test_migration.py b/backend/src/test/test_migration.py new file mode 100644 index 00000000..32c9688f --- /dev/null +++ b/backend/src/test/test_migration.py @@ -0,0 +1,455 @@ +"""Tests for config and database migration from 3.1.x to 3.2.x.""" +import json +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +from sqlalchemy import inspect, text +from sqlmodel import Session, SQLModel, create_engine + +from module.conf.config import Settings +from module.database.combine import Database +from module.models import Bangumi, RSSItem, Torrent, User + + +# --- Mock old 3.1.x config (as stored in config.json) --- +OLD_31X_CONFIG = { + "program": { + "sleep_time": 7200, + "times": 20, + "webui_port": 7892, + "data_version": 4.0, + }, + "downloader": { + "type": "qbittorrent", + "host": "192.168.1.100:8080", + "username": "admin", + "password": "mypassword", + "path": "/downloads/Bangumi", + "ssl": False, + }, + "rss_parser": { + "enable": True, + "type": "mikan", + "custom_url": "mikanani.me", + "token": "abc123token", + "enable_tmdb": True, + "filter": ["720", "\\d+-\\d+"], + "language": "zh", + }, + "bangumi_manage": { + "enable": True, + "eps_complete": False, + "rename_method": "pn", + "group_tag": True, + "remove_bad_torrent": False, + }, + "log": { + "debug_enable": True, + }, + "proxy": { + "enable": True, + "type": "http", + "host": "127.0.0.1", + "port": 7890, + "username": "", + "password": "", + }, + "notification": { + "enable": True, + "type": "telegram", + "token": "bot123456:ABC-DEF", + "chat_id": "123456789", + }, +} + + +class TestConfigMigration: + """Test that old 3.1.x config files are properly migrated.""" + + def test_migrate_old_config_renames_program_fields(self): + """sleep_time -> rss_time, times -> rename_time.""" + result = Settings._migrate_old_config(json.loads(json.dumps(OLD_31X_CONFIG))) + assert "rss_time" in result["program"] + assert result["program"]["rss_time"] == 7200 + assert "rename_time" in result["program"] + assert result["program"]["rename_time"] == 20 + assert "sleep_time" not in result["program"] + assert "times" not in result["program"] + + def test_migrate_old_config_removes_data_version(self): + """data_version field should be removed.""" + result = Settings._migrate_old_config(json.loads(json.dumps(OLD_31X_CONFIG))) + assert "data_version" not in result["program"] + + def test_migrate_old_config_removes_deprecated_rss_fields(self): + """type, custom_url, token, enable_tmdb should be removed from rss_parser.""" + result = Settings._migrate_old_config(json.loads(json.dumps(OLD_31X_CONFIG))) + assert "type" not in result["rss_parser"] + assert "custom_url" not in result["rss_parser"] + assert "token" not in result["rss_parser"] + assert "enable_tmdb" not in result["rss_parser"] + + def test_migrate_old_config_preserves_valid_fields(self): + """Valid fields like rss_parser.filter, downloader.host should be preserved.""" + result = Settings._migrate_old_config(json.loads(json.dumps(OLD_31X_CONFIG))) + assert result["rss_parser"]["enable"] is True + assert result["rss_parser"]["filter"] == ["720", "\\d+-\\d+"] + assert result["rss_parser"]["language"] == "zh" + assert result["downloader"]["host"] == "192.168.1.100:8080" + assert result["downloader"]["password"] == "mypassword" + assert result["notification"]["token"] == "bot123456:ABC-DEF" + assert result["bangumi_manage"]["group_tag"] is True + assert result["log"]["debug_enable"] is True + assert result["proxy"]["port"] == 7890 + + def test_migrate_new_config_no_change(self): + """A config already in 3.2 format should not be altered.""" + new_config = { + "program": { + "rss_time": 900, + "rename_time": 60, + "webui_port": 7892, + }, + "rss_parser": { + "enable": True, + "filter": ["720"], + "language": "zh", + }, + } + result = Settings._migrate_old_config(json.loads(json.dumps(new_config))) + assert result["program"]["rss_time"] == 900 + assert result["program"]["rename_time"] == 60 + + def test_migrate_does_not_overwrite_new_fields_with_old(self): + """If both old and new field names exist, keep the new one.""" + config = { + "program": { + "sleep_time": 7200, + "rss_time": 900, + "times": 20, + "rename_time": 60, + "webui_port": 7892, + }, + "rss_parser": {"enable": True, "filter": [], "language": "zh"}, + } + result = Settings._migrate_old_config(json.loads(json.dumps(config))) + assert result["program"]["rss_time"] == 900 + assert result["program"]["rename_time"] == 60 + assert "sleep_time" not in result["program"] + assert "times" not in result["program"] + + def test_load_old_config_file(self): + """Full integration: loading a 3.1.x config.json produces correct Settings.""" + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as f: + json.dump(OLD_31X_CONFIG, f) + config_path = Path(f.name) + + try: + with patch("module.conf.config.CONFIG_PATH", config_path): + settings = Settings() + # Verify migrated fields + assert settings.program.rss_time == 7200 + assert settings.program.rename_time == 20 + assert settings.program.webui_port == 7892 + # Verify preserved fields + assert settings.downloader.host_ == "192.168.1.100:8080" + assert settings.downloader.password_ == "mypassword" + assert settings.rss_parser.enable is True + assert settings.rss_parser.filter == ["720", "\\d+-\\d+"] + assert settings.notification.enable is True + assert settings.notification.token_ == "bot123456:ABC-DEF" + assert settings.bangumi_manage.group_tag is True + assert settings.log.debug_enable is True + assert settings.proxy.port == 7890 + # Verify experimental_openai gets defaults + assert settings.experimental_openai.enable is False + finally: + config_path.unlink() + + def test_load_old_config_saves_migrated_format(self): + """After loading old config, the saved file should use new field names.""" + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as f: + json.dump(OLD_31X_CONFIG, f) + config_path = Path(f.name) + + try: + with patch("module.conf.config.CONFIG_PATH", config_path): + Settings() + # Re-read saved config + with open(config_path) as f: + saved = json.load(f) + assert "rss_time" in saved["program"] + assert "rename_time" in saved["program"] + assert "sleep_time" not in saved["program"] + assert "times" not in saved["program"] + assert "data_version" not in saved["program"] + assert "type" not in saved["rss_parser"] + assert "custom_url" not in saved["rss_parser"] + assert "token" not in saved["rss_parser"] + assert "enable_tmdb" not in saved["rss_parser"] + finally: + config_path.unlink() + + +class TestDatabaseMigration: + """Test that old 3.1.x databases are properly migrated to 3.2.x schema.""" + + def _create_old_31x_database(self, engine): + """Create a database matching the 3.1.x schema (no air_weekday column).""" + with engine.connect() as conn: + # Create bangumi table WITHOUT air_weekday (3.1.x schema) + conn.execute(text(""" + CREATE TABLE bangumi ( + id INTEGER PRIMARY KEY, + official_title TEXT NOT NULL DEFAULT 'official_title', + year TEXT, + title_raw TEXT NOT NULL DEFAULT 'title_raw', + season INTEGER NOT NULL DEFAULT 1, + season_raw TEXT, + group_name TEXT, + dpi TEXT, + source TEXT, + subtitle TEXT, + eps_collect BOOLEAN NOT NULL DEFAULT 0, + "offset" INTEGER NOT NULL DEFAULT 0, + filter TEXT NOT NULL DEFAULT '720,\\d+-\\d+', + rss_link TEXT NOT NULL DEFAULT '', + poster_link TEXT, + added BOOLEAN NOT NULL DEFAULT 0, + rule_name TEXT, + save_path TEXT, + deleted BOOLEAN NOT NULL DEFAULT 0 + ) + """)) + # Create user table + conn.execute(text(""" + CREATE TABLE user ( + id INTEGER PRIMARY KEY, + username TEXT NOT NULL DEFAULT 'admin', + password TEXT NOT NULL DEFAULT 'adminadmin' + ) + """)) + # Create torrent table + conn.execute(text(""" + CREATE TABLE torrent ( + id INTEGER PRIMARY KEY, + bangumi_id INTEGER REFERENCES bangumi(id), + rss_id INTEGER REFERENCES rssitem(id), + name TEXT NOT NULL DEFAULT '', + url TEXT NOT NULL DEFAULT 'https://example.com/torrent', + homepage TEXT, + downloaded BOOLEAN NOT NULL DEFAULT 0 + ) + """)) + # Create rssitem table + conn.execute(text(""" + CREATE TABLE rssitem ( + id INTEGER PRIMARY KEY, + name TEXT, + url TEXT NOT NULL DEFAULT 'https://mikanani.me', + aggregate BOOLEAN NOT NULL DEFAULT 0, + parser TEXT NOT NULL DEFAULT 'mikan', + enabled BOOLEAN NOT NULL DEFAULT 1 + ) + """)) + conn.commit() + + def _insert_old_data(self, engine): + """Insert sample 3.1.x data.""" + with engine.connect() as conn: + conn.execute(text(""" + INSERT INTO user (username, password) VALUES ('admin', 'adminadmin') + """)) + conn.execute(text(""" + INSERT INTO bangumi ( + official_title, year, title_raw, season, group_name, + dpi, source, subtitle, eps_collect, "offset", + filter, rss_link, poster_link, added, deleted + ) VALUES ( + '无职转生', '2021', 'Mushoku Tensei', 1, 'Lilith-Raws', + '1080p', 'Baha', 'CHT', 0, 0, + '720,\\d+-\\d+', 'https://mikanani.me/RSS/Bangumi?bangumiId=2353', + 'https://mikanani.me/images/Bangumi/202101/test.jpg', 1, 0 + ) + """)) + conn.execute(text(""" + INSERT INTO bangumi ( + official_title, year, title_raw, season, group_name, + dpi, eps_collect, "offset", filter, rss_link, added, deleted + ) VALUES ( + '咒术回战', '2023', 'Jujutsu Kaisen', 2, 'ANi', + '1080p', 0, 0, '720', 'https://mikanani.me/RSS/Bangumi?bangumiId=2888', + 1, 0 + ) + """)) + conn.execute(text(""" + INSERT INTO rssitem (name, url, aggregate, parser, enabled) + VALUES ('Mikan', 'https://mikanani.me/RSS/MyBangumi?token=abc', 1, 'mikan', 1) + """)) + conn.execute(text(""" + INSERT INTO torrent (bangumi_id, rss_id, name, url, downloaded) + VALUES (1, 1, '[Lilith-Raws] Mushoku Tensei - 01.mkv', + 'https://example.com/torrent1', 1) + """)) + conn.commit() + + def test_migrate_adds_air_weekday_column(self): + """Migration should add air_weekday column to bangumi table.""" + engine = create_engine("sqlite://", echo=False) + self._create_old_31x_database(engine) + self._insert_old_data(engine) + + # Verify air_weekday does NOT exist before migration + inspector = inspect(engine) + columns = [col["name"] for col in inspector.get_columns("bangumi")] + assert "air_weekday" not in columns + + # Run migration + db = Database(engine) + db.create_table() + + # Verify air_weekday now exists + inspector = inspect(engine) + columns = [col["name"] for col in inspector.get_columns("bangumi")] + assert "air_weekday" in columns + + db.close() + + def test_migrate_preserves_existing_data(self): + """Migration should not lose existing bangumi data.""" + engine = create_engine("sqlite://", echo=False) + self._create_old_31x_database(engine) + self._insert_old_data(engine) + + # Run migration + db = Database(engine) + db.create_table() + + # Check data is preserved + bangumis = db.bangumi.search_all() + assert len(bangumis) == 2 + assert bangumis[0].official_title == "无职转生" + assert bangumis[0].year == "2021" + assert bangumis[0].season == 1 + assert bangumis[0].group_name == "Lilith-Raws" + assert bangumis[0].added is True + assert bangumis[0].air_weekday is None # New column, should be NULL + + assert bangumis[1].official_title == "咒术回战" + assert bangumis[1].season == 2 + + db.close() + + def test_migrate_preserves_user_data(self): + """User table should be intact after migration.""" + engine = create_engine("sqlite://", echo=False) + self._create_old_31x_database(engine) + self._insert_old_data(engine) + + db = Database(engine) + db.create_table() + + users = db.user.get_user("admin") + assert users is not None + assert users.username == "admin" + + db.close() + + def test_migrate_preserves_rss_data(self): + """RSS items should be preserved after migration.""" + engine = create_engine("sqlite://", echo=False) + self._create_old_31x_database(engine) + self._insert_old_data(engine) + + db = Database(engine) + db.create_table() + + rss = db.rss.search_id(1) + assert rss is not None + assert rss.url == "https://mikanani.me/RSS/MyBangumi?token=abc" + assert rss.aggregate is True + + db.close() + + def test_migrate_preserves_torrent_data(self): + """Torrent data should be preserved after migration.""" + engine = create_engine("sqlite://", echo=False) + self._create_old_31x_database(engine) + self._insert_old_data(engine) + + db = Database(engine) + db.create_table() + + torrent = db.torrent.search(1) + assert torrent is not None + assert "[Lilith-Raws]" in torrent.name + assert torrent.downloaded is True + + db.close() + + def test_migrate_idempotent(self): + """Running migration multiple times should not cause errors.""" + engine = create_engine("sqlite://", echo=False) + self._create_old_31x_database(engine) + self._insert_old_data(engine) + + # Run migration twice + db = Database(engine) + db.create_table() + db.create_table() # Should not fail + + bangumis = db.bangumi.search_all() + assert len(bangumis) == 2 + + db.close() + + def test_new_bangumi_with_air_weekday(self): + """After migration, new bangumi can be added with air_weekday.""" + engine = create_engine("sqlite://", echo=False) + self._create_old_31x_database(engine) + self._insert_old_data(engine) + + db = Database(engine) + db.create_table() + + new_bangumi = Bangumi( + official_title="葬送的芙莉莲", + year="2023", + title_raw="Sousou no Frieren", + season=1, + group_name="SubsPlease", + dpi="1080p", + rss_link="https://mikanani.me/RSS/test", + added=True, + air_weekday=5, # Friday + ) + db.bangumi.add(new_bangumi) + db.commit() + + result = db.bangumi.search_id(3) + assert result is not None + assert result.official_title == "葬送的芙莉莲" + assert result.air_weekday == 5 + + db.close() + + def test_passkey_table_created(self): + """Migration should create the new passkey table.""" + engine = create_engine("sqlite://", echo=False) + self._create_old_31x_database(engine) + self._insert_old_data(engine) + + db = Database(engine) + db.create_table() + + inspector = inspect(engine) + tables = inspector.get_table_names() + assert "passkey" in tables + + db.close() diff --git a/docs/changelog/3.2.md b/docs/changelog/3.2.md index 774c97ef..b2b9d903 100644 --- a/docs/changelog/3.2.md +++ b/docs/changelog/3.2.md @@ -40,6 +40,14 @@ - 修复 poster 端点路径检查错误拦截所有请求 - 修复 OpenAI 解析器安全问题 - 修复数据库测试使用异步会话与同步代码不匹配 +- 修复 3.1.x 升级 3.2 时配置字段冲突导致设置丢失的问题 + - `program.sleep_time` / `program.times` 自动迁移为 `rss_time` / `rename_time` + - 移除已废弃的 `rss_parser` 字段(`type`、`custom_url`、`token`、`enable_tmdb`) + - 修复 `ENV_TO_ATTR` 环境变量映射指向不存在的模型字段 + - 修复 `DEFAULT_SETTINGS` 与当前配置模型不一致 +- 修复版本升级时迁移逻辑错误(所有升级均调用 3.0→3.1 迁移函数) + - 新增版本感知的迁移调度,根据来源版本执行对应迁移 + - 新增 `from_31_to_32()` 迁移函数处理数据库 schema 变更 ## Frontend