From a85b4a65a3a143507f356dbbac247af1c3ca780d Mon Sep 17 00:00:00 2001 From: charlesxie <408737515@qq.com> Date: Mon, 8 May 2023 15:06:47 +0800 Subject: [PATCH] =?UTF-8?q?feature=EF=BC=9A=E5=BC=80=E5=8F=91subsonic=20ap?= =?UTF-8?q?i=20=E5=95=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- applications/music/admin.py | 165 +++++++++++++++ .../migrations/0005_auto_20230427_1349.py | 33 +++ .../migrations/0006_auto_20230427_1429.py | 18 ++ .../migrations/0007_folder_updated_at.py | 18 ++ applications/music/models.py | 14 +- applications/subsonic/serializers.py | 5 +- applications/subsonic/utils.py | 2 +- applications/task/services/scan_utils.py | 195 ++++++++++++++++++ applications/task/tasks.py | 20 +- 9 files changed, 449 insertions(+), 21 deletions(-) create mode 100644 applications/music/migrations/0005_auto_20230427_1349.py create mode 100644 applications/music/migrations/0006_auto_20230427_1429.py create mode 100644 applications/music/migrations/0007_folder_updated_at.py create mode 100644 applications/task/services/scan_utils.py diff --git a/applications/music/admin.py b/applications/music/admin.py index e69de29..74ea477 100644 --- a/applications/music/admin.py +++ b/applications/music/admin.py @@ -0,0 +1,165 @@ +# -*- coding: utf-8 -*- +from django.contrib import admin + +from .models import Album, Track, Artist, Genre, Attachment, Playlist, TrackFavorite, Folder + + +@admin.register(Album) +class AlbumAdmin(admin.ModelAdmin): + list_display = ( + 'id', + 'name', + 'artist', + 'all_artist_ids', + 'max_year', + 'song_count', + 'plays_count', + 'duration', + 'genre', + 'created_at', + 'updated_at', + 'accessed_date', + 'full_text', + 'size', + 'comment', + 'paths', + 'description', + 'attachment_cover', + 'mbz_album_id', + 'mbz_album_artist_id', + 'mbz_album_type', + 'mbz_album_comment', + 'external_url', + 'external_info_updated_at', + ) + list_filter = ( + 'created_at', + 'updated_at', + 'accessed_date', + 'external_info_updated_at', + ) + search_fields = ('name',) + date_hierarchy = 'created_at' + + +@admin.register(Track) +class TrackAdmin(admin.ModelAdmin): + list_display = ( + 'id', + 'name', + 'path', + 'album', + 'artist', + 'has_cover_art', + 'track_number', + 'disc_number', + 'plays_count', + 'year', + 'size', + 'suffix', + 'mimetype', + 'duration', + 'bit_rate', + 'genre', + 'created_at', + 'updated_at', + 'accessed_date', + 'full_text', + 'comment', + 'lyrics', + 'mbz_track_id', + 'mbz_album_id', + 'mbz_artist_id', + 'mbz_album_artist_id', + 'mbz_album_type', + 'mbz_album_comment', + 'mbz_release_track_id', + ) + list_filter = ( + 'album', + 'artist', + 'has_cover_art', + 'genre', + 'created_at', + 'updated_at', + 'accessed_date', + ) + search_fields = ('name',) + date_hierarchy = 'created_at' + + +@admin.register(Artist) +class ArtistAdmin(admin.ModelAdmin): + list_display = ( + 'id', + 'name', + 'album_count', + 'full_text', + 'song_count', + 'size', + 'mbz_artist_id', + 'attachment_cover', + 'similar_artists', + 'external_url', + 'external_info_updated_at', + ) + list_filter = ('attachment_cover', 'external_info_updated_at') + search_fields = ('name',) + + +@admin.register(Genre) +class GenreAdmin(admin.ModelAdmin): + list_display = ('id', 'name') + search_fields = ('name',) + + +@admin.register(Attachment) +class AttachmentAdmin(admin.ModelAdmin): + list_display = ( + 'id', + 'url', + 'creation_date', + 'last_fetch_date', + 'size', + 'mimetype', + 'file', + ) + list_filter = ('creation_date', 'last_fetch_date') + + +@admin.register(Playlist) +class PlaylistAdmin(admin.ModelAdmin): + list_display = ( + 'id', + 'name', + 'user', + 'creation_date', + 'modification_date', + 'privacy_level', + ) + list_filter = ('user', 'creation_date', 'modification_date') + search_fields = ('name',) + + +@admin.register(TrackFavorite) +class TrackFavoriteAdmin(admin.ModelAdmin): + list_display = ('id', 'creation_date', 'user', 'track') + list_filter = ('creation_date', 'user', 'track') + + +@admin.register(Folder) +class FolderAdmin(admin.ModelAdmin): + list_display = ( + 'id', + 'name', + 'path', + 'created_at', + 'last_scan_time', + 'file_type', + 'uid', + 'parent_id', + ) + list_filter = ('created_at', 'last_scan_time') + search_fields = ('name',) + date_hierarchy = 'created_at' + diff --git a/applications/music/migrations/0005_auto_20230427_1349.py b/applications/music/migrations/0005_auto_20230427_1349.py new file mode 100644 index 0000000..6aaa746 --- /dev/null +++ b/applications/music/migrations/0005_auto_20230427_1349.py @@ -0,0 +1,33 @@ +# Generated by Django 2.2.6 on 2023-04-27 13:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('music', '0004_auto_20230426_1732'), + ] + + operations = [ + migrations.RemoveField( + model_name='artist', + name='large_image_url', + ), + migrations.RemoveField( + model_name='artist', + name='medium_image_url', + ), + migrations.RemoveField( + model_name='artist', + name='order_artist_name', + ), + migrations.RemoveField( + model_name='artist', + name='small_image_url', + ), + migrations.RemoveField( + model_name='artist', + name='sort_artist_name', + ), + ] diff --git a/applications/music/migrations/0006_auto_20230427_1429.py b/applications/music/migrations/0006_auto_20230427_1429.py new file mode 100644 index 0000000..307171a --- /dev/null +++ b/applications/music/migrations/0006_auto_20230427_1429.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2023-04-27 14:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('music', '0005_auto_20230427_1349'), + ] + + operations = [ + migrations.AlterField( + model_name='track', + name='created_at', + field=models.DateTimeField(auto_now_add=True, null=True), + ), + ] diff --git a/applications/music/migrations/0007_folder_updated_at.py b/applications/music/migrations/0007_folder_updated_at.py new file mode 100644 index 0000000..14fcf11 --- /dev/null +++ b/applications/music/migrations/0007_folder_updated_at.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2023-05-08 11:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('music', '0006_auto_20230427_1429'), + ] + + operations = [ + migrations.AddField( + model_name='folder', + name='updated_at', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/applications/music/models.py b/applications/music/models.py index 5e6bd5f..81ebce3 100644 --- a/applications/music/models.py +++ b/applications/music/models.py @@ -21,7 +21,7 @@ class Album(models.Model): duration = models.FloatField("歌曲时长s", default=0, null=False) genre = models.ForeignKey('Genre', on_delete=models.SET_NULL, null=True, related_name='albums', db_constraint=False) created_at = models.DateTimeField(null=True) - updated_at = models.DateTimeField(null=True, auto_now=datetime.now) + updated_at = models.DateTimeField(null=True, auto_now=True) accessed_date = models.DateTimeField("访问时间", null=True) full_text = models.CharField(max_length=255, default='', null=True, blank=True) @@ -67,8 +67,8 @@ class Track(models.Model): duration = models.FloatField("歌曲时长s", default=0, null=False) bit_rate = models.IntegerField(default=0, null=True) genre = models.ForeignKey('Genre', on_delete=models.SET_NULL, null=True, related_name='tracks', db_constraint=False) - created_at = models.DateTimeField(null=True) - updated_at = models.DateTimeField(null=True, auto_now=datetime.now) + created_at = models.DateTimeField(null=True, auto_now_add=True) + updated_at = models.DateTimeField(null=True, auto_now=True) accessed_date = models.DateTimeField("访问时间", null=True) full_text = models.CharField(default='', max_length=255, null=True, blank=True) comment = models.TextField(null=True) @@ -94,16 +94,13 @@ class Artist(models.Model): name = models.CharField(max_length=255, default='', blank=False) album_count = models.IntegerField(default=0) full_text = models.CharField(max_length=255, default='', null=True, blank=True) - order_artist_name = models.CharField(max_length=255, null=True, blank=True) - sort_artist_name = models.CharField(max_length=255, null=True, blank=True) + song_count = models.IntegerField(default=0, null=True, blank=True) size = models.IntegerField(default=0, null=True, blank=True) mbz_artist_id = models.CharField(max_length=255, null=True, blank=True) attachment_cover = models.ForeignKey('Attachment', null=True, blank=True, on_delete=models.SET_NULL, related_name='artist_cover') - small_image_url = models.CharField(max_length=255, default='', null=True, blank=True) - medium_image_url = models.CharField(max_length=255, default='', null=True, blank=True) - large_image_url = models.CharField(max_length=255, default='', null=True, blank=True) + similar_artists = models.CharField(max_length=255, default='', null=True, blank=True) external_url = models.CharField(max_length=255, default='', null=True, blank=True) external_info_updated_at = models.DateTimeField(null=True, blank=True) @@ -204,6 +201,7 @@ class Folder(models.Model): path = models.TextField() created_at = models.DateTimeField(auto_now_add=True) last_scan_time = models.DateTimeField(auto_now=True) + updated_at = models.DateTimeField(auto_now=True) # 文件格式，例如：folder, music, image file_type = models.CharField(max_length=32, default='folder') uid = models.UUIDField(default=uuid.uuid4, editable=False) diff --git a/applications/subsonic/serializers.py b/applications/subsonic/serializers.py index 08c2a75..64c8051 100644 --- a/applications/subsonic/serializers.py +++ b/applications/subsonic/serializers.py @@ -47,7 +47,7 @@ def get_artist_data(artist_values): return { "id": artist_values["id"], "name": artist_values["name"], - "albumCount": artist_values["album_count"], + "albumCount": artist_values["_albums_count"], "coverArt": "ar-{}".format(artist_values["id"]), } @@ -56,7 +56,8 @@ class GetArtistsSerializer(serializers.Serializer): def to_representation(self, queryset): payload = {"ignoredArticles": "", "index": []} queryset = queryset.order_by(functions.Lower("name")) - values = queryset.values("id", "album_count", "name") + queryset = queryset.annotate(_albums_count=Count("albums")) + values = queryset.values("id", "_albums_count", "name") first_letter_mapping = collections.defaultdict(list) for artist in values: diff --git a/applications/subsonic/utils.py b/applications/subsonic/utils.py index d6ee1c4..61619fd 100644 --- a/applications/subsonic/utils.py +++ b/applications/subsonic/utils.py @@ -23,7 +23,7 @@ def handle_serve( now = datetime.now() track.accessed_date = now track.save(update_fields=["accessed_date"]) - file_path = track.path + file_path = track.path.replace(str(settings.BASE_DIR), "").encode("utf-8") mt = track.mimetype if mt: diff --git a/applications/task/services/scan_utils.py b/applications/task/services/scan_utils.py new file mode 100644 index 0000000..b9f65be --- /dev/null +++ b/applications/task/services/scan_utils.py @@ -0,0 +1,195 @@ +from applications.music.models import Folder, Attachment +import music_tag +from django.conf import settings +import time +from applications.music.models import Folder, Track, Album, Artist, Genre +from applications.subsonic.constants import AUDIO_EXTENSIONS_AND_MIMETYPE, COVER_TYPE +from django_vue_cli.celery_app import app +import os +import uuid +from django.core.files.base import ContentFile +import io +from django.core.files import File # you need this somewhere +from PIL import Image + + +class MusicInfo: + def __init__(self, folder: Folder): + self.file = music_tag.load_file(folder.path) + self.path = folder.path + self.folder_name = folder.name + self.file_type = folder.file_type + self.parent_id = folder.parent_id + + @property + def album_name(self): + return self.file["album"].value + + @property + def artist_name(self): + return self.file["artist"].value + + @property + def year(self): + return self.file["year"].value + + @property + def genre(self): + return self.file["genre"].value + + @property + def comment(self): + return self.file["comment"].value + + @property + def lyrics(self): + return self.file["lyrics"].value + + @property + def duration(self): + return self.file['#length'].value + + @property + def size(self): + return os.path.getsize(self.path) + + @property + def suffix(self): + return self.file['#codec'].value + + @property + def bit_rate(self): + return self.file['#bitrate'].value + + @property + def track_number(self): + return self.file['tracknumber'].value + + @property + def disc_number(self): + return self.file['discnumber'].value + + @property + def title(self): + return self.file['title'].value + + @property + def artwork(self): + try: + return self.file['artwork'].value + except Exception: + return "" + + def to_dict(self): + return { + "year": self.year, + "comment": self.comment, + "lyrics": self.lyrics, + "duration": self.duration, + "size": self.size, + "suffix": self.suffix, + "bit_rate": self.bit_rate, + "track_number": self.track_number, + "disc_number": self.disc_number, + "name": self.title, + } + + +class ScanMusic: + def __init__(self, path): + self.path = path + self.artist_map = dict(Artist.objects.values_list("name", "id")) + self.album_map = dict(Album.objects.values_list("full_text", "id")) + self.genre_map = dict(Genre.objects.values_list("name", "id")) + + def get_scan_list(self): + music_list = Folder.objects.filter(file_type="music").all() + return music_list + + def get_or_create_artist(self, music_info): + artist_name = music_info.artist_name + if artist_name not in self.artist_map: + artist = Artist.objects.create(**{ + "name": artist_name, + }) + self.artist_map[artist_name] = artist.id + return self.artist_map[artist_name] + + def get_or_create_album(self, music_info): + album_name = music_info.album_name + artist_name = music_info.artist_name + year = music_info.year + genre = music_info.genre + comment = music_info.comment + full_text = f"{album_name}-{artist_name}" + + if full_text not in self.album_map: + artist_id = self.artist_map[artist_name] + album = Album.objects.create(**{ + "name": album_name, + "artist_id": artist_id, + "max_year": year, + "genre_id": self.genre_map[genre], + "comment": comment, + "full_text": full_text + }) + self.album_map[full_text] = album.id + else: + album = Album.objects.filter(id=self.album_map[full_text]).first() + return album + + def get_or_create_genre(self, music_info): + genre = music_info.genre + + if genre not in self.genre_map: + genre = Genre.objects.create(**{ + "name": genre, + }) + self.genre_map[genre] = genre.id + return self.genre_map[genre] + + def get_or_create_attachment(self, music_info, album): + if album.attachment_cover is None: + folder = Folder.objects.filter(file_type="image", parent_id=music_info.parent_id, + name__startswith="cover.").first() + if folder: + image_path = folder.path + with open(image_path, "rb") as f: + image_data = f.read() + at = Attachment.objects.create(**{ + "size": len(image_data), + "mimetype": f"image/{folder.name.split('.')[-1]}", + }) + at.file.save(f"{album.name}.jpg", ContentFile(image_data), True) + album.attachment_cover = at + album.save() + else: + artwork = music_info.artwork + if artwork: + at = Attachment.objects.create(**{ + "size": len(artwork.raw), + "mimetype": artwork.mime, + }) + at.file.save(f"{album.name}.jpg", ContentFile(artwork.raw), True) + + album.attachment_cover = at + album.save() + + def update_or_create_track(self, music_info, album_id, artist_id, genre_id): + path = music_info.path + default_data = music_info.to_dict() + default_data["album_id"] = album_id + default_data["artist_id"] = artist_id + default_data["genre_id"] = genre_id + track, _ = Track.objects.update_or_create(path=path, defaults=default_data) + return track + + def scan(self): + folder_list = self.get_scan_list() + for folder in folder_list: + music_info = MusicInfo(folder) + artist_id = self.get_or_create_artist(music_info) + genre_id = self.get_or_create_genre(music_info) + album = self.get_or_create_album(music_info) + self.get_or_create_attachment(music_info, album) + self.update_or_create_track(music_info, album_id=album.id, artist_id=artist_id, genre_id=genre_id) diff --git a/applications/task/tasks.py b/applications/task/tasks.py index 02b2582..023b1e6 100644 --- a/applications/task/tasks.py +++ b/applications/task/tasks.py @@ -1,11 +1,13 @@ -import music_tag +import os +import time +import uuid + from django.conf import settings -from applications.music.models import Folder, Track +from applications.music.models import Folder from applications.subsonic.constants import AUDIO_EXTENSIONS_AND_MIMETYPE, COVER_TYPE +from applications.task.services.scan_utils import ScanMusic from django_vue_cli.celery_app import app -import os -import uuid def get_uuid(): @@ -13,7 +15,7 @@ def get_uuid(): @app.task -def full_scan(): +def full_scan_folder(): music_folder = os.path.join(settings.MEDIA_ROOT, "music") bulk_create = [] stack = [(None, music_folder)] @@ -64,8 +66,6 @@ def full_scan(): @app.task def scan_music_id3(): - music_list = Folder.objects.filter(file_type="music").all() - bulk_create = [] - for music in music_list: - f = music_tag.load_file(music.path) - Track.objects.filter() \ No newline at end of file + a = time.time() + ScanMusic("/").scan() + print(time.time() - a)