From 825130bd612e00dba18e7d558d9bd84737b6ba9f Mon Sep 17 00:00:00 2001 From: Robert Honz Date: Sun, 14 Jan 2024 17:29:39 +0100 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactoring=20of=20MPD=20d?= =?UTF-8?q?ownload.=20Metadata=20does=20not=20need=20an=20explicit=20`medi?= =?UTF-8?q?a=5Ftype`=20flag=20anymore.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 +- tidal_dl_ng/config.py | 5 +- tidal_dl_ng/constants.py | 8 +- tidal_dl_ng/download.py | 190 ++++++++++++++----------------------- tidal_dl_ng/gui.py | 1 + tidal_dl_ng/helper/path.py | 8 +- tidal_dl_ng/metadata.py | 36 +++---- tidal_dl_ng/model/cfg.py | 18 ++-- tidal_dl_ng/model/tidal.py | 2 +- 9 files changed, 111 insertions(+), 159 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e5d4731..1b56be4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,7 +171,7 @@ profile = "black" multi_line_output = 3 indent = 4 color_output = true -known_third_party = ["Crypto", "PySide6", "coloredlogs", "constants", "dataclasses_json", "ffmpeg", "helper", "m3u8", "model", "mutagen", "pathvalidate", "requests", "rich", "tidalapi", "typer"] +known_third_party = ["Crypto", "PySide6", "coloredlogs", "constants", "dataclasses_json", "ffmpeg", "helper", "m3u8", "model", "mpegdash", "mutagen", "pathvalidate", "requests", "rich", "tidalapi", "typer"] [tool.mypy] # https://mypy.readthedocs.io/en/latest/config_file.html#using-a-pyproject-toml-file diff --git a/tidal_dl_ng/config.py b/tidal_dl_ng/config.py index 4432d3c..3c5bebe 100644 --- a/tidal_dl_ng/config.py +++ b/tidal_dl_ng/config.py @@ -47,7 +47,10 @@ class BaseConfig: # Move the invalid config file to the backup location. shutil.move(path, path_bak) # TODO: Implement better global logger. - print(f"Something is wrong with your config. Maybe it is not compatible anymore due to a new app version. You can find a backup of your old config here: '{path_bak}'. A new default config was created.") + print( + "Something is wrong with your config. Maybe it is not compatible anymore due to a new app version." + f" You can find a backup of your old config here: '{path_bak}'. A new default config was created." + ) self.data = self.cls_model() diff --git a/tidal_dl_ng/constants.py b/tidal_dl_ng/constants.py index a98ce45..ad9bcca 100644 --- a/tidal_dl_ng/constants.py +++ b/tidal_dl_ng/constants.py @@ -27,5 +27,11 @@ class SkipExisting(Enum): class StreamManifestMimeType(Enum): MPD = "application/dash+xml" - JSON = "a" + BTS = "application/vnd.tidal.bts" VIDEO = "video/mp2t" + + +class CoverDimensions(Enum): + Px320 = "320x320" + Px640 = "640x640" + Px1280 = "1280x1280" diff --git a/tidal_dl_ng/download.py b/tidal_dl_ng/download.py index 40345ae..44ef1bc 100644 --- a/tidal_dl_ng/download.py +++ b/tidal_dl_ng/download.py @@ -14,13 +14,13 @@ import m3u8 import requests from helper.tidal import name_builder_item from model.tidal import StreamManifest -from requests.exceptions import HTTPError -from rich.progress import Progress -from tidalapi import Album, Mix, Playlist, Session, Track, UserPlaylist, Video from mpegdash.parser import MPEGDASHParser +from requests.exceptions import HTTPError +from rich.progress import Progress, TaskID +from tidalapi import Album, Mix, Playlist, Session, Track, UserPlaylist, Video from tidal_dl_ng.config import Settings -from tidal_dl_ng.constants import REQUESTS_TIMEOUT_SEC, MediaType, SkipExisting, StreamManifestMimeType +from tidal_dl_ng.constants import REQUESTS_TIMEOUT_SEC, CoverDimensions, MediaType, SkipExisting, StreamManifestMimeType from tidal_dl_ng.helper.decryption import decrypt_file, decrypt_security_token from tidal_dl_ng.helper.exceptions import MediaMissing, MediaUnknown, UnknownManifestFormat from tidal_dl_ng.helper.path import check_file_exists, format_path_media, path_file_sanitize @@ -52,10 +52,10 @@ class Download: self.session = session self.skip_existing = skip_existing - def _audio_stream( + def _download( self, fn_logger: Callable, - media: Track, + media: Track | Video, progress: Progress, progress_gui: ProgressBars, stream_manifest: StreamManifest, @@ -71,31 +71,49 @@ class Download: progress_gui.item_name.emit(media_name) try: - # Download the media as stream, so we can iterate over the response. - r = requests.get(stream_manifest.stream_urls, stream=True, timeout=REQUESTS_TIMEOUT_SEC) + # Compute total iterations for progress + urls_count: int = len(stream_manifest.urls) - r.raise_for_status() + if urls_count > 1: + progress_total: int = urls_count + block_size: int | None = None + else: + # Compute progress iterations based on the file size. + r = requests.get(stream_manifest.urls[0], stream=True, timeout=REQUESTS_TIMEOUT_SEC) - # Get file size and compute progress steps - total_size_in_bytes = int(r.headers.get("content-length", 0)) - block_size = 4096 - p_task = progress.add_task( - f"[blue]Item '{media_name[:30]}'", - total=total_size_in_bytes / block_size, + r.raise_for_status() + + # Get file size and compute progress steps + total_size_in_bytes: int = int(r.headers.get("content-length", 0)) + block_size: int | None = 4096 + progress_total: float = total_size_in_bytes / block_size + + # Create progress Task + p_task: TaskID = progress.add_task( + f"[blue]Item '{media_name[:20]}'", + total=progress_total, visible=progress_stdout, ) # Write content to file until progress is finished. while not progress.tasks[p_task].finished: with open(path_file, "wb") as f: - for data in r.iter_content(chunk_size=block_size): - f.write(data) - # Advance progress bar. - progress.advance(p_task) + for url in stream_manifest.urls: + # Create the request object with stream=True, so the content won't be loaded into memory at once. + r = requests.get(url, stream=True, timeout=REQUESTS_TIMEOUT_SEC) - # To send the progress to the GUI, we need to emit the percentage. - if not progress_stdout: - progress_gui.item.emit(progress.tasks[p_task].percentage) + r.raise_for_status() + + # Write the content to disk. If `chunk_size` is set to `None` the whole file will be written at once. + for data in r.iter_content(chunk_size=block_size): + f.write(data) + # Advance progress bar. + progress.advance(p_task) + + # To send the progress to the GUI, we need to emit the percentage. + if not progress_stdout: + # progress_gui.item.emit(progress.tasks[p_task].percentage) + pass except HTTPError as e: # TODO: Handle Exception... fn_logger(e) @@ -111,73 +129,11 @@ class Download: tmp_path_file_decrypted = path_file # Write metadata to file. - self.metadata_write(media, tmp_path_file_decrypted) + if not isinstance(media, Video): + self.metadata_write(media, tmp_path_file_decrypted) return tmp_path_file_decrypted - def _mpeg_segments( - self, - fn_logger: Callable, - media: Track, - progress: Progress, - progress_gui: ProgressBars, - stream_manifest: StreamManifest, - path_file: str, - ): - media_name: str = name_builder_item(media) - - # Set the correct progress output channel. - if progress_gui is None: - progress_stdout: bool = True - else: - progress_stdout: bool = False - progress_gui.item_name.emit(media_name) - - try: - total_iterations = stream_manifest.segments_count - p_task = progress.add_task( - f"[blue]Item '{media.name[:30]}'", - total=total_iterations, - visible=progress_stdout, - ) - - # Write content to file until progress is finished. - while not progress.tasks[p_task].finished: - with open(path_file, "wb") as f: - for index in range(total_iterations): - # Download the media. - segment_url = stream_manifest.stream_urls.replace('$Number$', str(index)) - r = requests.get(segment_url, timeout=REQUESTS_TIMEOUT_SEC) - - r.raise_for_status() - # Write data - f.write(r.content) - # Advance progress bar. - progress.advance(p_task) - - # To send the progress to the GUI, we need to emit the percentage. - if not progress_stdout: - progress_gui.item.emit(progress.tasks[p_task].percentage) - except HTTPError as e: - # TODO: Handle Exception... - fn_logger(e) - - return path_file - - def _video(self, video: Video, path_file: str) -> str | None: - result: str | None = None - - with open(path_file, "wb") as f: - for segment in m3u8_playlist.data["segments"]: - url = segment["uri"] - r = requests.get(url, timeout=REQUESTS_TIMEOUT_SEC) - - f.write(r.content) - - result = path_file - - return result - def instantiate_media( self, session: Session, media_type: type[MediaType.Track, MediaType.Video], id_media: str ) -> Track | Video: @@ -245,24 +201,15 @@ class Download: if not download_skip: # Create a temp directory and file. with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_path_dir: - tmp_path_file = os.path.join(tmp_path_dir, str(uuid4())) + tmp_path_file = os.path.join(tmp_path_dir, str(uuid4()) + stream_manifest.file_extension) + # Download media. + tmp_path_file = self._download(fn_logger, media, progress, progress_gui, stream_manifest, tmp_path_file) - if isinstance(media, Track): - if stream_manifest.segments_count > 0: - tmp_path_file = self._mpeg_segments(fn_logger, media, progress, progress_gui, stream_manifest, tmp_path_file) - else: - tmp_path_file = self._audio_stream( - fn_logger, media, progress, progress_gui, stream_manifest, tmp_path_file - ) - elif isinstance(media, Video): - tmp_path_file = self._video(media, tmp_path_file) - - # TODO: Check if is possible to write metadata to MPEG Transport Stream files. + if isinstance(media, Video): # TODO: Make optional. # Convert `*.ts` file to `*.mp4` using ffmpeg - if True: - tmp_path_file = self._video_convert(tmp_path_file) - path_file = os.path.splitext(path_file)[0] + ".mp4" + tmp_path_file = self._video_convert(tmp_path_file) + path_file = os.path.splitext(path_file)[0] + ".mp4" # Move final file to the configured destination directory. os.makedirs(os.path.dirname(path_file), exist_ok=True) @@ -272,18 +219,20 @@ class Download: return not download_skip, path_file - def cover_url(self, sid: str, width: int = 320, height: int = 320): + def cover_url(self, sid: str, dimension: CoverDimensions = CoverDimensions.Px320): if sid is None: return "" - return f"https://resources.tidal.com/images/{sid.replace('-', '/')}/{int(width)}x{int(height)}.jpg" + return f"https://resources.tidal.com/images/{sid.replace('-', '/')}/{dimension.value}.jpg" def metadata_write(self, track: Track, path_file: str): settings: Settings = Settings() result: bool = False - release_date: str = track.album.release_date.strftime("%Y-%m-%d") if track.album.release_date else "" - copy_right: str = track.copyright if track.copyright else "" - isrc: str = track.isrc if track.isrc else "" + release_date: str = ( + track.album.release_date.strftime("%Y-%m-%d") if track.album and track.album.release_date else "" + ) + copy_right: str = track.copyright if hasattr(track, "copyright") else "" + isrc: str = track.isrc if hasattr(track, "isrc") else "" try: lyrics: str = track.lyrics().subtitles if hasattr(track, "lyrics") else "" @@ -297,17 +246,15 @@ class Download: copy_right=copy_right, title=track.name, artists=[artist.name for artist in track.artists], - album=track.album.name, + album=track.album.name if track.album else "", tracknumber=track.track_num, date=release_date, isrc=isrc, - albumartist=track.artist.name, - totaltrack=track.album.num_tracks if track.album.num_tracks else 1, - totaldisc=track.album.num_volumes if track.album.num_volumes else 1, - discnumber=track.volume_num, - url_cover=self.cover_url( - track.album.cover, settings.data.metadata_cover_width, settings.data.metadata_cover_height - ), + albumartist=name_builder_item(track), + totaltrack=track.album.num_tracks if track.album and track.album.num_tracks else 1, + totaldisc=track.album.num_volumes if track.album and track.album.num_volumes else 1, + discnumber=track.volume_num if track.volume_num else 1, + url_cover=self.cover_url(track.album.cover, settings.data.metadata_cover_dimension) if track.album else "", ) m.save() @@ -379,7 +326,8 @@ class Download: progress.advance(p_task1) if not progress_stdout: - progress_gui.list_item.emit(progress.tasks[p_task1].percentage) + # progress_gui.list_item.emit(progress.tasks[p_task1].percentage) + pass if download_delay and status_download: time_sleep: float = round(random.SystemRandom().uniform(2, 5), 1) @@ -405,7 +353,7 @@ class Download: # else: # result = ".m4a" result: str = ".mp4" - if ".ts" in stream_url: + elif ".ts" in stream_url: result: str = ".ts" else: result: str = ".m4a" @@ -439,14 +387,14 @@ class Download: stream_urls: list[str] = [] for index in range(segments_count): - stream_urls.append(segment_template.media.replace('$Number$', str(index))) + stream_urls.append(segment_template.media.replace("$Number$", str(index))) - elif mime_type == StreamManifestMimeType.JSON.value: + elif mime_type == StreamManifestMimeType.BTS.value: # Stream Manifest is base64 encoded. manifest_parsed: str = base64.b64decode(manifest).decode("utf-8") # JSON string to object. stream_manifest = json.loads(manifest_parsed) - # TODO: Handle more than one dowload URL + # TODO: Handle more than one download URL stream_urls: str = stream_manifest["urls"] codecs: str = stream_manifest["codecs"] mime_type: str = stream_manifest["mimeType"] @@ -472,12 +420,12 @@ class Download: file_extension: str = self.get_file_extension(stream_urls[0], codecs) result: StreamManifest = StreamManifest( - stream_urls=stream_urls, + urls=stream_urls, codecs=codecs, file_extension=file_extension, encryption_type=encryption_type, encryption_key=encryption_key, - mime_type=mime_type + mime_type=mime_type, ) return result diff --git a/tidal_dl_ng/gui.py b/tidal_dl_ng/gui.py index 5c5061f..216faee 100644 --- a/tidal_dl_ng/gui.py +++ b/tidal_dl_ng/gui.py @@ -135,6 +135,7 @@ class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow): def populate_tree_lists(self, tidal: Tidal): # Start loading spinner + # TODO: Fix QPaint issue. App is crashing because of this. self.spinner_start.emit(self.tr_lists_user) self.tr_results.clear() diff --git a/tidal_dl_ng/helper/path.py b/tidal_dl_ng/helper/path.py index 61e17b1..708b569 100644 --- a/tidal_dl_ng/helper/path.py +++ b/tidal_dl_ng/helper/path.py @@ -127,7 +127,9 @@ def path_file_sanitize(path_file: str, adapt: bool = False) -> (bool, str): # Sanitize path try: - pathname_sanitized = sanitize_filepath(pathname, replacement_text=" ", validate_after_sanitize=True, platform="auto") + pathname_sanitized = sanitize_filepath( + pathname, replacement_text=" ", validate_after_sanitize=True, platform="auto" + ) except ValidationError: # If adaption of path is allowed in case of an error set path to HOME. if adapt: @@ -137,7 +139,9 @@ def path_file_sanitize(path_file: str, adapt: bool = False) -> (bool, str): # Sanitize filename try: - filename_sanitized = sanitize_filename(filename, replacement_text=" ", validate_after_sanitize=True, platform="auto") + filename_sanitized = sanitize_filename( + filename, replacement_text=" ", validate_after_sanitize=True, platform="auto" + ) filename_sanitized_extension = Path(filename_sanitized).suffix # Check if the file extension was removed by shortening the filename length diff --git a/tidal_dl_ng/metadata.py b/tidal_dl_ng/metadata.py index 651d8dd..fa6ff57 100644 --- a/tidal_dl_ng/metadata.py +++ b/tidal_dl_ng/metadata.py @@ -1,15 +1,12 @@ -import pathlib - import mutagen import requests from constants import REQUESTS_TIMEOUT_SEC -from mutagen import flac, mp4 +from mutagen import flac, id3, mp4 from mutagen.id3 import APIC, TALB, TCOM, TCOP, TDRC, TIT2, TOPE, TPE1, TRCK, TSRC, USLT class Metadata: path_file: str = None - type_audio: str = None title: str = None album: str = None albumartist: str = None @@ -31,7 +28,6 @@ class Metadata: def __init__( self, path_file: str, - type_audio: str = "", album: str = "", title: str = "", artists: list[str] | None = None, @@ -49,12 +45,6 @@ class Metadata: url_cover: str = "", ): self.path_file = path_file - - if type_audio: - self.type_audio = type_audio - else: - self.type_audio = pathlib.Path(self.path_file).suffix[1:] - self.title = title self.album = album self.albumartist = albumartist @@ -70,23 +60,24 @@ class Metadata: self.lyrics = lyrics self.path_cover = path_cover self.url_cover = url_cover - self.m: mutagen.MP4.MP4 | mutagen.m4a.M4A | mutagen.flac.FLAC | mutagen.mp3.MP3 = mutagen.File(self.path_file) + self.m: mutagen.mp4.MP4 | mutagen.flac.FLAC | mutagen.mp3.MP3 = mutagen.File(self.path_file) def _cover(self) -> bool: result: bool = False data_cover: str | bytes = self.cover_data(url=self.url_cover, path_file=self.path_cover) if data_cover: - if self.type_audio == "flac": + if isinstance(self.m, mutagen.flac.FLAC): flac_cover = flac.Picture() + flac_cover.type = id3.PictureType.COVER_FRONT flac_cover.data = data_cover flac_cover.mime = "image/jpeg" self.m.clear_pictures() self.m.add_picture(flac_cover) - elif self.type_audio == "mp3": + elif isinstance(self.m, mutagen.mp3.MP3): self.m.tags.add(APIC(encoding=3, data=data_cover)) - elif self.type_audio in ["mp4", "m4a"]: + elif isinstance(self.m, mutagen.mp4.MP4): cover_mp4 = mp4.MP4Cover(data_cover) self.m.tags["covr"] = [cover_mp4] @@ -95,11 +86,14 @@ class Metadata: return result def save(self): - if self.type_audio == "flac": + if not self.m.tags: + self.m.add_tags() + + if isinstance(self.m, mutagen.flac.FLAC): self.set_flac() - elif self.type_audio in ["mp3", "ts"]: + elif isinstance(self.m, mutagen.mp3.MP3): self.set_mp3() - elif self.type_audio in ["mp4", "m4a"]: + elif isinstance(self.m, mutagen.mp4.MP4): self.set_mp4() self._cover() @@ -108,9 +102,6 @@ class Metadata: return True def set_flac(self): - if not self.m.tags: - self.m.add_tags() - self.m.tags["title"] = self.title self.m.tags["album"] = self.album self.m.tags["albumartist"] = self.albumartist @@ -126,9 +117,6 @@ class Metadata: self.m.tags["lyrics"] = self.lyrics def set_mp3(self): - if not self.m.tags: - self.m.add_tags() - self.m.tags.add(TIT2(encoding=3, text=self.title)) self.m.tags.add(TALB(encoding=3, text=self.album)) self.m.tags.add(TOPE(encoding=3, text=self.albumartist)) diff --git a/tidal_dl_ng/model/cfg.py b/tidal_dl_ng/model/cfg.py index 2c93efc..1083132 100644 --- a/tidal_dl_ng/model/cfg.py +++ b/tidal_dl_ng/model/cfg.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from dataclasses_json import dataclass_json from tidalapi import Quality -from tidal_dl_ng.constants import QualityVideo, SkipExisting +from tidal_dl_ng.constants import CoverDimensions, QualityVideo, SkipExisting @dataclass_json @@ -28,15 +28,16 @@ class Settings: format_track: str = "Tracks/{artist_name} - {track_title}" format_video: str = "Videos/{artist_name} - {track_title}" video_convert_mp4: bool = True - metadata_cover_width: int = 320 - metadata_cover_height: int = 320 + metadata_cover_dimension: CoverDimensions = CoverDimensions.Px320 @dataclass class HelpSettings: - skip_existing: str = ("Do not download, if file already exists. Possible option false = do not skip, " - "'exact' = if filename already exists, 'extension_ignore' = skip even if a file with a " - "different file extension exists.") + skip_existing: str = ( + "Do not download, if file already exists. Possible option false = do not skip, " + "'exact' = if filename already exists, 'extension_ignore' = skip even if a file with a " + "different file extension exists." + ) album_cover_save: str = "Safe cover to album folder." lyrics_save: str = "Safe lyrics to audio file." api_key_index: str = "Set the device API KEY." @@ -64,8 +65,9 @@ class HelpSettings: "Videos are downloaded as MPEG Transport Stream (TS) files. With this option each video " "will be converted to MP4. FFMPEG must be installed and added to your 'PATH' variable." ) - metadata_cover_width: str = "The width of the cover image embedded into the track." - metadata_cover_height: str = "The height of the cover image embedded into the track." + metadata_cover_dimension: str = ( + "The dimensions of the cover image embedded into the track. Possible values: 320x320, 640x640x 1280x1280." + ) @dataclass_json diff --git a/tidal_dl_ng/model/tidal.py b/tidal_dl_ng/model/tidal.py index 2b83270..a5a18cc 100644 --- a/tidal_dl_ng/model/tidal.py +++ b/tidal_dl_ng/model/tidal.py @@ -5,7 +5,7 @@ from dataclasses import dataclass class StreamManifest: codecs: str mime_type: str - stream_urls: [str] + urls: [str] file_extension: str encryption_type: str | None = None encryption_key: str | None = None