From c1a102f646665d3570fe2f40e6d43f845c9ec55a Mon Sep 17 00:00:00 2001 From: Robert Honz Date: Wed, 13 Nov 2024 09:09:00 +0100 Subject: [PATCH 1/6] =?UTF-8?q?=E2=9C=A8=20Removed=20complex=20skipping=20?= =?UTF-8?q?options=20in=20favor=20of=20a=20faster=20duplicates=20check=20/?= =?UTF-8?q?=20skipping=20of=20already=20present=20items=20without=20downlo?= =?UTF-8?q?ad=20delay.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tidal_dl_ng/constants.py | 7 -- tidal_dl_ng/dialog.py | 4 +- tidal_dl_ng/download.py | 119 ++++++++++++++++-------------- tidal_dl_ng/helper/wrapper.py | 3 + tidal_dl_ng/model/cfg.py | 10 +-- tidal_dl_ng/model/downloader.py | 3 + tidal_dl_ng/ui/dialog_settings.py | 52 ++++++------- tidal_dl_ng/ui/dialog_settings.ui | 51 +++++-------- 8 files changed, 116 insertions(+), 133 deletions(-) diff --git a/tidal_dl_ng/constants.py b/tidal_dl_ng/constants.py index ad8d2ee..10fb93d 100644 --- a/tidal_dl_ng/constants.py +++ b/tidal_dl_ng/constants.py @@ -27,13 +27,6 @@ class MediaType(StrEnum): ARTIST: str = "artist" -class SkipExisting(StrEnum): - Disabled: str = "False" - Filename: str = "exact" - ExtensionIgnore: str = "extension_ignore" - Append: str = "append" - - class CoverDimensions(StrEnum): Px80: str = "80" Px160: str = "160" diff --git a/tidal_dl_ng/dialog.py b/tidal_dl_ng/dialog.py index c035867..92656dc 100644 --- a/tidal_dl_ng/dialog.py +++ b/tidal_dl_ng/dialog.py @@ -9,7 +9,7 @@ from tidalapi import Quality as QualityAudio from tidal_dl_ng import __version__ from tidal_dl_ng.config import Settings -from tidal_dl_ng.constants import CoverDimensions, QualityVideo, SkipExisting +from tidal_dl_ng.constants import CoverDimensions, QualityVideo from tidal_dl_ng.model.cfg import HelpSettings from tidal_dl_ng.model.cfg import Settings as ModelSettings from tidal_dl_ng.model.meta import ReleaseLatest @@ -157,7 +157,6 @@ class DialogPreferences(QtWidgets.QDialog): def _init_comboboxes(self): self.parameters_combo = [ - ("skip_existing", SkipExisting), ("quality_audio", QualityAudio), ("quality_video", QualityVideo), ("metadata_cover_dimension", CoverDimensions), @@ -173,6 +172,7 @@ class DialogPreferences(QtWidgets.QDialog): "extract_flac", "metadata_cover_embed", "cover_album_file", + "skip_existing", ] def gui_populate(self): diff --git a/tidal_dl_ng/download.py b/tidal_dl_ng/download.py index b1ff2c6..a9348f3 100644 --- a/tidal_dl_ng/download.py +++ b/tidal_dl_ng/download.py @@ -15,10 +15,11 @@ from constants import CHUNK_SIZE, COVER_NAME from requests.exceptions import HTTPError from rich.progress import Progress, TaskID from tidalapi import Album, Mix, Playlist, Session, Track, UserPlaylist, Video +from tidalapi.exceptions import TooManyRequests from tidalapi.media import AudioExtensions, Codec, Quality, StreamManifest, VideoExtensions from tidal_dl_ng.config import Settings -from tidal_dl_ng.constants import EXTENSION_LYRICS, REQUESTS_TIMEOUT_SEC, MediaType, QualityVideo, SkipExisting +from tidal_dl_ng.constants import EXTENSION_LYRICS, REQUESTS_TIMEOUT_SEC, MediaType, QualityVideo from tidal_dl_ng.helper.decryption import decrypt_file, decrypt_security_token from tidal_dl_ng.helper.exceptions import MediaMissing from tidal_dl_ng.helper.path import check_file_exists, format_path_media, path_file_sanitize, url_to_filename @@ -53,7 +54,7 @@ class RequestsClient: class Download: settings: Settings session: Session - skip_existing: SkipExisting = SkipExisting.Disabled + skip_existing: bool = False fn_logger: Callable progress_gui: ProgressBars progress: Progress @@ -63,7 +64,7 @@ class Download: session: Session, path_base: str, fn_logger: Callable, - skip_existing: SkipExisting = SkipExisting.Disabled, + skip_existing: bool = False, progress_gui: ProgressBars = None, progress: Progress = None, ): @@ -209,6 +210,7 @@ class Download: path_segment: pathlib.Path = path_base / url_to_filename(url) # Calculate the segment ID based on the file name within the URL. id_segment: int = int(path_segment.stem) + error: HTTPError | None = None try: # Create the request object with stream=True, so the content won't be loaded into memory at once. @@ -224,15 +226,18 @@ class Download: self.progress.advance(p_task) result = True - except HTTPError: - # TODO: Maybe return e as well? + except HTTPError as e: + error = e self.progress.advance(p_task) - finally: - # To send the progress to the GUI, we need to emit the percentage. - if not progress_to_stdout: - self.progress_gui.item.emit(self.progress.tasks[p_task].percentage) + self.fn_logger.exception(e.charachter_written) - return DownloadSegmentResult(result=result, url=url, path_segment=path_segment, id_segment=id_segment) + # To send the progress to the GUI, we need to emit the percentage. + if not progress_to_stdout: + self.progress_gui.item.emit(self.progress.tasks[p_task].percentage) + + return DownloadSegmentResult( + result=result, url=url, path_segment=path_segment, id_segment=id_segment, error=error + ) def item( self, @@ -273,50 +278,56 @@ class Download: return False, "" - # Get extension. - file_extension: str - do_flac_extract = False - - if isinstance(media, Track): - # If a quality is explicitly set, change it. - if quality_audio: - quality_audio_old: Quality = self.adjust_quality_audio(quality_audio) - - file_extension = media.get_stream().get_stream_manifest().file_extension - # Use M4A extension for MP4 audio tracks, because it looks better and is completely interchangeable. - file_extension = AudioExtensions.M4A if file_extension == AudioExtensions.MP4 else file_extension - - if self.settings.data.extract_flac: - if ( - media.get_stream().get_stream_manifest().codecs.upper() == Codec.FLAC - and file_extension != AudioExtensions.FLAC - ): - file_extension = AudioExtensions.FLAC - do_flac_extract = True - elif isinstance(media, Video): - if quality_video: - quality_video_old: QualityVideo = self.adjust_quality_video(quality_video) - - file_extension = AudioExtensions.MP4 if self.settings.data.video_convert_mp4 else VideoExtensions.TS - # Create file name and path + file_extension_dummy: str = AudioExtensions.FLAC file_name_relative = format_path_media(file_template, media) path_media_dst = os.path.abspath( os.path.normpath(os.path.join(os.path.expanduser(self.path_base), file_name_relative)) ) # Sanitize final path_file to fit into OS boundaries. - uniquify: bool = self.skip_existing == SkipExisting.Append - path_media_dst = path_file_sanitize(path_media_dst + file_extension, adapt=True, uniquify=uniquify) + path_media_dst = path_file_sanitize(path_media_dst + file_extension_dummy, adapt=True) # Compute if and how downloads need to be skipped. - if self.skip_existing in (SkipExisting.ExtensionIgnore, SkipExisting.Filename): - extension_ignore: bool = self.skip_existing == SkipExisting.ExtensionIgnore - file_exists: bool = check_file_exists(path_media_dst, extension_ignore=extension_ignore) + if self.skip_existing: + skip_file: bool = check_file_exists(path_media_dst, extension_ignore=True) else: - file_exists: bool = False + skip_file: bool = False + + if not skip_file: + # Get extension. + file_extension: str + do_flac_extract = False + # If a quality is explicitly set, change it and remember the previously set quality. + quality_audio_old: Quality = self.adjust_quality_audio(quality_audio) + quality_video_old: QualityVideo = self.adjust_quality_video(quality_video) + + if isinstance(media, Track): + try: + media_stream = media.get_stream() + except TooManyRequests: + self.fn_logger.exception( + f"Too many requests against TIDAL backend. Skipping '{name_builder_item(media)}'. " + f"Consider to activate delay between downloads." + ) + + return False, "" + + file_extension = media_stream.get_stream_manifest().file_extension + # Use M4A extension for MP4 audio tracks, because it looks better and is completely interchangeable. + file_extension = AudioExtensions.M4A if file_extension == AudioExtensions.MP4 else file_extension + + if self.settings.data.extract_flac and ( + media_stream.get_stream_manifest().codecs.upper() == Codec.FLAC + and file_extension != AudioExtensions.FLAC + ): + file_extension = AudioExtensions.FLAC + do_flac_extract = True + elif isinstance(media, Video): + file_extension = AudioExtensions.MP4 if self.settings.data.video_convert_mp4 else VideoExtensions.TS + + # TODO: correct extension - if not file_exists: # Create a temp directory and file. with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_path_dir: tmp_path_file: pathlib.Path = pathlib.Path(tmp_path_dir) / str(uuid4()) @@ -356,22 +367,22 @@ class Download: # Move cover file if self.settings.data.cover_album_file: self._move_cover(tmp_path_cover, path_media_dst) + + if quality_audio: + # Set quality back to the global user value + self.adjust_quality_audio(quality_audio_old) + + if quality_video: + # Set quality back to the global user value + self.adjust_quality_video(quality_video_old) else: self.fn_logger.debug(f"Download skipped, since file exists: '{path_media_dst}'") - status_download: bool = not file_exists - - if quality_audio: - # Set quality back to the global user value - self.adjust_quality_audio(quality_audio_old) - - if quality_video: - # Set quality back to the global user value - self.adjust_quality_video(quality_video_old) + status_download: bool = not skip_file # Whether a file was downloaded or skipped and the download delay is enabled, wait until the next download. # Only use this, if you have a list of several Track items. - if download_delay: + if download_delay and not skip_file: time_sleep: float = round(random.SystemRandom().uniform(2, 5), 1) self.fn_logger.debug(f"Next download will start in {time_sleep} seconds.") @@ -575,7 +586,7 @@ class Download: while not self.progress.finished: for item_media in items: # Download the item. - status_download, result_path_file = self.item( + status, result_path_file = self.item( media=item_media, file_template=file_name_relative, quality_audio=quality_audio, diff --git a/tidal_dl_ng/helper/wrapper.py b/tidal_dl_ng/helper/wrapper.py index d8ffb09..cd8e6ba 100644 --- a/tidal_dl_ng/helper/wrapper.py +++ b/tidal_dl_ng/helper/wrapper.py @@ -21,3 +21,6 @@ class LoggerWrapped: def critical(self, value): self.fn_print(value) + + def exception(self, value): + self.fn_print(value) diff --git a/tidal_dl_ng/model/cfg.py b/tidal_dl_ng/model/cfg.py index 6e9b991..631a2f2 100644 --- a/tidal_dl_ng/model/cfg.py +++ b/tidal_dl_ng/model/cfg.py @@ -3,13 +3,13 @@ from dataclasses import dataclass from dataclasses_json import dataclass_json from tidalapi import Quality -from tidal_dl_ng.constants import CoverDimensions, QualityVideo, SkipExisting +from tidal_dl_ng.constants import CoverDimensions, QualityVideo @dataclass_json @dataclass class Settings: - skip_existing: SkipExisting = SkipExisting.Disabled + skip_existing: bool = True lyrics_embed: bool = False lyrics_file: bool = False # TODO: Implement API KEY selection. @@ -43,11 +43,7 @@ class Settings: @dataclass_json @dataclass class HelpSettings: - skip_existing: str = ( - "Do not download, if file already exists. Possible option false = do not skip, " - "'exact' = if filename already exists, 'extension_ignore' = skip even if a file with a " - "different file extension exists." - ) + skip_existing: str = "Skip download if file already exists." album_cover_save: str = "Safe cover to album folder." lyrics_embed: str = "Embed lyrics in audio file, if lyrics are available." lyrics_file: str = "Save lyrics to separate *.lrc file, if lyrics are available." diff --git a/tidal_dl_ng/model/downloader.py b/tidal_dl_ng/model/downloader.py index 52c8e30..3f6151f 100644 --- a/tidal_dl_ng/model/downloader.py +++ b/tidal_dl_ng/model/downloader.py @@ -1,6 +1,8 @@ import pathlib from dataclasses import dataclass +from requests import HTTPError + @dataclass class DownloadSegmentResult: @@ -8,3 +10,4 @@ class DownloadSegmentResult: url: str path_segment: pathlib.Path id_segment: int + error: HTTPError | None = None diff --git a/tidal_dl_ng/ui/dialog_settings.py b/tidal_dl_ng/ui/dialog_settings.py index ad45f1d..a624e34 100644 --- a/tidal_dl_ng/ui/dialog_settings.py +++ b/tidal_dl_ng/ui/dialog_settings.py @@ -153,6 +153,24 @@ class Ui_DialogSettings: self.lv_flags.addLayout(self.lh_flags_4) + self.horizontalLayout = QHBoxLayout() + self.horizontalLayout.setObjectName("horizontalLayout") + self.lv_flag_skip_existing = QVBoxLayout() + self.lv_flag_skip_existing.setObjectName("lv_flag_skip_existing") + self.cb_skip_existing = QCheckBox(self.gb_flags) + self.cb_skip_existing.setObjectName("cb_skip_existing") + + self.lv_flag_skip_existing.addWidget(self.cb_skip_existing) + + self.horizontalLayout.addLayout(self.lv_flag_skip_existing) + + self.verticalLayout_3 = QVBoxLayout() + self.verticalLayout_3.setObjectName("verticalLayout_3") + + self.horizontalLayout.addLayout(self.verticalLayout_3) + + self.lv_flags.addLayout(self.horizontalLayout) + self.lv_main.addWidget(self.gb_flags) self.gb_choices = QGroupBox(DialogSettings) @@ -164,38 +182,13 @@ class Ui_DialogSettings: self.gb_choices.setSizePolicy(sizePolicy3) self.lv_choices = QVBoxLayout(self.gb_choices) self.lv_choices.setObjectName("lv_choices") - self.lh_choices_skip_existing = QHBoxLayout() - self.lh_choices_skip_existing.setObjectName("lh_choices_skip_existing") - self.l_icon_skip_existing = QLabel(self.gb_choices) - self.l_icon_skip_existing.setObjectName("l_icon_skip_existing") - sizePolicy4 = QSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed) - sizePolicy4.setHorizontalStretch(0) - sizePolicy4.setVerticalStretch(0) - sizePolicy4.setHeightForWidth(self.l_icon_skip_existing.sizePolicy().hasHeightForWidth()) - self.l_icon_skip_existing.setSizePolicy(sizePolicy4) - - self.lh_choices_skip_existing.addWidget(self.l_icon_skip_existing) - - self.l_skip_existing = QLabel(self.gb_choices) - self.l_skip_existing.setObjectName("l_skip_existing") - sizePolicy4.setHeightForWidth(self.l_skip_existing.sizePolicy().hasHeightForWidth()) - self.l_skip_existing.setSizePolicy(sizePolicy4) - - self.lh_choices_skip_existing.addWidget(self.l_skip_existing) - - self.c_skip_existing = QComboBox(self.gb_choices) - self.c_skip_existing.setObjectName("c_skip_existing") - - self.lh_choices_skip_existing.addWidget(self.c_skip_existing) - - self.lh_choices_skip_existing.setStretch(2, 50) - - self.lv_choices.addLayout(self.lh_choices_skip_existing) - self.lh_choices_quality_audio = QHBoxLayout() self.lh_choices_quality_audio.setObjectName("lh_choices_quality_audio") self.l_icon_quality_audio = QLabel(self.gb_choices) self.l_icon_quality_audio.setObjectName("l_icon_quality_audio") + sizePolicy4 = QSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed) + sizePolicy4.setHorizontalStretch(0) + sizePolicy4.setVerticalStretch(0) sizePolicy4.setHeightForWidth(self.l_icon_quality_audio.sizePolicy().hasHeightForWidth()) self.l_icon_quality_audio.setSizePolicy(sizePolicy4) @@ -532,9 +525,8 @@ class Ui_DialogSettings: self.cb_extract_flac.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None)) self.cb_metadata_cover_embed.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None)) self.cb_cover_album_file.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None)) + self.cb_skip_existing.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None)) self.gb_choices.setTitle(QCoreApplication.translate("DialogSettings", "Choices", None)) - self.l_icon_skip_existing.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None)) - self.l_skip_existing.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None)) self.l_icon_quality_audio.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None)) self.l_quality_audio.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None)) self.l_icon_quality_video.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None)) diff --git a/tidal_dl_ng/ui/dialog_settings.ui b/tidal_dl_ng/ui/dialog_settings.ui index 9bc894d..4fa07cf 100644 --- a/tidal_dl_ng/ui/dialog_settings.ui +++ b/tidal_dl_ng/ui/dialog_settings.ui @@ -210,6 +210,24 @@ + + + + + + + + CheckBox + + + + + + + + + + @@ -225,39 +243,6 @@ Choices - - - - - - - 0 - 0 - - - - TextLabel - - - - - - - - 0 - 0 - - - - TextLabel - - - - - - - - From 3171b1570be4fe5a520bda45049a50a34037fbbb Mon Sep 17 00:00:00 2001 From: Robert Honz Date: Wed, 13 Nov 2024 09:49:33 +0100 Subject: [PATCH 2/6] =?UTF-8?q?=E2=9C=A8=20Do=20not=20process=20tracks=20w?= =?UTF-8?q?ith=20missing=20segments.=20Refactored=20some=20method=20to=20`?= =?UTF-8?q?pathlib.Path`.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tidal_dl_ng/download.py | 90 +++++++++++++++++++------------------- tidal_dl_ng/helper/path.py | 12 ++--- 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/tidal_dl_ng/download.py b/tidal_dl_ng/download.py index a9348f3..98a7c75 100644 --- a/tidal_dl_ng/download.py +++ b/tidal_dl_ng/download.py @@ -92,7 +92,7 @@ class Download: self, media: Track | Video, path_file: pathlib.Path, - ) -> pathlib.Path: + ) -> (bool, pathlib.Path): media_name: str = name_builder_item(media) urls: [str] path_base: pathlib.Path = path_file.parent @@ -165,21 +165,20 @@ class Download: # mark the whole thing as corrupt. result_segments = False - # Bring list into right order, so segments can be easily merged. - dl_segment_results.sort(key=lambda x: x.id_segment) - # TODO: Implement error handling on corrupt segments (also in the following method. - reult_merge: bool = self._segments_merge(path_file, dl_segment_results) - tmp_path_file_decrypted: pathlib.Path = pathlib.Path("") + tmp_path_file_decrypted: pathlib.Path = path_file - if reult_merge: - if isinstance(media, Track) and stream_manifest.is_encrypted: + # Only if no error happened while downloading. + if result_segments: + # Bring list into right order, so segments can be easily merged. + dl_segment_results.sort(key=lambda x: x.id_segment) + reult_merge: bool = self._segments_merge(path_file, dl_segment_results) + + if reult_merge and isinstance(media, Track) and stream_manifest.is_encrypted: key, nonce = decrypt_security_token(stream_manifest.encryption_key) - tmp_path_file_decrypted = path_file + "_decrypted" + tmp_path_file_decrypted = path_file.with_suffix("_decrypted") decrypt_file(path_file, tmp_path_file_decrypted, key, nonce) - else: - tmp_path_file_decrypted = path_file - return tmp_path_file_decrypted + return result_segments, tmp_path_file_decrypted def _segments_merge(self, path_file, dl_segment_results) -> bool: result: bool @@ -229,7 +228,7 @@ class Download: except HTTPError as e: error = e self.progress.advance(p_task) - self.fn_logger.exception(e.charachter_written) + self.fn_logger.exception(e) # To send the progress to the GUI, we need to emit the percentage. if not progress_to_stdout: @@ -281,12 +280,14 @@ class Download: # Create file name and path file_extension_dummy: str = AudioExtensions.FLAC file_name_relative = format_path_media(file_template, media) - path_media_dst = os.path.abspath( - os.path.normpath(os.path.join(os.path.expanduser(self.path_base), file_name_relative)) + path_media_dst: pathlib.Path = ( + (pathlib.Path(self.path_base).expanduser() / file_name_relative).resolve().absolute() ) # Sanitize final path_file to fit into OS boundaries. - path_media_dst = path_file_sanitize(path_media_dst + file_extension_dummy, adapt=True) + path_media_dst = pathlib.Path( + path_file_sanitize(str(path_media_dst.with_suffix(file_extension_dummy)), adapt=True) + ) # Compute if and how downloads need to be skipped. if self.skip_existing: @@ -336,38 +337,39 @@ class Download: tmp_path_file.touch() # Download media. - tmp_path_file = self._download(media=media, path_file=tmp_path_file) + result_download, tmp_path_file = self._download(media=media, path_file=tmp_path_file) - # Convert video from TS to MP4 - if isinstance(media, Video) and self.settings.data.video_convert_mp4: - # Convert `*.ts` file to `*.mp4` using ffmpeg - tmp_path_file = self._video_convert(tmp_path_file) + if result_download: + # Convert video from TS to MP4 + if isinstance(media, Video) and self.settings.data.video_convert_mp4: + # Convert `*.ts` file to `*.mp4` using ffmpeg + tmp_path_file = self._video_convert(tmp_path_file) - # Extract FLAC from MP4 container using ffmpeg - if isinstance(media, Track) and self.settings.data.extract_flac and do_flac_extract: - tmp_path_file = self._extract_flac(tmp_path_file) + # Extract FLAC from MP4 container using ffmpeg + if isinstance(media, Track) and self.settings.data.extract_flac and do_flac_extract: + tmp_path_file = self._extract_flac(tmp_path_file) - tmp_path_lyrics: pathlib.Path | None = None - tmp_path_cover: pathlib.Path | None = None + tmp_path_lyrics: pathlib.Path | None = None + tmp_path_cover: pathlib.Path | None = None - # Write metadata to file. - if not isinstance(media, Video): - result_metadata, tmp_path_lyrics, tmp_path_cover = self.metadata_write( - media, tmp_path_file, is_parent_album - ) + # Write metadata to file. + if not isinstance(media, Video): + result_metadata, tmp_path_lyrics, tmp_path_cover = self.metadata_write( + media, tmp_path_file, is_parent_album + ) + + # Move lyrics file + if self.settings.data.lyrics_file and not isinstance(media, Video): + self._move_lyrics(tmp_path_lyrics, path_media_dst) + + # Move cover file + if self.settings.data.cover_album_file: + self._move_cover(tmp_path_cover, path_media_dst) # Move final file to the configured destination directory. - os.makedirs(os.path.dirname(path_media_dst), exist_ok=True) + os.makedirs(path_media_dst.parent, exist_ok=True) shutil.move(tmp_path_file, path_media_dst) - # Move lyrics file - if self.settings.data.lyrics_file and not isinstance(media, Video): - self._move_lyrics(tmp_path_lyrics, path_media_dst) - - # Move cover file - if self.settings.data.cover_album_file: - self._move_cover(tmp_path_cover, path_media_dst) - if quality_audio: # Set quality back to the global user value self.adjust_quality_audio(quality_audio_old) @@ -418,16 +420,16 @@ class Download: return result - def _move_lyrics(self, path_lyrics: pathlib.Path, file_media_dst: str) -> bool: + def _move_lyrics(self, path_lyrics: pathlib.Path, file_media_dst: pathlib.Path) -> bool: # Build tmp lyrics filename - path_file_lyrics: str = os.path.splitext(file_media_dst)[0] + EXTENSION_LYRICS + path_file_lyrics: pathlib.Path = file_media_dst.with_suffix(EXTENSION_LYRICS) result: bool = self._move_file(path_lyrics, path_file_lyrics) return result - def _move_cover(self, path_cover: pathlib.Path, file_media_dst: str) -> bool: + def _move_cover(self, path_cover: pathlib.Path, file_media_dst: pathlib.Path) -> bool: # Build tmp lyrics filename - path_file_cover: pathlib.Path = pathlib.Path(file_media_dst).parent.absolute() / COVER_NAME + path_file_cover: pathlib.Path = file_media_dst.parent / COVER_NAME result: bool = self._move_file(path_cover, path_file_cover) return result diff --git a/tidal_dl_ng/helper/path.py b/tidal_dl_ng/helper/path.py index e6f5b14..627f7c1 100644 --- a/tidal_dl_ng/helper/path.py +++ b/tidal_dl_ng/helper/path.py @@ -1,9 +1,9 @@ import math import os +import pathlib import posixpath import re import sys -from pathlib import Path, PosixPath from urllib.parse import unquote, urlsplit from pathvalidate import sanitize_filename, sanitize_filepath @@ -197,7 +197,7 @@ def get_format_template( def path_file_sanitize(path_file: str, adapt: bool = False, uniquify: bool = False) -> (bool, str): # Split into path and filename pathname, filename = os.path.split(path_file) - file_extension: str = Path(path_file).suffix + file_extension: str = pathlib.Path(path_file).suffix # Sanitize path try: @@ -207,7 +207,7 @@ def path_file_sanitize(path_file: str, adapt: bool = False, uniquify: bool = Fal except ValidationError: # If adaption of path is allowed in case of an error set path to HOME. if adapt: - pathname_sanitized: str = Path.home() + pathname_sanitized: str = str(pathlib.Path.home()) else: raise @@ -274,10 +274,10 @@ def file_unique_suffix(path_file: str, seperator: str = "_") -> str: return unique_suffix -def check_file_exists(path_file: str, extension_ignore: bool = False) -> bool: +def check_file_exists(path_file: pathlib.Path, extension_ignore: bool = False) -> bool: if extension_ignore: - path_file_stem: str = Path(path_file).stem - path_parent: PosixPath = Path(path_file).parent + path_file_stem: str = pathlib.Path(path_file).stem + path_parent: pathlib.Path = pathlib.Path(path_file).parent path_files: [str] = [] for extension in AudioExtensions: From 0eea0f35484786a549eaf77574f4f4742464af5a Mon Sep 17 00:00:00 2001 From: Robert Honz Date: Wed, 13 Nov 2024 10:37:29 +0100 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F=20Fixed=20suffix=20ha?= =?UTF-8?q?ndling=20with=20`pathlib.Path`.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tidal_dl_ng/download.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tidal_dl_ng/download.py b/tidal_dl_ng/download.py index 98a7c75..2329381 100644 --- a/tidal_dl_ng/download.py +++ b/tidal_dl_ng/download.py @@ -281,13 +281,13 @@ class Download: file_extension_dummy: str = AudioExtensions.FLAC file_name_relative = format_path_media(file_template, media) path_media_dst: pathlib.Path = ( - (pathlib.Path(self.path_base).expanduser() / file_name_relative).resolve().absolute() + (pathlib.Path(self.path_base).expanduser() / (file_name_relative + file_extension_dummy)) + .resolve() + .absolute() ) # Sanitize final path_file to fit into OS boundaries. - path_media_dst = pathlib.Path( - path_file_sanitize(str(path_media_dst.with_suffix(file_extension_dummy)), adapt=True) - ) + path_media_dst = pathlib.Path(path_file_sanitize(str(path_media_dst), adapt=True)) # Compute if and how downloads need to be skipped. if self.skip_existing: @@ -327,7 +327,9 @@ class Download: elif isinstance(media, Video): file_extension = AudioExtensions.MP4 if self.settings.data.video_convert_mp4 else VideoExtensions.TS - # TODO: correct extension + # Compute file name and create destination directory + path_media_dst = path_media_dst.with_suffix(file_extension) + os.makedirs(path_media_dst.parent, exist_ok=True) # Create a temp directory and file. with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_path_dir: @@ -359,15 +361,14 @@ class Download: ) # Move lyrics file - if self.settings.data.lyrics_file and not isinstance(media, Video): + if self.settings.data.lyrics_file and not isinstance(media, Video) and tmp_path_lyrics: self._move_lyrics(tmp_path_lyrics, path_media_dst) # Move cover file - if self.settings.data.cover_album_file: + if self.settings.data.cover_album_file and tmp_path_cover: self._move_cover(tmp_path_cover, path_media_dst) # Move final file to the configured destination directory. - os.makedirs(path_media_dst.parent, exist_ok=True) shutil.move(tmp_path_file, path_media_dst) if quality_audio: @@ -410,7 +411,7 @@ class Download: result: bool # Check if the file was downloaded - if path_file_source and os.path.isfile(path_file_source): + if path_file_source and path_file_source.is_file(): # Move it. shutil.move(path_file_source, path_file_destination) From f3be5f30140b5604460e06f09abd2405771cf682 Mon Sep 17 00:00:00 2001 From: Robert Honz Date: Wed, 13 Nov 2024 10:58:52 +0100 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F=20Fixed=20ffmpeg=20ou?= =?UTF-8?q?tput=20file=20to=20`str`.=20Fixed=20segment=20parsing=20for=20v?= =?UTF-8?q?ideos=20while=20downloading=20segments.=20Lowered=20time=20dela?= =?UTF-8?q?y=20span.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tidal_dl_ng/download.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tidal_dl_ng/download.py b/tidal_dl_ng/download.py index 2329381..26d6f84 100644 --- a/tidal_dl_ng/download.py +++ b/tidal_dl_ng/download.py @@ -208,7 +208,7 @@ class Download: result: bool = False path_segment: pathlib.Path = path_base / url_to_filename(url) # Calculate the segment ID based on the file name within the URL. - id_segment: int = int(path_segment.stem) + id_segment: int = int(str(path_segment.stem).split("_")[-1]) error: HTTPError | None = None try: @@ -386,7 +386,7 @@ class Download: # Whether a file was downloaded or skipped and the download delay is enabled, wait until the next download. # Only use this, if you have a list of several Track items. if download_delay and not skip_file: - time_sleep: float = round(random.SystemRandom().uniform(2, 5), 1) + time_sleep: float = round(random.SystemRandom().uniform(1.5, 4), 1) self.fn_logger.debug(f"Next download will start in {time_sleep} seconds.") time.sleep(time_sleep) @@ -608,7 +608,7 @@ class Download: path_file_out: pathlib.Path = path_file.with_suffix(AudioExtensions.MP4) result, _ = ( ffmpeg.input(path_file) - .output(path_file_out, map=0, c="copy", loglevel="quiet") + .output(str(path_file_out), map=0, c="copy", loglevel="quiet") .run(cmd=self.settings.data.path_binary_ffmpeg) ) @@ -619,7 +619,12 @@ class Download: result, _ = ( ffmpeg.input(path_media_src) .output( - path_media_out, map=0, movflags="use_metadata_tags", acodec="copy", map_metadata="0:g", loglevel="quiet" + str(path_media_out), + map=0, + movflags="use_metadata_tags", + acodec="copy", + map_metadata="0:g", + loglevel="quiet", ) .run(cmd=self.settings.data.path_binary_ffmpeg) ) From ae2dd4fb2b4738cf73d2a0d1b0d94e7410b5ca65 Mon Sep 17 00:00:00 2001 From: Robert Honz Date: Wed, 13 Nov 2024 12:04:24 +0100 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F=20Do=20not=20attempt?= =?UTF-8?q?=20to=20set=20quality=20if=20it=20is=20set=20to=20`None`.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tidal_dl_ng/download.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tidal_dl_ng/download.py b/tidal_dl_ng/download.py index 26d6f84..6c637f9 100644 --- a/tidal_dl_ng/download.py +++ b/tidal_dl_ng/download.py @@ -300,8 +300,10 @@ class Download: file_extension: str do_flac_extract = False # If a quality is explicitly set, change it and remember the previously set quality. - quality_audio_old: Quality = self.adjust_quality_audio(quality_audio) - quality_video_old: QualityVideo = self.adjust_quality_video(quality_video) + quality_audio_old: Quality = self.adjust_quality_audio(quality_audio) if quality_audio else quality_audio + quality_video_old: QualityVideo = ( + self.adjust_quality_video(quality_video) if quality_video else quality_video + ) if isinstance(media, Track): try: @@ -312,6 +314,10 @@ class Download: f"Consider to activate delay between downloads." ) + return False, "" + except: + self.fn_logger.exception(f"Something went wrong. Skipping '{name_builder_item(media)}'.") + return False, "" file_extension = media_stream.get_stream_manifest().file_extension From f5ed1c6554dbe5e7dda8ce662ac7ee49919cece3 Mon Sep 17 00:00:00 2001 From: Robert Honz Date: Wed, 13 Nov 2024 12:36:01 +0100 Subject: [PATCH 6/6] =?UTF-8?q?=E2=9C=A8=20Time=20exponential=20retries=20?= =?UTF-8?q?on=20segment=20download=20errors.=20Fixes=20#196?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tidal_dl_ng/download.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tidal_dl_ng/download.py b/tidal_dl_ng/download.py index 6c637f9..6f039a8 100644 --- a/tidal_dl_ng/download.py +++ b/tidal_dl_ng/download.py @@ -12,6 +12,7 @@ import ffmpeg import m3u8 import requests from constants import CHUNK_SIZE, COVER_NAME +from requests.adapters import HTTPAdapter, Retry from requests.exceptions import HTTPError from rich.progress import Progress, TaskID from tidalapi import Album, Mix, Playlist, Session, Track, UserPlaylist, Video @@ -164,6 +165,7 @@ class Download: # If this is NOT the case, but any other URL has resulted in an error, # mark the whole thing as corrupt. result_segments = False + self.fn_logger.error(f"Something went wrong while downloading {media_name}. File is corrupt!") tmp_path_file_decrypted: pathlib.Path = path_file @@ -211,9 +213,15 @@ class Download: id_segment: int = int(str(path_segment.stem).split("_")[-1]) error: HTTPError | None = None + # Retry download on failed segments, with an exponential delay between retries + s = requests.Session() + retries = Retry(total=5, backoff_factor=1) # , status_forcelist=[ 502, 503, 504 ]) + + s.mount("https://", HTTPAdapter(max_retries=retries)) + try: # Create the request object with stream=True, so the content won't be loaded into memory at once. - r = requests.get(url, stream=True, timeout=REQUESTS_TIMEOUT_SEC) + r = s.get(url, stream=True, timeout=REQUESTS_TIMEOUT_SEC) r.raise_for_status() @@ -225,7 +233,7 @@ class Download: self.progress.advance(p_task) result = True - except HTTPError as e: + except Exception as e: error = e self.progress.advance(p_task) self.fn_logger.exception(e)