6FPS/tidal-dl
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨ Removed complex skipping options in favor of a faster duplicates check / skipping of already present items without download delay.

Browse Source
This commit is contained in:
Robert Honz
2024-11-13 09:09:00 +01:00
parent beea8f1fe7
commit c1a102f646
8 changed files with 116 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tidal_dl_ng/constants.py
-7
View File
@@ -27,13 +27,6 @@ class MediaType(StrEnum):
ARTIST: str = "artist"
class SkipExisting(StrEnum):
Disabled: str = "False"
Filename: str = "exact"
ExtensionIgnore: str = "extension_ignore"
Append: str = "append"
class CoverDimensions(StrEnum):
Px80: str = "80"
Px160: str = "160"
tidal_dl_ng/dialog.py
+2 -2
View File
@@ -9,7 +9,7 @@ from tidalapi import Quality as QualityAudio
from tidal_dl_ng import __version__
from tidal_dl_ng.config import Settings
from tidal_dl_ng.constants import CoverDimensions, QualityVideo, SkipExisting
from tidal_dl_ng.constants import CoverDimensions, QualityVideo
from tidal_dl_ng.model.cfg import HelpSettings
from tidal_dl_ng.model.cfg import Settings as ModelSettings
from tidal_dl_ng.model.meta import ReleaseLatest
@@ -157,7 +157,6 @@ class DialogPreferences(QtWidgets.QDialog):
def _init_comboboxes(self):
self.parameters_combo = [
("skip_existing", SkipExisting),
("quality_audio", QualityAudio),
("quality_video", QualityVideo),
("metadata_cover_dimension", CoverDimensions),
@@ -173,6 +172,7 @@ class DialogPreferences(QtWidgets.QDialog):
"extract_flac",
"metadata_cover_embed",
"cover_album_file",
"skip_existing",
]
def gui_populate(self):
tidal_dl_ng/download.py
+65 -54
View File
@@ -15,10 +15,11 @@ from constants import CHUNK_SIZE, COVER_NAME
from requests.exceptions import HTTPError
from rich.progress import Progress, TaskID
from tidalapi import Album, Mix, Playlist, Session, Track, UserPlaylist, Video
from tidalapi.exceptions import TooManyRequests
from tidalapi.media import AudioExtensions, Codec, Quality, StreamManifest, VideoExtensions
from tidal_dl_ng.config import Settings
from tidal_dl_ng.constants import EXTENSION_LYRICS, REQUESTS_TIMEOUT_SEC, MediaType, QualityVideo, SkipExisting
from tidal_dl_ng.constants import EXTENSION_LYRICS, REQUESTS_TIMEOUT_SEC, MediaType, QualityVideo
from tidal_dl_ng.helper.decryption import decrypt_file, decrypt_security_token
from tidal_dl_ng.helper.exceptions import MediaMissing
from tidal_dl_ng.helper.path import check_file_exists, format_path_media, path_file_sanitize, url_to_filename
@@ -53,7 +54,7 @@ class RequestsClient:
class Download:
settings: Settings
session: Session
skip_existing: SkipExisting = SkipExisting.Disabled
skip_existing: bool = False
fn_logger: Callable
progress_gui: ProgressBars
progress: Progress
@@ -63,7 +64,7 @@ class Download:
session: Session,
path_base: str,
fn_logger: Callable,
skip_existing: SkipExisting = SkipExisting.Disabled,
skip_existing: bool = False,
progress_gui: ProgressBars = None,
progress: Progress = None,
):
@@ -209,6 +210,7 @@ class Download:
path_segment: pathlib.Path = path_base / url_to_filename(url)
# Calculate the segment ID based on the file name within the URL.
id_segment: int = int(path_segment.stem)
error: HTTPError | None = None
try:
# Create the request object with stream=True, so the content won't be loaded into memory at once.
@@ -224,15 +226,18 @@ class Download:
self.progress.advance(p_task)
result = True
except HTTPError:
# TODO: Maybe return e as well?
except HTTPError as e:
error = e
self.progress.advance(p_task)
finally:
# To send the progress to the GUI, we need to emit the percentage.
if not progress_to_stdout:
self.progress_gui.item.emit(self.progress.tasks[p_task].percentage)
self.fn_logger.exception(e.charachter_written)
return DownloadSegmentResult(result=result, url=url, path_segment=path_segment, id_segment=id_segment)
# To send the progress to the GUI, we need to emit the percentage.
if not progress_to_stdout:
self.progress_gui.item.emit(self.progress.tasks[p_task].percentage)
return DownloadSegmentResult(
result=result, url=url, path_segment=path_segment, id_segment=id_segment, error=error
)
def item(
self,
@@ -273,50 +278,56 @@ class Download:
return False, ""
# Get extension.
file_extension: str
do_flac_extract = False
if isinstance(media, Track):
# If a quality is explicitly set, change it.
if quality_audio:
quality_audio_old: Quality = self.adjust_quality_audio(quality_audio)
file_extension = media.get_stream().get_stream_manifest().file_extension
# Use M4A extension for MP4 audio tracks, because it looks better and is completely interchangeable.
file_extension = AudioExtensions.M4A if file_extension == AudioExtensions.MP4 else file_extension
if self.settings.data.extract_flac:
if (
media.get_stream().get_stream_manifest().codecs.upper() == Codec.FLAC
and file_extension != AudioExtensions.FLAC
):
file_extension = AudioExtensions.FLAC
do_flac_extract = True
elif isinstance(media, Video):
if quality_video:
quality_video_old: QualityVideo = self.adjust_quality_video(quality_video)
file_extension = AudioExtensions.MP4 if self.settings.data.video_convert_mp4 else VideoExtensions.TS
# Create file name and path
file_extension_dummy: str = AudioExtensions.FLAC
file_name_relative = format_path_media(file_template, media)
path_media_dst = os.path.abspath(
os.path.normpath(os.path.join(os.path.expanduser(self.path_base), file_name_relative))
)
# Sanitize final path_file to fit into OS boundaries.
uniquify: bool = self.skip_existing == SkipExisting.Append
path_media_dst = path_file_sanitize(path_media_dst + file_extension, adapt=True, uniquify=uniquify)
path_media_dst = path_file_sanitize(path_media_dst + file_extension_dummy, adapt=True)
# Compute if and how downloads need to be skipped.
if self.skip_existing in (SkipExisting.ExtensionIgnore, SkipExisting.Filename):
extension_ignore: bool = self.skip_existing == SkipExisting.ExtensionIgnore
file_exists: bool = check_file_exists(path_media_dst, extension_ignore=extension_ignore)
if self.skip_existing:
skip_file: bool = check_file_exists(path_media_dst, extension_ignore=True)
else:
file_exists: bool = False
skip_file: bool = False
if not skip_file:
# Get extension.
file_extension: str
do_flac_extract = False
# If a quality is explicitly set, change it and remember the previously set quality.
quality_audio_old: Quality = self.adjust_quality_audio(quality_audio)
quality_video_old: QualityVideo = self.adjust_quality_video(quality_video)
if isinstance(media, Track):
try:
media_stream = media.get_stream()
except TooManyRequests:
self.fn_logger.exception(
f"Too many requests against TIDAL backend. Skipping '{name_builder_item(media)}'. "
f"Consider to activate delay between downloads."
)
return False, ""
file_extension = media_stream.get_stream_manifest().file_extension
# Use M4A extension for MP4 audio tracks, because it looks better and is completely interchangeable.
file_extension = AudioExtensions.M4A if file_extension == AudioExtensions.MP4 else file_extension
if self.settings.data.extract_flac and (
media_stream.get_stream_manifest().codecs.upper() == Codec.FLAC
and file_extension != AudioExtensions.FLAC
):
file_extension = AudioExtensions.FLAC
do_flac_extract = True
elif isinstance(media, Video):
file_extension = AudioExtensions.MP4 if self.settings.data.video_convert_mp4 else VideoExtensions.TS
# TODO: correct extension
if not file_exists:
# Create a temp directory and file.
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_path_dir:
tmp_path_file: pathlib.Path = pathlib.Path(tmp_path_dir) / str(uuid4())
@@ -356,22 +367,22 @@ class Download:
# Move cover file
if self.settings.data.cover_album_file:
self._move_cover(tmp_path_cover, path_media_dst)
if quality_audio:
# Set quality back to the global user value
self.adjust_quality_audio(quality_audio_old)
if quality_video:
# Set quality back to the global user value
self.adjust_quality_video(quality_video_old)
else:
self.fn_logger.debug(f"Download skipped, since file exists: '{path_media_dst}'")
status_download: bool = not file_exists
if quality_audio:
# Set quality back to the global user value
self.adjust_quality_audio(quality_audio_old)
if quality_video:
# Set quality back to the global user value
self.adjust_quality_video(quality_video_old)
status_download: bool = not skip_file
# Whether a file was downloaded or skipped and the download delay is enabled, wait until the next download.
# Only use this, if you have a list of several Track items.
if download_delay:
if download_delay and not skip_file:
time_sleep: float = round(random.SystemRandom().uniform(2, 5), 1)
self.fn_logger.debug(f"Next download will start in {time_sleep} seconds.")
@@ -575,7 +586,7 @@ class Download:
while not self.progress.finished:
for item_media in items:
# Download the item.
status_download, result_path_file = self.item(
status, result_path_file = self.item(
media=item_media,
file_template=file_name_relative,
quality_audio=quality_audio,
tidal_dl_ng/helper/wrapper.py
+3
View File
@@ -21,3 +21,6 @@ class LoggerWrapped:
def critical(self, value):
self.fn_print(value)
def exception(self, value):
self.fn_print(value)
tidal_dl_ng/model/cfg.py
+3 -7
View File
@@ -3,13 +3,13 @@ from dataclasses import dataclass
from dataclasses_json import dataclass_json
from tidalapi import Quality
from tidal_dl_ng.constants import CoverDimensions, QualityVideo, SkipExisting
from tidal_dl_ng.constants import CoverDimensions, QualityVideo
@dataclass_json
@dataclass
class Settings:
skip_existing: SkipExisting = SkipExisting.Disabled
skip_existing: bool = True
lyrics_embed: bool = False
lyrics_file: bool = False
# TODO: Implement API KEY selection.
@@ -43,11 +43,7 @@ class Settings:
@dataclass_json
@dataclass
class HelpSettings:
skip_existing: str = (
"Do not download, if file already exists. Possible option false = do not skip, "
"'exact' = if filename already exists, 'extension_ignore' = skip even if a file with a "
"different file extension exists."
)
skip_existing: str = "Skip download if file already exists."
album_cover_save: str = "Safe cover to album folder."
lyrics_embed: str = "Embed lyrics in audio file, if lyrics are available."
lyrics_file: str = "Save lyrics to separate *.lrc file, if lyrics are available."
tidal_dl_ng/model/downloader.py
+3
View File
@@ -1,6 +1,8 @@
import pathlib
from dataclasses import dataclass
from requests import HTTPError
@dataclass
class DownloadSegmentResult:
@@ -8,3 +10,4 @@ class DownloadSegmentResult:
url: str
path_segment: pathlib.Path
id_segment: int
error: HTTPError | None = None
tidal_dl_ng/ui/dialog_settings.py
+22 -30
View File
@@ -153,6 +153,24 @@ class Ui_DialogSettings:
self.lv_flags.addLayout(self.lh_flags_4)
self.horizontalLayout = QHBoxLayout()
self.horizontalLayout.setObjectName("horizontalLayout")
self.lv_flag_skip_existing = QVBoxLayout()
self.lv_flag_skip_existing.setObjectName("lv_flag_skip_existing")
self.cb_skip_existing = QCheckBox(self.gb_flags)
self.cb_skip_existing.setObjectName("cb_skip_existing")
self.lv_flag_skip_existing.addWidget(self.cb_skip_existing)
self.horizontalLayout.addLayout(self.lv_flag_skip_existing)
self.verticalLayout_3 = QVBoxLayout()
self.verticalLayout_3.setObjectName("verticalLayout_3")
self.horizontalLayout.addLayout(self.verticalLayout_3)
self.lv_flags.addLayout(self.horizontalLayout)
self.lv_main.addWidget(self.gb_flags)
self.gb_choices = QGroupBox(DialogSettings)
@@ -164,38 +182,13 @@ class Ui_DialogSettings:
self.gb_choices.setSizePolicy(sizePolicy3)
self.lv_choices = QVBoxLayout(self.gb_choices)
self.lv_choices.setObjectName("lv_choices")
self.lh_choices_skip_existing = QHBoxLayout()
self.lh_choices_skip_existing.setObjectName("lh_choices_skip_existing")
self.l_icon_skip_existing = QLabel(self.gb_choices)
self.l_icon_skip_existing.setObjectName("l_icon_skip_existing")
sizePolicy4 = QSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed)
sizePolicy4.setHorizontalStretch(0)
sizePolicy4.setVerticalStretch(0)
sizePolicy4.setHeightForWidth(self.l_icon_skip_existing.sizePolicy().hasHeightForWidth())
self.l_icon_skip_existing.setSizePolicy(sizePolicy4)
self.lh_choices_skip_existing.addWidget(self.l_icon_skip_existing)
self.l_skip_existing = QLabel(self.gb_choices)
self.l_skip_existing.setObjectName("l_skip_existing")
sizePolicy4.setHeightForWidth(self.l_skip_existing.sizePolicy().hasHeightForWidth())
self.l_skip_existing.setSizePolicy(sizePolicy4)
self.lh_choices_skip_existing.addWidget(self.l_skip_existing)
self.c_skip_existing = QComboBox(self.gb_choices)
self.c_skip_existing.setObjectName("c_skip_existing")
self.lh_choices_skip_existing.addWidget(self.c_skip_existing)
self.lh_choices_skip_existing.setStretch(2, 50)
self.lv_choices.addLayout(self.lh_choices_skip_existing)
self.lh_choices_quality_audio = QHBoxLayout()
self.lh_choices_quality_audio.setObjectName("lh_choices_quality_audio")
self.l_icon_quality_audio = QLabel(self.gb_choices)
self.l_icon_quality_audio.setObjectName("l_icon_quality_audio")
sizePolicy4 = QSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed)
sizePolicy4.setHorizontalStretch(0)
sizePolicy4.setVerticalStretch(0)
sizePolicy4.setHeightForWidth(self.l_icon_quality_audio.sizePolicy().hasHeightForWidth())
self.l_icon_quality_audio.setSizePolicy(sizePolicy4)
@@ -532,9 +525,8 @@ class Ui_DialogSettings:
self.cb_extract_flac.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None))
self.cb_metadata_cover_embed.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None))
self.cb_cover_album_file.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None))
self.cb_skip_existing.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None))
self.gb_choices.setTitle(QCoreApplication.translate("DialogSettings", "Choices", None))
self.l_icon_skip_existing.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None))
self.l_skip_existing.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None))
self.l_icon_quality_audio.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None))
self.l_quality_audio.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None))
self.l_icon_quality_video.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None))
tidal_dl_ng/ui/dialog_settings.ui
+18 -33
View File
@@ -210,6 +210,24 @@
</item>
</layout>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<layout class="QVBoxLayout" name="lv_flag_skip_existing">
<item>
<widget class="QCheckBox" name="cb_skip_existing">
<property name="text">
<string>CheckBox</string>
</property>
</widget>
</item>
</layout>
</item>
<item>
<layout class="QVBoxLayout" name="verticalLayout_3"/>
</item>
</layout>
</item>
</layout>
</widget>
</item>
@@ -225,39 +243,6 @@
<string>Choices</string>
</property>
<layout class="QVBoxLayout" name="lv_choices">
<item>
<layout class="QHBoxLayout" name="lh_choices_skip_existing" stretch="0,0,50">
<item>
<widget class="QLabel" name="l_icon_skip_existing">
<property name="sizePolicy">
<sizepolicy hsizetype="Fixed" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>TextLabel</string>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="l_skip_existing">
<property name="sizePolicy">
<sizepolicy hsizetype="Fixed" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>TextLabel</string>
</property>
</widget>
</item>
<item>
<widget class="QComboBox" name="c_skip_existing"/>
</item>
</layout>
</item>
<item>
<layout class="QHBoxLayout" name="lh_choices_quality_audio" stretch="0,0,50">
<item>