6FPS/tidal-dl
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

♻️ Refactoring of MPD download. Metadata does not need an explicit media_type flag anymore.

Browse Source
This commit is contained in:
Robert Honz
2024-01-14 17:29:39 +01:00
parent b59cba77c3
commit 825130bd61
9 changed files with 111 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyproject.toml
+1 -1
View File
@@ -171,7 +171,7 @@ profile = "black"
multi_line_output = 3
indent = 4
color_output = true
known_third_party = ["Crypto", "PySide6", "coloredlogs", "constants", "dataclasses_json", "ffmpeg", "helper", "m3u8", "model", "mutagen", "pathvalidate", "requests", "rich", "tidalapi", "typer"]
known_third_party = ["Crypto", "PySide6", "coloredlogs", "constants", "dataclasses_json", "ffmpeg", "helper", "m3u8", "model", "mpegdash", "mutagen", "pathvalidate", "requests", "rich", "tidalapi", "typer"]
[tool.mypy]
# https://mypy.readthedocs.io/en/latest/config_file.html#using-a-pyproject-toml-file
tidal_dl_ng/config.py
+4 -1
View File
@@ -47,7 +47,10 @@ class BaseConfig:
# Move the invalid config file to the backup location.
shutil.move(path, path_bak)
# TODO: Implement better global logger.
print(f"Something is wrong with your config. Maybe it is not compatible anymore due to a new app version. You can find a backup of your old config here: '{path_bak}'. A new default config was created.")
print(
"Something is wrong with your config. Maybe it is not compatible anymore due to a new app version."
f" You can find a backup of your old config here: '{path_bak}'. A new default config was created."
)
self.data = self.cls_model()
tidal_dl_ng/constants.py
+7 -1
View File
@@ -27,5 +27,11 @@ class SkipExisting(Enum):
class StreamManifestMimeType(Enum):
MPD = "application/dash+xml"
JSON = "a"
BTS = "application/vnd.tidal.bts"
VIDEO = "video/mp2t"
class CoverDimensions(Enum):
Px320 = "320x320"
Px640 = "640x640"
Px1280 = "1280x1280"
tidal_dl_ng/download.py
+58 -110
View File
@@ -14,13 +14,13 @@ import m3u8
import requests
from helper.tidal import name_builder_item
from model.tidal import StreamManifest
from requests.exceptions import HTTPError
from rich.progress import Progress
from tidalapi import Album, Mix, Playlist, Session, Track, UserPlaylist, Video
from mpegdash.parser import MPEGDASHParser
from requests.exceptions import HTTPError
from rich.progress import Progress, TaskID
from tidalapi import Album, Mix, Playlist, Session, Track, UserPlaylist, Video
from tidal_dl_ng.config import Settings
from tidal_dl_ng.constants import REQUESTS_TIMEOUT_SEC, MediaType, SkipExisting, StreamManifestMimeType
from tidal_dl_ng.constants import REQUESTS_TIMEOUT_SEC, CoverDimensions, MediaType, SkipExisting, StreamManifestMimeType
from tidal_dl_ng.helper.decryption import decrypt_file, decrypt_security_token
from tidal_dl_ng.helper.exceptions import MediaMissing, MediaUnknown, UnknownManifestFormat
from tidal_dl_ng.helper.path import check_file_exists, format_path_media, path_file_sanitize
@@ -52,10 +52,10 @@ class Download:
self.session = session
self.skip_existing = skip_existing
def _audio_stream(
def _download(
self,
fn_logger: Callable,
media: Track,
media: Track | Video,
progress: Progress,
progress_gui: ProgressBars,
stream_manifest: StreamManifest,
@@ -71,23 +71,40 @@ class Download:
progress_gui.item_name.emit(media_name)
try:
# Download the media as stream, so we can iterate over the response.
r = requests.get(stream_manifest.stream_urls, stream=True, timeout=REQUESTS_TIMEOUT_SEC)
# Compute total iterations for progress
urls_count: int = len(stream_manifest.urls)
if urls_count > 1:
progress_total: int = urls_count
block_size: int | None = None
else:
# Compute progress iterations based on the file size.
r = requests.get(stream_manifest.urls[0], stream=True, timeout=REQUESTS_TIMEOUT_SEC)
r.raise_for_status()
# Get file size and compute progress steps
total_size_in_bytes = int(r.headers.get("content-length", 0))
block_size = 4096
p_task = progress.add_task(
f"[blue]Item '{media_name[:30]}'",
total=total_size_in_bytes / block_size,
total_size_in_bytes: int = int(r.headers.get("content-length", 0))
block_size: int | None = 4096
progress_total: float = total_size_in_bytes / block_size
# Create progress Task
p_task: TaskID = progress.add_task(
f"[blue]Item '{media_name[:20]}'",
total=progress_total,
visible=progress_stdout,
)
# Write content to file until progress is finished.
while not progress.tasks[p_task].finished:
with open(path_file, "wb") as f:
for url in stream_manifest.urls:
# Create the request object with stream=True, so the content won't be loaded into memory at once.
r = requests.get(url, stream=True, timeout=REQUESTS_TIMEOUT_SEC)
r.raise_for_status()
# Write the content to disk. If `chunk_size` is set to `None` the whole file will be written at once.
for data in r.iter_content(chunk_size=block_size):
f.write(data)
# Advance progress bar.
@@ -95,7 +112,8 @@ class Download:
# To send the progress to the GUI, we need to emit the percentage.
if not progress_stdout:
progress_gui.item.emit(progress.tasks[p_task].percentage)
# progress_gui.item.emit(progress.tasks[p_task].percentage)
pass
except HTTPError as e:
# TODO: Handle Exception...
fn_logger(e)
@@ -111,73 +129,11 @@ class Download:
tmp_path_file_decrypted = path_file
# Write metadata to file.
if not isinstance(media, Video):
self.metadata_write(media, tmp_path_file_decrypted)
return tmp_path_file_decrypted
def _mpeg_segments(
self,
fn_logger: Callable,
media: Track,
progress: Progress,
progress_gui: ProgressBars,
stream_manifest: StreamManifest,
path_file: str,
):
media_name: str = name_builder_item(media)
# Set the correct progress output channel.
if progress_gui is None:
progress_stdout: bool = True
else:
progress_stdout: bool = False
progress_gui.item_name.emit(media_name)
try:
total_iterations = stream_manifest.segments_count
p_task = progress.add_task(
f"[blue]Item '{media.name[:30]}'",
total=total_iterations,
visible=progress_stdout,
)
# Write content to file until progress is finished.
while not progress.tasks[p_task].finished:
with open(path_file, "wb") as f:
for index in range(total_iterations):
# Download the media.
segment_url = stream_manifest.stream_urls.replace('$Number$', str(index))
r = requests.get(segment_url, timeout=REQUESTS_TIMEOUT_SEC)
r.raise_for_status()
# Write data
f.write(r.content)
# Advance progress bar.
progress.advance(p_task)
# To send the progress to the GUI, we need to emit the percentage.
if not progress_stdout:
progress_gui.item.emit(progress.tasks[p_task].percentage)
except HTTPError as e:
# TODO: Handle Exception...
fn_logger(e)
return path_file
def _video(self, video: Video, path_file: str) -> str | None:
result: str | None = None
with open(path_file, "wb") as f:
for segment in m3u8_playlist.data["segments"]:
url = segment["uri"]
r = requests.get(url, timeout=REQUESTS_TIMEOUT_SEC)
f.write(r.content)
result = path_file
return result
def instantiate_media(
self, session: Session, media_type: type[MediaType.Track, MediaType.Video], id_media: str
) -> Track | Video:
@@ -245,22 +201,13 @@ class Download:
if not download_skip:
# Create a temp directory and file.
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_path_dir:
tmp_path_file = os.path.join(tmp_path_dir, str(uuid4()))
tmp_path_file = os.path.join(tmp_path_dir, str(uuid4()) + stream_manifest.file_extension)
# Download media.
tmp_path_file = self._download(fn_logger, media, progress, progress_gui, stream_manifest, tmp_path_file)
if isinstance(media, Track):
if stream_manifest.segments_count > 0:
tmp_path_file = self._mpeg_segments(fn_logger, media, progress, progress_gui, stream_manifest, tmp_path_file)
else:
tmp_path_file = self._audio_stream(
fn_logger, media, progress, progress_gui, stream_manifest, tmp_path_file
)
elif isinstance(media, Video):
tmp_path_file = self._video(media, tmp_path_file)
# TODO: Check if is possible to write metadata to MPEG Transport Stream files.
if isinstance(media, Video):
# TODO: Make optional.
# Convert `*.ts` file to `*.mp4` using ffmpeg
if True:
tmp_path_file = self._video_convert(tmp_path_file)
path_file = os.path.splitext(path_file)[0] + ".mp4"
@@ -272,18 +219,20 @@ class Download:
return not download_skip, path_file
def cover_url(self, sid: str, width: int = 320, height: int = 320):
def cover_url(self, sid: str, dimension: CoverDimensions = CoverDimensions.Px320):
if sid is None:
return ""
return f"https://resources.tidal.com/images/{sid.replace('-', '/')}/{int(width)}x{int(height)}.jpg"
return f"https://resources.tidal.com/images/{sid.replace('-', '/')}/{dimension.value}.jpg"
def metadata_write(self, track: Track, path_file: str):
settings: Settings = Settings()
result: bool = False
release_date: str = track.album.release_date.strftime("%Y-%m-%d") if track.album.release_date else ""
copy_right: str = track.copyright if track.copyright else ""
isrc: str = track.isrc if track.isrc else ""
release_date: str = (
track.album.release_date.strftime("%Y-%m-%d") if track.album and track.album.release_date else ""
)
copy_right: str = track.copyright if hasattr(track, "copyright") else ""
isrc: str = track.isrc if hasattr(track, "isrc") else ""
try:
lyrics: str = track.lyrics().subtitles if hasattr(track, "lyrics") else ""
@@ -297,17 +246,15 @@ class Download:
copy_right=copy_right,
title=track.name,
artists=[artist.name for artist in track.artists],
album=track.album.name,
album=track.album.name if track.album else "",
tracknumber=track.track_num,
date=release_date,
isrc=isrc,
albumartist=track.artist.name,
totaltrack=track.album.num_tracks if track.album.num_tracks else 1,
totaldisc=track.album.num_volumes if track.album.num_volumes else 1,
discnumber=track.volume_num,
url_cover=self.cover_url(
track.album.cover, settings.data.metadata_cover_width, settings.data.metadata_cover_height
),
albumartist=name_builder_item(track),
totaltrack=track.album.num_tracks if track.album and track.album.num_tracks else 1,
totaldisc=track.album.num_volumes if track.album and track.album.num_volumes else 1,
discnumber=track.volume_num if track.volume_num else 1,
url_cover=self.cover_url(track.album.cover, settings.data.metadata_cover_dimension) if track.album else "",
)
m.save()
@@ -379,7 +326,8 @@ class Download:
progress.advance(p_task1)
if not progress_stdout:
progress_gui.list_item.emit(progress.tasks[p_task1].percentage)
# progress_gui.list_item.emit(progress.tasks[p_task1].percentage)
pass
if download_delay and status_download:
time_sleep: float = round(random.SystemRandom().uniform(2, 5), 1)
@@ -405,7 +353,7 @@ class Download:
# else:
# result = ".m4a"
result: str = ".mp4"
if ".ts" in stream_url:
elif ".ts" in stream_url:
result: str = ".ts"
else:
result: str = ".m4a"
@@ -439,14 +387,14 @@ class Download:
stream_urls: list[str] = []
for index in range(segments_count):
stream_urls.append(segment_template.media.replace('$Number$', str(index)))
stream_urls.append(segment_template.media.replace("$Number$", str(index)))
elif mime_type == StreamManifestMimeType.JSON.value:
elif mime_type == StreamManifestMimeType.BTS.value:
# Stream Manifest is base64 encoded.
manifest_parsed: str = base64.b64decode(manifest).decode("utf-8")
# JSON string to object.
stream_manifest = json.loads(manifest_parsed)
# TODO: Handle more than one dowload URL
# TODO: Handle more than one download URL
stream_urls: str = stream_manifest["urls"]
codecs: str = stream_manifest["codecs"]
mime_type: str = stream_manifest["mimeType"]
@@ -472,12 +420,12 @@ class Download:
file_extension: str = self.get_file_extension(stream_urls[0], codecs)
result: StreamManifest = StreamManifest(
stream_urls=stream_urls,
urls=stream_urls,
codecs=codecs,
file_extension=file_extension,
encryption_type=encryption_type,
encryption_key=encryption_key,
mime_type=mime_type
mime_type=mime_type,
)
return result
tidal_dl_ng/gui.py
+1
View File
@@ -135,6 +135,7 @@ class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow):
def populate_tree_lists(self, tidal: Tidal):
# Start loading spinner
# TODO: Fix QPaint issue. App is crashing because of this.
self.spinner_start.emit(self.tr_lists_user)
self.tr_results.clear()
tidal_dl_ng/helper/path.py
+6 -2
View File
@@ -127,7 +127,9 @@ def path_file_sanitize(path_file: str, adapt: bool = False) -> (bool, str):
# Sanitize path
try:
pathname_sanitized = sanitize_filepath(pathname, replacement_text=" ", validate_after_sanitize=True, platform="auto")
pathname_sanitized = sanitize_filepath(
pathname, replacement_text=" ", validate_after_sanitize=True, platform="auto"
)
except ValidationError:
# If adaption of path is allowed in case of an error set path to HOME.
if adapt:
@@ -137,7 +139,9 @@ def path_file_sanitize(path_file: str, adapt: bool = False) -> (bool, str):
# Sanitize filename
try:
filename_sanitized = sanitize_filename(filename, replacement_text=" ", validate_after_sanitize=True, platform="auto")
filename_sanitized = sanitize_filename(
filename, replacement_text=" ", validate_after_sanitize=True, platform="auto"
)
filename_sanitized_extension = Path(filename_sanitized).suffix
# Check if the file extension was removed by shortening the filename length
tidal_dl_ng/metadata.py
+12 -24
View File
@@ -1,15 +1,12 @@
import pathlib
import mutagen
import requests
from constants import REQUESTS_TIMEOUT_SEC
from mutagen import flac, mp4
from mutagen import flac, id3, mp4
from mutagen.id3 import APIC, TALB, TCOM, TCOP, TDRC, TIT2, TOPE, TPE1, TRCK, TSRC, USLT
class Metadata:
path_file: str = None
type_audio: str = None
title: str = None
album: str = None
albumartist: str = None
@@ -31,7 +28,6 @@ class Metadata:
def __init__(
self,
path_file: str,
type_audio: str = "",
album: str = "",
title: str = "",
artists: list[str] | None = None,
@@ -49,12 +45,6 @@ class Metadata:
url_cover: str = "",
):
self.path_file = path_file
if type_audio:
self.type_audio = type_audio
else:
self.type_audio = pathlib.Path(self.path_file).suffix[1:]
self.title = title
self.album = album
self.albumartist = albumartist
@@ -70,23 +60,24 @@ class Metadata:
self.lyrics = lyrics
self.path_cover = path_cover
self.url_cover = url_cover
self.m: mutagen.MP4.MP4 | mutagen.m4a.M4A | mutagen.flac.FLAC | mutagen.mp3.MP3 = mutagen.File(self.path_file)
self.m: mutagen.mp4.MP4 | mutagen.flac.FLAC | mutagen.mp3.MP3 = mutagen.File(self.path_file)
def _cover(self) -> bool:
result: bool = False
data_cover: str | bytes = self.cover_data(url=self.url_cover, path_file=self.path_cover)
if data_cover:
if self.type_audio == "flac":
if isinstance(self.m, mutagen.flac.FLAC):
flac_cover = flac.Picture()
flac_cover.type = id3.PictureType.COVER_FRONT
flac_cover.data = data_cover
flac_cover.mime = "image/jpeg"
self.m.clear_pictures()
self.m.add_picture(flac_cover)
elif self.type_audio == "mp3":
elif isinstance(self.m, mutagen.mp3.MP3):
self.m.tags.add(APIC(encoding=3, data=data_cover))
elif self.type_audio in ["mp4", "m4a"]:
elif isinstance(self.m, mutagen.mp4.MP4):
cover_mp4 = mp4.MP4Cover(data_cover)
self.m.tags["covr"] = [cover_mp4]
@@ -95,11 +86,14 @@ class Metadata:
return result
def save(self):
if self.type_audio == "flac":
if not self.m.tags:
self.m.add_tags()
if isinstance(self.m, mutagen.flac.FLAC):
self.set_flac()
elif self.type_audio in ["mp3", "ts"]:
elif isinstance(self.m, mutagen.mp3.MP3):
self.set_mp3()
elif self.type_audio in ["mp4", "m4a"]:
elif isinstance(self.m, mutagen.mp4.MP4):
self.set_mp4()
self._cover()
@@ -108,9 +102,6 @@ class Metadata:
return True
def set_flac(self):
if not self.m.tags:
self.m.add_tags()
self.m.tags["title"] = self.title
self.m.tags["album"] = self.album
self.m.tags["albumartist"] = self.albumartist
@@ -126,9 +117,6 @@ class Metadata:
self.m.tags["lyrics"] = self.lyrics
def set_mp3(self):
if not self.m.tags:
self.m.add_tags()
self.m.tags.add(TIT2(encoding=3, text=self.title))
self.m.tags.add(TALB(encoding=3, text=self.album))
self.m.tags.add(TOPE(encoding=3, text=self.albumartist))
tidal_dl_ng/model/cfg.py
+9 -7
View File
@@ -3,7 +3,7 @@ from dataclasses import dataclass
from dataclasses_json import dataclass_json
from tidalapi import Quality
from tidal_dl_ng.constants import QualityVideo, SkipExisting
from tidal_dl_ng.constants import CoverDimensions, QualityVideo, SkipExisting
@dataclass_json
@@ -28,15 +28,16 @@ class Settings:
format_track: str = "Tracks/{artist_name} - {track_title}"
format_video: str = "Videos/{artist_name} - {track_title}"
video_convert_mp4: bool = True
metadata_cover_width: int = 320
metadata_cover_height: int = 320
metadata_cover_dimension: CoverDimensions = CoverDimensions.Px320
@dataclass
class HelpSettings:
skip_existing: str = ("Do not download, if file already exists. Possible option false = do not skip, "
skip_existing: str = (
"Do not download, if file already exists. Possible option false = do not skip, "
"'exact' = if filename already exists, 'extension_ignore' = skip even if a file with a "
"different file extension exists.")
"different file extension exists."
)
album_cover_save: str = "Safe cover to album folder."
lyrics_save: str = "Safe lyrics to audio file."
api_key_index: str = "Set the device API KEY."
@@ -64,8 +65,9 @@ class HelpSettings:
"Videos are downloaded as MPEG Transport Stream (TS) files. With this option each video "
"will be converted to MP4. FFMPEG must be installed and added to your 'PATH' variable."
)
metadata_cover_width: str = "The width of the cover image embedded into the track."
metadata_cover_height: str = "The height of the cover image embedded into the track."
metadata_cover_dimension: str = (
"The dimensions of the cover image embedded into the track. Possible values: 320x320, 640x640x 1280x1280."
)
@dataclass_json
tidal_dl_ng/model/tidal.py
+1 -1
View File
@@ -5,7 +5,7 @@ from dataclasses import dataclass
class StreamManifest:
codecs: str
mime_type: str
stream_urls: [str]
urls: [str]
file_extension: str
encryption_type: str | None = None
encryption_key: str | None = None