From e21348a8c0dd5d9760d780dfa4221eb2c9a8cbd4 Mon Sep 17 00:00:00 2001 From: Ivan Zverev Date: Thu, 8 Apr 2021 12:22:16 +0300 Subject: [PATCH 1/4] mp3 support with mp3splt, flac currupted header mutagen error handler, change virtual file extension for source --- Dockerfile | 2 +- Dockerfile.local | 50 ++++++++++++++++++++++++++++++++++++++++++ README.md | 5 +++-- README.py.md | 5 +++-- trackfs/albuminfo.py | 7 ++++-- trackfs/cuesheet.py | 3 +++ trackfs/flactracks.py | 51 ++++++++++++++++++++++++++++++++++++++----- trackfs/fusepath.py | 13 ++++------- 8 files changed, 114 insertions(+), 22 deletions(-) create mode 100644 Dockerfile.local diff --git a/Dockerfile b/Dockerfile index 4f64fb5..78497ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ FROM docker.io/python:3.8-alpine as builder # install dependencies RUN \ - apk --no-cache add fuse fuse-dev flac \ + apk --no-cache add fuse fuse-dev flac mp3splt \ && /usr/local/bin/python -m pip install --upgrade pip # enable non-root users to make FUSE fs non-private diff --git a/Dockerfile.local b/Dockerfile.local new file mode 100644 index 0000000..2345249 --- /dev/null +++ b/Dockerfile.local @@ -0,0 +1,50 @@ +# ================================= +# Dockerfile for run trackfs from source locally +# +# Copyright 2020 by Andreas Schmidt +# All rights reserved. +# This file is part of the trackfs project +# and licensed under the terms of the GNU Lesser General Public License v3.0. +# See https://github.com/andresch/trackfs for details. +# +# ================================= + +FROM docker.io/python:3.8-alpine as builder + +# install dependencies +RUN \ + apk --no-cache add fuse fuse-dev flac mp3splt \ + && /usr/local/bin/python -m pip install --upgrade pip + +# enable non-root users to make FUSE fs non-private +RUN echo "user_allow_other" >> /etc/fuse.conf + +# FUSE requires that the user that mounts the FUSE filesystem +# has an entry in /etc/passwd +# Since we want to allow (and encourage) the usage of docker's +# --user option to run the container as non-root user, +# and with that don't know the uid of the user at build time +# we can't create the entry for that user at build time +# and also can't use adduser command during runtime as this would +# require root privileges. +# Instead we open /etc/passwd for writing. +# As /ets/shadow is still protected this should not cause harm, +# even if some attacker finds a way to take over the container + +RUN chmod 666 /etc/passwd + +COPY ./ /usr/src/trackfs +RUN pip install /usr/src/trackfs + +# source directory containing flac+cue files +VOLUME /src + +# mount point where to generate the tracks from the flac+cue files +VOLUME /dst + +COPY launcher.sh /usr/local/bin/ +RUN chmod 555 /usr/local/bin/launcher.sh + +ENTRYPOINT ["/usr/local/bin/launcher.sh", "/src", "/dst"] + + diff --git a/README.md b/README.md index 0fa8fd6..d928408 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ * FLAC files with embedded cuesheets (FLAC+CUE) * FLAC files with accompanying cuesheet * WAVE files with accompanying cuesheet +* MP3 files with accompanying cuesheet (embedded cuesheet not tested) Accompanying cuesheets must use the fileextension `.cue` and have the same basename as album file. @@ -38,7 +39,7 @@ Replace `/path/to/yourmusiclibrary` with the root directory where `trackfs` scan Once started you will find all directories and files from your music library also in the `trackfs`-filesystem. Only album files got replaced: Instead of a single album file you will find individual FLAC files for each track found in the cue sheet. The track-files will have the following names: - {album-file}.#-#.{tracknumber}.{track-title}.flac + {album-file}.#-#.{tracknumber}.{track-title}.{source_ext} While the tracks can be used like regular files, they don't exist in the physical file system on your machine. Instead `trackfs` creates them on the fly whenever an application starts loading any of the track files. This usually takes (depending on your system) a few seconds. @@ -82,7 +83,7 @@ Instead it is recommended to let `trackfs` run as a regular user. For that to wo `trackfs` provides a few options that allow you to tweak its default behavior: -* `-e EXTENSION`, `--extension EXTENSION` (default: "(\\.flac|\\.wav)") : +* `-e EXTENSION`, `--extension EXTENSION` (default: "(\\.flac|\\.wav|\\.mp3)") : A regular expression matching file extension of album files in the music library * `-s SEPARATOR`, `--separator SEPARATOR` (default: ".#-#."): The separator used inside the name of the track-files. Must never occur in regular filenames diff --git a/README.py.md b/README.py.md index 16b1d55..7ce9b6e 100644 --- a/README.py.md +++ b/README.py.md @@ -40,17 +40,18 @@ preconditions / has the following software installed: * **[python](https://www.python.org/)**: use recent a python version (>=3.8) (trackfs is developped and tested with 3.8), including pip * **[fuse](https://github.com/libfuse/libfuse)**: make sure that you have FUSE support enabled in your kernal and the FUSE libraries installed * **[flac](https://xiph.org/flac/)**: make sure you have official flac binaries (flac and metaflac) installed and on your path +* **[mp3splt](http://mp3splt.sourceforge.net/)**: make sure you have official mp3splt binaries installed and on your path On most recent debian based system you should get all dependencies with ``` -sudo apt-get install python3 python3-pip fuse libfuse-dev flac +sudo apt-get install python3 python3-pip fuse libfuse-dev flac mp3splt ``` On alpine linux (used for the dockerized version of `trackfs`) you would use ``` -sudo apk add python3 py3-pip fuse fuse-dev flac +sudo apk add python3 py3-pip fuse fuse-dev flac mp3splt ``` #### Verify that you have the expected python version diff --git a/trackfs/albuminfo.py b/trackfs/albuminfo.py index 8052fb8..6528a08 100644 --- a/trackfs/albuminfo.py +++ b/trackfs/albuminfo.py @@ -13,7 +13,7 @@ from functools import lru_cache, cached_property from typing import List, Optional -from mutagen import File +from mutagen import File, MutagenError import chardet from . import cuesheet @@ -35,7 +35,10 @@ def __init__(self, path): @cached_property def meta(self) -> File: - return File(self.path) + try: + return File(self.path) + except MutagenError: + return type("DummyType", (object,), {'tags': None})() def format(self) -> str: return type(self.meta).__name__.upper() diff --git a/trackfs/cuesheet.py b/trackfs/cuesheet.py index bfa2c34..696879a 100644 --- a/trackfs/cuesheet.py +++ b/trackfs/cuesheet.py @@ -258,6 +258,9 @@ def seconds(self): def flac_time(self): return f'{self.mm:02d}:{self.ss:02d}.{int(100.0 / 75.0 * self.ff):02d}' + def mp3split_time(self): + return f'{self.mm:02d}.{self.ss:02d}.{int(100.0 / 75.0 * self.ff):02d}' + def __repr__(self): return '%02d%02d%02d' % (self.mm, self.ss, self.ff) diff --git a/trackfs/flactracks.py b/trackfs/flactracks.py index 0ba1b42..d0208a6 100644 --- a/trackfs/flactracks.py +++ b/trackfs/flactracks.py @@ -249,6 +249,35 @@ def _extract_wave_track(self, path: os.PathLike, fp: FusePath, album_info: album return track_file + def _extract_mp3_track(self, path: os.PathLike, fp: FusePath, album_info: albuminfo.AlbumInfo) -> os.PathLike: + """creates a real file for a given virtual track file + + extracts the track from the underlying MP3+CUE file into + a temporary file and then opens the temporary file""" + log.info(f'open track "{path}"') + + track_file = self._new_temp_filename() + + track = album_info.track(fp.num) + mp3_cmd = ( + f'mp3splt -Q -d {os.path.dirname(track_file)} -o {os.path.splitext(os.path.basename(track_file))[0]} ' + f'-g "r%[@o,@n={track.num},@t={shlex.quote(track.title)},@a={shlex.quote(" ".join(track.artists))}]" ' + f'{track.start.mp3split_time()} {track.end.mp3split_time()} ' + f'"{fp.source}" && mv {track_file}.* {track_file}' + ) + log.debug(f'extracting track with command: "{mp3_cmd}"') + rc = run(mp3_cmd, shell=True, stdout=None, stderr=DEVNULL).returncode + with self.rwlock: + if rc != 0: + err_msg = f'failed to extract track #{fp.num} from file "{fp.source}"' + log.error(err_msg) + os.remove(track_file) + del self[path] + raise FlacSplitException(err_msg) + else: + self._add(path, track_file) + return track_file + def prepare_track(self, path: os.PathLike, fp: FusePath) -> os.PathLike: log.info(f'prepare track "{path}"') assert fp.is_track @@ -271,6 +300,8 @@ def prepare_track(self, path: os.PathLike, fp: FusePath) -> os.PathLike: return self._extract_wave_track(path, fp, album_info) elif audio_format == 'FLAC': return self._extract_flac_track(path, fp, album_info) + elif audio_format == 'MP3': + return self._extract_mp3_track(path, fp, album_info) else: err_msg = f'unexpected audio format "{audio_format}"; can\'t proceed' log.error(err_msg) @@ -354,12 +385,20 @@ def estimate_track_file_size(self, path: os.PathLike, fp: FusePath) -> int: album_info = albuminfo.get(fp.source) meta = album_info.meta track = album_info.track(fp.num) - return int( - (track.end - track.start).seconds() - * meta.info.channels - * (meta.info.bits_per_sample / 8) - * meta.info.sample_rate - ) + if getattr(meta.info, 'bits_per_sample', None): # flac + return int( + (track.end - track.start).seconds() + * meta.info.channels + * (meta.info.bits_per_sample / 8) + * meta.info.sample_rate + ) + elif getattr(meta.info, 'bitrate', None): # mp3 + return int( + (track.end - track.start).seconds() + * meta.info.bitrate / 8 + ) + else: + return 0 else: # use the actual size of the track-file return os.stat(track_info.temp_file_path).st_size diff --git a/trackfs/fusepath.py b/trackfs/fusepath.py index 602539a..1796a05 100644 --- a/trackfs/fusepath.py +++ b/trackfs/fusepath.py @@ -29,10 +29,9 @@ DEFAULT_TRACK_SEPARATOR : str = '.#-#.' DEFAULT_MAX_TITLE_LEN : int = 20 -DEFAULT_ALBUM_EXTENSION : str = '(?i:\\.flac|\\.wav)' +DEFAULT_ALBUM_EXTENSION : str = '(?i:\\.flac|\\.wav|\\.mp3)' DEFAULT_VALID_CHARS : str = "-_() " + string.ascii_letters + string.digits DEFAULT_KEEP_ALBUM : bool = False -DEFAULT_TRACK_EXTENSION : str = '.flac' @dataclass(frozen=True) class Factory: @@ -43,16 +42,14 @@ class Factory: album_extension : str = DEFAULT_ALBUM_EXTENSION valid_filename_chars : str = DEFAULT_VALID_CHARS keep_album : bool = DEFAULT_KEEP_ALBUM - track_extension : bool = DEFAULT_TRACK_EXTENSION - + @cached_property def track_file_regex(self): separator_rex = self.track_separator.replace('.','\\.') - track_exentension_rex = self.track_extension.replace('.','\\.') flac_cue_rex = ( '^(?P.*)(?P'+self.album_extension+')'+separator_rex + '(?P\\d+)(?P(\\.[^\\.]{,'+str(self.max_title_len) - + '}?)?)'+track_exentension_rex+'$' + + '}?)?)\\.[^.]+$' ) log.debug("Factory.track_file_regex: "+flac_cue_rex) return re.compile(flac_cue_rex) @@ -108,8 +105,6 @@ def track_file_regex(self): return self._factory.track_file_regex def album_ext_regex(self): return self._factory.album_ext_regex @property def keep_album(self): return self._factory.keep_album - @property - def track_extension(self): return self._factory.track_extension @cached_property def source(self): @@ -129,7 +124,7 @@ def vpath(self): if(self.is_track): return ( f'{self.source_root}{self.extension}{self.track_separator}{self.num:03d}' - f'{self.title_fragment}{self.track_extension}' + f'{self.title_fragment}{self.extension}' ) else: return self.source From 4db181a0a84cbc06c4e38483fba7fc977f6d9c25 Mon Sep 17 00:00:00 2001 From: Ivan Zverev <izverev@gmail.com> Date: Thu, 8 Apr 2021 15:06:57 +0300 Subject: [PATCH 2/4] version fix --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 53a75d6..967b33f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.6 +0.2.7 \ No newline at end of file From b84be42c81880cc8931caea23db5a947529276b1 Mon Sep 17 00:00:00 2001 From: Ivan Zverev <izverev@gmail.com> Date: Mon, 12 Apr 2021 21:20:27 +0300 Subject: [PATCH 3/4] flac & mp3 shell extraction refactoring + VEXTENSION support --- VERSION | 2 +- trackfs/flactracks.py | 58 ++++++++++++++++++++++++------------------- trackfs/fusepath.py | 8 ++++-- 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/VERSION b/VERSION index 967b33f..9325c3c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.7 \ No newline at end of file +0.3.0 \ No newline at end of file diff --git a/trackfs/flactracks.py b/trackfs/flactracks.py index d0208a6..06c93e0 100644 --- a/trackfs/flactracks.py +++ b/trackfs/flactracks.py @@ -15,7 +15,7 @@ from dataclasses import dataclass from math import trunc from tempfile import mkstemp -from subprocess import DEVNULL, run +from subprocess import DEVNULL, PIPE, Popen, run from concurrent.futures import ThreadPoolExecutor from typing import Tuple, Dict, Optional from threading import RLock, Thread @@ -138,8 +138,8 @@ def track_tags_as_flac_args(album_info, num): return ' '.join([TrackManager._tag_as_flac_arg(k, v) for k, vs in tags.items() for v in vs]) @staticmethod - def _new_temp_filename() -> os.PathLike: - (fh, temp_file) = mkstemp() + def _new_temp_filename(**kwargs) -> os.PathLike: + (fh, temp_file) = mkstemp(**kwargs) # we don't want to process the file in python; just want a unique filename # that we let flac write the track into # => close right away @@ -166,26 +166,33 @@ def _extract_flac_track(self, path: os.PathLike, fp: FusePath, album_info: album # extract picture from flac if available picture_file = self._new_temp_filename() - metaflac_cmd = f'metaflac --export-picture-to="{picture_file}" "{fp.source}"' - log.debug(f'extracting picture with command: "{metaflac_cmd}"') - rc = run(metaflac_cmd, shell=True, stdout=None, stderr=DEVNULL).returncode + metaflac_cmd = ['metaflac', '--export-picture-to', picture_file, fp.source] + log.debug(f'extracting picture with command: "{" ".join(metaflac_cmd)}"') + rc = run(metaflac_cmd, stdout=None, stderr=DEVNULL).returncode picture_arg = "" if rc == 0: - picture_arg = f' --picture="{picture_file}"' + picture_arg = picture_file else: local_album_art = self._find_albmum_art(fp) if local_album_art: - picture_arg = f' --picture="{local_album_art}"' + picture_arg = local_album_art track = album_info.track(fp.num) - flac_cmd = ( - f'flac -d --silent --stdout --skip={track.start.flac_time()}' - f' --until={track.end.flac_time()} "{fp.source}" ' - f'| flac --silent -f --fast' - f' {self.track_tags_as_flac_args(album_info,fp.num)}{picture_arg} -o {track_file} -' - ) - log.debug(f'extracting track with command: "{flac_cmd}"') - rc = run(flac_cmd, shell=True, stdout=None, stderr=DEVNULL).returncode + flac_cmd_in = ['flac', '-d', '--silent', '--stdout', + '--skip', track.start.flac_time(), '--until', track.end.flac_time(), fp.source] + ps = Popen(flac_cmd_in, stdout=PIPE) + + flac_cmd_out = ['flac', '--silent', '-f', '--fast'] + for k, vs in album_info.track_tags(fp.num).items(): + for v in vs: + flac_cmd_out.extend(['--tag', f'{str(k).replace("=", " ")}={str(v).replace("=", " ")}']) + if picture_arg: + flac_cmd_out.extend(['--picture', picture_arg]) + flac_cmd_out.extend(['-o', track_file, '-']) + + log.debug(f'extracting track with command: "{" ".join(flac_cmd_in)} | {" ".join(flac_cmd_out)}"') + rc = run(flac_cmd_out, stdin=ps.stdout, stdout=None, stderr=DEVNULL).returncode + os.remove(picture_file) with self.rwlock: if rc != 0: @@ -256,17 +263,18 @@ def _extract_mp3_track(self, path: os.PathLike, fp: FusePath, album_info: albumi a temporary file and then opens the temporary file""" log.info(f'open track "{path}"') - track_file = self._new_temp_filename() + track_file = self._new_temp_filename(suffix='.mp3') # mp3splt adds .mp3 ext track = album_info.track(fp.num) - mp3_cmd = ( - f'mp3splt -Q -d {os.path.dirname(track_file)} -o {os.path.splitext(os.path.basename(track_file))[0]} ' - f'-g "r%[@o,@n={track.num},@t={shlex.quote(track.title)},@a={shlex.quote(" ".join(track.artists))}]" ' - f'{track.start.mp3split_time()} {track.end.mp3split_time()} ' - f'"{fp.source}" && mv {track_file}.* {track_file}' - ) - log.debug(f'extracting track with command: "{mp3_cmd}"') - rc = run(mp3_cmd, shell=True, stdout=None, stderr=DEVNULL).returncode + mp3_cmd = [ + 'mp3splt', '-Q', '-d', os.path.dirname(track_file), + '-o', os.path.splitext(os.path.basename(track_file))[0], + '-g', f'r%[@o,@n={track.num},@t={track.title},@a={" ".join(track.artists)}]', + track.start.mp3split_time(), track.end.mp3split_time(), fp.source, + ] + + log.debug(f'extracting track with command: "{" ".join(mp3_cmd)}"') + rc = run(mp3_cmd, stdout=None, stderr=DEVNULL).returncode with self.rwlock: if rc != 0: err_msg = f'failed to extract track #{fp.num} from file "{fp.source}"' diff --git a/trackfs/fusepath.py b/trackfs/fusepath.py index 1796a05..0292ebc 100644 --- a/trackfs/fusepath.py +++ b/trackfs/fusepath.py @@ -83,6 +83,10 @@ def from_track(self, source_root, extension, track): @dataclass(frozen=True) class FusePath: + VEXTENSION = { + # 'mpeg3': 'mp3', possible use + 'wav': 'flac' # all wav files saves with flac extension to preserve metadata + } ''' represents an entry in the virtual trackfs filesystem''' source_root : str extension : str @@ -124,9 +128,9 @@ def vpath(self): if(self.is_track): return ( f'{self.source_root}{self.extension}{self.track_separator}{self.num:03d}' - f'{self.title_fragment}{self.extension}' + f'{self.title_fragment}{self.VEXTENSION.get(self.extension, self.extension)}' ) - else: + else: return self.source def dirname(self): From 0afab48f1ceda5ec4e3065e32fb4b6d3837d74ec Mon Sep 17 00:00:00 2001 From: Ivan Zverev <izverev@gmail.com> Date: Mon, 12 Feb 2024 14:46:43 +0300 Subject: [PATCH 4/4] track manager artists fix --- trackfs/flactracks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trackfs/flactracks.py b/trackfs/flactracks.py index 06c93e0..00652c2 100644 --- a/trackfs/flactracks.py +++ b/trackfs/flactracks.py @@ -269,7 +269,7 @@ def _extract_mp3_track(self, path: os.PathLike, fp: FusePath, album_info: albumi mp3_cmd = [ 'mp3splt', '-Q', '-d', os.path.dirname(track_file), '-o', os.path.splitext(os.path.basename(track_file))[0], - '-g', f'r%[@o,@n={track.num},@t={track.title},@a={" ".join(track.artists)}]', + '-g', f'r%[@o,@n={track.num},@t={track.title},@a={" ".join(track.artists or [])}]', track.start.mp3split_time(), track.end.mp3split_time(), fp.source, ]