From 5cf9db9589f24dadec64dd011fad19a302f2dfa2 Mon Sep 17 00:00:00 2001 From: KemoNine Date: Sun, 2 Jun 2024 10:43:45 -0400 Subject: [PATCH] initial repo creation & code import --- Dockerfile | 58 +++++++++++++ README.md | 5 ++ _notes.org | 170 ++++++++++++++++++++++++++++++++++++++ _template_beets.org | 91 ++++++++++++++++++++ bash_aliases | 4 + beets_config/config.yaml | 160 +++++++++++++++++++++++++++++++++++ beets_config/library.yaml | 5 ++ build.sh | 9 ++ duplicate_alternatives.py | 20 +++++ run.sh | 17 ++++ 10 files changed, 539 insertions(+) create mode 100644 Dockerfile create mode 100755 README.md create mode 100755 _notes.org create mode 100755 _template_beets.org create mode 100644 bash_aliases create mode 100644 beets_config/config.yaml create mode 100644 beets_config/library.yaml create mode 100755 build.sh create mode 100755 duplicate_alternatives.py create mode 100755 run.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ab9592e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,58 @@ +# static ffmpeg to use (built in other docker container) +# https://github.com/jrottenberg/ffmpeg +FROM my-ffmpeg-static:latest as builder + +# main os for image +FROM ubuntu:20.04 + +# statically built ffmpeg +COPY --from=builder /ffmpeg /usr/local/bin/ +COPY --from=builder /ffprobe /usr/local/bin/ + +# base os packages needed +ENV DEBIAN_FRONTEND=noninteractive +RUN apt update && apt upgrade -y \ + && apt install -y wget nano sqlite jq less git imagemagick python3 python3-pip \ + && apt install -y libqt5concurrent5 libqt5core5a libtag1v5 python3-dev libchromaprint-dev libeigen3-dev libfftw3-dev libsamplerate0 libyaml-dev libavformat58 libavfilter7 libswresample3 libavcodec58 libswscale5 libavdevice58 libavutil56 \ + && mkdir /opt/tmp/ \ + && cd /opt/tmp/ \ + && wget https://github.com/acoustid/chromaprint/releases/download/v1.5.1/chromaprint-fpcalc-1.5.1-linux-x86_64.tar.gz \ + && tar -xzf chromaprint-fpcalc-1.5.1-linux-x86_64.tar.gz \ + && mv chromaprint*/fpcalc /usr/local/bin \ + && wget https://github.com/doctorfree/mpplus-essentia/releases/download/v1.0.1r2/mpplus-essentia_1.0.1-2.amd64.deb \ + && dpkg -i mpplus-essentia_1.0.1-2.amd64.deb + +# add beets user +RUN useradd -u 1000 -m -d /home/beets -s /bin/bash beets +# ensure perms on /opt are proper +RUN chown -R 1000 /opt +# flip to non-root user and setup beets as a user app/install +USER 1000 +RUN pip install --user -U numpy \ + && pip install --user -U flask pyacoustid pylast requests pillow \ + && pip install --user -U git+https://github.com/beetbox/beets.git \ + && pip install --user -U beets-xtractor beets-describe beets-alternatives \ + && pip install --user -U git+https://github.com/steven-murray/beet-summarize.git + +# general env stuff +ENV BEETSDIR="/opt/music/beets/" +ENV EDITOR="nano" +RUN echo "export PATH=/home/beets/.local/bin:${PATH}" >> /home/beets/.bashrc +COPY bash_aliases /home/beets/.bash_aliases +COPY duplicate_alternatives.py /usr/local/bin/duplicate_alternatives.py +WORKDIR /opt/music + +# volumes +VOLUME /opt/music/beets +VOLUME /opt/music/library +VOLUME /opt/music/to_import +VOLUME /opt/music/alternatives +VOLUME /opt/music/unimported +VOLUME /opt/music/missing +VOLUME /opt/music/dupes + +# port for beets web ui +EXPOSE 8337 + +# just run bash, let folk do whatever they need and however they want +ENTRYPOINT ["/bin/bash"] diff --git a/README.md b/README.md new file mode 100755 index 0000000..c65cd88 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# beets music library management + +A generalized container setup for beets and some org-mode notes on using beets to manage a large music library. An org-capture template is also included for those doing regular imports/adds of new music. + +See the `_notes.org` and `_template_beets.org` files for specifics. diff --git a/_notes.org b/_notes.org new file mode 100755 index 0000000..a34c4b5 --- /dev/null +++ b/_notes.org @@ -0,0 +1,170 @@ +* beets + +** container build & run + +#+begin_src sh + +#+end_src + +** misc commands + + - ~beets web # web ui~ + - ~beets info --library | less # library data~ + - ~beets info | less # file data~ + - ~beets replaygain # run replay gain checks on all tracks~ + - ~beets xtractor # run xtractor on all tracks~ + - ~beets-flac xt 'mb_trackid::^.+$'~ + - ~jq '.metadata.tags' /opt/music/beets/xtraction_data/[uid].json | less~ + - ~beets-flac ls -f '$artist - $title - $mb_trackid' 'mb_trackid::^.+$'~ + +** tagging and searches + +https://beets.readthedocs.io/en/stable/reference/query.html + +- use carat (^) to negate queries +- ~beets fields~ to print database fields +- ~beets describe~ to aggregate queries +- ~beets ls -f '$albumartist - $title - is_mail: $is_male'~ + - ~-f~ is for format strings and can be used adjust output string + - can print any fields/tags from library database +- ~beets ls [-a] trust in trance~ + - ~-a~ prints album info using passed query string ; no ~-a~ prints tracks +- ~beets ls -p trust in trance~ + - prints paths + - can be used to print album folder path, not just track paths +- ~beets-library info -l -a albumartist:Shinedown sound madness~ + - show library data for an album + - remove ~-a~ to operate at item level + +- ~beets ls -f '$id - $albumartist - $album - $path' path:/opt/music/beets/dupes/~ + - finds items at a specific path + - use this to cleanup bad items (deleted dupes/similar) + - follow up with the following to remove items at the path + - ~beets rm path:/opt/music/beets/dupes/~ + +- ~beets-library ls -f '$albumartist - $album - $title - $mb_albumid' mb_albumid::^$~ + - find all tracks without music brains id + - add ~-a~ flag to get albums + +- ~beets ls -a 'added:2024-05-08'~ + - show all albums added on a specific day + - remove ~-a~ to search by item +- ~beets ls -a 'added:-1w..'~ + - show all albums added in the last week + - remove ~-a~ to search by item +- ~beets ls -f '$added - $albumartist - $album %ifdef{title, - $title}'~ + - list when all tracks were added to library + - using ~-a~ will show for album (this can differ from tracks if 'missing' were filled in later) +- ~beets modify [-a] [query] [field=value] [field!]~ + - ~-a~ operates at album level + - ~field=value~ to set field values + - ~field!~ to remove field from items +- ~beets ls my_seed:True~ + - show all items with custom tag set to true +- ~beets ls ^my_seed:True~ + - show all items missing custom tag or having it set to false +- ~beets ls my_import:..2~ + - show all items with int field less than or equal to 2 + - use ~n..~ for greater than + - omit ~..~ and just give ~n~ to match explicit value + +- commands related to replay gain + - ~beets-library ls rg_album_gain::^$~ + - ~beets-library ls rg_track_gain::^$~ + #+begin_src sh + beets-library ls -a -f '$r128_album_gain - $rg_album_gain' \ + my_import:[n] ^rg_album_gain::^$ , my_import:[n] ^r128_album_gain::^$ + beets-library ls -f '$r128_album_gain - $r128_track_gain - $rg_album_gain - $rg_track_gain' \ + my_import:[n] ^rg_track_gain::^$ , my_import:[n] ^r128_track_gain::^$ + #+end_src +- command to set music brains id and update Metadata accordingly + - ~beets modify -a $id mb_albumid=blah && beets mbsync...~ + - ~beets modify $id mb_trackid=blah && beets mbsync...~ +** importing new files/tracks/etc (full order of operations) + +#+begin_src sh +# beets commands to run, in order for every import/addition to beets +beets-library unimported # list all the files in the library folder which are not in the beets database +beets-library describe my_import # check to figure out what the next serial number is for custom import tag +#beets-library import --set my_import=[n] --set my_seed=[True|False] /opt/music/to_import # main import +beets-library import -l /opt/music/beets/import.log --set my_import=[n] --set my_seed=True /opt/music/to_import +#beets-library stats # check library data is ok or at least 'looks reasonable plus or minus dupes or bad tracks' +#beets-library info --library | less # show file metadata (in bulk) +beets-library duplicates | uniq -d # check for dupes +beets-library duplicates --move /opt/music/dupes # remove dupes from lib and move dupes for verification / follow up as appropriate +beets-library unimported # ensure no dangling files in library prior to missing track cleanup +beets-library unimported | xargs -I{} mv {} /opt/music/unimported/ # move unimported 'cruft' out of library for future follow up +beets-library missing # check for missing tracks +beets-library ls -f '$albumartist - $album - $path' -a missing:1.. # list all ablums with one or more missing tracks +beets-library remove -a missing:1.. # does not change filesystem ; removes albums with missing tracks from library +beets-library unimported # verify this looks appropriate and matches the output from missing +beets-library unimported | xargs -I{} mv {} /opt/music/missing/ # move missing out of library for future follow up +beets-library bad my_import:[n] # bad/missing files check +beets-library move # move files to library directory +beets-library describe my_import # figure out what imorts to process - use as query (filter) with remaining commands +# use my_import:[n].. to filter on import n and later, see query ranges for added options +# best to add the import filter to below commands to save processing time +#beets-library mbsync my_import:[n] # update music brainz data (not stricly required but smart if re-importing or re-exporting in bulk) +beets-library ftintitle my_import:[n] # move 'featured by' to track title +beets-library modify -a my_import:[n] r128_album_gain! rg_album_gain! # wipe album replay gain info in full +beets-library modify my_import:[n] r128_album_gain! r128_track_gain! rg_album_gain! rg_track_gain! # wipe track replay gain info in full +beets-library lastgenre my_import:[n] # pull down last.fm genre stuff +beets-library fetchart my_import:[n] # fetch cover art +beets-library fingerprint my_import:[n] # chromaprint analysis +beets-library ls -f '$artist - $title - $acoustid_fingerprint' | less # verify chromaprint worked (optional) +beets-library xtractor my_import:[n] # essentia data extraction -- music analysis (long runtime) +beets-library ls -f '$artist - $title - $mood_aggressive - $mood_electronic' | less # verify xtractor worked (optional) +#\beet --config /opt/music/beets/library.yaml ls -a -f '$albumartist' | uniq | sort > /opt/music/to_import/_artists_for_rg.txt +#cat /opt/music/to_import/_artists_for_rg.txt | xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain -w -f {} +# replay gain with -a and non -a form of below for /full/ library replay gain analysis +#beets-library -v replaygain -f -a my_import:[n] # do replaygain analysis on albums +#beets-library -v replaygain -f my_import:[n] # do replaygain analysis on tracks +# replay gain albums, can be resumed +beets-library ls -a -f '$id' my_import:[n] rg_album_gain::^$ | \ +xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain -a id:{} +# replay gain tracks, can be resumed +beets-library ls -f '$id' my_import:[n] rg_track_gain::^$ | \ +xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain id:{} +beets-library ls -f '$artist - $title - $rg_album_gain - $rg_track_gain' | less # verify replay gain worked (optional / limited test) +beets-library scrub my_import:[n] # scrub file tags & write only beets tracked metadata to files +#beets-library embedart # not needed if calling scrub -- embed cover art in files +#beets-library write # not needed if calling scrub -- write changes to files +beets-library move my_import:[n] # re-organize files based on latest metadata +beets-library alt update airsonic # run conversion for airsonic alternatives profile -- manage the 'compressed audio library' automagically +duplicate_alternatives # check for dupes in 'alternatives' area +#+end_src + +** xtractor json decode exception monkey patch + +#+begin_src python +#~/.local/lib/python3.8/site-packages/beetsplug/xtractor/helper.py +def extract_from_output(output_path, target_map: Subview): + """extracts data from the json file as mapped out in the + `low_level_targets` / `high_level_targets` configuration keys + """ + data = {} + + if os.path.isfile(output_path): + with open(output_path, "r") as json_file: + audiodata = None + try: + audiodata = json.load(json_file) + except json.decoder.JSONDecodeError: + print('Error processing ', output_path) + from pathlib import Path + Path(output_path).unlink(missing_ok=True) + for key in target_map.keys(): + data[key] = None + return data + for key in target_map.keys(): + try: + val = extract_value_from_audiodata(audiodata, target_map[key]) + except AttributeError: + val = None + + data[key] = val + else: + raise FileNotFoundError("Output file({}) not found!".format(output_path)) + + return data +#+end_src diff --git a/_template_beets.org b/_template_beets.org new file mode 100755 index 0000000..54b636c --- /dev/null +++ b/_template_beets.org @@ -0,0 +1,91 @@ +* TODO %^{Title} +%?########## +# import +########## +# figure out what import number this is +beets-library describe my_import +# import tracks (watch out for ~my_seed~) +beets-library import -l /opt/music/beets/import.log --set my_import=[n] [--set my_seed=True] /opt/music/to_import + +########## +# initial processing / data move +########## +# move 'featured by' to track title prior to move so move is run only once +beets-library ftintitle my_import:[n] +# move files to library directory +beets-library move my_import:[n] + +########## +# cleanup jpg files & empty folders +########## +find /tank/Music/to_import -type f -iname \*.jpg -delete +find /tank/Music/to_import -depth -empty -type d -delete + +########## +# misc cleanup / bad file check +########## +# pre-flight 'junk' move prior to delete +mkdir /opt/music/to_import/_delete_me +mv /opt/music/to_import/* /opt/music/to_import/_delete_me/ +# bad/missing files check +beets-library bad my_import:[n] +# delete 'junk' files from import zone +rm -r /opt/music/to_import/_delete_me + +########## +# litmus tests (optional) +########## +# check for dupes +beets-library duplicates | uniq -d +# remove dupes from lib and move dupes for verification / follow up as appropriate +beets-library duplicates --move /opt/music/dupes +# ensure no dangling files in library prior to missing track cleanup +beets-library unimported +beets-library unimported | xargs -I{} mv {} /opt/music/unimported/ +# check for missing tracks +beets-library missing +# list all ablums with one or more missing tracks +beets-library ls -f '$albumartist - $album - $path' -a missing:1.. +# does not change filesystem ; removes albums with missing tracks from library +beets-library remove -a missing:1.. +# verify this looks appropriate and matches the output from missing +beets-library unimported +# move missing out of library for future follow up +beets-library unimported | xargs -I{} mv {} /opt/music/missing/ + +########## +# main processing +########## +# wipe replay gain values & process fresh (essentially: ignore values on incoming tracks) +beets-library modify -a my_import:[n] r128_album_gain! rg_album_gain! +beets-library modify my_import:[n] r128_album_gain! r128_track_gain! rg_album_gain! rg_track_gain! +# pull down last.fm genre stuff +beets-library lastgenre my_import:[n] +# fetch cover art +beets-library fetchart my_import:[n] +# chromaprint analysis +beets-library fingerprint my_import:[n] +# essentia data extraction -- music analysis (long runtime) +beets-library xtractor my_import:[n] +# replay gain for albums (long runtime, can be resumed if above clear command run first) +beets-library ls -a -f '$id' my_import:[n] rg_album_gain::^$ | \ +xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain -a id:{} +# replay gain for tracks (long runtime, can be resumed if above clear command run first) +beets-library ls -f '$id' my_import:[n] rg_track_gain::^$ | \ +xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain id:{} + +########## +# cleanup files & generate files for music server +########## +# scrub file tags & write only beets tracked metadata to files +beets-library scrub my_import:[n] +# run conversion for airsonic alternatives profile -- manage the 'compressed audio library' automagically +beets-library alt update airsonic +# check for dupes in 'alternatives' area +duplicate_alternatives + +########## +# litmus test +########## +# check library data is ok or at least 'looks reasonable plus or minus dupes or bad tracks' +beets-library stats \ No newline at end of file diff --git a/bash_aliases b/bash_aliases new file mode 100644 index 0000000..6433cba --- /dev/null +++ b/bash_aliases @@ -0,0 +1,4 @@ +alias duplicate_alternatives="/usr/local/bin/duplicate_alternatives.py" +alias beet="echo 'Please use beets-[library]' aliases" +alias beets="echo 'Please use beets-[library]' aliases" +alias beets-library="\beet --config /opt/music/beets/library.yaml" diff --git a/beets_config/config.yaml b/beets_config/config.yaml new file mode 100644 index 0000000..f9f60ff --- /dev/null +++ b/beets_config/config.yaml @@ -0,0 +1,160 @@ +########## +# /opt/music/beets/config.yaml +# this is /always/ read by beets ; leave it at this location +# NO need for include in main library configs +# export BEETSDIR="/opt/music/beets" +# beet --config /opt/music/beets/[library].yaml + +########## +# in case of future need(s) +# https://github.com/adammillerio/beets-copyartifacts + +plugins: xtractor replaygain web types describe info chroma summarize missing duplicates inline scrub ftintitle mbsync lastgenre embedart fetchart edit unimported badfiles convert alternatives + +threaded: yes +ignore_hidden: true +asciify_paths: yes +original_date: yes + +ui: + color: yes + +web: + host: 0.0.0.0 + +import: + autotag: yes + timid: yes + write: no + copy: no + move: no + log: /opt/tmp/beetslog.txt + +paths: + default: $albumartist/$album%aunique{}/%if{$multidisc,$disc - }$track - $title + singleton: $albumartist/[non-album tracks]/$title + comp: Various Artists/$album%aunique{}/%if{$multidisc,$disc - }/$track - $title + albumtype:soundtrack: Soundtrack/$album%aunique{}/%if{$multidisc,$disc - }/$track - $title + +item_fields: + multidisc: 1 if disctotal > 1 else 0 + +types: + my_import: int + my_seed: bool + +unimported: + ignore_extensions: jpg png + +badfiles: + check_on_import: no + commands: + flac: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);' + m4a: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);' + mp3: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);' + aac: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);' + ape: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);' + +duplicates: + format: $albumartist - $album - $track - $path + full: yes + +match: + preferred: + countries: [ 'US', 'XW' ] + media: ['CD', 'Digital Media|File'] + original_year: yes + +ftintitle: + auto: no + drop: no + format: "feat. {0}" + +embedart: + auto: no + +scrub: + auto: no + +lastgenre: + count: 1 + prefer_specific: yes + source: album + +fetchart: + auto: no + google_key: "" + lastfm_key: "" + sources: + #- filesystem + - coverart: release releasegroup + - itunes + - lastfm + store_source: no + min_width: 1000 + max_width: 1500 + +chroma: + auto: no + +replaygain: + auto: no + write: no + threads: 10 # change me + parallel_on_import: yes + backend: ffmpeg + overwrite: no + +convert: + delete_originals: false + auto: no + copy_album_art: yes + embed: yes + never_convert_lossy_files: yes + threads: 10 # change me + format: m4a + formats: + m4a: + extension: m4a + command: ffmpeg -i $source -y -c:v copy -c:a libfdk_aac -vbr 5 $dest + aac: + extension: aac + command: ffmpeg -i $source -y -c:a libfdk_aac -vbr 5 $dest + # unused / i prefer vbr fdk_aac for compressed audio ; below command is what i used for mp3 prior to using beets + #mp3: + # extension: mp3 + # command: lame -V 0 -q 0 -m s $source $dest + +alternatives: + airsonic: + directory: /opt/music/alternatives + formats: m4a aac mp3 + query: "" + removable: false + +xtractor: + auto: yes + dry-run: no + write: no + threads: 10 # change me + force: no + quiet: no + keep_output: yes + keep_profiles: no + output_path: /opt/music/beets/xtraction_data + essentia_extractor: /usr/bin/essentia_streaming_extractor_music + extractor_profile: + highlevel: + svm_models: + - /usr/share/mpplus-essentia/svm_models/danceability.history + - /usr/share/mpplus-essentia/svm_models/gender.history + - /usr/share/mpplus-essentia/svm_models/genre_rosamerica.history + - /usr/share/mpplus-essentia/svm_models/mood_acoustic.history + - /usr/share/mpplus-essentia/svm_models/mood_aggressive.history + - /usr/share/mpplus-essentia/svm_models/mood_electronic.history + - /usr/share/mpplus-essentia/svm_models/mood_happy.history + - /usr/share/mpplus-essentia/svm_models/mood_sad.history + - /usr/share/mpplus-essentia/svm_models/mood_party.history + - /usr/share/mpplus-essentia/svm_models/mood_relaxed.history + - /usr/share/mpplus-essentia/svm_models/voice_instrumental.history + - /usr/share/mpplus-essentia/svm_models/moods_mirex.history diff --git a/beets_config/library.yaml b/beets_config/library.yaml new file mode 100644 index 0000000..7bbef12 --- /dev/null +++ b/beets_config/library.yaml @@ -0,0 +1,5 @@ +########## +# /opt/music/beets/library.yaml +statefile: /opt/music/beets/library.state.pickle +library: /opt/music/beets/library.db +directory: /opt/music/library diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..3e2ed88 --- /dev/null +++ b/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# git clone https://github.com/wader/static-ffmpeg.git ./wader-static-ffmpeg +cd ./wader-static-ffmpeg +git pull +docker build --pull --no-cache --build-arg ENABLE_FDKAAC=1 -t my-ffmpeg-static:latest . + +cd ../ +docker pull ubuntu:20.04 +docker build --no-cache --tag beets:latest . diff --git a/duplicate_alternatives.py b/duplicate_alternatives.py new file mode 100755 index 0000000..339edcf --- /dev/null +++ b/duplicate_alternatives.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import os +import os.path +import pprint +from collections import Counter + +base_path = '/opt/music/alternatives' + +possible_dupes = Counter({}) + +for sub_path in os.listdir(base_path): + for root, dirs, files in os.walk(os.path.join(base_path, sub_path)): + for file in files: + to_check = os.path.join(root, os.path.splitext(file)[0]) + to_check = to_check.lower() # make the check case insensitive + possible_dupes.update([to_check,]) # ensure counter doesnt unpack a path string into distinct letters + +duplicates = {key:value for key, value in possible_dupes.items() if value > 1} +pprint.pprint(duplicates) diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..4fe9094 --- /dev/null +++ b/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# build latest image +./build.sh + +# run with local storage mounted in container +# uses host networking to ensure web ui is visible outside container +docker run --rm -itu 1000 --name beets \ + --net host \ + -v /tank/Music/alternatives:/opt/music/alternatives \ + -v /tank/Music/beets:/opt/music/beets \ + -v /tank/Music/dupes:/opt/music/dupes \ + -v /tank/Music/library:/opt/music/library \ + -v /tank/Music/missing:/opt/music/missing \ + -v /tank/Music/to_import:/opt/music/to_import \ + -v /tank/Music/unimported:/opt/music/unimported \ + beets:latest