initial repo creation & code import

Dockerfile

@@ -0,0 +1,58 @@
+# static ffmpeg to use (built in other docker container)
+# https://github.com/jrottenberg/ffmpeg
+FROM my-ffmpeg-static:latest as builder
+
+# main os for image
+FROM ubuntu:20.04
+
+# statically built ffmpeg
+COPY --from=builder /ffmpeg /usr/local/bin/
+COPY --from=builder /ffprobe /usr/local/bin/
+
+# base os packages needed
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt update && apt upgrade -y \
+    && apt install -y wget nano sqlite jq less git imagemagick python3 python3-pip \
+    && apt install -y libqt5concurrent5 libqt5core5a libtag1v5 python3-dev libchromaprint-dev libeigen3-dev libfftw3-dev libsamplerate0 libyaml-dev libavformat58 libavfilter7 libswresample3 libavcodec58 libswscale5 libavdevice58 libavutil56 \
+    && mkdir /opt/tmp/ \
+    && cd /opt/tmp/ \
+    && wget https://github.com/acoustid/chromaprint/releases/download/v1.5.1/chromaprint-fpcalc-1.5.1-linux-x86_64.tar.gz \
+    && tar -xzf chromaprint-fpcalc-1.5.1-linux-x86_64.tar.gz \
+    && mv chromaprint*/fpcalc /usr/local/bin \
+    && wget https://github.com/doctorfree/mpplus-essentia/releases/download/v1.0.1r2/mpplus-essentia_1.0.1-2.amd64.deb \
+    && dpkg -i mpplus-essentia_1.0.1-2.amd64.deb
+
+# add beets user
+RUN useradd -u 1000 -m -d /home/beets -s /bin/bash beets
+# ensure perms on /opt are proper
+RUN chown -R 1000 /opt
+# flip to non-root user and setup beets as a user app/install
+USER 1000
+RUN pip install --user -U numpy \
+    && pip install --user -U flask pyacoustid pylast requests pillow \
+    && pip install --user -U git+https://github.com/beetbox/beets.git \
+    && pip install --user -U beets-xtractor beets-describe beets-alternatives \
+    && pip install --user -U git+https://github.com/steven-murray/beet-summarize.git
+
+# general env stuff
+ENV BEETSDIR="/opt/music/beets/"
+ENV EDITOR="nano"
+RUN echo "export PATH=/home/beets/.local/bin:${PATH}" >> /home/beets/.bashrc
+COPY bash_aliases /home/beets/.bash_aliases
+COPY duplicate_alternatives.py /usr/local/bin/duplicate_alternatives.py
+WORKDIR /opt/music
+
+# volumes
+VOLUME /opt/music/beets
+VOLUME /opt/music/library
+VOLUME /opt/music/to_import
+VOLUME /opt/music/alternatives
+VOLUME /opt/music/unimported
+VOLUME /opt/music/missing
+VOLUME /opt/music/dupes
+
+# port for beets web ui
+EXPOSE 8337
+
+# just run bash, let folk do whatever they need and however they want
+ENTRYPOINT ["/bin/bash"]

README.md

@@ -0,0 +1,5 @@
+# beets music library management
+
+A generalized container setup for beets and some org-mode notes on using beets to manage a large music library. An org-capture template is also included for those doing regular imports/adds of new music.
+
+See the `_notes.org` and `_template_beets.org` files for specifics.

_notes.org

@@ -0,0 +1,170 @@
+* beets
+
+** container build & run
+
+#+begin_src sh
+
+#+end_src
+
+** misc commands
+
+   - ~beets web # web ui~
+   - ~beets info --library | less # library data~
+   - ~beets info | less # file data~
+   - ~beets replaygain # run replay gain checks on all tracks~
+   - ~beets xtractor # run xtractor on all tracks~
+   - ~beets-flac xt 'mb_trackid::^.+$'~
+   - ~jq '.metadata.tags' /opt/music/beets/xtraction_data/[uid].json | less~
+   - ~beets-flac ls -f '$artist - $title - $mb_trackid' 'mb_trackid::^.+$'~
+
+** tagging and searches
+
+https://beets.readthedocs.io/en/stable/reference/query.html
+
+- use carat (^) to negate queries
+- ~beets fields~ to print database fields
+- ~beets describe~ to aggregate queries
+- ~beets ls -f '$albumartist - $title - is_mail: $is_male'~
+   - ~-f~ is for format strings and can be used adjust output string 
+   - can print any fields/tags from library database
+- ~beets ls [-a] trust in trance~
+   - ~-a~ prints album info using passed query string ; no ~-a~ prints tracks
+- ~beets ls -p trust in trance~
+   - prints paths
+   - can be used to print album folder path, not just track paths
+- ~beets-library info -l -a albumartist:Shinedown sound madness~
+   - show library data for an album
+   - remove ~-a~ to operate at item level
+
+- ~beets ls -f '$id - $albumartist - $album - $path' path:/opt/music/beets/dupes/~
+   - finds items at a specific path
+   - use this to cleanup bad items (deleted dupes/similar)
+   - follow up with the following to remove items at the path
+      - ~beets rm path:/opt/music/beets/dupes/~
+
+- ~beets-library ls -f '$albumartist - $album - $title - $mb_albumid' mb_albumid::^$~
+   - find all tracks without music brains id
+   - add ~-a~ flag to get albums
+
+- ~beets ls -a 'added:2024-05-08'~
+   - show all albums added on a specific day
+   - remove ~-a~ to search by item
+- ~beets ls -a 'added:-1w..'~
+   - show all albums added in the last week
+   - remove ~-a~ to search by item
+- ~beets ls -f '$added - $albumartist - $album %ifdef{title, - $title}'~
+   - list when all tracks were added to library
+   - using ~-a~ will show for album (this can differ from tracks if 'missing' were filled in later)
+- ~beets modify [-a] [query] [field=value] [field!]~
+   - ~-a~ operates at album level
+   - ~field=value~ to set field values
+   - ~field!~ to remove field from items
+- ~beets ls my_seed:True~
+   - show all items with custom tag set to true
+- ~beets ls ^my_seed:True~
+   - show all items missing custom tag or having it set to false
+- ~beets ls my_import:..2~
+   - show all items with int field less than or equal to 2
+   - use ~n..~ for greater than
+   - omit ~..~ and just give ~n~ to match explicit value
+
+- commands related to replay gain
+   - ~beets-library ls rg_album_gain::^$~
+   - ~beets-library ls rg_track_gain::^$~
+   #+begin_src sh
+   beets-library ls -a -f '$r128_album_gain - $rg_album_gain' \
+      my_import:[n] ^rg_album_gain::^$ , my_import:[n] ^r128_album_gain::^$
+   beets-library ls -f '$r128_album_gain - $r128_track_gain - $rg_album_gain - $rg_track_gain' \
+      my_import:[n] ^rg_track_gain::^$ , my_import:[n] ^r128_track_gain::^$
+   #+end_src
+- command to set music brains id and update Metadata accordingly
+   - ~beets modify -a $id mb_albumid=blah && beets mbsync...~
+   - ~beets modify $id mb_trackid=blah && beets mbsync...~
+** importing new files/tracks/etc (full order of operations)
+
+#+begin_src sh
+# beets commands to run, in order for every import/addition to beets
+beets-library unimported # list all the files in the library folder which are not in the beets database
+beets-library describe my_import # check to figure out what the next serial number is for custom import tag
+#beets-library import --set my_import=[n] --set my_seed=[True|False] /opt/music/to_import # main import
+beets-library import -l /opt/music/beets/import.log --set my_import=[n] --set my_seed=True /opt/music/to_import
+#beets-library stats # check library data is ok or at least 'looks reasonable plus or minus dupes or bad tracks'
+#beets-library info --library | less # show file metadata (in bulk)
+beets-library duplicates | uniq -d # check for dupes
+beets-library duplicates --move /opt/music/dupes # remove dupes from lib and move dupes for verification / follow up as appropriate
+beets-library unimported # ensure no dangling files in library prior to missing track cleanup
+beets-library unimported | xargs -I{} mv {} /opt/music/unimported/ # move unimported 'cruft' out of library for future follow up
+beets-library missing # check for missing tracks
+beets-library ls -f '$albumartist - $album - $path' -a missing:1.. # list all ablums with one or more missing tracks
+beets-library remove -a missing:1.. # does not change filesystem ; removes albums with missing tracks from library
+beets-library unimported # verify this looks appropriate and matches the output from missing
+beets-library unimported | xargs -I{} mv {} /opt/music/missing/ # move missing out of library for future follow up
+beets-library bad my_import:[n] # bad/missing files check
+beets-library move # move files to library directory
+beets-library describe my_import # figure out what imorts to process - use as query (filter) with remaining commands
+# use my_import:[n].. to filter on import n and later, see query ranges for added options
+#    best to add the import filter to below commands to save processing time
+#beets-library mbsync my_import:[n] # update music brainz data (not stricly required but smart if re-importing or re-exporting in bulk)
+beets-library ftintitle my_import:[n] # move 'featured by' to track title
+beets-library modify -a my_import:[n] r128_album_gain! rg_album_gain! # wipe album replay gain info in full
+beets-library modify my_import:[n] r128_album_gain! r128_track_gain! rg_album_gain! rg_track_gain! # wipe track replay gain info in full
+beets-library lastgenre my_import:[n] # pull down last.fm genre stuff
+beets-library fetchart  my_import:[n] # fetch cover art
+beets-library fingerprint my_import:[n] # chromaprint analysis
+beets-library ls -f '$artist - $title - $acoustid_fingerprint' | less # verify chromaprint worked (optional)
+beets-library xtractor my_import:[n] # essentia data extraction -- music analysis (long runtime)
+beets-library ls -f '$artist - $title - $mood_aggressive - $mood_electronic' | less # verify xtractor worked (optional)
+#\beet --config /opt/music/beets/library.yaml ls -a -f '$albumartist' | uniq | sort > /opt/music/to_import/_artists_for_rg.txt
+#cat /opt/music/to_import/_artists_for_rg.txt | xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain -w -f {}
+# replay gain with -a and non -a form of below for /full/ library replay gain analysis
+#beets-library -v replaygain -f -a my_import:[n] # do replaygain analysis on albums
+#beets-library -v replaygain -f my_import:[n] # do replaygain analysis on tracks
+# replay gain albums, can be resumed
+beets-library ls -a -f '$id' my_import:[n] rg_album_gain::^$ | \
+xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain -a id:{}
+# replay gain tracks, can be resumed
+beets-library ls -f '$id' my_import:[n] rg_track_gain::^$ | \
+xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain id:{}
+beets-library ls -f '$artist - $title - $rg_album_gain - $rg_track_gain' | less # verify replay gain worked (optional / limited test)
+beets-library scrub my_import:[n] # scrub file tags & write only beets tracked metadata to files
+#beets-library embedart # not needed if calling scrub -- embed cover art in files
+#beets-library write # not needed if calling scrub -- write changes to files
+beets-library move my_import:[n] # re-organize files based on latest metadata
+beets-library alt update airsonic # run conversion for airsonic alternatives profile -- manage the 'compressed audio library' automagically
+duplicate_alternatives # check for dupes in 'alternatives' area
+#+end_src
+
+** xtractor json decode exception monkey patch
+
+#+begin_src python
+#~/.local/lib/python3.8/site-packages/beetsplug/xtractor/helper.py
+def extract_from_output(output_path, target_map: Subview):
+    """extracts data from the json file as mapped out in the
+    `low_level_targets` / `high_level_targets` configuration keys
+    """
+    data = {}
+
+    if os.path.isfile(output_path):
+        with open(output_path, "r") as json_file:
+            audiodata = None
+            try:
+                audiodata = json.load(json_file)
+            except json.decoder.JSONDecodeError:
+               print('Error processing ', output_path)
+               from pathlib import Path
+               Path(output_path).unlink(missing_ok=True)
+               for key in target_map.keys():
+                   data[key] = None
+               return data
+            for key in target_map.keys():
+                try:
+                    val = extract_value_from_audiodata(audiodata, target_map[key])
+                except AttributeError:
+                    val = None
+
+                data[key] = val
+    else:
+        raise FileNotFoundError("Output file({}) not found!".format(output_path))
+
+    return data
+#+end_src

_template_beets.org

@@ -0,0 +1,91 @@
+* TODO %^{Title}
+%?##########
+# import
+##########
+# figure out what import number this is
+beets-library describe my_import
+# import tracks (watch out for ~my_seed~)
+beets-library import -l /opt/music/beets/import.log --set my_import=[n] [--set my_seed=True] /opt/music/to_import
+
+##########
+# initial processing / data move
+##########
+# move 'featured by' to track title prior to move so move is run only once
+beets-library ftintitle my_import:[n]
+# move files to library directory
+beets-library move my_import:[n]
+
+##########
+# cleanup jpg files & empty folders
+##########
+find /tank/Music/to_import -type f -iname \*.jpg -delete
+find /tank/Music/to_import -depth -empty -type d -delete
+
+##########
+# misc cleanup / bad file check
+##########
+# pre-flight 'junk' move prior to delete
+mkdir /opt/music/to_import/_delete_me
+mv /opt/music/to_import/* /opt/music/to_import/_delete_me/
+# bad/missing files check 
+beets-library bad my_import:[n]
+# delete 'junk' files from import zone
+rm -r /opt/music/to_import/_delete_me
+
+##########
+# litmus tests (optional)
+##########
+# check for dupes
+beets-library duplicates | uniq -d
+# remove dupes from lib and move dupes for verification / follow up as appropriate
+beets-library duplicates --move /opt/music/dupes
+# ensure no dangling files in library prior to missing track cleanup
+beets-library unimported
+beets-library unimported | xargs -I{} mv {} /opt/music/unimported/
+# check for missing tracks
+beets-library missing
+# list all ablums with one or more missing tracks
+beets-library ls -f '$albumartist - $album - $path' -a missing:1..
+# does not change filesystem ; removes albums with missing tracks from library
+beets-library remove -a missing:1..
+# verify this looks appropriate and matches the output from missing
+beets-library unimported
+# move missing out of library for future follow up
+beets-library unimported | xargs -I{} mv {} /opt/music/missing/
+
+##########
+# main processing
+##########
+# wipe replay gain values & process fresh (essentially: ignore values on incoming tracks)
+beets-library modify -a my_import:[n] r128_album_gain! rg_album_gain!
+beets-library modify my_import:[n] r128_album_gain! r128_track_gain! rg_album_gain! rg_track_gain!
+# pull down last.fm genre stuff
+beets-library lastgenre my_import:[n]
+# fetch cover art
+beets-library fetchart my_import:[n]
+# chromaprint analysis
+beets-library fingerprint my_import:[n]
+# essentia data extraction -- music analysis (long runtime)
+beets-library xtractor my_import:[n]
+# replay gain for albums (long runtime, can be resumed if above clear command run first)
+beets-library ls -a -f '$id' my_import:[n] rg_album_gain::^$ | \
+xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain -a id:{}
+# replay gain for tracks (long runtime, can be resumed if above clear command run first)
+beets-library ls -f '$id' my_import:[n] rg_track_gain::^$ | \
+xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain id:{}
+
+##########
+# cleanup files & generate files for music server
+##########
+# scrub file tags & write only beets tracked metadata to files
+beets-library scrub my_import:[n]
+# run conversion for airsonic alternatives profile -- manage the 'compressed audio library' automagically
+beets-library alt update airsonic
+# check for dupes in 'alternatives' area
+duplicate_alternatives
+
+##########
+# litmus test
+##########
+# check library data is ok or at least 'looks reasonable plus or minus dupes or bad tracks'
+beets-library stats

bash_aliases

@@ -0,0 +1,4 @@
+alias duplicate_alternatives="/usr/local/bin/duplicate_alternatives.py"
+alias beet="echo 'Please use beets-[library]' aliases"
+alias beets="echo 'Please use beets-[library]' aliases"
+alias beets-library="\beet --config /opt/music/beets/library.yaml"

beets_config/config.yaml

@@ -0,0 +1,160 @@
+##########
+# /opt/music/beets/config.yaml
+# this is /always/ read by beets ; leave it at this location
+#    NO need for include in main library configs
+# export BEETSDIR="/opt/music/beets"
+# beet --config /opt/music/beets/[library].yaml
+
+##########
+# in case of future need(s)
+# https://github.com/adammillerio/beets-copyartifacts
+
+plugins: xtractor replaygain web types describe info chroma summarize missing duplicates inline scrub ftintitle mbsync lastgenre embedart fetchart edit unimported badfiles convert alternatives
+
+threaded: yes
+ignore_hidden: true
+asciify_paths: yes
+original_date: yes
+
+ui:
+  color: yes
+
+web:
+  host: 0.0.0.0
+
+import:
+  autotag: yes
+  timid: yes
+  write: no
+  copy: no
+  move: no
+  log: /opt/tmp/beetslog.txt
+
+paths:
+  default: $albumartist/$album%aunique{}/%if{$multidisc,$disc - }$track - $title
+  singleton: $albumartist/[non-album tracks]/$title
+  comp: Various Artists/$album%aunique{}/%if{$multidisc,$disc - }/$track - $title
+  albumtype:soundtrack: Soundtrack/$album%aunique{}/%if{$multidisc,$disc - }/$track - $title
+
+item_fields:
+  multidisc: 1 if disctotal > 1 else 0
+
+types:
+  my_import: int
+  my_seed: bool
+
+unimported:
+  ignore_extensions: jpg png
+
+badfiles:
+  check_on_import: no
+  commands:
+    flac: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);'
+    m4a: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);'
+    mp3: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);'
+    aac: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);'
+    ape: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);'
+
+duplicates:
+  format: $albumartist - $album - $track - $path
+  full: yes
+
+match:
+  preferred:
+    countries: [ 'US', 'XW' ]
+    media: ['CD', 'Digital Media|File']
+    original_year: yes
+
+ftintitle:
+  auto: no
+  drop: no
+  format: "feat. {0}"
+
+embedart:
+  auto: no
+
+scrub:
+  auto: no
+
+lastgenre:
+  count: 1
+  prefer_specific: yes
+  source: album
+
+fetchart:
+  auto: no
+  google_key: ""
+  lastfm_key: ""
+  sources:
+    #- filesystem
+    - coverart: release releasegroup
+    - itunes
+    - lastfm
+  store_source: no
+  min_width: 1000
+  max_width: 1500
+
+chroma:
+  auto: no
+
+replaygain:
+  auto: no
+  write: no
+  threads: 10 # change me
+  parallel_on_import: yes
+  backend: ffmpeg
+  overwrite: no
+
+convert:
+  delete_originals: false
+  auto: no
+  copy_album_art: yes
+  embed: yes
+  never_convert_lossy_files: yes
+  threads: 10 # change me
+  format: m4a
+  formats:
+    m4a:
+      extension: m4a
+      command: ffmpeg -i $source -y -c:v copy -c:a libfdk_aac -vbr 5 $dest
+    aac:
+      extension: aac
+      command: ffmpeg -i $source -y -c:a libfdk_aac -vbr 5 $dest
+    # unused / i prefer vbr fdk_aac for compressed audio ; below command is what i used for mp3 prior to using beets
+    #mp3:
+    #  extension: mp3
+    #  command: lame -V 0 -q 0 -m s $source $dest
+
+alternatives:
+  airsonic:
+    directory: /opt/music/alternatives
+    formats: m4a aac mp3
+    query: ""
+    removable: false
+
+xtractor:
+  auto: yes
+  dry-run: no
+  write: no
+  threads: 10 # change me
+  force: no
+  quiet: no
+  keep_output: yes
+  keep_profiles: no
+  output_path: /opt/music/beets/xtraction_data
+  essentia_extractor: /usr/bin/essentia_streaming_extractor_music
+  extractor_profile:
+    highlevel:
+      svm_models:
+        - /usr/share/mpplus-essentia/svm_models/danceability.history
+        - /usr/share/mpplus-essentia/svm_models/gender.history
+        - /usr/share/mpplus-essentia/svm_models/genre_rosamerica.history
+        - /usr/share/mpplus-essentia/svm_models/mood_acoustic.history
+        - /usr/share/mpplus-essentia/svm_models/mood_aggressive.history
+        - /usr/share/mpplus-essentia/svm_models/mood_electronic.history
+        - /usr/share/mpplus-essentia/svm_models/mood_happy.history
+        - /usr/share/mpplus-essentia/svm_models/mood_sad.history
+        - /usr/share/mpplus-essentia/svm_models/mood_party.history
+        - /usr/share/mpplus-essentia/svm_models/mood_relaxed.history
+        - /usr/share/mpplus-essentia/svm_models/voice_instrumental.history
+        - /usr/share/mpplus-essentia/svm_models/moods_mirex.history

beets_config/library.yaml

@@ -0,0 +1,5 @@
+##########
+# /opt/music/beets/library.yaml
+statefile: /opt/music/beets/library.state.pickle
+library: /opt/music/beets/library.db
+directory: /opt/music/library

build.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# git clone https://github.com/wader/static-ffmpeg.git ./wader-static-ffmpeg
+cd ./wader-static-ffmpeg
+git pull
+docker build --pull --no-cache --build-arg ENABLE_FDKAAC=1 -t my-ffmpeg-static:latest .
+
+cd ../
+docker pull ubuntu:20.04
+docker build --no-cache --tag beets:latest .

duplicate_alternatives.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+import os
+import os.path
+import pprint
+from collections import Counter
+
+base_path = '/opt/music/alternatives'
+
+possible_dupes = Counter({})
+
+for sub_path in os.listdir(base_path):
+	for root, dirs, files in os.walk(os.path.join(base_path, sub_path)):
+		for file in files:
+			to_check = os.path.join(root, os.path.splitext(file)[0])
+			to_check = to_check.lower() # make the check case insensitive
+			possible_dupes.update([to_check,]) # ensure counter doesnt unpack a path string into distinct letters
+
+duplicates = {key:value for key, value in possible_dupes.items() if value > 1}
+pprint.pprint(duplicates)

run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# build latest image
+./build.sh
+
+# run with local storage mounted in container
+#     uses host networking to ensure web ui is visible outside container
+docker run --rm -itu 1000 --name beets \
+    --net host \
+    -v /tank/Music/alternatives:/opt/music/alternatives \
+    -v /tank/Music/beets:/opt/music/beets \
+    -v /tank/Music/dupes:/opt/music/dupes \
+    -v /tank/Music/library:/opt/music/library \
+    -v /tank/Music/missing:/opt/music/missing \
+    -v /tank/Music/to_import:/opt/music/to_import \
+    -v /tank/Music/unimported:/opt/music/unimported \
+    beets:latest