1
0
Fork 0

initial repo creation & code import

This commit is contained in:
KemoNine 2024-06-02 10:43:45 -04:00
commit 5cf9db9589
10 changed files with 539 additions and 0 deletions

58
Dockerfile Normal file
View file

@ -0,0 +1,58 @@
# static ffmpeg to use (built in other docker container)
# https://github.com/jrottenberg/ffmpeg
FROM my-ffmpeg-static:latest as builder
# main os for image
FROM ubuntu:20.04
# statically built ffmpeg
COPY --from=builder /ffmpeg /usr/local/bin/
COPY --from=builder /ffprobe /usr/local/bin/
# base os packages needed
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt upgrade -y \
&& apt install -y wget nano sqlite jq less git imagemagick python3 python3-pip \
&& apt install -y libqt5concurrent5 libqt5core5a libtag1v5 python3-dev libchromaprint-dev libeigen3-dev libfftw3-dev libsamplerate0 libyaml-dev libavformat58 libavfilter7 libswresample3 libavcodec58 libswscale5 libavdevice58 libavutil56 \
&& mkdir /opt/tmp/ \
&& cd /opt/tmp/ \
&& wget https://github.com/acoustid/chromaprint/releases/download/v1.5.1/chromaprint-fpcalc-1.5.1-linux-x86_64.tar.gz \
&& tar -xzf chromaprint-fpcalc-1.5.1-linux-x86_64.tar.gz \
&& mv chromaprint*/fpcalc /usr/local/bin \
&& wget https://github.com/doctorfree/mpplus-essentia/releases/download/v1.0.1r2/mpplus-essentia_1.0.1-2.amd64.deb \
&& dpkg -i mpplus-essentia_1.0.1-2.amd64.deb
# add beets user
RUN useradd -u 1000 -m -d /home/beets -s /bin/bash beets
# ensure perms on /opt are proper
RUN chown -R 1000 /opt
# flip to non-root user and setup beets as a user app/install
USER 1000
RUN pip install --user -U numpy \
&& pip install --user -U flask pyacoustid pylast requests pillow \
&& pip install --user -U git+https://github.com/beetbox/beets.git \
&& pip install --user -U beets-xtractor beets-describe beets-alternatives \
&& pip install --user -U git+https://github.com/steven-murray/beet-summarize.git
# general env stuff
ENV BEETSDIR="/opt/music/beets/"
ENV EDITOR="nano"
RUN echo "export PATH=/home/beets/.local/bin:${PATH}" >> /home/beets/.bashrc
COPY bash_aliases /home/beets/.bash_aliases
COPY duplicate_alternatives.py /usr/local/bin/duplicate_alternatives.py
WORKDIR /opt/music
# volumes
VOLUME /opt/music/beets
VOLUME /opt/music/library
VOLUME /opt/music/to_import
VOLUME /opt/music/alternatives
VOLUME /opt/music/unimported
VOLUME /opt/music/missing
VOLUME /opt/music/dupes
# port for beets web ui
EXPOSE 8337
# just run bash, let folk do whatever they need and however they want
ENTRYPOINT ["/bin/bash"]

5
README.md Executable file
View file

@ -0,0 +1,5 @@
# beets music library management
A generalized container setup for beets and some org-mode notes on using beets to manage a large music library. An org-capture template is also included for those doing regular imports/adds of new music.
See the `_notes.org` and `_template_beets.org` files for specifics.

170
_notes.org Executable file
View file

@ -0,0 +1,170 @@
* beets
** container build & run
#+begin_src sh
#+end_src
** misc commands
- ~beets web # web ui~
- ~beets info --library | less # library data~
- ~beets info | less # file data~
- ~beets replaygain # run replay gain checks on all tracks~
- ~beets xtractor # run xtractor on all tracks~
- ~beets-flac xt 'mb_trackid::^.+$'~
- ~jq '.metadata.tags' /opt/music/beets/xtraction_data/[uid].json | less~
- ~beets-flac ls -f '$artist - $title - $mb_trackid' 'mb_trackid::^.+$'~
** tagging and searches
https://beets.readthedocs.io/en/stable/reference/query.html
- use carat (^) to negate queries
- ~beets fields~ to print database fields
- ~beets describe~ to aggregate queries
- ~beets ls -f '$albumartist - $title - is_mail: $is_male'~
- ~-f~ is for format strings and can be used adjust output string
- can print any fields/tags from library database
- ~beets ls [-a] trust in trance~
- ~-a~ prints album info using passed query string ; no ~-a~ prints tracks
- ~beets ls -p trust in trance~
- prints paths
- can be used to print album folder path, not just track paths
- ~beets-library info -l -a albumartist:Shinedown sound madness~
- show library data for an album
- remove ~-a~ to operate at item level
- ~beets ls -f '$id - $albumartist - $album - $path' path:/opt/music/beets/dupes/~
- finds items at a specific path
- use this to cleanup bad items (deleted dupes/similar)
- follow up with the following to remove items at the path
- ~beets rm path:/opt/music/beets/dupes/~
- ~beets-library ls -f '$albumartist - $album - $title - $mb_albumid' mb_albumid::^$~
- find all tracks without music brains id
- add ~-a~ flag to get albums
- ~beets ls -a 'added:2024-05-08'~
- show all albums added on a specific day
- remove ~-a~ to search by item
- ~beets ls -a 'added:-1w..'~
- show all albums added in the last week
- remove ~-a~ to search by item
- ~beets ls -f '$added - $albumartist - $album %ifdef{title, - $title}'~
- list when all tracks were added to library
- using ~-a~ will show for album (this can differ from tracks if 'missing' were filled in later)
- ~beets modify [-a] [query] [field=value] [field!]~
- ~-a~ operates at album level
- ~field=value~ to set field values
- ~field!~ to remove field from items
- ~beets ls my_seed:True~
- show all items with custom tag set to true
- ~beets ls ^my_seed:True~
- show all items missing custom tag or having it set to false
- ~beets ls my_import:..2~
- show all items with int field less than or equal to 2
- use ~n..~ for greater than
- omit ~..~ and just give ~n~ to match explicit value
- commands related to replay gain
- ~beets-library ls rg_album_gain::^$~
- ~beets-library ls rg_track_gain::^$~
#+begin_src sh
beets-library ls -a -f '$r128_album_gain - $rg_album_gain' \
my_import:[n] ^rg_album_gain::^$ , my_import:[n] ^r128_album_gain::^$
beets-library ls -f '$r128_album_gain - $r128_track_gain - $rg_album_gain - $rg_track_gain' \
my_import:[n] ^rg_track_gain::^$ , my_import:[n] ^r128_track_gain::^$
#+end_src
- command to set music brains id and update Metadata accordingly
- ~beets modify -a $id mb_albumid=blah && beets mbsync...~
- ~beets modify $id mb_trackid=blah && beets mbsync...~
** importing new files/tracks/etc (full order of operations)
#+begin_src sh
# beets commands to run, in order for every import/addition to beets
beets-library unimported # list all the files in the library folder which are not in the beets database
beets-library describe my_import # check to figure out what the next serial number is for custom import tag
#beets-library import --set my_import=[n] --set my_seed=[True|False] /opt/music/to_import # main import
beets-library import -l /opt/music/beets/import.log --set my_import=[n] --set my_seed=True /opt/music/to_import
#beets-library stats # check library data is ok or at least 'looks reasonable plus or minus dupes or bad tracks'
#beets-library info --library | less # show file metadata (in bulk)
beets-library duplicates | uniq -d # check for dupes
beets-library duplicates --move /opt/music/dupes # remove dupes from lib and move dupes for verification / follow up as appropriate
beets-library unimported # ensure no dangling files in library prior to missing track cleanup
beets-library unimported | xargs -I{} mv {} /opt/music/unimported/ # move unimported 'cruft' out of library for future follow up
beets-library missing # check for missing tracks
beets-library ls -f '$albumartist - $album - $path' -a missing:1.. # list all ablums with one or more missing tracks
beets-library remove -a missing:1.. # does not change filesystem ; removes albums with missing tracks from library
beets-library unimported # verify this looks appropriate and matches the output from missing
beets-library unimported | xargs -I{} mv {} /opt/music/missing/ # move missing out of library for future follow up
beets-library bad my_import:[n] # bad/missing files check
beets-library move # move files to library directory
beets-library describe my_import # figure out what imorts to process - use as query (filter) with remaining commands
# use my_import:[n].. to filter on import n and later, see query ranges for added options
# best to add the import filter to below commands to save processing time
#beets-library mbsync my_import:[n] # update music brainz data (not stricly required but smart if re-importing or re-exporting in bulk)
beets-library ftintitle my_import:[n] # move 'featured by' to track title
beets-library modify -a my_import:[n] r128_album_gain! rg_album_gain! # wipe album replay gain info in full
beets-library modify my_import:[n] r128_album_gain! r128_track_gain! rg_album_gain! rg_track_gain! # wipe track replay gain info in full
beets-library lastgenre my_import:[n] # pull down last.fm genre stuff
beets-library fetchart my_import:[n] # fetch cover art
beets-library fingerprint my_import:[n] # chromaprint analysis
beets-library ls -f '$artist - $title - $acoustid_fingerprint' | less # verify chromaprint worked (optional)
beets-library xtractor my_import:[n] # essentia data extraction -- music analysis (long runtime)
beets-library ls -f '$artist - $title - $mood_aggressive - $mood_electronic' | less # verify xtractor worked (optional)
#\beet --config /opt/music/beets/library.yaml ls -a -f '$albumartist' | uniq | sort > /opt/music/to_import/_artists_for_rg.txt
#cat /opt/music/to_import/_artists_for_rg.txt | xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain -w -f {}
# replay gain with -a and non -a form of below for /full/ library replay gain analysis
#beets-library -v replaygain -f -a my_import:[n] # do replaygain analysis on albums
#beets-library -v replaygain -f my_import:[n] # do replaygain analysis on tracks
# replay gain albums, can be resumed
beets-library ls -a -f '$id' my_import:[n] rg_album_gain::^$ | \
xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain -a id:{}
# replay gain tracks, can be resumed
beets-library ls -f '$id' my_import:[n] rg_track_gain::^$ | \
xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain id:{}
beets-library ls -f '$artist - $title - $rg_album_gain - $rg_track_gain' | less # verify replay gain worked (optional / limited test)
beets-library scrub my_import:[n] # scrub file tags & write only beets tracked metadata to files
#beets-library embedart # not needed if calling scrub -- embed cover art in files
#beets-library write # not needed if calling scrub -- write changes to files
beets-library move my_import:[n] # re-organize files based on latest metadata
beets-library alt update airsonic # run conversion for airsonic alternatives profile -- manage the 'compressed audio library' automagically
duplicate_alternatives # check for dupes in 'alternatives' area
#+end_src
** xtractor json decode exception monkey patch
#+begin_src python
#~/.local/lib/python3.8/site-packages/beetsplug/xtractor/helper.py
def extract_from_output(output_path, target_map: Subview):
"""extracts data from the json file as mapped out in the
`low_level_targets` / `high_level_targets` configuration keys
"""
data = {}
if os.path.isfile(output_path):
with open(output_path, "r") as json_file:
audiodata = None
try:
audiodata = json.load(json_file)
except json.decoder.JSONDecodeError:
print('Error processing ', output_path)
from pathlib import Path
Path(output_path).unlink(missing_ok=True)
for key in target_map.keys():
data[key] = None
return data
for key in target_map.keys():
try:
val = extract_value_from_audiodata(audiodata, target_map[key])
except AttributeError:
val = None
data[key] = val
else:
raise FileNotFoundError("Output file({}) not found!".format(output_path))
return data
#+end_src

91
_template_beets.org Executable file
View file

@ -0,0 +1,91 @@
* TODO %^{Title}
%?##########
# import
##########
# figure out what import number this is
beets-library describe my_import
# import tracks (watch out for ~my_seed~)
beets-library import -l /opt/music/beets/import.log --set my_import=[n] [--set my_seed=True] /opt/music/to_import
##########
# initial processing / data move
##########
# move 'featured by' to track title prior to move so move is run only once
beets-library ftintitle my_import:[n]
# move files to library directory
beets-library move my_import:[n]
##########
# cleanup jpg files & empty folders
##########
find /tank/Music/to_import -type f -iname \*.jpg -delete
find /tank/Music/to_import -depth -empty -type d -delete
##########
# misc cleanup / bad file check
##########
# pre-flight 'junk' move prior to delete
mkdir /opt/music/to_import/_delete_me
mv /opt/music/to_import/* /opt/music/to_import/_delete_me/
# bad/missing files check
beets-library bad my_import:[n]
# delete 'junk' files from import zone
rm -r /opt/music/to_import/_delete_me
##########
# litmus tests (optional)
##########
# check for dupes
beets-library duplicates | uniq -d
# remove dupes from lib and move dupes for verification / follow up as appropriate
beets-library duplicates --move /opt/music/dupes
# ensure no dangling files in library prior to missing track cleanup
beets-library unimported
beets-library unimported | xargs -I{} mv {} /opt/music/unimported/
# check for missing tracks
beets-library missing
# list all ablums with one or more missing tracks
beets-library ls -f '$albumartist - $album - $path' -a missing:1..
# does not change filesystem ; removes albums with missing tracks from library
beets-library remove -a missing:1..
# verify this looks appropriate and matches the output from missing
beets-library unimported
# move missing out of library for future follow up
beets-library unimported | xargs -I{} mv {} /opt/music/missing/
##########
# main processing
##########
# wipe replay gain values & process fresh (essentially: ignore values on incoming tracks)
beets-library modify -a my_import:[n] r128_album_gain! rg_album_gain!
beets-library modify my_import:[n] r128_album_gain! r128_track_gain! rg_album_gain! rg_track_gain!
# pull down last.fm genre stuff
beets-library lastgenre my_import:[n]
# fetch cover art
beets-library fetchart my_import:[n]
# chromaprint analysis
beets-library fingerprint my_import:[n]
# essentia data extraction -- music analysis (long runtime)
beets-library xtractor my_import:[n]
# replay gain for albums (long runtime, can be resumed if above clear command run first)
beets-library ls -a -f '$id' my_import:[n] rg_album_gain::^$ | \
xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain -a id:{}
# replay gain for tracks (long runtime, can be resumed if above clear command run first)
beets-library ls -f '$id' my_import:[n] rg_track_gain::^$ | \
xargs -d "\n" -n1 -o -t -I{} beet --config /opt/music/beets/library.yaml -v replaygain id:{}
##########
# cleanup files & generate files for music server
##########
# scrub file tags & write only beets tracked metadata to files
beets-library scrub my_import:[n]
# run conversion for airsonic alternatives profile -- manage the 'compressed audio library' automagically
beets-library alt update airsonic
# check for dupes in 'alternatives' area
duplicate_alternatives
##########
# litmus test
##########
# check library data is ok or at least 'looks reasonable plus or minus dupes or bad tracks'
beets-library stats

4
bash_aliases Normal file
View file

@ -0,0 +1,4 @@
alias duplicate_alternatives="/usr/local/bin/duplicate_alternatives.py"
alias beet="echo 'Please use beets-[library]' aliases"
alias beets="echo 'Please use beets-[library]' aliases"
alias beets-library="\beet --config /opt/music/beets/library.yaml"

160
beets_config/config.yaml Normal file
View file

@ -0,0 +1,160 @@
##########
# /opt/music/beets/config.yaml
# this is /always/ read by beets ; leave it at this location
# NO need for include in main library configs
# export BEETSDIR="/opt/music/beets"
# beet --config /opt/music/beets/[library].yaml
##########
# in case of future need(s)
# https://github.com/adammillerio/beets-copyartifacts
plugins: xtractor replaygain web types describe info chroma summarize missing duplicates inline scrub ftintitle mbsync lastgenre embedart fetchart edit unimported badfiles convert alternatives
threaded: yes
ignore_hidden: true
asciify_paths: yes
original_date: yes
ui:
color: yes
web:
host: 0.0.0.0
import:
autotag: yes
timid: yes
write: no
copy: no
move: no
log: /opt/tmp/beetslog.txt
paths:
default: $albumartist/$album%aunique{}/%if{$multidisc,$disc - }$track - $title
singleton: $albumartist/[non-album tracks]/$title
comp: Various Artists/$album%aunique{}/%if{$multidisc,$disc - }/$track - $title
albumtype:soundtrack: Soundtrack/$album%aunique{}/%if{$multidisc,$disc - }/$track - $title
item_fields:
multidisc: 1 if disctotal > 1 else 0
types:
my_import: int
my_seed: bool
unimported:
ignore_extensions: jpg png
badfiles:
check_on_import: no
commands:
flac: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);'
m4a: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);'
mp3: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);'
aac: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);'
ape: python3 -c 'import sys ; import os.path ; val = 0 if os.path.isfile(sys.argv[1]) else 1; sys.exit(val);'
duplicates:
format: $albumartist - $album - $track - $path
full: yes
match:
preferred:
countries: [ 'US', 'XW' ]
media: ['CD', 'Digital Media|File']
original_year: yes
ftintitle:
auto: no
drop: no
format: "feat. {0}"
embedart:
auto: no
scrub:
auto: no
lastgenre:
count: 1
prefer_specific: yes
source: album
fetchart:
auto: no
google_key: ""
lastfm_key: ""
sources:
#- filesystem
- coverart: release releasegroup
- itunes
- lastfm
store_source: no
min_width: 1000
max_width: 1500
chroma:
auto: no
replaygain:
auto: no
write: no
threads: 10 # change me
parallel_on_import: yes
backend: ffmpeg
overwrite: no
convert:
delete_originals: false
auto: no
copy_album_art: yes
embed: yes
never_convert_lossy_files: yes
threads: 10 # change me
format: m4a
formats:
m4a:
extension: m4a
command: ffmpeg -i $source -y -c:v copy -c:a libfdk_aac -vbr 5 $dest
aac:
extension: aac
command: ffmpeg -i $source -y -c:a libfdk_aac -vbr 5 $dest
# unused / i prefer vbr fdk_aac for compressed audio ; below command is what i used for mp3 prior to using beets
#mp3:
# extension: mp3
# command: lame -V 0 -q 0 -m s $source $dest
alternatives:
airsonic:
directory: /opt/music/alternatives
formats: m4a aac mp3
query: ""
removable: false
xtractor:
auto: yes
dry-run: no
write: no
threads: 10 # change me
force: no
quiet: no
keep_output: yes
keep_profiles: no
output_path: /opt/music/beets/xtraction_data
essentia_extractor: /usr/bin/essentia_streaming_extractor_music
extractor_profile:
highlevel:
svm_models:
- /usr/share/mpplus-essentia/svm_models/danceability.history
- /usr/share/mpplus-essentia/svm_models/gender.history
- /usr/share/mpplus-essentia/svm_models/genre_rosamerica.history
- /usr/share/mpplus-essentia/svm_models/mood_acoustic.history
- /usr/share/mpplus-essentia/svm_models/mood_aggressive.history
- /usr/share/mpplus-essentia/svm_models/mood_electronic.history
- /usr/share/mpplus-essentia/svm_models/mood_happy.history
- /usr/share/mpplus-essentia/svm_models/mood_sad.history
- /usr/share/mpplus-essentia/svm_models/mood_party.history
- /usr/share/mpplus-essentia/svm_models/mood_relaxed.history
- /usr/share/mpplus-essentia/svm_models/voice_instrumental.history
- /usr/share/mpplus-essentia/svm_models/moods_mirex.history

View file

@ -0,0 +1,5 @@
##########
# /opt/music/beets/library.yaml
statefile: /opt/music/beets/library.state.pickle
library: /opt/music/beets/library.db
directory: /opt/music/library

9
build.sh Executable file
View file

@ -0,0 +1,9 @@
#!/bin/bash
# git clone https://github.com/wader/static-ffmpeg.git ./wader-static-ffmpeg
cd ./wader-static-ffmpeg
git pull
docker build --pull --no-cache --build-arg ENABLE_FDKAAC=1 -t my-ffmpeg-static:latest .
cd ../
docker pull ubuntu:20.04
docker build --no-cache --tag beets:latest .

20
duplicate_alternatives.py Executable file
View file

@ -0,0 +1,20 @@
#!/usr/bin/env python3
import os
import os.path
import pprint
from collections import Counter
base_path = '/opt/music/alternatives'
possible_dupes = Counter({})
for sub_path in os.listdir(base_path):
for root, dirs, files in os.walk(os.path.join(base_path, sub_path)):
for file in files:
to_check = os.path.join(root, os.path.splitext(file)[0])
to_check = to_check.lower() # make the check case insensitive
possible_dupes.update([to_check,]) # ensure counter doesnt unpack a path string into distinct letters
duplicates = {key:value for key, value in possible_dupes.items() if value > 1}
pprint.pprint(duplicates)

17
run.sh Executable file
View file

@ -0,0 +1,17 @@
#!/bin/bash
# build latest image
./build.sh
# run with local storage mounted in container
# uses host networking to ensure web ui is visible outside container
docker run --rm -itu 1000 --name beets \
--net host \
-v /tank/Music/alternatives:/opt/music/alternatives \
-v /tank/Music/beets:/opt/music/beets \
-v /tank/Music/dupes:/opt/music/dupes \
-v /tank/Music/library:/opt/music/library \
-v /tank/Music/missing:/opt/music/missing \
-v /tank/Music/to_import:/opt/music/to_import \
-v /tank/Music/unimported:/opt/music/unimported \
beets:latest