add monit, thin html dashboard on port 80, additional notes, secrets storage, msmtp-mta for email notifications

.gitignore

@@ -0,0 +1 @@
+secrets.sh

Dockerfile

@@ -1,9 +1,16 @@
 FROM ubuntu:latest
 
-# Munin specific
+# Munin
 VOLUME /opt/munin
 EXPOSE 2813
 
+# Monit
+VOLUME /opt/monit
+EXPOSE 2812/tcp
+
+# Dashboard
+EXPOSE 80
+
 # Run s6-overlay as the init so we get services and similar
 ENTRYPOINT [ "/init" ]
 
@@ -15,7 +22,7 @@ WORKDIR /opt/
 
 # Install and base setup all the things
 RUN apt update && apt upgrade -y && \
-  apt install -y curl jq xz-utils lm-sensors && \
+  apt install -y curl jq xz-utils lm-sensors nano && \
   S6_RELEASE=$(curl -sX GET "https://api.github.com/repos/just-containers/s6-overlay/tags" \
 	  | jq -r .[0].name); \
   echo "**** s6-overlay release: ${S6_RELEASE} ****" && \
@@ -28,6 +35,10 @@ RUN apt update && apt upgrade -y && \
   tar -C / -Jxpf /opt/s6-overlay-noarch.tar.xz && \
   echo "**** munin related ****" && \
   apt install -y munin lighttpd && \
+  echo "**** monit related ***" && \
+  apt install -y monit && \
+  echo "**** dashboard related ****" && \
+  apt install -y lighttpd && \
   echo "**** cleanup ****" && \
   rm /opt/s6-overlay*.tar.xz && \
   apt remove --purge -y jq xz-utils && \

README.md

@@ -1,6 +1,32 @@
-Simple container with munin leveraging s6 init for services/cron
+# What is this?
 
-Host config:
+Simple container proving munin and monit with msmtp-mta for email notifications
+
+# Security Considerations
+
+- This container does **NOT** setup SSL
+- This container does **NOT** setup any kind of proper auth for munin
+- Use a reverse proxy as appropriate for SSL and auth needs
+  - Take note of the `-p` directives in `run.sh` to ensure each service endpoint is properly proxied
+  - It is probably wise to remove the `-p` directives when using a reverse proxy for access
+
+# Container config
+
+- Create `secrets.sh` as appropriate, see `secrets.sh.example` for reference
+- (optoinal) Add `-v /var/containerized-monitoring/misc:/opt/misc` to `run.sh` if you want persistent storage for msmtp-mta config
+- (optional) Add `-v /var/containerized-monitoring/dashboard:/opt/dashboard` to `run.sh` if you want persistent storage for the simple html dashboard files
+
+# Container notes
+
+- I've included `nano` (<1Mb package) to facilitate debugging and tuning from directly within the container
+
+# Monit
+
+The included a default setup will perform base monitoring of the container (host). You'll want to source m/monit or similar for monitoring more than the local container/machine
+
+# Munin 
+
+## Host config (optional)
 
 ```
 pacman -S lm_sensors
@@ -8,4 +34,7 @@ sensors-detect # walk through this completely
 systemctl enable --now lm_sensors
 ```
 
-**This will throw errors for 5 minutes at first run due to lack of data, it'll self-correct when the munin cron job runs once**
+## Notes
+
+- This will throw errors for 5 minutes (max) at first run due to lack of data, it'll self-correct when the munin cron job runs the first time
+- The web front end for munin will be broken for 5 minutes (max) first run due to the way html rendering works in munin, it'll self-correct when the munin cron job runs the first time

build.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 git pull
-sudo docker build --no-cache --pull -t munin:latest -f Dockerfile .
+sudo docker build -t containerized-monitoring:latest -f Dockerfile .

root/etc/s6-overlay/s6-rc.d/dashboard/run

@@ -0,0 +1,3 @@
+#!/command/with-contenv bash
+
+/usr/sbin/lighttpd -D -f /opt/dashboard/lighttpd.conf

root/etc/s6-overlay/s6-rc.d/dashboard/type

@@ -0,0 +1 @@
+longrun

root/etc/s6-overlay/s6-rc.d/monit/run

@@ -0,0 +1,5 @@
+#!/command/with-contenv bash
+
+cd /opt/monit
+/usr/bin/monit -I \
+    -c /opt/monit/monitrc

root/etc/s6-overlay/s6-rc.d/monit/type

@@ -0,0 +1 @@
+longrun

root/etc/s6-overlay/s6-rc.d/setup/setup

@@ -1,5 +1,79 @@
 #!/command/with-contenv bash
 
+####################
+# Email notifications
+####################
+echo "---------- Setting up email notifications (msmtp-mta) ----------"
+if [ ! -d "/opt/misc" ] ; then
+  mkdir /opt/misc
+fi
+if [ ! -f "/opt/misc/msmtp.log" ] ; then
+touch /opt/misc/msmtp.log
+chown root: /opt/misc/msmtp.log
+chmod 666 /opt/misc/msmtp.log
+fi
+if [ ! -f "/opt/misc/aliases" ] ; then
+cat > /opt/misc/aliases <<EOF
+root: $CONFIG_NOTIFY_EMAIL
+cron: $CONFIG_NOTIFY_EMAIL
+default: $CONFIG_NOTIFY_EMAIL
+EOF
+fi
+if [ ! -f "/opt/misc/msmtprc" ] ; then
+cat > /opt/misc/msmtprc <<EOF
+# Accounts will inherit settings from this section
+defaults
+auth             on
+tls              on
+tls_trust_file   /etc/ssl/certs/ca-certificates.crt
+
+logfile        /opt/misc/msmtp.log
+
+from           $CONFIG_NOTIFY_SMTP_USER
+keepbcc        on
+
+account        containermonitoring
+host           $CONFIG_NOTIFY_SMTP_SERVER
+port           587
+auth           on
+user           $CONFIG_NOTIFY_SMTP_USER
+password       $CONFIG_NOTIFY_SMTP_PASS
+
+# Set a default account
+account default : containermonitoring
+
+aliases /etc/aliases
+EOF
+fi
+ln -sf /opt/misc/aliases /etc/aliases
+ln -sf /opt/misc/msmtprc /etc/msmtprc
+
+####################
+# Monit
+####################
+echo "---------- Setting up monit ----------"
+if [ ! -d "/opt/monit/conf.d" ] ; then
+  mkdir /opt/monit/conf.d
+fi
+if [ ! -f "/opt/monit/monitrc" ] ; then
+cat > /opt/monit/monitrc <<EOF
+set daemon 120
+set log /opt/monit/monit.log
+set idfile /opt/monit/id
+set statefile /opt/monit/state
+set eventqueue
+    basedir /opt/monit/events # set the base directory where events will be stored
+    slots 100                     # optionally limit the queue size
+
+set httpd port 2812 and
+    use address 0.0.0.0
+    allow ${CONFIG_MONIT_USER}:${CONFIG_MONIT_PASS}
+
+include /opt/monit/conf.d/*
+EOF
+chmod 600 /opt/monit/monitrc
+fi
+
 ####################
 # Munin
 ####################
@@ -157,3 +231,71 @@ echo "---------- Munin node setup ----------"
 rm /etc/munin/plugin-conf.d/*
 munin-node-configure --shell --remove-also --debug | sh -x
 #su - munin --shell=/bin/sh /usr/bin/munin-cron
+
+####################
+# Dashboard
+####################
+echo "---------- Setting up dashboard ----------"
+if [ ! -d "/opt/dashboard" ] ; then
+  mkdir /opt/dashboard
+fi
+if [ ! -f "/opt/dashboard/lighttpd.conf" ] ; then
+cat > /opt/dashboard/lighttpd.conf <<EOF
+server.username = "www-data"
+server.groupname = "www-data"
+
+server.document-root = "/opt/dashboard/www"
+server.port = 80
+
+server.errorlog         = "/dev/stdout"
+accesslog.filename      = "/dev/stdout"
+dir-listing.activate    = "disable"
+server.modules          = (
+                            "mod_access",
+                            "mod_accesslog",
+                            "mod_alias",
+                            "mod_rewrite",
+                            "mod_redirect",
+                            "mod_cgi",
+                            "mod_fastcgi",
+                            "mod_auth",
+                            "mod_authn_file",
+                            )
+server.pid-file         = "/run/lighttpd.pid"
+server.follow-symlink = "enable"
+index-file.names = ( "index.html", "index.htm" )
+
+mimetype.assign         = (
+                                ".html" => "text/html",
+                                ".txt" => "text/plain",
+                                ".css" => "text/css",
+                                ".js" => "application/x-javascript",
+                                ".jpg" => "image/jpeg",
+                                ".jpeg" => "image/jpeg",
+                                ".gif" => "image/gif",
+                                ".png" => "image/png",
+                                "" => "application/octet-stream"
+                        )
+EOF
+fi
+if [ ! -d "/opt/dashboard/www" ] ; then
+mkdir /opt/dashboard/www
+chown www-data /opt/dashboard/www
+fi
+if [ ! -f "/opt/dashboard/www/index.html" ] ; then
+cat > /opt/dashboard/www/index.html <<EOF
+<html>
+
+<head>
+<title>containerized-monitoring</title>
+</head>
+
+<body>
+<p>This page uses javascript to avoid manual entry of an ip for the href tag. Ports for the service have been listed below in case you do not have a javascript enabled browser.</p>
+<p><a href="#" onclick="javascript:window.location.port=2812">Monit Monitoring (port 2812)</a></p>
+<p><a href="#" onclick="javascript:window.location.port=2813">Munin Monitoring (port 2813)</a></p>
+</body>
+</html>
+EOF
+chown www-data /opt/dashboard/www/index.html
+fi

run.sh

@@ -1,24 +1,38 @@
 #!/bin/bash
 
-echo "**** Building latest Munin container ****"
+echo "**** Building latest monitoring container ****"
 $(pwd)/build.sh
 
-echo "**** Deleting Original Container ****"
-sudo docker rm -f munin
+echo "**** Loading secrets.sh ****"
+source ./secrets.sh
 
-echo "**** Running Munin ****"
+echo "**** Deleting existing container ****"
+sudo docker rm -f containerized-monitoring
+
+echo "**** Running monitoring ****"
 sudo docker run -itd \
     --restart unless-stopped \
     --network services \
-    --name munin \
+    --name containerized-monitoring \
     -e TZ=UTC \
     -e DEBUG=1 \
+    -e CONFIG_NOTIFY_EMAIL=$CONFIG_NOTIFY_EMAIL \
+    -e CONFIG_NOTIFY_SMTP_SERVER=$CONFIG_NOTIFY_SMTP_SERVER \
+    -e CONFIG_NOTIFY_SMTP_USER=$CONFIG_NOTIFY_SMTP_USER \
+    -e CONFIG_NOTIFY_SMTP_PASS=$CONFIG_NOTIFY_SMTP_PASS \
+    -e CONFIG_MONIT_USER=$CONFIG_MONIT_USER \
+    -e CONFIG_MONIT_PASS=$CONFIG_MONIT_PASS \
+    -p 2812:2812/tcp \
     -p 2813:2813/tcp \
+    -p 80:80/tcp \
     -l diun.enable=true \
     -l traefik.enable=false \
-    -v /var/munin:/opt/munin \
+    -v /var/containerized-monitoring/monit:/opt/monit \
+    -v /var/containerized-monitoring/munin:/opt/munin \
     --mount type=tmpfs,destination=/opt/munin/www \
     --mount type=tmpfs,destination=/opt/munin/log \
     --mount type=tmpfs,destination=/opt/munin/run \
     --mount type=tmpfs,destination=/opt/munin/cgi-tmp \
-    munin:latest
+    containerized-monitoring:latest
+
+docker logs -f --since=1m containerized-monitoring

secrets.sh.example

@@ -0,0 +1,15 @@
+#!/bin/bash
+##########
+# monit
+##########
+CONFIG_MONIT_USER=admin
+CONFIG_MONIT_PASS=apassword
+
+##########
+# email notifications via msmtp-mta
+#   optional
+##########
+#CONFIG_NOTIFY_EMAIL=user@domain.tld
+#CONFIG_NOTIFY_SMTP_SERVER=email.domain.tld
+#CONFIG_NOTIFY_SMTP_USER=user@domain.tld
+#CONFIG_NOTIFY_SMTP_PASS=apassword

test.sh

@@ -1,23 +1,36 @@
 #!/bin/bash
 
-echo "**** Building latest Munin container ****"
+echo "**** Building latest monitoring container ****"
 $(pwd)/build.sh
 
-echo "**** Deleting Original Container ****"
-sudo docker rm -f munin
+echo "**** Loading secrets.sh ****"
+source ./secrets.sh
 
-echo "**** Running Munin ****"
-sudo docker run -it \
+echo "**** Deleting existing container ****"
+sudo docker rm -f containerized-monitoring
+
+echo "**** Running monitoring ****"
+sudo docker run -itd \
     --restart unless-stopped \
     --network services \
-    --name munin \
+    --name containerized-monitoring \
     -e TZ=UTC \
     -e DEBUG=1 \
+    -e CONFIG_NOTIFY_EMAIL=$CONFIG_NOTIFY_EMAIL \
+    -e CONFIG_NOTIFY_SMTP_SERVER=$CONFIG_NOTIFY_SMTP_SERVER \
+    -e CONFIG_NOTIFY_SMTP_USER=$CONFIG_NOTIFY_SMTP_USER \
+    -e CONFIG_NOTIFY_SMTP_PASS=$CONFIG_NOTIFY_SMTP_PASS \
+    -e CONFIG_MONIT_USER=$CONFIG_MONIT_USER \
+    -e CONFIG_MONIT_PASS=$CONFIG_MONIT_PASS \
+    -p 2812:2812/tcp \
     -p 2813:2813/tcp \
+    -p 80:80/tcp \
     -l diun.enable=true \
     -l traefik.enable=false \
     --mount type=tmpfs,destination=/opt/munin/www \
     --mount type=tmpfs,destination=/opt/munin/log \
     --mount type=tmpfs,destination=/opt/munin/run \
     --mount type=tmpfs,destination=/opt/munin/cgi-tmp \
-    munin:latest
+    containerized-monitoring:latest
+
+docker logs -f --since=1m containerized-monitoring