Implement metrics with Grafana and Prometheus

This commit adds two services, `grafana` and `prometheus`, and sets up some existing services (`dovecot` and `prosody`) to expose metrics into Grafana. In addition, systemd services have been added to facilitate registering metrics for services into Prometheus, as well as automatically provisioning Grafana dashboards based on static JSON representations. This work will continue to evolve as more services gain proper Grafana dashboards, and Loki is also integrated for access to the systemd journal.
2022-04-25 20:12:06 +01:00 · 2022-04-25 20:12:06 +01:00 · 011650105b
parent 4aba8f73c2
commit 011650105b
24 changed files with 179 additions and 5 deletions
--- a/host/lhr01nuc/spec.bu
+++ b/host/lhr01nuc/spec.bu
@ -20,6 +20,8 @@ ignition:
      - local: service/navidrome/spec.ign
      - local: service/rclone/spec.ign
      - local: service/hugo/spec.ign
+      - local: service/prometheus/spec.ign
+      - local: service/grafana/spec.ign
      - local: private/spec.ign

 passwd:
--- a/host/lhr01srv/spec.bu
+++ b/host/lhr01srv/spec.bu
@ -21,6 +21,8 @@ ignition:
      - local: service/navidrome/spec.ign
      - local: service/rclone/spec.ign
      - local: service/hugo/spec.ign
+      - local: service/prometheus/spec.ign
+      - local: service/grafana/spec.ign
      - local: private/spec.ign

 passwd:
--- a/host/virtual/spec.bu
+++ b/host/virtual/spec.bu
@ -23,6 +23,8 @@ ignition:
      - local: service/coturn/spec.ign
      - local: service/rclone/spec.ign
      - local: service/hugo/spec.ign
+      - local: service/prometheus/spec.ign
+      - local: service/grafana/spec.ign

 passwd:
  users:
@ -117,6 +119,14 @@ systemd:
            [Service]
            Environment=UPSTREAM_HOST=writefreely UPSTREAM_PORT=8080

+    - name: nginx-proxy-http@metrics.localhost.service
+      enabled: true
+      dropins:
+        - name: grafana-upstream.conf
+          contents: |
+            [Service]
+            Environment=UPSTREAM_HOST=grafana UPSTREAM_PORT=8080
+
    - name: letsencrypt-dns-register@localhost.service
      enabled: true
      dropins:
--- a/service/dovecot/container/config/conf.d/90-stats.conf
+++ b/service/dovecot/container/config/conf.d/90-stats.conf
@ -0,0 +1,5 @@
+service stats {
+    inet_listener http {
+        port = 9900
+    }
+}
--- a/service/dovecot/systemd/dovecot.service
+++ b/service/dovecot/systemd/dovecot.service
@ -1,7 +1,8 @@
 [Unit]
 Description=Dovecot POP3/IMAP Server
-Wants=container-build@%N.service container-volume@%N.service mariadb.service rspamd.service
+Wants=container-build@%N.service container-volume@%N.service mariadb.service rspamd.service grafana-dashboard@dovecot.service
 After=container-build@%N.service container-volume@%N.service mariadb.service rspamd.service
+Before=grafana-dashboard@dovecot.service

 [Service]
 Type=notify
--- a/service/dovecot/systemd/grafana-dashboard@dovecot.service.d/after-prometheus.conf
+++ b/service/dovecot/systemd/grafana-dashboard@dovecot.service.d/after-prometheus.conf
@ -0,0 +1,3 @@
+[Unit]
+Wants=prometheus-service-register@dovecot:9900.service
+After=prometheus-service-register@dovecot:9900.service
--- a/service/grafana/Containerfile
+++ b/service/grafana/Containerfile
@ -0,0 +1,2 @@
+FROM docker.io/grafana/grafana:8.5.0
+COPY container/config /etc/grafana
--- a/service/grafana/container/config/grafana.ini
+++ b/service/grafana/container/config/grafana.ini
@ -0,0 +1,15 @@
+[paths]
+# Folder used for static provisioning of dependencies.
+provisioning = /etc/grafana/provisioning
+
+[server]
+# The HTTP port to use.
+http_port = 8080
+
+[log]
+# Only log to standard output.
+mode = console
+
+[analytics]
+# Disable anonymous usage collection.
+reporting_enabled = false
--- a/service/grafana/container/config/provisioning/dashboards/services.yaml
+++ b/service/grafana/container/config/provisioning/dashboards/services.yaml
@ -0,0 +1,10 @@
+# Service dashboards provisioned by services.
+apiVersion: 1
+providers:
+  - name: 'Services'
+    folder: 'Services'
+    type: file
+    disableDeletion: true
+    allowUiUpdates: false
+    options:
+      path: /var/lib/grafana/dashboards
--- a/service/grafana/container/config/provisioning/datasources/prometheus.yaml
+++ b/service/grafana/container/config/provisioning/datasources/prometheus.yaml
@ -0,0 +1,7 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
--- a/service/grafana/spec.bu
+++ b/service/grafana/spec.bu
@ -0,0 +1,12 @@
+variant: fcos
+version: 1.3.0
+storage:
+  trees:
+    - path: /etc/coreos-home-server/grafana
+      local: service/grafana/
+    - path: /etc/systemd/system
+      local: service/grafana/systemd/
+systemd:
+  units:
+    - name: grafana.service
+      enabled: true
--- a/service/grafana/systemd/grafana-dashboard@.service
+++ b/service/grafana/systemd/grafana-dashboard@.service
@ -0,0 +1,16 @@
+[Unit]
+Description=Register Grafana Dashboard for %I
+Wants=grafana.service
+After=grafana.service
+ConditionPathExists=%E/coreos-home-server/%i/service/%p.json
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+SyslogIdentifier=%N
+ExecStartPre=/bin/podman exec grafana mkdir -p /var/lib/grafana/dashboards
+ExecStart=/bin/podman cp %E/coreos-home-server/%i/service/%p.json grafana:/var/lib/grafana/dashboards/%i.json
+ExecStop=/bin/podman exec grafana rm -f /var/lib/grafana/dashboards/%i.json
+
+[Install]
+WantedBy=multi-user.target
--- a/service/grafana/systemd/grafana.service
+++ b/service/grafana/systemd/grafana.service
@ -0,0 +1,17 @@
+[Unit]
+Description=Grafana Visualization and Analytics
+Wants=container-build@%N.service container-volume@%N.service prometheus.service
+After=container-build@%N.service container-volume@%N.service prometheus.service
+
+[Service]
+Type=notify
+NotifyAccess=all
+SyslogIdentifier=%N
+Restart=on-failure
+Environment=PODMAN_SYSTEMD_UNIT=%n
+ExecStart=/bin/podman run --replace --name %N --net internal --sdnotify=conmon --volume %N:/var/lib/%N:z localhost/%N:latest
+ExecStop=/bin/podman stop --ignore --time 10 %N
+ExecStopPost=/bin/podman rm --ignore --force %N
+
+[Install]
+WantedBy=multi-user.target
--- a/service/postfix/container/config/main.cf.template
+++ b/service/postfix/container/config/main.cf.template
@ -160,7 +160,7 @@ virtual_mailbox_domains = mysql:/etc/postfix/mysql-domains.cf
 virtual_uid_maps = static:5000
 virtual_gid_maps = static:5000

-# Pass messages through 'opendkim' and 'rmilter'.
+# Pass messages through 'rspamd'.
 smtpd_milters = ${POSTFIX_MAIL_MILTER_PATH}
 milter_protocol = 6
 milter_mail_macros = i {mail_addr} {client_addr} {client_name} {auth_authen}
--- a/service/prometheus/Containerfile
+++ b/service/prometheus/Containerfile
@ -0,0 +1,2 @@
+FROM docker.io/prom/prometheus:v2.35.0
+COPY container/config /etc/prometheus
--- a/service/prometheus/container/config/prometheus.yml
+++ b/service/prometheus/container/config/prometheus.yml
@ -0,0 +1,6 @@
+scrape_configs:
+  - job_name: prometheus
+    honor_labels: true
+    file_sd_configs:
+      - files:
+          - /etc/prometheus/service.d/*.yml
--- a/service/prometheus/container/config/service.d/README.md
+++ b/service/prometheus/container/config/service.d/README.md
@ -0,0 +1,4 @@
+# Service Discovery Configuration for Prometheus
+
+Any file with an extension of `.yml` placed in this directory will be included as part of the
+Prometheus service discovery configuration.
--- a/service/prometheus/service/prometheus-service-register.yml.template
+++ b/service/prometheus/service/prometheus-service-register.yml.template
@ -0,0 +1,3 @@
+# Service discovery configuration for dynamic host.
+- targets:
+  - "${SERVICE_HOST}"
--- a/service/prometheus/spec.bu
+++ b/service/prometheus/spec.bu
@ -0,0 +1,12 @@
+variant: fcos
+version: 1.3.0
+storage:
+  trees:
+    - path: /etc/coreos-home-server/prometheus
+      local: service/prometheus/
+    - path: /etc/systemd/system
+      local: service/prometheus/systemd/
+systemd:
+  units:
+    - name: prometheus.service
+      enabled: true
--- a/service/prometheus/systemd/prometheus-service-register@.service
+++ b/service/prometheus/systemd/prometheus-service-register@.service
@ -0,0 +1,19 @@
+[Unit]
+Description=Register Prometheus Metrics for %I
+Wants=prometheus.service
+After=prometheus.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+SyslogIdentifier=%N
+Environment=SERVICE_HOST=%i
+Environment=PROMETHEUS_CONF=%E/coreos-home-server/prometheus/service/%p.yml.template
+ExecStart=/bin/sh -c "envsubst '$SERVICE_HOST' < ${PROMETHEUS_CONF} > /tmp/%i.yml"
+ExecStartPost=/bin/sh -c 'podman cp --archive=false /tmp/%i.yml prometheus:/etc/prometheus/service.d/%i.yml && rm -f /tmp/%i.yml'
+ExecStartPost=/bin/podman exec prometheus sh -c 'kill -HUP $(pidof prometheus)'
+ExecStop=/bin/podman exec --user=root prometheus rm -f /etc/prometheus/service.d/%i.yml
+ExecStopPost=/bin/podman exec prometheus sh -c 'kill -HUP $(pidof prometheus)'
+
+[Install]
+WantedBy=multi-user.target
--- a/service/prometheus/systemd/prometheus.service
+++ b/service/prometheus/systemd/prometheus.service
@ -0,0 +1,20 @@
+[Unit]
+Description=Prometheus Systems and Service Monitoring
+Wants=container-build@%N.service container-volume@%N.service
+After=container-build@%N.service container-volume@%N.service
+
+[Service]
+Type=notify
+NotifyAccess=all
+SyslogIdentifier=%N
+Restart=on-failure
+Environment=PODMAN_SYSTEMD_UNIT=%n
+ExecStart=/bin/podman run --replace --name %N --net internal --sdnotify=conmon \
+                          --volume %N:/%N:z \
+                          --volume %N-service:/etc/%N/service.d:z \
+                          localhost/%N:latest
+ExecStop=/bin/podman stop --ignore --time 10 %N
+ExecStopPost=/bin/podman rm --ignore --force %N
+
+[Install]
+WantedBy=multi-user.target
--- a/service/prosody/container/config/prosody.cfg.lua
+++ b/service/prosody/container/config/prosody.cfg.lua
@ -68,6 +68,7 @@ modules_enabled = {

    -- HTTP modules
    "websocket"; -- XMPP over WebSockets
+    "http_openmetrics"; -- Enables metrics collection in OpenMetrics-compatible format.

    -- Other specific functionality
    "conversejs"; -- Web-based frontend for XMPP
@ -169,7 +170,9 @@ turn_external_secret = os.getenv("PROSODY_TURN_SECRET") or ""

 -- Uncomment to enable statistics
 -- For more info see https://prosody.im/doc/statistics
-- statistics = "internal"
+statistics = "internal"
+statistics_interval = "manual"
+openmetrics_allow_cidr = "10.89.0.0/16"

 -- Certificates
 -- Every virtual host and component needs a certificate so that clients and
--- a/service/prosody/systemd/grafana-dashboard@prosody.service.d/after-prometheus.conf
+++ b/service/prosody/systemd/grafana-dashboard@prosody.service.d/after-prometheus.conf
@ -0,0 +1,3 @@
+[Unit]
+Wants=prometheus-service-register@prosody:5280.service
+After=prometheus-service-register@prosody:5280.service
--- a/service/prosody/systemd/prosody.service
+++ b/service/prosody/systemd/prosody.service
@ -1,7 +1,7 @@
 [Unit]
 Description=Prosody XMPP server
-Wants=container-build@%N.service container-volume@%N.service dovecot.service
-After=container-build@%N.service container-volume@%N.service dovecot.service
+Wants=container-build@%N.service container-volume@%N.service dovecot.service grafana-dashboard@prosody.service
+After=container-build@%N.service container-volume@%N.service dovecot.service grafana-dashboard@prosody.service

 [Service]
 Type=notify