diff --git a/service/grafana/systemd/grafana-dashboard@.service b/service/grafana/systemd/grafana-dashboard@.service index a657378..ac9dc53 100644 --- a/service/grafana/systemd/grafana-dashboard@.service +++ b/service/grafana/systemd/grafana-dashboard@.service @@ -8,8 +8,9 @@ ConditionPathExists=%E/coreos-home-server/%i/service/%p.json Type=oneshot RemainAfterExit=true SyslogIdentifier=%N +Environment=DASHBOARD_CONF=%E/coreos-home-server/%i/service/%p.json ExecStartPre=/bin/podman exec grafana mkdir -p /var/lib/grafana/dashboards -ExecStart=/bin/podman cp %E/coreos-home-server/%i/service/%p.json grafana:/var/lib/grafana/dashboards/%i.json +ExecStart=/bin/podman cp ${DASHBOARD_CONF} grafana:/var/lib/grafana/dashboards/%i.json ExecStop=/bin/podman exec grafana rm -f /var/lib/grafana/dashboards/%i.json [Install] diff --git a/service/prometheus/Containerfile b/service/prometheus/Containerfile index 566b886..c8e3cb4 100644 --- a/service/prometheus/Containerfile +++ b/service/prometheus/Containerfile @@ -1,8 +1,18 @@ +FROM docker.io/golang:1.21-bookworm AS podman-exporter-builder + +RUN apt-get update -y && apt-get upgrade -y && \ + apt-get install -y --no-install-recommends libgpgme-dev libbtrfs-dev libdevmapper-dev libassuan-dev pkg-config + +ARG PODMAN_EXPORTER_VERSION=1.6.0 +RUN git clone --branch v${PODMAN_EXPORTER_VERSION} --depth 1 https://github.com/containers/prometheus-podman-exporter.git /src && \ + cd /src && make binary + FROM docker.io/debian:bookworm-slim ARG VERSION=2.48.0 RUN apt-get update -y && apt-get upgrade -y && \ - apt-get install -y --no-install-recommends curl ca-certificates + apt-get install -y --no-install-recommends curl ca-certificates libgpgme11 libbtrfs0 \ + libdevmapper1.02.1 libassuan0 RUN addgroup --system --gid 10000 prometheus RUN adduser --system --uid 10000 --ingroup prometheus --home /var/lib/prometheus prometheus @@ -19,6 +29,8 @@ RUN curl --silent --show-error -L https://github.com/prometheus/node_exporter/re mv /tmp/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64/node_exporter /bin/node_exporter && \ rmdir /tmp/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64 +COPY --from=podman-exporter-builder /src/bin/prometheus-podman-exporter /bin/podman_exporter + COPY container/config /etc/prometheus USER prometheus diff --git a/service/prometheus/quadlet/prometheus-node-exporter.container b/service/prometheus/quadlet/prometheus-node-exporter.container index 3ed098c..5bb0108 100644 --- a/service/prometheus/quadlet/prometheus-node-exporter.container +++ b/service/prometheus/quadlet/prometheus-node-exporter.container @@ -1,7 +1,7 @@ [Unit] Description=Prometheus Node Metrics Exporter -Wants=grafana-dashboard@prometheus.service -After=grafana-dashboard@prometheus.service +Wants=grafana-dashboard@prometheus-node-exporter.service +After=grafana-dashboard@prometheus-node-exporter.service [Container] AutoUpdate=local diff --git a/service/prometheus/quadlet/prometheus-podman-exporter.container b/service/prometheus/quadlet/prometheus-podman-exporter.container new file mode 100644 index 0000000..bb876ad --- /dev/null +++ b/service/prometheus/quadlet/prometheus-podman-exporter.container @@ -0,0 +1,23 @@ +[Unit] +Description=Prometheus Podman Metrics Exporter +Wants=podman.socket grafana-dashboard@prometheus-podman-exporter.service +After=podman.socket grafana-dashboard@prometheus-podman-exporter.service + +[Container] +AutoUpdate=local +ContainerName=%N +Environment=CONTAINER_HOST=unix:///run/podman/podman.sock +Exec=--collector.enable-all +HealthCmd=timeout 5s curl --fail --silent --show-error -o /dev/null localhost:9882 +Image=localhost/prometheus:latest +Network=internal +PodmanArgs=--sdnotify=healthy --entrypoint /bin/podman_exporter +SecurityLabelDisable=true +User=root +Volume=/run/podman/podman.sock:/run/podman/podman.sock:ro + +[Service] +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/service/prometheus/service/grafana-dashboard.json b/service/prometheus/service/grafana-dashboard-node-exporter.json similarity index 100% rename from service/prometheus/service/grafana-dashboard.json rename to service/prometheus/service/grafana-dashboard-node-exporter.json diff --git a/service/prometheus/service/grafana-dashboard-podman-exporter.json b/service/prometheus/service/grafana-dashboard-podman-exporter.json new file mode 100644 index 0000000..c983797 --- /dev/null +++ b/service/prometheus/service/grafana-dashboard-podman-exporter.json @@ -0,0 +1,771 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "purple", + "index": 4, + "text": "Created ?" + }, + "1": { + "color": "red", + "index": 3, + "text": "Error ❌" + }, + "2": { + "color": "light-red", + "index": 2, + "text": "Exited ❗️" + }, + "3": { + "color": "light-yellow", + "index": 1, + "text": "Paused ⏸" + }, + "4": { + "color": "light-green", + "index": 0, + "text": "Running ?" + }, + "5": { + "color": "#808080", + "index": 6, + "text": "Degraded ?" + }, + "6": { + "color": "super-light-red", + "index": 7, + "text": "Stopped ?" + }, + "-1": { + "color": "orange", + "index": 5, + "text": "Unkown ❔" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "text", + "index": 8, + "text": "No value " + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(name) (podman_pod_info * on(id) group_right(name) podman_pod_state)", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Services uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "Sates", + "axisPlacement": "auto", + "axisSoftMax": 6, + "axisSoftMin": -1, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "color": "purple", + "index": 4, + "text": "Created" + }, + "1": { + "color": "red", + "index": 3, + "text": "Error" + }, + "2": { + "color": "light-red", + "index": 2, + "text": "Exited" + }, + "3": { + "color": "light-yellow", + "index": 1, + "text": "Paused" + }, + "4": { + "color": "light-green", + "index": 0, + "text": "Running" + }, + "5": { + "color": "#808080", + "index": 6, + "text": "Degraded" + }, + "6": { + "color": "super-light-red", + "index": 7, + "text": "Stopped" + }, + "-1": { + "color": "orange", + "index": 5, + "text": "Unkown" + } + }, + "type": "value" + } + ], + "noValue": "-1", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(name) (podman_pod_info * on(id) group_right(name) podman_pod_state)", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Services uptime", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + { + "options": { + "0": { + "color": "purple", + "index": 4, + "text": "Created" + }, + "1": { + "color": "red", + "index": 3, + "text": "Error" + }, + "2": { + "color": "light-red", + "index": 2, + "text": "Exited" + }, + "3": { + "color": "light-yellow", + "index": 1, + "text": "Paused" + }, + "4": { + "color": "light-green", + "index": 0, + "text": "Running" + }, + "5": { + "color": "#808080", + "index": 6, + "text": "Degraded" + }, + "6": { + "color": "super-light-red", + "index": 7, + "text": "Stopped" + }, + "-1": { + "color": "orange", + "index": 5, + "text": "Unkown" + } + }, + "type": "value" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(name) (podman_container_info{name!~\".+infra\"} * on(id) group_right(name) podman_container_mem_usage_bytes / 1024 /1024)", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + { + "options": { + "0": { + "color": "purple", + "index": 4, + "text": "Created" + }, + "1": { + "color": "red", + "index": 3, + "text": "Error" + }, + "2": { + "color": "light-red", + "index": 2, + "text": "Exited" + }, + "3": { + "color": "light-yellow", + "index": 1, + "text": "Paused" + }, + "4": { + "color": "light-green", + "index": 0, + "text": "Running" + }, + "5": { + "color": "#808080", + "index": 6, + "text": "Degraded" + }, + "6": { + "color": "super-light-red", + "index": 7, + "text": "Stopped" + }, + "-1": { + "color": "orange", + "index": 5, + "text": "Unkown" + } + }, + "type": "value" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(name) (podman_container_info{name!~\".+infra\"} * on(id) group_right(name) rate(podman_container_cpu_seconds_total[1m]))", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(name) (podman_container_info{name!~\".+infra\"} * on(id) group_right(name) rate(podman_container_net_output_total[1m]) / 1024)", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Network Output", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(name) (podman_container_info{name!~\".+infra\"} * on(id) group_right(name) rate(podman_container_net_input_total[1m]) / 1024)", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Network input", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Podman", + "uid": "cUITC74Vk", + "version": 10, + "weekStart": "", + "gnetId": 17138, + "description": "Podman exporter dashboard \r\nhttps://github.com/containers/prometheus-podman-exporter" +} diff --git a/service/prometheus/systemd/grafana-dashboard@prometheus-node-exporter.service.d/after-prometheus.conf b/service/prometheus/systemd/grafana-dashboard@prometheus-node-exporter.service.d/after-prometheus.conf new file mode 100644 index 0000000..1e79aa9 --- /dev/null +++ b/service/prometheus/systemd/grafana-dashboard@prometheus-node-exporter.service.d/after-prometheus.conf @@ -0,0 +1,8 @@ +[Unit] +Wants=prometheus-service-register@host.containers.internal:9100.service +After=prometheus-service-register@host.containers.internal:9100.service +ConditionPathExists= +ConditionPathExists=%E/coreos-home-server/prometheus/service/grafana-dashboard-node-exporter.json + +[Service] +Environment=DASHBOARD_CONF=%E/coreos-home-server/prometheus/service/grafana-dashboard-node-exporter.json diff --git a/service/prometheus/systemd/grafana-dashboard@prometheus-podman-exporter.service.d/after-prometheus.conf b/service/prometheus/systemd/grafana-dashboard@prometheus-podman-exporter.service.d/after-prometheus.conf new file mode 100644 index 0000000..1c2c811 --- /dev/null +++ b/service/prometheus/systemd/grafana-dashboard@prometheus-podman-exporter.service.d/after-prometheus.conf @@ -0,0 +1,8 @@ +[Unit] +Wants=prometheus-service-register@prometheus-podman-exporter:9882.service +After=prometheus-service-register@prometheus-podman-exporter:9882.service +ConditionPathExists= +ConditionPathExists=%E/coreos-home-server/prometheus/service/grafana-dashboard-podman-exporter.json + +[Service] +Environment=DASHBOARD_CONF=%E/coreos-home-server/prometheus/service/grafana-dashboard-podman-exporter.json diff --git a/service/prometheus/systemd/grafana-dashboard@prometheus.service.d/after-prometheus.conf b/service/prometheus/systemd/grafana-dashboard@prometheus.service.d/after-prometheus.conf deleted file mode 100644 index 230a5eb..0000000 --- a/service/prometheus/systemd/grafana-dashboard@prometheus.service.d/after-prometheus.conf +++ /dev/null @@ -1,3 +0,0 @@ -[Unit] -Wants=prometheus-service-register@host.containers.internal:9100.service -After=prometheus-service-register@host.containers.internal:9100.service