deuill/coreos-home-server
1
0
Fork 0
mirror of https://github.com/deuill/coreos-home-server.git synced 2024-09-21 05:30:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

prometheus: Add exporter for container metrics

Browse Source 
This also adds a basic Grafana dashboard for Podman, to be updated later.
This commit is contained in:
Alex Palaistras 2023-12-04 10:44:21 +00:00
parent b123008850
commit 6199425956
9 changed files with 827 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
service/grafana/systemd/grafana-dashboard@.service
View File

@ -8,8 +8,9 @@ ConditionPathExists=%E/coreos-home-server/%i/service/%p.json
Type=oneshot
RemainAfterExit=true
SyslogIdentifier=%N
Environment=DASHBOARD_CONF=%E/coreos-home-server/%i/service/%p.json
ExecStartPre=/bin/podman exec grafana mkdir -p /var/lib/grafana/dashboards
ExecStart=/bin/podman cp %E/coreos-home-server/%i/service/%p.json grafana:/var/lib/grafana/dashboards/%i.json
ExecStart=/bin/podman cp ${DASHBOARD_CONF} grafana:/var/lib/grafana/dashboards/%i.json
ExecStop=/bin/podman exec grafana rm -f /var/lib/grafana/dashboards/%i.json
[Install]

14
service/prometheus/Containerfile
View File

@ -1,8 +1,18 @@
FROM docker.io/golang:1.21-bookworm AS podman-exporter-builder
RUN apt-get update -y && apt-get upgrade -y && \
apt-get install -y --no-install-recommends libgpgme-dev libbtrfs-dev libdevmapper-dev libassuan-dev pkg-config
ARG PODMAN_EXPORTER_VERSION=1.6.0
RUN git clone --branch v${PODMAN_EXPORTER_VERSION} --depth 1 https://github.com/containers/prometheus-podman-exporter.git /src && \
cd /src && make binary
FROM docker.io/debian:bookworm-slim
ARG VERSION=2.48.0
RUN apt-get update -y && apt-get upgrade -y && \
apt-get install -y --no-install-recommends curl ca-certificates
apt-get install -y --no-install-recommends curl ca-certificates libgpgme11 libbtrfs0 \
libdevmapper1.02.1 libassuan0
RUN addgroup --system --gid 10000 prometheus
RUN adduser --system --uid 10000 --ingroup prometheus --home /var/lib/prometheus prometheus
@ -19,6 +29,8 @@ RUN curl --silent --show-error -L https://github.com/prometheus/node_exporter/re
mv /tmp/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64/node_exporter /bin/node_exporter && \
rmdir /tmp/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64
COPY --from=podman-exporter-builder /src/bin/prometheus-podman-exporter /bin/podman_exporter
COPY container/config /etc/prometheus
USER prometheus

4
service/prometheus/quadlet/prometheus-node-exporter.container
View File

@ -1,7 +1,7 @@
[Unit]
Description=Prometheus Node Metrics Exporter
Wants=grafana-dashboard@prometheus.service
After=grafana-dashboard@prometheus.service
Wants=grafana-dashboard@prometheus-node-exporter.service
After=grafana-dashboard@prometheus-node-exporter.service
[Container]
AutoUpdate=local

23
service/prometheus/quadlet/prometheus-podman-exporter.container Normal file
View File

@ -0,0 +1,23 @@
[Unit]
Description=Prometheus Podman Metrics Exporter
Wants=podman.socket grafana-dashboard@prometheus-podman-exporter.service
After=podman.socket grafana-dashboard@prometheus-podman-exporter.service
[Container]
AutoUpdate=local
ContainerName=%N
Environment=CONTAINER_HOST=unix:///run/podman/podman.sock
Exec=--collector.enable-all
HealthCmd=timeout 5s curl --fail --silent --show-error -o /dev/null localhost:9882
Image=localhost/prometheus:latest
Network=internal
PodmanArgs=--sdnotify=healthy --entrypoint /bin/podman_exporter
SecurityLabelDisable=true
User=root
Volume=/run/podman/podman.sock:/run/podman/podman.sock:ro
[Service]
Restart=on-failure
[Install]
WantedBy=multi-user.target

0
service/prometheus/service/grafana-dashboard.json → service/prometheus/service/grafana-dashboard-node-exporter.json
View File

771
service/prometheus/service/grafana-dashboard-podman-exporter.json Normal file
View File

@ -0,0 +1,771 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"options": {
"0": {
"color": "purple",
"index": 4,
"text": "Created ?"
},
"1": {
"color": "red",
"index": 3,
"text": "Error ❌"
},
"2": {
"color": "light-red",
"index": 2,
"text": "Exited ❗️"
},
"3": {
"color": "light-yellow",
"index": 1,
"text": "Paused ⏸"
},
"4": {
"color": "light-green",
"index": 0,
"text": "Running ?"
},
"5": {
"color": "#808080",
"index": 6,
"text": "Degraded ?"
},
"6": {
"color": "super-light-red",
"index": 7,
"text": "Stopped ?"
},
"-1": {
"color": "orange",
"index": 5,
"text": "Unkown ❔"
}
},
"type": "value"
},
{
"options": {
"match": "null",
"result": {
"color": "text",
"index": 8,
"text": "No value "
}
},
"type": "special"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"last"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.5.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum by(name) (podman_pod_info * on(id) group_right(name) podman_pod_state)",
"legendFormat": "{{name}}",
"range": true,
"refId": "A"
}
],
"title": "Services uptime",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisGridShow": true,
"axisLabel": "Sates",
"axisPlacement": "auto",
"axisSoftMax": 6,
"axisSoftMin": -1,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "stepBefore",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 8,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"mappings": [
{
"options": {
"0": {
"color": "purple",
"index": 4,
"text": "Created"
},
"1": {
"color": "red",
"index": 3,
"text": "Error"
},
"2": {
"color": "light-red",
"index": 2,
"text": "Exited"
},
"3": {
"color": "light-yellow",
"index": 1,
"text": "Paused"
},
"4": {
"color": "light-green",
"index": 0,
"text": "Running"
},
"5": {
"color": "#808080",
"index": 6,
"text": "Degraded"
},
"6": {
"color": "super-light-red",
"index": 7,
"text": "Stopped"
},
"-1": {
"color": "orange",
"index": 5,
"text": "Unkown"
}
},
"type": "value"
}
],
"noValue": "-1",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 24,
"x": 0,
"y": 12
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum by(name) (podman_pod_info * on(id) group_right(name) podman_pod_state)",
"legendFormat": "{{name}}",
"range": true,
"refId": "A"
}
],
"title": "Services uptime",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 12,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [
{
"options": {
"0": {
"color": "purple",
"index": 4,
"text": "Created"
},
"1": {
"color": "red",
"index": 3,
"text": "Error"
},
"2": {
"color": "light-red",
"index": 2,
"text": "Exited"
},
"3": {
"color": "light-yellow",
"index": 1,
"text": "Paused"
},
"4": {
"color": "light-green",
"index": 0,
"text": "Running"
},
"5": {
"color": "#808080",
"index": 6,
"text": "Degraded"
},
"6": {
"color": "super-light-red",
"index": 7,
"text": "Stopped"
},
"-1": {
"color": "orange",
"index": 5,
"text": "Unkown"
}
},
"type": "value"
}
],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "decmbytes"
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 12,
"x": 0,
"y": 23
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum by(name) (podman_container_info{name!~\".+infra\"} * on(id) group_right(name) podman_container_mem_usage_bytes / 1024 /1024)",
"legendFormat": "{{name}}",
"range": true,
"refId": "A"
}
],
"title": "Memory usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 12,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [
{
"options": {
"0": {
"color": "purple",
"index": 4,
"text": "Created"
},
"1": {
"color": "red",
"index": 3,
"text": "Error"
},
"2": {
"color": "light-red",
"index": 2,
"text": "Exited"
},
"3": {
"color": "light-yellow",
"index": 1,
"text": "Paused"
},
"4": {
"color": "light-green",
"index": 0,
"text": "Running"
},
"5": {
"color": "#808080",
"index": 6,
"text": "Degraded"
},
"6": {
"color": "super-light-red",
"index": 7,
"text": "Stopped"
},
"-1": {
"color": "orange",
"index": 5,
"text": "Unkown"
}
},
"type": "value"
}
],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 12,
"x": 12,
"y": 23
},
"id": 7,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum by(name) (podman_container_info{name!~\".+infra\"} * on(id) group_right(name) rate(podman_container_cpu_seconds_total[1m]))",
"legendFormat": "{{name}}",
"range": true,
"refId": "A"
}
],
"title": "CPU",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 12,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 34
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum by(name) (podman_container_info{name!~\".+infra\"} * on(id) group_right(name) rate(podman_container_net_output_total[1m]) / 1024)",
"legendFormat": "{{name}}",
"range": true,
"refId": "A"
}
],
"title": "Network Output",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 12,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 34
},
"id": 5,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.1.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum by(name) (podman_container_info{name!~\".+infra\"} * on(id) group_right(name) rate(podman_container_net_input_total[1m]) / 1024)",
"legendFormat": "{{name}}",
"range": true,
"refId": "A"
}
],
"title": "Network input",
"type": "timeseries"
}
],
"refresh": "",
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Podman",
"uid": "cUITC74Vk",
"version": 10,
"weekStart": "",
"gnetId": 17138,
"description": "Podman exporter dashboard \r\nhttps://github.com/containers/prometheus-podman-exporter"
}

8
service/prometheus/systemd/grafana-dashboard@prometheus-node-exporter.service.d/after-prometheus.conf Normal file
View File

@ -0,0 +1,8 @@
[Unit]
Wants=prometheus-service-register@host.containers.internal:9100.service
After=prometheus-service-register@host.containers.internal:9100.service
ConditionPathExists=
ConditionPathExists=%E/coreos-home-server/prometheus/service/grafana-dashboard-node-exporter.json
[Service]
Environment=DASHBOARD_CONF=%E/coreos-home-server/prometheus/service/grafana-dashboard-node-exporter.json

8
service/prometheus/systemd/grafana-dashboard@prometheus-podman-exporter.service.d/after-prometheus.conf Normal file
View File

@ -0,0 +1,8 @@
[Unit]
Wants=prometheus-service-register@prometheus-podman-exporter:9882.service
After=prometheus-service-register@prometheus-podman-exporter:9882.service
ConditionPathExists=
ConditionPathExists=%E/coreos-home-server/prometheus/service/grafana-dashboard-podman-exporter.json
[Service]
Environment=DASHBOARD_CONF=%E/coreos-home-server/prometheus/service/grafana-dashboard-podman-exporter.json

3
service/prometheus/systemd/grafana-dashboard@prometheus.service.d/after-prometheus.conf
View File

@ -1,3 +0,0 @@
[Unit]
Wants=prometheus-service-register@host.containers.internal:9100.service
After=prometheus-service-register@host.containers.internal:9100.service