RWejlgaard/pez-infra
1
0
Fork 0
mirror of https://github.com/RWejlgaard/pez-infra.git synced 2026-05-06 04:14:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Grafana Cloud Migration (#94)
Some checks are pending
Deploy (on merge) / Discover hosts (push) Waiting to run
Details
Deploy (on merge) / Deploy → (push) Blocked by required conditions
Details
Terraform / Plan (push) Waiting to run
Details
Terraform / Apply (push) Blocked by required conditions
Details

Browse source 
* Grafana Cloud migration, adding dashboards, fleet, alloy and synthetics

* modulize stuff now that we have multiple substantial things in here

* provider updates and new secrets

* remove grafana and prometheus from ansible
This commit is contained in:
Rasmus Wejlgaard 2026-05-04 13:40:30 +01:00 • committed by GitHub
parent 83f023aedd
commit 043c783361
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
43 changed files with 4340 additions and 20209 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
ansible/services/README.md
View file

@ -34,19 +34,12 @@ services/
| Caddy | `caddy.service` | enabled, stock | Installed via package manager. Config at `/etc/caddy/Caddyfile`. | | Caddy | `caddy.service` | enabled, stock | Installed via package manager. Config at `/etc/caddy/Caddyfile`. |
| thiswebsitedoesnotexist | `thiswebsitedoesnotexist.service` | enabled, custom | Node.js app. Env vars in `/opt/thiswebsitedoesnotexist/.env`. | | thiswebsitedoesnotexist | `thiswebsitedoesnotexist.service` | enabled, custom | Node.js app. Env vars in `/opt/thiswebsitedoesnotexist/.env`. |
### london-a (FreeBSD) ### london-a (Linux)
No custom rc.d scripts — all services installed via `pkg`. The `rc.conf` captures all enabled services: No custom rc.d scripts — all services installed via `pkg`. The `rc.conf` captures all enabled services:
| Service | rc.conf variable | Notes | | Service | Unit | Notes |
|---------|-----------------|-------| |---------|-----------------|-------|
| Grafana | `grafana_enable="YES"` | Monitoring dashboards | | libvirtd | `libvirtd.service` | Virtualisation daemon |
| Prometheus | `prometheus_enable="YES"` | Metrics collection |
| node_exporter | `node_exporter_enable="YES"` | Host metrics exporter |
| Tailscale | `tailscaled_enable="YES"` | Mesh VPN |
| InfluxDB | `influxd_enable="YES"` | Time-series database |
| libvirtd | `libvirtd_enable="YES"` | Virtualisation daemon |
| Redis | `redis_enable="YES"` | In-memory data store |
| PostgreSQL | `postgresql_enable="YES"` | Relational database |

168
ansible/services/authelia/configuration.yml
View file

@ -12,12 +12,12 @@
# This file is deployed to /root/authelia/config/configuration.yml # This file is deployed to /root/authelia/config/configuration.yml
server: server:
address: 'tcp://:9091/' address: "tcp://:9091/"
log: log:
level: 'info' level: "info"
format: 'text' format: "text"
file_path: '/config/authelia.log' file_path: "/config/authelia.log"
keep_stdout: true keep_stdout: true
identity_validation: identity_validation:
@ -28,113 +28,101 @@ identity_validation:
## ##
authentication_backend: authentication_backend:
ldap: ldap:
address: 'ldap://lldap:3890' address: "ldap://lldap:3890"
implementation: 'lldap' implementation: "lldap"
timeout: '20 seconds' timeout: "20 seconds"
start_tls: false start_tls: false
base_dn: 'dc=pez,dc=sh' base_dn: "dc=pez,dc=sh"
additional_users_dn: 'ou=people' additional_users_dn: "ou=people"
additional_groups_dn: 'ou=groups' additional_groups_dn: "ou=groups"
user: 'cn=admin,ou=people,dc=pez,dc=sh' user: "cn=admin,ou=people,dc=pez,dc=sh"
# Password provided via AUTHELIA_AUTHENTICATION_BACKEND_LDAP_PASSWORD_FILE env var # Password provided via AUTHELIA_AUTHENTICATION_BACKEND_LDAP_PASSWORD_FILE env var
## ##
## Access Control — default deny, per-service groups ## Access Control — default deny, per-service groups
## ##
access_control: access_control:
default_policy: 'deny' default_policy: "deny"
rules: rules:
# pez.sh domains # pez.sh domains
- domain: 'grafana.pez.sh' - domain: "radarr.pez.sh"
subject: 'group:pez_grafana_users' subject: "group:pez_radarr_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'prometheus.pez.sh' - domain: "sonarr.pez.sh"
subject: 'group:pez_prometheus_users' subject: "group:pez_sonarr_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'radarr.pez.sh' - domain: "lidarr.pez.sh"
subject: 'group:pez_radarr_users' subject: "group:pez_lidarr_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'sonarr.pez.sh' - domain: "readarr.pez.sh"
subject: 'group:pez_sonarr_users' subject: "group:pez_readarr_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'lidarr.pez.sh' - domain: "download.pez.sh"
subject: 'group:pez_lidarr_users' subject: "group:pez_download_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'readarr.pez.sh' - domain: "rss.pez.sh"
subject: 'group:pez_readarr_users' subject: "group:pez_rss_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'download.pez.sh' - domain: "soulseek.pez.sh"
subject: 'group:pez_download_users' subject: "group:pez_soulseek_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'rss.pez.sh' - domain: "prowlarr.pez.sh"
subject: 'group:pez_rss_users' subject: "group:pez_prowlarr_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'soulseek.pez.sh' - domain: "git.pez.sh"
subject: 'group:pez_soulseek_users' subject: "group:pez_git_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'prowlarr.pez.sh'
subject: 'group:pez_prowlarr_users'
policy: 'one_factor'
- domain: 'git.pez.sh'
subject: 'group:pez_git_users'
policy: 'one_factor'
# pez.solutions domains (mirrors) # pez.solutions domains (mirrors)
- domain: 'grafana.pez.solutions' - domain: "radarr.pez.solutions"
subject: 'group:pez_grafana_users' subject: "group:pez_radarr_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'prometheus.pez.solutions' - domain: "sonarr.pez.solutions"
subject: 'group:pez_prometheus_users' subject: "group:pez_sonarr_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'radarr.pez.solutions' - domain: "lidarr.pez.solutions"
subject: 'group:pez_radarr_users' subject: "group:pez_lidarr_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'sonarr.pez.solutions' - domain: "readarr.pez.solutions"
subject: 'group:pez_sonarr_users' subject: "group:pez_readarr_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'lidarr.pez.solutions' - domain: "download.pez.solutions"
subject: 'group:pez_lidarr_users' subject: "group:pez_download_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'readarr.pez.solutions' - domain: "soulseek.pez.solutions"
subject: 'group:pez_readarr_users' subject: "group:pez_soulseek_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'download.pez.solutions' - domain: "prowlarr.pez.solutions"
subject: 'group:pez_download_users' subject: "group:pez_prowlarr_users"
policy: 'one_factor' policy: "one_factor"
- domain: 'soulseek.pez.solutions'
subject: 'group:pez_soulseek_users'
policy: 'one_factor'
- domain: 'prowlarr.pez.solutions'
subject: 'group:pez_prowlarr_users'
policy: 'one_factor'
# Shared apps portals # Shared apps portals
- domain: 'apps.pez.sh' - domain: "apps.pez.sh"
subject: 'group:pez_plebs' subject: "group:pez_plebs"
policy: 'one_factor' policy: "one_factor"
- domain: 'apps.pez.solutions' - domain: "apps.pez.solutions"
subject: 'group:pez_plebs' subject: "group:pez_plebs"
policy: 'one_factor' policy: "one_factor"
## ##
## Session — cookie domains ## Session — cookie domains
## ##
session: session:
cookies: cookies:
- domain: 'pez.sh' - domain: "pez.sh"
authelia_url: 'https://auth.pez.sh' authelia_url: "https://auth.pez.sh"
- domain: 'pez.solutions' - domain: "pez.solutions"
authelia_url: 'https://auth.pez.solutions' authelia_url: "https://auth.pez.solutions"
## ##
## Storage — MariaDB ## Storage — MariaDB
## ##
storage: storage:
mysql: mysql:
address: 'tcp://mariadb:3306' address: "tcp://mariadb:3306"
database: 'authelia' database: "authelia"
username: 'authelia' username: "authelia"
timeout: '10 seconds' timeout: "10 seconds"
# Password provided via AUTHELIA_STORAGE_MYSQL_PASSWORD_FILE env var # Password provided via AUTHELIA_STORAGE_MYSQL_PASSWORD_FILE env var
## ##
@ -143,9 +131,9 @@ storage:
notifier: notifier:
disable_startup_check: true disable_startup_check: true
smtp: smtp:
address: 'smtp://mail.pez.sh' address: "smtp://mail.pez.sh"
username: 'pez' username: "pez"
# Password provided via AUTHELIA_NOTIFIER_SMTP_PASSWORD_FILE env var # Password provided via AUTHELIA_NOTIFIER_SMTP_PASSWORD_FILE env var
sender: 'Authelia <pez@pez.sh>' sender: "Authelia <pez@pez.sh>"
tls: tls:
server_name: 'mail.pez.sh' server_name: "mail.pez.sh"

1
ansible/services/caddy/README.md
View file

@ -77,7 +77,6 @@ forward_auth localhost:9091 {
| Service | Auth | Reason | | Service | Auth | Reason |
|---------|------|--------| |---------|------|--------|
| Grafana, Prometheus | Authelia | Admin dashboards |
| Radarr, Sonarr, Lidarr, Readarr | Authelia | Media management | | Radarr, Sonarr, Lidarr, Readarr | Authelia | Media management |
| Prowlarr, Transmission (download) | Authelia | Download tools | | Prowlarr, Transmission (download) | Authelia | Download tools |
| slskd (Soulseek) | Authelia | P2P client | | slskd (Soulseek) | Authelia | P2P client |

62
ansible/services/grafana/README.md
View file

@ -1,62 +0,0 @@
# Grafana
Grafana dashboards, alerting rules, and provisioning config for the homelab/cloud stack.
Runs on **london-a** (FreeBSD, `100.122.219.41`) as a native service (not Docker).
Migrated from the standalone `pez-grafana` repo.
## Structure
```
services/grafana/
├── dashboards/ # Dashboard JSON files
│ ├── infrastructure.json # Infrastructure overview (linux hosts)
│ ├── living-room-display.json # Kiosk/TV dashboard
│ ├── node-exporter-full.json # Full node exporter metrics
│ └── traffic-slo.json # Traffic / SLO tracking
└── provisioning/ # Grafana provisioning files
├── alerting/
│ ├── contact-points.yml # Alert receivers (PagerDuty, email)
│ ├── notification-policy.yml # Routing: critical → PagerDuty, warning → email
│ ├── rules-critical.yml # Tier 1: pages PagerDuty immediately
│ └── rules-warning.yml # Tier 2: email only
├── dashboards/
│ └── dashboards.yml # Dashboard file provider config
└── datasources/
└── datasources.json # Prometheus datasource (localhost:9090)
```
## Alert Tiers
| Tier | Routing | Examples |
|----------|------------|--------------------------------------------|
| Critical | PagerDuty | Host down, disk >95%, memory >95% |
| Warning | Email | Disk >80%, memory >85%, high load/CPU |
## Deployment
Deployed via the monorepo's `ansible/deploy.yml` (Stage 4e: Monitoring stack).
```bash
cd ansible
ansible-playbook deploy.yml --limit london-a --tags monitoring
```
Provisioning files are synced to `/usr/local/etc/grafana/provisioning/` and dashboards
to `/usr/local/etc/grafana/dashboards/` on london-a. Grafana is restarted after changes.
### Notes
- The old `pez-grafana` repo deployed provisioning to `/usr/local/share/grafana/conf/provisioning/`.
The monorepo uses `/usr/local/etc/grafana/` — verify the correct path on london-a before first deploy.
- PagerDuty integration key is referenced via `${PAGERDUTY_INTEGRATION_KEY}` env var (not stored in repo).
- Grafana password is not committed; pass via `--extra-vars` or env.
## Importing Dashboards Manually
```bash
curl -X POST -H "Content-Type: application/json" \
-u admin:password \
-d "{\"dashboard\": $(cat dashboards/infrastructure.json), \"overwrite\": true}" \
http://localhost:3000/api/dashboards/db
```

762
ansible/services/grafana/dashboards/energy.json
View file

@ -1,762 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 10,
"links": [],
"liveNow": true,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "#EAB839",
"value": 1000
},
{
"color": "red",
"value": 2000
}
]
},
"unit": "watt"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 0,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "inverted",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": true,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.3",
"targets": [
{
"editorMode": "code",
"expr": "avg_over_time(octopus_electricity_demand_watts{}[30m])",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"timeFrom": "30m",
"title": "Average (30 min)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "#EAB839",
"value": 1000
},
{
"color": "red",
"value": 2000
}
]
},
"unit": "watt"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 6,
"y": 0
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "inverted",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": true,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.3",
"targets": [
{
"editorMode": "code",
"expr": "avg_over_time(octopus_electricity_demand_watts{}[6h])",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"timeFrom": "6h",
"title": "Average (6 hours)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "#EAB839",
"value": 1000
},
{
"color": "red",
"value": 2000
}
]
},
"unit": "watt"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 12,
"y": 0
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "inverted",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": true,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.3",
"targets": [
{
"editorMode": "code",
"expr": "avg_over_time(octopus_electricity_demand_watts{}[24h])",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"timeFrom": "1d",
"title": "Average (1 day)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "#EAB839",
"value": 1000
},
{
"color": "red",
"value": 2000
}
]
},
"unit": "watt"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 18,
"y": 0
},
"id": 5,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "inverted",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": true,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.3",
"targets": [
{
"editorMode": "code",
"expr": "avg_over_time(octopus_electricity_demand_watts{}[7d])",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"timeFrom": "7d",
"title": "Average (7 days)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMin": 0,
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 18,
"x": 0,
"y": 6
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "builder",
"expr": "octopus_electricity_demand_watts",
"legendFormat": "Watts",
"range": true,
"refId": "A"
}
],
"title": "Wattage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"max": 5000,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "yellow",
"value": 1000
},
{
"color": "red",
"value": 2000
}
]
},
"unit": "watt"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 6,
"x": 18,
"y": 6
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.3",
"targets": [
{
"editorMode": "builder",
"exemplar": false,
"expr": "octopus_electricity_demand_watts",
"instant": true,
"legendFormat": "__auto",
"range": false,
"refId": "A"
}
],
"title": "Current Watts",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 2,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "currencyGBP"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 0,
"y": 13
},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "inverted",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": true,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.3",
"targets": [
{
"editorMode": "code",
"expr": "sum_over_time((octopus_electricity_consumption_kwh * on() octopus_electricity_unit_rate_pence)[$__range:30m]) / 100",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"timeFrom": "1h",
"title": "Price",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 2,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "currencyGBP"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 6,
"y": 13
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "inverted",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": true,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.3",
"targets": [
{
"editorMode": "code",
"expr": "sum_over_time((octopus_electricity_consumption_kwh * on() octopus_electricity_unit_rate_pence)[$__range:30m]) / 100",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"timeFrom": "1d",
"title": "Price",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 2,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "currencyGBP"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 12,
"y": 13
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "inverted",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": true,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.3",
"targets": [
{
"editorMode": "code",
"expr": "sum_over_time((octopus_electricity_consumption_kwh * on() octopus_electricity_unit_rate_pence)[$__range:30m]) / 100",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"timeFrom": "1w",
"title": "Price",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 2,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "currencyGBP"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 18,
"y": 13
},
"id": 10,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "inverted",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": true,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.3",
"targets": [
{
"editorMode": "code",
"expr": "sum_over_time((octopus_electricity_consumption_kwh * on() octopus_electricity_unit_rate_pence)[$__range:30m]) / 100",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"timeFrom": "30d",
"title": "Price",
"type": "stat"
}
],
"preload": false,
"refresh": "1m",
"schemaVersion": 42,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Energy",
"uid": "5101a7c4-e5cd-4178-8acf-320588a7a25e",
"version": 5
}

1034
ansible/services/grafana/dashboards/infrastructure.json
View file

File diff suppressed because it is too large Load diff

959
ansible/services/grafana/dashboards/living-room-display.json
View file

@ -1,959 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": true,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 1,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.4.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"expr": "100 * (1 - avg(rate(node_cpu_seconds_total{server=\"london-b\", mode=\"idle\"}[$__rate_interval])))",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "CPU",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "yellow",
"value": 1000
},
{
"color": "red",
"value": 2000
}
]
},
"unit": "watt"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 6,
"x": 5,
"y": 0
},
"id": 14,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.4.2",
"targets": [
{
"editorMode": "code",
"expr": "octopus_electricity_demand_watts",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Watts",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 7,
"x": 11,
"y": 0
},
"id": 4,
"options": {
"displayLabels": [],
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"sort": "desc",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.4.2",
"targets": [
{
"editorMode": "code",
"expr": "node_filesystem_avail_bytes{server=\"london-b\",mountpoint=\"/hdd\"}",
"legendFormat": "Available",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"expr": "node_filesystem_size_bytes{server=\"london-b\",mountpoint=\"/hdd\"} - node_filesystem_avail_bytes{server=\"london-b\",mountpoint=\"/hdd\"}",
"legendFormat": "Used",
"range": true,
"refId": "B"
}
],
"title": "HDD Space",
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 6,
"x": 18,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "/.*/",
"values": false
},
"showPercentChange": false,
"textMode": "name",
"wideLayout": true
},
"pluginVersion": "12.4.2",
"targets": [
{
"editorMode": "code",
"exemplar": false,
"expr": "node_zfs_zpool_state{server=\"london-b\",zpool=\"hdd\", state=\"online\"} > 0",
"instant": true,
"legendFormat": "Online",
"range": false,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"exemplar": false,
"expr": "node_zfs_zpool_state{server=\"london-b\",zpool=\"hdd\", state=\"degraded\"} > 0",
"instant": true,
"legendFormat": "Degraded",
"range": false,
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"exemplar": false,
"expr": "node_zfs_zpool_state{server=\"london-b\",zpool=\"hdd\", state=\"faulted\"} > 0",
"instant": true,
"legendFormat": "Faulted",
"range": false,
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"exemplar": false,
"expr": "node_zfs_zpool_state{server=\"london-b\",zpool=\"hdd\", state=\"offline\"} > 0",
"instant": true,
"legendFormat": "Offline",
"range": false,
"refId": "D"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"exemplar": false,
"expr": "node_zfs_zpool_state{server=\"london-b\",zpool=\"hdd\", state=\"removed\"} > 0",
"instant": true,
"legendFormat": "Removed",
"range": false,
"refId": "E"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"exemplar": false,
"expr": "node_zfs_zpool_state{server=\"london-b\",zpool=\"hdd\", state=\"suspended\"} > 0",
"instant": true,
"legendFormat": "Suspended",
"range": false,
"refId": "F"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"exemplar": false,
"expr": "node_zfs_zpool_state{server=\"london-b\",zpool=\"hdd\", state=\"unavail\"} > 0",
"instant": true,
"legendFormat": "Unavailable",
"range": false,
"refId": "G"
}
],
"title": "HDD State",
"transformations": [
{
"id": "filterFieldsByName",
"options": {
"include": {
"names": [
"Online"
]
}
}
}
],
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 1,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "#EAB839",
"value": 80
},
{
"color": "red",
"value": 90
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 0,
"y": 5
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.4.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"expr": "clamp_min((1 - (node_memory_MemAvailable_bytes{server=\"london-b\"} / node_memory_MemTotal_bytes{server=\"london-b\"})) * 100, 0)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Memory",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "currencyGBP"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 6,
"x": 5,
"y": 5
},
"id": 15,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.4.2",
"targets": [
{
"editorMode": "code",
"expr": "octopus_account_balance_pence / 100 * -1",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Octopus Account Bill",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 1,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 18,
"y": 5
},
"id": 5,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.4.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"expr": "node_filesystem_avail_bytes{server=\"london-b\",mountpoint=\"/hdd\"}",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Available Space",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 1,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 21,
"y": 5
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.4.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"expr": "node_filesystem_size_bytes{server=\"london-b\",mountpoint=\"/hdd\"} - node_filesystem_avail_bytes{server=\"london-b\",mountpoint=\"/hdd\"}",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Used Space",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"filterable": false,
"footer": {
"reducers": []
},
"inspect": false
},
"mappings": [
{
"options": {
"Mak999": {
"index": 4,
"text": "Amar"
},
"Malene Wejlgaard Knudsen": {
"index": 5,
"text": "Malene"
},
"d.han81": {
"index": 2,
"text": "Han"
},
"er1227": {
"index": 1,
"text": "Erik"
},
"guykeren437": {
"index": 15,
"text": "Guy"
},
"isab579": {
"index": 3,
"text": "Scoulers Daughter"
},
"naveen.629": {
"index": 6,
"text": "Naveen"
},
"pe423": {
"index": 13,
"text": "Living Room"
},
"praczyk.": {
"index": 7,
"text": "Trevor"
},
"pravee63": {
"index": 8,
"text": "Praveen"
},
"scou210": {
"index": 9,
"text": "Scouler"
},
"sorghumc": {
"index": 10,
"text": "Anton"
},
"theonet5": {
"index": 11,
"text": "Trevor"
},
"theonetb": {
"index": 12,
"text": "Trevor"
},
"wooley_82": {
"index": 0,
"text": "Wooly"
},
"yp2xc": {
"index": 14,
"text": "Trevor"
}
},
"type": "value"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 20,
"x": 0,
"y": 10
},
"id": 12,
"options": {
"cellHeight": "sm",
"showHeader": true
},
"pluginVersion": "12.4.2",
"targets": [
{
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "plays_total{user!=\"Rasmus\"}",
"format": "table",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "User",
"range": false,
"refId": "A",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(plays_total) by (title)",
"format": "table",
"hide": true,
"instant": true,
"legendFormat": "Title",
"range": false,
"refId": "B"
}
],
"title": "Current Activity",
"transformations": [
{
"id": "filterFieldsByName",
"options": {
"include": {
"names": [
"child_title",
"stream_resolution",
"stream_type",
"title",
"user",
"grandchild_title"
]
}
}
},
{
"id": "organize",
"options": {
"excludeByName": {},
"includeByName": {},
"indexByName": {
"child_title": 2,
"grandchild_title": 3,
"stream_resolution": 5,
"stream_type": 4,
"title": 1,
"user": 0
},
"renameByName": {
"child_title": "Season",
"grandchild_title": "Episode Title",
"stream_resolution": "Resolution",
"stream_type": "Stream",
"title": "Title",
"user": "User"
}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"fieldMinMax": false,
"mappings": [
{
"options": {
"wooley_82": {
"index": 0,
"text": "Wooly"
}
},
"type": "value"
}
],
"min": 0,
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 4,
"x": 20,
"y": 10
},
"id": 13,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.4.2",
"targets": [
{
"disableTextWrap": false,
"editorMode": "code",
"exemplar": false,
"expr": "count(plays_total{user!=\"Rasmus\"})",
"format": "table",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "__auto",
"range": false,
"refId": "A",
"useBackend": false
}
],
"title": "Active Streams",
"type": "stat"
}
],
"preload": false,
"refresh": "30s",
"schemaVersion": 42,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {
"nowDelay": "0m"
},
"timezone": "browser",
"title": "Living Room Display",
"uid": "a68bd259-c836-4fad-b33e-98f1a52a5eb9",
"version": 19,
"weekStart": ""
}

15726
ansible/services/grafana/dashboards/node-exporter-full.json
View file

File diff suppressed because it is too large Load diff

587
ansible/services/grafana/dashboards/traffic-slo.json
View file

@ -1,587 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 3,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": 0
},
{
"color": "yellow",
"value": 99.9
},
{
"color": "green",
"value": 99.99
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 4,
"x": 0,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.1.0",
"targets": [
{
"disableTextWrap": false,
"editorMode": "code",
"expr": "clamp_max(\n (sum(caddy_http_response_size_bytes_count{host=~\".*(pez.solutions|pez.sh)\", code!~\"5.*\"}) / (sum(caddy_http_response_size_bytes_count{host=~\".*(pez.solutions|pez.sh)\"}))) * 100,\n 99.999\n)",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "SLI",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 3,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": 0
},
{
"color": "yellow",
"value": 99.9
},
{
"color": "green",
"value": 99.99
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 20,
"x": 4,
"y": 0
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.1.0",
"targets": [
{
"disableTextWrap": false,
"editorMode": "code",
"expr": "clamp_max(\n (\n sum(\n label_replace(\n caddy_http_response_size_bytes_count{host=~\".*(pez.solutions|pez.sh)\", host!~\"(pez.sh|pez.solutions)\", code!~\"5.*\"},\n \"host_prefix\",\n \"$1\",\n \"host\",\n \"([^.]+)\\\\..*\"\n )\n ) by (host_prefix)\n /\n sum(\n label_replace(\n caddy_http_response_size_bytes_count{host=~\".*(pez.solutions|pez.sh)\", host!~\"(pez.sh|pez.solutions)\"},\n \"host_prefix\",\n \"$1\",\n \"host\",\n \"([^.]+)\\\\..*\"\n )\n ) by (host_prefix)\n ) * 100,\n 99.999\n)",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "SLI by Host",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 19,
"x": 0,
"y": 10
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.1.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum(\n label_replace(\n rate(caddy_http_requests_total{handler!=\"metrics\", host=~\".*(pez.solutions|pez.sh)\"}[$__rate_interval]),\n \"host_prefix\",\n \"$1\",\n \"host\",\n \"([^.]+)\\\\..*\"\n )\n) by (host_prefix)",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "{{host}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Traffic Rate by Service",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 5,
"x": 19,
"y": 10
},
"id": 6,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "req/s"
}
]
},
"pluginVersion": "12.1.0",
"targets": [
{
"editorMode": "code",
"exemplar": false,
"expr": "sum(\n label_replace(\n rate(caddy_http_requests_total{handler!=\"metrics\", host=~\".*(pez.solutions|pez.sh)\", host!~\"(pez.sh|pez.solutions)\"}[$__rate_interval]),\n \"host_prefix\",\n \"$1\",\n \"host\",\n \"([^.]+)\\\\..*\"\n )\n) by (host_prefix) > 0",
"format": "table",
"instant": true,
"legendFormat": "__auto",
"range": false,
"refId": "A"
}
],
"title": "Active Services",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true
},
"includeByName": {},
"indexByName": {},
"renameByName": {
"Value": "req/s",
"host_prefix": "Service"
}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 12,
"x": 0,
"y": 21
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.1.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum(rate(caddy_http_response_duration_seconds_count{code!~\"5.*\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "Good",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Response Codes (Good)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMin": 0,
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 12,
"x": 12,
"y": 21
},
"id": 5,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.1.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bezqqznn81wqof"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum(rate(caddy_http_response_duration_seconds_count{code=~\"5.*\"}[$__rate_interval])) by (code, host) > 0",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "{{code}} - {{host}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Response Codes (Bad)",
"type": "timeseries"
}
],
"preload": false,
"refresh": "5s",
"schemaVersion": 41,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Traffic / SLO",
"uid": "384f28fe-2435-480f-a0f0-723ccdcf8b3b"
}

24
ansible/services/grafana/provisioning/alerting/contact-points.yml
View file

@ -1,24 +0,0 @@
---
apiVersion: 1
# Contact points — defines where alerts are sent.
# PagerDuty key is managed via Grafana UI / environment variable; do not commit secrets here.
contactPoints:
- orgId: 1
name: PagerDuty
receivers:
- uid: bf0ukmhpefshsc
type: pagerduty
settings:
integrationKey: "{{ grafana_pagerduty_integration_key }}"
disableResolveMessage: false
- orgId: 1
name: email
receivers:
- uid: email-receiver
type: email
settings:
addresses: pez@pez.sh
disableResolveMessage: false

32
ansible/services/grafana/provisioning/alerting/notification-policy.yml
View file

@ -1,32 +0,0 @@
---
apiVersion: 1
# Notification routing policy.
# Critical alerts (severity=critical) → PagerDuty.
# Warning alerts (severity=warning) → email.
policies:
- orgId: 1
receiver: PagerDuty
group_by:
- alertname
- server
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
routes:
- receiver: PagerDuty
matchers:
- severity = critical
group_wait: 0s
group_interval: 1m
repeat_interval: 1h
continue: false
- receiver: email
matchers:
- severity = warning
group_wait: 2m
group_interval: 10m
repeat_interval: 8h
continue: false

413
ansible/services/grafana/provisioning/alerting/rules-critical.yml
View file

@ -1,413 +0,0 @@
---
apiVersion: 1
# Tier 1 — Critical alerts. These page PagerDuty.
# Datasource UID: bezqqznn81wqof (Prometheus on london-a)
# All alerts use reduce+threshold (not classic_conditions) so $labels.* and $value work in annotations.
groups:
- orgId: 1
name: critical-availability
folder: Alerting
interval: 1m
rules:
- uid: cff6uy1tufj0ge
title: Host Down
condition: C
data:
- refId: A
datasourceUid: bezqqznn81wqof
relativeTimeRange:
from: 600
to: 0
model:
expr: up{job="node_exporter"}
instant: true
intervalMs: 1000
maxDataPoints: 43200
refId: A
- refId: B
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
datasource:
type: __expr__
uid: __expr__
expression: A
reducer: last
settings:
mode: ""
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
conditions:
- evaluator:
params: [1]
type: lt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
refId: C
type: threshold
noDataState: Alerting
execErrState: Alerting
for: 2m
annotations:
summary: "Host {{ $labels.server }} is down"
description: "Node exporter on {{ $labels.server }} ({{ $labels.instance }}) has been unreachable for 2+ minutes."
labels:
severity: critical
isPaused: false
- uid: aff6uy1vxchdse
title: Disk Usage Critical (>95%)
condition: C
data:
- refId: A
datasourceUid: bezqqznn81wqof
relativeTimeRange:
from: 600
to: 0
model:
expr: |
(
node_filesystem_size_bytes{job="node_exporter", fstype!~"tmpfs|overlay|squashfs|devtmpfs"}
- node_filesystem_avail_bytes{job="node_exporter", fstype!~"tmpfs|overlay|squashfs|devtmpfs"}
)
/ node_filesystem_size_bytes{job="node_exporter", fstype!~"tmpfs|overlay|squashfs|devtmpfs"}
* 100
instant: true
intervalMs: 1000
maxDataPoints: 43200
refId: A
- refId: B
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
datasource:
type: __expr__
uid: __expr__
expression: A
reducer: last
settings:
mode: ""
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
conditions:
- evaluator:
params: [95]
type: gt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
refId: C
type: threshold
noDataState: NoData
execErrState: Error
for: 5m
annotations:
summary: "Disk critically full on {{ $labels.server }}"
description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.server }} is over 95% full (currently {{ $value | printf \"%.1f\" }}%)."
labels:
severity: critical
isPaused: false
- uid: aff6uy1xq9udca
title: Memory Usage Critical (>95%)
condition: C
data:
- refId: A
datasourceUid: bezqqznn81wqof
relativeTimeRange:
from: 600
to: 0
model:
expr: |
(1 - (node_memory_MemAvailable_bytes{job="node_exporter"} / node_memory_MemTotal_bytes{job="node_exporter"})) * 100
instant: true
intervalMs: 1000
maxDataPoints: 43200
refId: A
- refId: B
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
datasource:
type: __expr__
uid: __expr__
expression: A
reducer: last
settings:
mode: ""
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
conditions:
- evaluator:
params: [95]
type: gt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
refId: C
type: threshold
noDataState: NoData
execErrState: Error
for: 5m
annotations:
summary: "Memory critically low on {{ $labels.server }}"
description: "Memory usage on {{ $labels.server }} ({{ $labels.instance }}) is above 95% for 5+ minutes."
labels:
severity: critical
isPaused: false
- uid: fff6uy219mo00e
title: SMART Disk Health Failure (london-b)
condition: C
data:
- refId: A
datasourceUid: bezqqznn81wqof
relativeTimeRange:
from: 600
to: 0
model:
expr: smartctl_device_smart_status{job="smartmontools"}
instant: true
intervalMs: 1000
maxDataPoints: 43200
refId: A
- refId: B
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
datasource:
type: __expr__
uid: __expr__
expression: A
reducer: last
settings:
mode: ""
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
conditions:
- evaluator:
params: [1]
type: lt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
refId: C
type: threshold
noDataState: NoData
execErrState: Error
for: 0m
annotations:
summary: "Disk SMART health failure on london-b"
description: "Drive {{ $labels.device }} on london-b reports SMART health failure. Check immediately."
labels:
severity: critical
isPaused: false
- orgId: 1
name: critical-caddy
folder: Alerting
interval: 1m
rules:
- uid: fff6uy1zgpb0gd
title: Caddy Down (helsinki-a)
condition: C
data:
- refId: A
datasourceUid: bezqqznn81wqof
relativeTimeRange:
from: 600
to: 0
model:
expr: up{job="caddy"}
instant: true
intervalMs: 1000
maxDataPoints: 43200
refId: A
- refId: B
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
datasource:
type: __expr__
uid: __expr__
expression: A
reducer: last
settings:
mode: ""
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
conditions:
- evaluator:
params: [1]
type: lt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
refId: C
type: threshold
noDataState: Alerting
execErrState: Alerting
for: 1m
annotations:
summary: "Caddy is down on helsinki-a"
description: "Caddy (main reverse proxy) on helsinki-a unreachable. External services likely down."
labels:
severity: critical
isPaused: false
- orgId: 1
name: critical-services
folder: Alerting
interval: 1m
rules:
- uid: bff6uy2a2rrwgb
title: Plex Down (london-b)
condition: C
data:
- refId: A
datasourceUid: bezqqznn81wqof
relativeTimeRange:
from: 600
to: 0
model:
expr: up{job="plex"}
instant: true
intervalMs: 1000
maxDataPoints: 43200
refId: A
- refId: B
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
datasource:
type: __expr__
uid: __expr__
expression: A
reducer: last
settings:
mode: ""
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
conditions:
- evaluator:
params: [1]
type: lt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
refId: C
type: threshold
noDataState: Alerting
execErrState: Alerting
for: 5m
annotations:
summary: "Plex is down on london-b"
description: "The Plex exporter on london-b has been unreachable for 5+ minutes."
labels:
severity: critical
isPaused: false

278
ansible/services/grafana/provisioning/alerting/rules-warning.yml
View file

@ -1,278 +0,0 @@
---
apiVersion: 1
# Tier 2 — Warning alerts. These send email only (non-paging).
# Datasource UID: bezqqznn81wqof (Prometheus on london-a)
# All alerts use reduce+threshold (not classic_conditions) so $labels.* and $value work in annotations.
groups:
- orgId: 1
name: warning-resources
folder: Alerting
interval: 2m
rules:
- uid: cff6uy23024n4c
title: Disk Usage Warning (>80%)
condition: C
data:
- refId: A
datasourceUid: bezqqznn81wqof
relativeTimeRange:
from: 600
to: 0
model:
expr: |
(
node_filesystem_size_bytes{job="node_exporter", fstype!~"tmpfs|overlay|squashfs|devtmpfs"}
- node_filesystem_avail_bytes{job="node_exporter", fstype!~"tmpfs|overlay|squashfs|devtmpfs"}
)
/ node_filesystem_size_bytes{job="node_exporter", fstype!~"tmpfs|overlay|squashfs|devtmpfs"}
* 100
instant: true
intervalMs: 1000
maxDataPoints: 43200
refId: A
- refId: B
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
datasource:
type: __expr__
uid: __expr__
expression: A
reducer: last
settings:
mode: ""
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
conditions:
- evaluator:
params: [80]
type: gt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
refId: C
type: threshold
noDataState: NoData
execErrState: Error
for: 10m
annotations:
summary: "Disk usage high on {{ $labels.server }}"
description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.server }} is over 80% full (currently {{ $value | printf \"%.1f\" }}%)."
labels:
severity: warning
isPaused: false
- uid: dff6uy24szhmod
title: Memory Usage Warning (>85%)
condition: C
data:
- refId: A
datasourceUid: bezqqznn81wqof
relativeTimeRange:
from: 600
to: 0
model:
expr: |
(1 - (node_memory_MemAvailable_bytes{job="node_exporter"} / node_memory_MemTotal_bytes{job="node_exporter"})) * 100
instant: true
intervalMs: 1000
maxDataPoints: 43200
refId: A
- refId: B
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
datasource:
type: __expr__
uid: __expr__
expression: A
reducer: last
settings:
mode: ""
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
conditions:
- evaluator:
params: [85]
type: gt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
refId: C
type: threshold
noDataState: NoData
execErrState: Error
for: 10m
annotations:
summary: "Memory usage high on {{ $labels.server }}"
description: "Memory usage on {{ $labels.server }} ({{ $labels.instance }}) is above 85% for 10+ minutes."
labels:
severity: warning
isPaused: false
- uid: cff6uy26jey9sd
title: CPU Usage High (>85%)
condition: C
data:
- refId: A
datasourceUid: bezqqznn81wqof
relativeTimeRange:
from: 600
to: 0
model:
expr: |
100 - (avg by (server, instance) (rate(node_cpu_seconds_total{job="node_exporter", mode="idle"}[5m])) * 100)
instant: true
intervalMs: 1000
maxDataPoints: 43200
refId: A
- refId: B
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
datasource:
type: __expr__
uid: __expr__
expression: A
reducer: last
settings:
mode: ""
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
conditions:
- evaluator:
params: [85]
type: gt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
refId: C
type: threshold
noDataState: NoData
execErrState: Error
for: 15m
annotations:
summary: "CPU usage sustained high on {{ $labels.server }}"
description: "CPU on {{ $labels.server }} has been above 85% for 15+ minutes (currently {{ $value | printf \"%.1f\" }}%)."
labels:
severity: warning
isPaused: false
- uid: eff6uy289uewwb
title: System Load High (>2x CPUs)
condition: C
data:
- refId: A
datasourceUid: bezqqznn81wqof
relativeTimeRange:
from: 600
to: 0
model:
# Compare 15-minute load against number of CPUs
expr: |
node_load15{job="node_exporter"} / on(instance) group_left() count by (instance) (node_cpu_seconds_total{job="node_exporter", mode="idle"})
instant: true
intervalMs: 1000
maxDataPoints: 43200
refId: A
- refId: B
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
datasource:
type: __expr__
uid: __expr__
expression: A
reducer: last
settings:
mode: ""
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
relativeTimeRange:
from: 0
to: 0
model:
conditions:
- evaluator:
params: [2]
type: gt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
refId: C
type: threshold
noDataState: NoData
execErrState: Error
for: 15m
annotations:
summary: "High system load on {{ $labels.server }}"
description: "15-minute load average on {{ $labels.server }} is {{ $value | printf \"%.2f\" }}x the CPU count (threshold: 2x)."
labels:
severity: warning
isPaused: false

16
ansible/services/grafana/provisioning/dashboards/dashboards.yml
View file

@ -1,16 +0,0 @@
---
apiVersion: 1
# Dashboard provisioning — tells Grafana where to find dashboard JSON files.
# Path is relative to the Grafana installation on london-a (FreeBSD).
providers:
- name: default
orgId: 1
folder: ""
type: file
disableDeletion: false
updateIntervalSeconds: 30
options:
path: /usr/local/etc/grafana/dashboards
foldersFromFilesStructure: false

30
ansible/services/grafana/provisioning/datasources/datasources.json
View file

@ -1,30 +0,0 @@
[
{
"uid": "bezqqznn81wqof",
"name": "prometheus",
"type": "prometheus",
"typeName": "Prometheus",
"typeLogoUrl": "public/plugins/prometheus/img/prometheus_logo.svg",
"access": "proxy",
"url": "http://localhost:9090",
"user": "",
"database": "",
"basicAuth": false,
"isDefault": true,
"jsonData": {
"pdcInjected": false
}
},
{
"uid": "loki_london_a",
"name": "Loki",
"type": "loki",
"access": "proxy",
"url": "http://localhost:3100",
"basicAuth": false,
"isDefault": false,
"jsonData": {
"maxLines": 1000
}
}
]

54
ansible/services/prometheus/README.md
View file

@ -1,54 +0,0 @@
# Prometheus
Runs on **london-a** (FreeBSD, 100.122.219.41).
## Service Details
- **Binary:** `/usr/local/bin/prometheus`
- **Config:** `/usr/local/etc/prometheus.yml`
- **Data:** `/var/db/prometheus`
- **Web UI:** `http://london-a:9090`
- **Runs as:** `prometheus` user via daemon(8)
## Scrape Targets
| Job | Target | Host | Port | What it scrapes |
|-----|--------|------|------|-----------------|
| `prometheus` | localhost:9090 | london-a | 9090 | Prometheus self-metrics |
| `node_exporter` | 192.168.1.254:9100 | london-a | 9100 | OS metrics (FreeBSD) |
| `node_exporter` | 192.168.1.253:9100 | london-b | 9100 | OS metrics (Linux) |
| `node_exporter` | 100.89.206.60:9100 | copenhagen-a | 9100 | OS metrics (Linux) |
| `node_exporter` | 100.115.45.53:9100 | copenhagen-c | 9100 | OS metrics (Linux) |
| `node_exporter` | 100.117.235.28:9100 | nuremberg-a | 9100 | OS metrics (Alpine) |
| `node_exporter` | 100.67.6.27:9100 | helsinki-a | 9100 | OS metrics (Linux) |
| `smartmontools` | 192.168.1.253:9633 | london-b | 9633 | SMART disk health (smartctl_exporter) |
| `plex` | 192.168.1.253:9000 | london-b | 9000 | Plex media server metrics |
| `caddy` | 100.67.6.27:2019 | helsinki-a | 2019 | Caddy admin API / metrics |
### Network Notes
- London hosts (london-a, london-b) use **LAN IPs** (192.168.1.x) since Prometheus runs locally in the London rack
- Remote hosts (copenhagen, nuremberg, helsinki) use **Tailscale IPs** (100.x.x.x)
## Alerting Rules
### `rules/node-exporter.rules`
Sourced from pez-ansible. Currently all rules are **commented out** — only a placeholder `ServerRunningBtrfs` alert exists (disabled). No active alerting rules loaded by Prometheus. Alerting is handled exclusively by **Grafana** (not Alertmanager).
## What's Not Configured
- **Rule files** — referenced lines in `prometheus.yml` are commented out (rules exist in `rules/` but aren't loaded)
- **Recording rules** — none
## Deployment
Config is managed manually on london-a. To deploy changes:
```bash
# Copy config to london-a
scp prometheus.yml root@100.122.219.41:/usr/local/etc/prometheus.yml
# Reload (graceful, no restart needed)
ssh root@100.122.219.41 "kill -HUP $(pgrep prometheus)"
```

91
ansible/services/prometheus/prometheus.yml.j2
View file

@ -1,91 +0,0 @@
---
# Ansible managed — generated from prometheus.yml.j2
# Config file location on london-a: /usr/local/etc/prometheus.yml
# Prometheus runs as: /usr/local/bin/prometheus --config.file=/usr/local/etc/prometheus.yml
# Data directory: /var/db/prometheus
global:
scrape_interval: 15s
evaluation_interval: 15s
# Alerting notifications are handled by Grafana (unified alerting with
# PagerDuty + email contact points), not Alertmanager. No alerting:
# section is needed here. Prometheus still evaluates these rule_files
# so the ALERTS / ALERTS_FOR_STATE metrics are available for queries.
rule_files:
- /usr/local/etc/prometheus/rules/*.rules
scrape_configs:
- job_name: "prometheus"
static_configs:
- targets: ["localhost:9090"]
- job_name: "octopus_exporter"
static_configs:
{% for host in groups['all'] | sort %}
{% set h = hostvars[host] %}
{% if 'octopus-exporter' in (h.docker_services | default([])) %}
- targets: ["{{ h.ansible_host }}:9359"]
labels:
location: {{ h.prometheus_location }}
server: {{ host }}
{% endif %}
{% endfor %}
- job_name: "node_exporter"
static_configs:
{% for host in groups['all'] | sort %}
{% set h = hostvars[host] %}
{% if h.ansible_host is defined %}
- targets: ["{{ h.ansible_host }}:9100"]
labels:
location: {{ h.prometheus_location }}
server: {{ host }}
{% endif %}
{% endfor %}
- job_name: "smartmontools"
static_configs:
{% for host in groups['all'] | sort %}
{% set h = hostvars[host] %}
{% if 'smartctl-exporter' in (h.docker_services | default([])) %}
- targets: ["{{ h.ansible_host }}:9633"]
labels:
location: {{ h.prometheus_location }}
server: {{ host }}
{% endif %}
{% endfor %}
- job_name: "plex"
static_configs:
{% for host in groups['all'] | sort %}
{% set h = hostvars[host] %}
{% if 'plex-exporter' in (h.docker_services | default([])) %}
- targets: ["{{ h.ansible_host }}:9000"]
labels:
location: {{ h.prometheus_location }}
server: {{ host }}
{% endif %}
{% endfor %}
- job_name: "systemd_exporter"
static_configs:
{% for host in groups['systemd_exporter_hosts'] | sort %}
{% set h = hostvars[host] %}
- targets: ["{{ h.ansible_host }}:9558"]
labels:
location: {{ h.prometheus_location }}
server: {{ host }}
{% endfor %}
- job_name: "caddy"
static_configs:
{% for host in groups['all'] | sort %}
{% set h = hostvars[host] %}
{% if h.caddy_config_src is defined %}
- targets: ["{{ h.ansible_host }}:2019"]
labels:
location: {{ h.prometheus_location }}
server: {{ host }}
{% endif %}
{% endfor %}

29
ansible/services/prometheus/rules/zfs.rules
View file

@ -1,29 +0,0 @@
groups:
- name: zfs
rules:
- alert: ZfsPoolDegraded
expr: node_zfs_zpool_state{state="degraded"} == 1
for: 5m
labels:
severity: warning
annotations:
summary: "ZFS pool {{ $labels.zpool }} is degraded on {{ $labels.instance }}"
description: "Pool {{ $labels.zpool }} on {{ $labels.instance }} has entered a degraded state. Check disk health immediately."
- alert: ZfsPoolFaulted
expr: node_zfs_zpool_state{state="faulted"} == 1
for: 1m
labels:
severity: critical
annotations:
summary: "ZFS pool {{ $labels.zpool }} is FAULTED on {{ $labels.instance }}"
description: "Pool {{ $labels.zpool }} on {{ $labels.instance }} is faulted. Data may be at risk."
- alert: ZfsPoolOffline
expr: node_zfs_zpool_state{state="offline"} == 1
for: 5m
labels:
severity: warning
annotations:
summary: "ZFS pool {{ $labels.zpool }} is offline on {{ $labels.instance }}"
description: "Pool {{ $labels.zpool }} on {{ $labels.instance }} is offline."

16
terraform/grafana/dashboards.tf Normal file
View file

@ -0,0 +1,16 @@
resource "grafana_dashboard" "energy" {
config_json = file("${path.module}/dashboards/energy.json")
}
resource "grafana_dashboard" "grafana_cloud_usage" {
config_json = file("${path.module}/dashboards/grafana_cloud_usage.json")
}
resource "grafana_dashboard" "living_room_display" {
config_json = file("${path.module}/dashboards/living_room_display.json")
}
resource "grafana_dashboard" "traffic_slo" {
org_id = 0
config_json = file("${path.module}/dashboards/traffic_slo.json")
}

1140
terraform/grafana/dashboards/energy.json Normal file
View file

File diff suppressed because it is too large Load diff

231
terraform/grafana/dashboards/grafana_cloud_usage.json Normal file
View file

@ -0,0 +1,231 @@
{
"apiVersion": "dashboard.grafana.app/v2",
"kind": "Dashboard",
"metadata": {
"name": "rwx88sh",
"namespace": "stacks-1621083",
"uid": "339a5668-ef0b-4d4d-ab4a-552787f13239",
"resourceVersion": "1777799491857762",
"generation": 2,
"creationTimestamp": "2026-05-03T09:04:14Z",
"labels": {
"grafana.app/deprecatedInternalID": "1726166845087744"
},
"annotations": {
"grafana.app/createdBy": "user:ffkviz5ugketce",
"grafana.app/folder": "",
"grafana.app/saved-from-ui": "Grafana Cloud",
"grafana.app/updatedBy": "user:ffkviz5ugketce",
"grafana.app/updatedTimestamp": "2026-05-03T09:11:31Z"
}
},
"spec": {
"annotations": [
{
"kind": "AnnotationQuery",
"spec": {
"query": {
"kind": "DataQuery",
"group": "grafana",
"version": "v0",
"datasource": {
"name": "-- Grafana --"
},
"spec": {}
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"builtIn": true
}
}
],
"cursorSync": "Off",
"editable": true,
"elements": {
"panel-1": {
"kind": "Panel",
"spec": {
"id": 1,
"title": "Active Series",
"description": "",
"links": [],
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"query": {
"kind": "DataQuery",
"group": "prometheus",
"version": "v0",
"datasource": {
"name": "grafanacloud-usage"
},
"spec": {
"editorMode": "builder",
"expr": "grafanacloud_instance_active_series",
"legendFormat": "Active Series",
"range": true
}
},
"refId": "A",
"hidden": false
}
}
],
"transformations": [],
"queryOptions": {}
}
},
"vizConfig": {
"kind": "VizConfig",
"group": "timeseries",
"version": "13.1.0-25098815508",
"spec": {
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "percentage",
"steps": [
{
"value": 0,
"color": "green"
},
{
"value": 40,
"color": "#EAB839"
},
{
"value": 60,
"color": "red"
}
]
},
"color": {
"mode": "palette-classic",
"seriesBy": "last"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 27,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
}
},
"overrides": []
}
}
}
}
}
},
"layout": {
"kind": "AutoGridLayout",
"spec": {
"maxColumnCount": 3,
"columnWidthMode": "standard",
"rowHeightMode": "standard",
"items": [
{
"kind": "AutoGridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-1"
}
}
}
]
}
},
"links": [],
"liveNow": false,
"preload": false,
"tags": [],
"timeSettings": {
"timezone": "browser",
"from": "now-6h",
"to": "now",
"autoRefresh": "",
"autoRefreshIntervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"hideTimepicker": false,
"fiscalYearStartMonth": 0
},
"title": "Grafana Cloud Usage",
"variables": [],
"preferences": {
"layout": {
"kind": "AutoGridLayout",
"spec": {
"maxColumnCount": 3,
"columnWidthMode": "standard",
"rowHeightMode": "standard",
"items": []
}
}
}
}
}

1401
terraform/grafana/dashboards/living_room_display.json Normal file
View file

File diff suppressed because it is too large Load diff

826
terraform/grafana/dashboards/traffic_slo.json Normal file
View file

@ -0,0 +1,826 @@
{
"apiVersion": "dashboard.grafana.app/v2",
"kind": "Dashboard",
"metadata": {
"name": "384f28fe-2435-480f-a0f0-723ccdcf8b3b",
"namespace": "stacks-1621083",
"uid": "aef74b06-a1af-4a8e-ad1d-88d1208aeb6d",
"resourceVersion": "1777831196986612",
"generation": 2,
"creationTimestamp": "2026-05-03T17:57:39Z",
"labels": {
"grafana.app/deprecatedInternalID": "1860405031632896"
},
"annotations": {
"grafana.app/createdBy": "user:ffkviz5ugketce",
"grafana.app/folder": "",
"grafana.app/saved-from-ui": "Grafana Cloud",
"grafana.app/updatedBy": "user:ffkviz5ugketce",
"grafana.app/updatedTimestamp": "2026-05-03T17:59:56Z"
}
},
"spec": {
"annotations": [
{
"kind": "AnnotationQuery",
"spec": {
"query": {
"kind": "DataQuery",
"group": "grafana",
"version": "v0",
"datasource": {
"name": "-- Grafana --"
},
"spec": {}
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"builtIn": true
}
}
],
"cursorSync": "Off",
"editable": true,
"elements": {
"panel-1": {
"kind": "Panel",
"spec": {
"id": 1,
"title": "Traffic Rate by Service",
"description": "",
"links": [],
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"query": {
"kind": "DataQuery",
"group": "prometheus",
"version": "v0",
"datasource": {
"name": "grafanacloud-prom"
},
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum(\n label_replace(\n rate(caddy_http_request_duration_seconds_count{handler!=\"metrics\", host=~\".*(pez.solutions|pez.sh)\"}[$__rate_interval]),\n \"host_prefix\",\n \"$1\",\n \"host\",\n \"([^.]+)\\\\..*\"\n )\n) by (host_prefix)",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "{{host}}",
"range": true,
"useBackend": false
}
},
"refId": "A",
"hidden": false
}
}
],
"transformations": [],
"queryOptions": {}
}
},
"vizConfig": {
"kind": "VizConfig",
"group": "timeseries",
"version": "13.1.0-25153870157",
"spec": {
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": 0,
"color": "green"
},
{
"value": 80,
"color": "red"
}
]
},
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
}
},
"overrides": []
}
}
}
}
},
"panel-2": {
"kind": "Panel",
"spec": {
"id": 2,
"title": "Response Codes (Good)",
"description": "",
"links": [],
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"query": {
"kind": "DataQuery",
"group": "prometheus",
"version": "v0",
"datasource": {
"name": "grafanacloud-prom"
},
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum(rate(caddy_http_request_duration_seconds_count{code!~\"5.*\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "Good",
"range": true,
"useBackend": false
}
},
"refId": "A",
"hidden": false
}
}
],
"transformations": [],
"queryOptions": {}
}
},
"vizConfig": {
"kind": "VizConfig",
"group": "timeseries",
"version": "13.1.0-25153870157",
"spec": {
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": 0,
"color": "green"
},
{
"value": 80,
"color": "red"
}
]
},
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
}
},
"overrides": []
}
}
}
}
},
"panel-3": {
"kind": "Panel",
"spec": {
"id": 3,
"title": "SLI",
"description": "",
"links": [],
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"query": {
"kind": "DataQuery",
"group": "prometheus",
"version": "v0",
"datasource": {
"name": "grafanacloud-prom"
},
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "clamp_max(\n (sum(caddy_http_request_duration_seconds_count{host=~\".*(pez.solutions|pez.sh)\", code!~\"5.*\"}) / (sum(caddy_http_request_duration_seconds_count{host=~\".*(pez.solutions|pez.sh)\"}))) * 100,\n 99.999\n)",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
}
},
"refId": "A",
"hidden": false
}
}
],
"transformations": [],
"queryOptions": {}
}
},
"vizConfig": {
"kind": "VizConfig",
"group": "stat",
"version": "13.1.0-25153870157",
"spec": {
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"decimals": 3,
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": 0,
"color": "red"
},
{
"value": 99.9,
"color": "yellow"
},
{
"value": 99.99,
"color": "green"
}
]
},
"color": {
"mode": "thresholds"
}
},
"overrides": []
}
}
}
}
},
"panel-4": {
"kind": "Panel",
"spec": {
"id": 4,
"title": "SLI by Host",
"description": "",
"links": [],
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"query": {
"kind": "DataQuery",
"group": "prometheus",
"version": "v0",
"datasource": {
"name": "grafanacloud-prom"
},
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "clamp_max(\n (\n sum(\n label_replace(\n caddy_http_request_duration_seconds_count{host=~\".*(pez.solutions|pez.sh)\", host!~\"(pez.sh|pez.solutions)\", code!~\"5.*\"},\n \"host_prefix\",\n \"$1\",\n \"host\",\n \"([^.]+)\\\\..*\"\n )\n ) by (host_prefix)\n /\n sum(\n label_replace(\n caddy_http_request_duration_seconds_count{host=~\".*(pez.solutions|pez.sh)\", host!~\"(pez.sh|pez.solutions)\"},\n \"host_prefix\",\n \"$1\",\n \"host\",\n \"([^.]+)\\\\..*\"\n )\n ) by (host_prefix)\n ) * 100,\n 99.999\n)",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
}
},
"refId": "A",
"hidden": false
}
}
],
"transformations": [],
"queryOptions": {}
}
},
"vizConfig": {
"kind": "VizConfig",
"group": "stat",
"version": "13.1.0-25153870157",
"spec": {
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"decimals": 3,
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": 0,
"color": "red"
},
{
"value": 99.9,
"color": "yellow"
},
{
"value": 99.99,
"color": "green"
}
]
},
"color": {
"mode": "thresholds"
}
},
"overrides": []
}
}
}
}
},
"panel-5": {
"kind": "Panel",
"spec": {
"id": 5,
"title": "Response Codes (Bad)",
"description": "",
"links": [],
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"query": {
"kind": "DataQuery",
"group": "prometheus",
"version": "v0",
"datasource": {
"name": "grafanacloud-prom"
},
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "(sum(rate(caddy_http_request_duration_seconds_count{code=~\"5.*\"}[$__rate_interval])) by (code, host) > 0) or vector(0)",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "{{code}} - {{host}}",
"range": true,
"useBackend": false
}
},
"refId": "A",
"hidden": false
}
}
],
"transformations": [],
"queryOptions": {}
}
},
"vizConfig": {
"kind": "VizConfig",
"group": "timeseries",
"version": "13.1.0-25153870157",
"spec": {
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": 0,
"color": "green"
},
{
"value": 80,
"color": "red"
}
]
},
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMin": 0,
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
}
},
"overrides": []
}
}
}
}
},
"panel-6": {
"kind": "Panel",
"spec": {
"id": 6,
"title": "Active Services",
"description": "",
"links": [],
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"query": {
"kind": "DataQuery",
"group": "prometheus",
"version": "v0",
"datasource": {
"name": "grafanacloud-prom"
},
"spec": {
"editorMode": "code",
"exemplar": false,
"expr": "sum(\n label_replace(\n rate(caddy_http_request_duration_seconds_count{handler!=\"metrics\", host=~\".*(pez.solutions|pez.sh)\", host!~\"(pez.sh|pez.solutions)\"}[$__rate_interval]),\n \"host_prefix\",\n \"$1\",\n \"host\",\n \"([^.]+)\\\\..*\"\n )\n) by (host_prefix) > 0",
"format": "table",
"instant": true,
"legendFormat": "__auto",
"range": false
}
},
"refId": "A",
"hidden": false
}
}
],
"transformations": [
{
"kind": "Transformation",
"group": "organize",
"spec": {
"options": {
"excludeByName": {
"Time": true
},
"includeByName": {},
"indexByName": {},
"renameByName": {
"Value": "req/s",
"host_prefix": "Service"
}
}
}
}
],
"queryOptions": {}
}
},
"vizConfig": {
"kind": "VizConfig",
"group": "table",
"version": "13.1.0-25153870157",
"spec": {
"options": {
"cellHeight": "sm",
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "req/s"
}
]
},
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": 0,
"color": "green"
},
{
"value": 80,
"color": "red"
}
]
},
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"footer": {
"reducers": []
},
"inspect": false
}
},
"overrides": []
}
}
}
}
}
},
"layout": {
"kind": "GridLayout",
"spec": {
"items": [
{
"kind": "GridLayoutItem",
"spec": {
"x": 0,
"y": 0,
"width": 4,
"height": 10,
"element": {
"kind": "ElementReference",
"name": "panel-3"
}
}
},
{
"kind": "GridLayoutItem",
"spec": {
"x": 4,
"y": 0,
"width": 20,
"height": 10,
"element": {
"kind": "ElementReference",
"name": "panel-4"
}
}
},
{
"kind": "GridLayoutItem",
"spec": {
"x": 0,
"y": 10,
"width": 19,
"height": 11,
"element": {
"kind": "ElementReference",
"name": "panel-1"
}
}
},
{
"kind": "GridLayoutItem",
"spec": {
"x": 19,
"y": 10,
"width": 5,
"height": 11,
"element": {
"kind": "ElementReference",
"name": "panel-6"
}
}
},
{
"kind": "GridLayoutItem",
"spec": {
"x": 0,
"y": 21,
"width": 12,
"height": 11,
"element": {
"kind": "ElementReference",
"name": "panel-2"
}
}
},
{
"kind": "GridLayoutItem",
"spec": {
"x": 12,
"y": 21,
"width": 12,
"height": 11,
"element": {
"kind": "ElementReference",
"name": "panel-5"
}
}
}
]
}
},
"links": [],
"liveNow": false,
"preload": false,
"tags": [],
"timeSettings": {
"timezone": "browser",
"from": "now-24h",
"to": "now",
"autoRefresh": "5s",
"autoRefreshIntervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"hideTimepicker": false,
"fiscalYearStartMonth": 0
},
"title": "Traffic / SLO",
"variables": []
}
}

48
terraform/grafana/fleet_collectors.tf Normal file
View file

@ -0,0 +1,48 @@
resource "grafana_fleet_management_collector" "london_a" {
id = "london-a"
remote_attributes = {
location = "london"
}
}
resource "grafana_fleet_management_collector" "london_b" {
id = "london-b"
remote_attributes = {
location = "london"
}
}
resource "grafana_fleet_management_collector" "london_c" {
id = "london-c"
remote_attributes = {
location = "london"
}
}
resource "grafana_fleet_management_collector" "copenhagen_a" {
id = "copenhagen-a"
remote_attributes = {
location = "copenhagen"
}
}
resource "grafana_fleet_management_collector" "copenhagen_c" {
id = "copenhagen-c"
remote_attributes = {
location = "copenhagen"
}
}
resource "grafana_fleet_management_collector" "helsinki_a" {
id = "helsinki-a"
remote_attributes = {
location = "cloud"
}
}
resource "grafana_fleet_management_collector" "nuremberg_a" {
id = "nuremberg-a"
remote_attributes = {
location = "cloud"
}
}

29
terraform/grafana/fleet_pipelines.tf Normal file
View file

@ -0,0 +1,29 @@
resource "grafana_fleet_management_pipeline" "linux_node_linux" {
name = "linux_node_linux"
matchers = ["collector.os=\"linux\""]
contents = file("${path.module}/fleet_pipelines/linux_node_linux.alloy")
}
resource "grafana_fleet_management_pipeline" "octopus_exporter" {
name = "octopus_exporter"
matchers = ["collector.ID=\"london-c\""]
contents = file("${path.module}/fleet_pipelines/octopus_exporter.alloy")
}
resource "grafana_fleet_management_pipeline" "plex" {
name = "plex"
matchers = ["collector.ID=\"london-b\""]
contents = file("${path.module}/fleet_pipelines/plex.alloy")
}
resource "grafana_fleet_management_pipeline" "caddy_linux" {
name = "caddy_linux"
matchers = ["collector.ID=\"helsinki-a\""]
contents = file("${path.module}/fleet_pipelines/caddy_linux.alloy")
}
resource "grafana_fleet_management_pipeline" "docker_linux" {
name = "docker_linux"
matchers = ["collector.os=\"linux\""]
contents = file("${path.module}/fleet_pipelines/docker_linux.alloy")
}

37
terraform/grafana/fleet_pipelines/caddy_linux.alloy Normal file
View file

@ -0,0 +1,37 @@
discovery.relabel "metrics_integrations_integrations_caddy" {
targets = [{
__address__ = "localhost:2019",
}]
rule {
target_label = "instance"
replacement = constants.hostname
}
}
prometheus.scrape "metrics_integrations_integrations_caddy" {
targets = discovery.relabel.metrics_integrations_integrations_caddy.output
forward_to = [prometheus.relabel.metrics_integrations_integrations_caddy.receiver]
job_name = "integrations/caddy"
}
prometheus.relabel "metrics_integrations_integrations_caddy" {
forward_to = [prometheus.remote_write.metrics_service.receiver]
rule {
source_labels = ["__name__"]
regex = "up|caddy_http_request_duration_seconds_bucket|caddy_http_request_duration_seconds_count|caddy_http_requests_in_flight|caddy_http_requests_total"
action = "keep"
}
}
prometheus.remote_write "metrics_service" {
endpoint {
url = "https://prometheus-prod-55-prod-gb-south-1.grafana.net/api/prom/push"
basic_auth {
username = "3166394"
password = sys.env("GCLOUD_RW_API_KEY")
}
}
}

92
terraform/grafana/fleet_pipelines/docker_linux.alloy Normal file
View file

@ -0,0 +1,92 @@
prometheus.exporter.cadvisor "integrations_cadvisor" {
docker_only = true
}
discovery.relabel "integrations_cadvisor" {
targets = prometheus.exporter.cadvisor.integrations_cadvisor.targets
rule {
target_label = "job"
replacement = "integrations/docker"
}
rule {
target_label = "instance"
replacement = constants.hostname
}
}
prometheus.relabel "integrations_cadvisor" {
forward_to = [prometheus.remote_write.metrics_service.receiver]
rule {
source_labels = ["__name__"]
regex = "container_cpu_usage_seconds_total|container_fs_reads_total|container_fs_usage_bytes|container_fs_writes_total|container_last_seen|container_memory_usage_bytes|container_network_receive_bytes_total|container_network_receive_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_bytes_total|container_network_transmit_errors_total|container_network_transmit_packets_dropped_total|container_spec_memory_reservation_limit_bytes|machine_memory_bytes|machine_scrape_error|up"
action = "keep"
}
}
prometheus.scrape "integrations_cadvisor" {
targets = discovery.relabel.integrations_cadvisor.output
forward_to = [prometheus.relabel.integrations_cadvisor.receiver]
}
prometheus.remote_write "metrics_service" {
endpoint {
url = "https://prometheus-prod-55-prod-gb-south-1.grafana.net/api/prom/push"
basic_auth {
username = "3166394"
password = sys.env("GCLOUD_RW_API_KEY")
}
}
}
discovery.docker "logs_integrations_docker" {
host = "unix:///var/run/docker.sock"
refresh_interval = "5s"
}
discovery.relabel "logs_integrations_docker" {
targets = []
rule {
target_label = "job"
replacement = "integrations/docker"
}
rule {
target_label = "instance"
replacement = constants.hostname
}
rule {
source_labels = ["__meta_docker_container_name"]
regex = "/(.*)"
target_label = "container"
}
rule {
source_labels = ["__meta_docker_container_log_stream"]
target_label = "stream"
}
}
loki.source.docker "logs_integrations_docker" {
host = "unix:///var/run/docker.sock"
targets = discovery.docker.logs_integrations_docker.targets
forward_to = [loki.write.grafana_cloud_loki.receiver]
relabel_rules = discovery.relabel.logs_integrations_docker.rules
refresh_interval = "5s"
}
loki.write "grafana_cloud_loki" {
endpoint {
url = "https://logs-prod-035.grafana.net/loki/api/v1/push"
basic_auth {
username = "1578872"
password = sys.env("GCLOUD_RW_API_KEY")
}
}
}

126
terraform/grafana/fleet_pipelines/linux_node_linux.alloy Normal file
View file

@ -0,0 +1,126 @@
discovery.relabel "integrations_node_exporter" {
targets = prometheus.exporter.unix.integrations_node_exporter.targets
rule {
target_label = "instance"
replacement = constants.hostname
}
rule {
target_label = "job"
replacement = "integrations/node_exporter"
}
}
prometheus.exporter.unix "integrations_node_exporter" {
disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs"]
filesystem {
fs_types_exclude = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
mount_points_exclude = "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)"
mount_timeout = "5s"
}
netclass {
ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
}
netdev {
device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
}
}
prometheus.scrape "integrations_node_exporter" {
targets = discovery.relabel.integrations_node_exporter.output
forward_to = [prometheus.relabel.integrations_node_exporter.receiver]
}
prometheus.relabel "integrations_node_exporter" {
forward_to = [prometheus.remote_write.metrics_service.receiver]
rule {
source_labels = ["__name__"]
regex = "up|node_arp_entries|node_boot_time_seconds|node_context_switches_total|node_cpu_seconds_total|node_disk_io_time_seconds_total|node_disk_io_time_weighted_seconds_total|node_disk_read_bytes_total|node_disk_read_time_seconds_total|node_disk_reads_completed_total|node_disk_write_time_seconds_total|node_disk_writes_completed_total|node_disk_written_bytes_total|node_filefd_allocated|node_filefd_maximum|node_filesystem_avail_bytes|node_filesystem_device_error|node_filesystem_files|node_filesystem_files_free|node_filesystem_readonly|node_filesystem_size_bytes|node_intr_total|node_load1|node_load15|node_load5|node_md_disks|node_md_disks_required|node_memory_Active_anon_bytes|node_memory_Active_bytes|node_memory_Active_file_bytes|node_memory_AnonHugePages_bytes|node_memory_AnonPages_bytes|node_memory_Bounce_bytes|node_memory_Buffers_bytes|node_memory_Cached_bytes|node_memory_CommitLimit_bytes|node_memory_Committed_AS_bytes|node_memory_DirectMap1G_bytes|node_memory_DirectMap2M_bytes|node_memory_DirectMap4k_bytes|node_memory_Dirty_bytes|node_memory_HugePages_Free|node_memory_HugePages_Rsvd|node_memory_HugePages_Surp|node_memory_HugePages_Total|node_memory_Hugepagesize_bytes|node_memory_Inactive_anon_bytes|node_memory_Inactive_bytes|node_memory_Inactive_file_bytes|node_memory_Mapped_bytes|node_memory_MemAvailable_bytes|node_memory_MemFree_bytes|node_memory_MemTotal_bytes|node_memory_SReclaimable_bytes|node_memory_SUnreclaim_bytes|node_memory_ShmemHugePages_bytes|node_memory_ShmemPmdMapped_bytes|node_memory_Shmem_bytes|node_memory_Slab_bytes|node_memory_SwapTotal_bytes|node_memory_VmallocChunk_bytes|node_memory_VmallocTotal_bytes|node_memory_VmallocUsed_bytes|node_memory_WritebackTmp_bytes|node_memory_Writeback_bytes|node_netstat_Icmp6_InErrors|node_netstat_Icmp6_InMsgs|node_netstat_Icmp6_OutMsgs|node_netstat_Icmp_InErrors|node_netstat_Icmp_InMsgs|node_netstat_Icmp_OutMsgs|node_netstat_IpExt_InOctets|node_netstat_IpExt_OutOctets|node_netstat_TcpExt_ListenDrops|node_netstat_TcpExt_ListenOverflows|node_netstat_TcpExt_TCPSynRetrans|node_netstat_Tcp_InErrs|node_netstat_Tcp_InSegs|node_netstat_Tcp_OutRsts|node_netstat_Tcp_OutSegs|node_netstat_Tcp_RetransSegs|node_netstat_Udp6_InDatagrams|node_netstat_Udp6_InErrors|node_netstat_Udp6_NoPorts|node_netstat_Udp6_OutDatagrams|node_netstat_Udp6_RcvbufErrors|node_netstat_Udp6_SndbufErrors|node_netstat_UdpLite_InErrors|node_netstat_Udp_InDatagrams|node_netstat_Udp_InErrors|node_netstat_Udp_NoPorts|node_netstat_Udp_OutDatagrams|node_netstat_Udp_RcvbufErrors|node_netstat_Udp_SndbufErrors|node_network_carrier|node_network_info|node_network_mtu_bytes|node_network_receive_bytes_total|node_network_receive_compressed_total|node_network_receive_drop_total|node_network_receive_errs_total|node_network_receive_fifo_total|node_network_receive_multicast_total|node_network_receive_packets_total|node_network_speed_bytes|node_network_transmit_bytes_total|node_network_transmit_compressed_total|node_network_transmit_drop_total|node_network_transmit_errs_total|node_network_transmit_fifo_total|node_network_transmit_multicast_total|node_network_transmit_packets_total|node_network_transmit_queue_length|node_network_up|node_nf_conntrack_entries|node_nf_conntrack_entries_limit|node_os_info|node_sockstat_FRAG6_inuse|node_sockstat_FRAG_inuse|node_sockstat_RAW6_inuse|node_sockstat_RAW_inuse|node_sockstat_TCP6_inuse|node_sockstat_TCP_alloc|node_sockstat_TCP_inuse|node_sockstat_TCP_mem|node_sockstat_TCP_mem_bytes|node_sockstat_TCP_orphan|node_sockstat_TCP_tw|node_sockstat_UDP6_inuse|node_sockstat_UDPLITE6_inuse|node_sockstat_UDPLITE_inuse|node_sockstat_UDP_inuse|node_sockstat_UDP_mem|node_sockstat_UDP_mem_bytes|node_sockstat_sockets_used|node_softnet_dropped_total|node_softnet_processed_total|node_softnet_times_squeezed_total|node_systemd_unit_state|node_textfile_scrape_error|node_time_zone_offset_seconds|node_timex_estimated_error_seconds|node_timex_maxerror_seconds|node_timex_offset_seconds|node_timex_sync_status|node_uname_info|node_vmstat_oom_kill|node_vmstat_pgfault|node_vmstat_pgmajfault|node_vmstat_pgpgin|node_vmstat_pgpgout|node_vmstat_pswpin|node_vmstat_pswpout|process_max_fds|process_open_fds|node_zfs_zpool_state"
action = "keep"
}
}
prometheus.remote_write "metrics_service" {
endpoint {
url = "https://prometheus-prod-55-prod-gb-south-1.grafana.net/api/prom/push"
basic_auth {
username = "3166394"
password = sys.env("GCLOUD_RW_API_KEY")
}
}
}
loki.relabel "integrations_node_exporter" {
forward_to = [loki.write.grafana_cloud_loki.receiver]
rule {
target_label = "job"
replacement = "integrations/node_exporter"
}
rule {
target_label = "instance"
replacement = constants.hostname
}
}
journal_module "integrations_node_exporter" {
forward_to = [loki.relabel.integrations_node_exporter.receiver]
}
//JOURNAL
declare "journal_module" {
argument "forward_to" {
optional = false
}
loki.source.journal "default" {
max_age = "12h0m0s"
forward_to = [loki.process.default.receiver]
relabel_rules = loki.relabel.default.rules
}
loki.relabel "default" {
rule {
source_labels = ["__journal__systemd_unit"]
target_label = "unit"
}
rule {
source_labels = ["__journal__boot_id"]
target_label = "boot_id"
}
rule {
source_labels = ["__journal__transport"]
target_label = "transport"
}
rule {
source_labels = ["__journal_priority_keyword"]
target_label = "level"
}
forward_to = []
}
loki.process "default" {
forward_to = argument.forward_to.value
}
}
loki.write "grafana_cloud_loki" {
endpoint {
url = "https://logs-prod-035.grafana.net/loki/api/v1/push"
basic_auth {
username = "1578872"
password = sys.env("GCLOUD_RW_API_KEY")
}
}
}

20
terraform/grafana/fleet_pipelines/octopus_exporter.alloy Normal file
View file

@ -0,0 +1,20 @@
prometheus.remote_write "grafana_cloud" {
endpoint {
url = "https://prometheus-prod-55-prod-gb-south-1.grafana.net/api/prom/push"
basic_auth {
username = "3166394"
password = sys.env("GCLOUD_RW_API_KEY")
}
}
}
prometheus.scrape "octopus_exporter" {
targets = [
{
"__address__" = "localhost:9359",
"job" = "octopus_exporter",
},
]
forward_to = [prometheus.remote_write.grafana_cloud.receiver]
}

20
terraform/grafana/fleet_pipelines/plex.alloy Normal file
View file

@ -0,0 +1,20 @@
prometheus.remote_write "grafana_cloud" {
endpoint {
url = "https://prometheus-prod-55-prod-gb-south-1.grafana.net/api/prom/push"
basic_auth {
username = "3166394"
password = sys.env("GCLOUD_RW_API_KEY")
}
}
}
prometheus.scrape "plex" {
targets = [
{
"__address__" = "localhost:9000",
"job" = "plex",
},
]
forward_to = [prometheus.remote_write.grafana_cloud.receiver]
}

8
terraform/grafana/providers.tf Normal file
View file

@ -0,0 +1,8 @@
terraform {
required_providers {
grafana = {
source = "grafana/grafana"
version = "~> 4.35"
}
}
}

5
terraform/grafana/stack.tf Normal file
View file

@ -0,0 +1,5 @@
resource "grafana_cloud_stack" "pez" {
name = "pez.grafana.net"
slug = "pez"
region_slug = "prod-gb-south-1"
}

72
terraform/grafana/synthetic_check_alerts.tf Normal file
View file

@ -0,0 +1,72 @@
resource "grafana_synthetic_monitoring_check_alerts" "pez_sh" {
check_id = grafana_synthetic_monitoring_check.pez_sh.id
alerts = [
{
name = "ProbeFailedExecutionsTooHigh"
threshold = 3
period = "30m"
runbook_url = ""
}
]
}
resource "grafana_synthetic_monitoring_check_alerts" "pez_solutions" {
check_id = grafana_synthetic_monitoring_check.pez_solutions.id
alerts = [
{
name = "ProbeFailedExecutionsTooHigh"
threshold = 3
period = "30m"
runbook_url = ""
}
]
}
resource "grafana_synthetic_monitoring_check_alerts" "jellyfin" {
check_id = grafana_synthetic_monitoring_check.jellyfin.id
alerts = [
{
name = "ProbeFailedExecutionsTooHigh"
threshold = 3
period = "30m"
runbook_url = ""
}
]
}
resource "grafana_synthetic_monitoring_check_alerts" "plex" {
check_id = grafana_synthetic_monitoring_check.plex.id
alerts = [
{
name = "ProbeFailedExecutionsTooHigh"
threshold = 3
period = "30m"
runbook_url = ""
}
]
}
resource "grafana_synthetic_monitoring_check_alerts" "request" {
check_id = grafana_synthetic_monitoring_check.request.id
alerts = [
{
name = "ProbeFailedExecutionsTooHigh"
threshold = 3
period = "30m"
runbook_url = ""
}
]
}
resource "grafana_synthetic_monitoring_check_alerts" "jellyfin-requests" {
check_id = grafana_synthetic_monitoring_check.jellyfin-requests.id
alerts = [
{
name = "ProbeFailedExecutionsTooHigh"
threshold = 3
period = "30m"
runbook_url = ""
}
]
}

132
terraform/grafana/synthetic_checks.tf Normal file
View file

@ -0,0 +1,132 @@
resource "grafana_synthetic_monitoring_check" "pez_sh" {
job = "pez.sh"
target = "https://pez.sh"
enabled = true
probes = [14] # 14 = London, UK
settings {
http {
method = "GET"
compression = "none"
fail_if_not_ssl = true
ip_version = "V4"
valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
valid_status_codes = ["200"]
}
}
frequency = 600000
timeout = 3000
lifecycle {
ignore_changes = [settings]
}
}
resource "grafana_synthetic_monitoring_check" "pez_solutions" {
job = "pez.solutions"
target = "https://pez.solutions"
enabled = true
probes = [14] # 14 = London, UK
settings {
http {
method = "GET"
compression = "none"
fail_if_not_ssl = true
ip_version = "V4"
valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
valid_status_codes = ["200"]
}
}
frequency = 600000
timeout = 3000
lifecycle {
ignore_changes = [settings]
}
}
resource "grafana_synthetic_monitoring_check" "jellyfin" {
job = "jellyfin.pez.sh"
target = "https://jellyfin.pez.sh"
enabled = true
probes = [14] # 14 = London, UK
settings {
http {
method = "GET"
compression = "none"
fail_if_not_ssl = true
ip_version = "V4"
valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
valid_status_codes = ["200"]
}
}
frequency = 600000
timeout = 3000
lifecycle {
ignore_changes = [settings]
}
}
resource "grafana_synthetic_monitoring_check" "plex" {
job = "plex.pez.sh"
target = "https://plex.pez.sh"
enabled = true
probes = [14] # 14 = London, UK
settings {
http {
method = "GET"
headers = ["X-Plex-Token:${var.plex_token}"]
compression = "none"
fail_if_not_ssl = true
ip_version = "V4"
valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
valid_status_codes = ["200"]
}
}
frequency = 600000
timeout = 3000
lifecycle {
ignore_changes = [settings]
}
}
resource "grafana_synthetic_monitoring_check" "request" {
job = "request.pez.sh"
target = "https://request.pez.sh"
enabled = true
probes = [14] # 14 = London, UK
settings {
http {
method = "GET"
compression = "none"
fail_if_not_ssl = true
ip_version = "V4"
valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
valid_status_codes = ["200"]
}
}
frequency = 600000
timeout = 3000
lifecycle {
ignore_changes = [settings]
}
}
resource "grafana_synthetic_monitoring_check" "jellyfin-requests" {
job = "jellyfin-requests.pez.sh"
target = "https://jellyfin-requests.pez.sh"
enabled = true
probes = [14] # 14 = London, UK
settings {
http {
method = "GET"
compression = "none"
fail_if_not_ssl = true
ip_version = "V4"
valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
valid_status_codes = ["200"]
}
}
frequency = 600000
timeout = 3000
lifecycle {
ignore_changes = [settings]
}
}

4
terraform/grafana/vars.tf Normal file
View file

@ -0,0 +1,4 @@
variable "plex_token" {
type = string
sensitive = true
}

0
terraform/hetzner_compute.tf → terraform/hetzner/hetzner_compute.tf
View file

0
terraform/hetzner_dns.tf → terraform/hetzner/hetzner_dns.tf
View file

0
terraform/hetzner_firewall.tf → terraform/hetzner/hetzner_firewall.tf
View file

0
terraform/hetzner_ssh_keys.tf → terraform/hetzner/hetzner_ssh_keys.tf
View file

8
terraform/hetzner/providers.tf Normal file
View file

@ -0,0 +1,8 @@
terraform {
required_providers {
hcloud = {
source = "hetznercloud/hcloud"
version = "~> 1.45"
}
}
}

14
terraform/main.tf Normal file
View file

@ -0,0 +1,14 @@
module "hetzner" {
source = "./hetzner"
providers = {
hcloud = hcloud
}
}
module "grafana" {
source = "./grafana"
providers = {
grafana = grafana
}
plex_token = local.secrets["plex_token"]
}

14
terraform/providers.tf
View file

@ -6,6 +6,10 @@ terraform {
source = "hetznercloud/hcloud" source = "hetznercloud/hcloud"
version = "~> 1.45" version = "~> 1.45"
} }
grafana = {
source = "grafana/grafana"
version = "~> 4.35"
}
} }
@ -24,3 +28,13 @@ provider "hcloud" {
token = local.secrets["hetzner_token"] token = local.secrets["hetzner_token"]
} }
provider "grafana" {
cloud_access_policy_token = local.secrets["grafana_cloud_access_policy"]
sm_url = "https://synthetic-monitoring-api-gb-south-1.grafana.net"
sm_access_token = local.secrets["grafana_synthetic_monitoring_access_token"]
fleet_management_url = "https://fleet-management-prod-023.grafana.net"
fleet_management_auth = local.secrets["grafana_fleet_management_auth"]
url = "https://pez.grafana.net"
auth = local.secrets["grafana_service_account_token"]
}

27
terraform/secrets.enc.yaml
View file

@ -1,19 +1,24 @@
backblaze_keyID: ENC[AES256_GCM,data:7u0zAFOt1uKDNK/jFl+HLVBUVWd06fiQjQ==,iv:f+Mh38+Vo0JI1tLByjL3we3hOCXLhDtPZim/QIsO1vQ=,tag:WOHEj0ND3xnIOANwBj2y/g==,type:str] backblaze_keyID: ENC[AES256_GCM,data:Me1rx0EjrZHCP/F8iZy9Q35W2PVGNx/pNg==,iv:Pt19nfFkv5eLarnmQWcc/3XHjho4glYJHcsa5aMxsvc=,tag:K1pbJKKyfu4Bcl6acpj8Dw==,type:str]
backblaze_keyName: ENC[AES256_GCM,data:dt0YrkYmG+qIFlDMWsugvpU=,iv:Z8pZ38Wr5RxrI/LczeE3OMdTfPcfsOeTa/q2wdd3cc8=,tag:i2qlvue4tbVTuwwZli/qUA==,type:str] backblaze_keyName: ENC[AES256_GCM,data:o9Mkn3LwCjzlqNvMnfUayiM=,iv:XitvLpzdBm5Kbe3SGoJMSb76bpCQrOuKHoQuNAYVO6c=,tag:dKa3rMwCQd1OSjQ77qa2oQ==,type:str]
backblaze_applicationKey: ENC[AES256_GCM,data:uo7tQmDsunxuCd9nhATy/4rOjgDfz5Lhpn9wsyZdKA==,iv:RTsSAkU9X7IcpMYu+Qa/+lQ/H1ICp2BBFKGA8C9bl9Y=,tag:YnJRmQ/C2AAbzmkuS1lFpA==,type:str] backblaze_applicationKey: ENC[AES256_GCM,data:l+TOnls8JolenrPV8JrAWGQv2bf8dE33cnvUjllWrw==,iv:vv50XWpLjZo6GgqJSFFTpKWHHwCDfFDbOYmi7ekMDlk=,tag:ptfxYUcCyptPI4WsWqfXxw==,type:str]
hetzner_token: ENC[AES256_GCM,data:9oBDjMvpiiiY1+vN3cTdoPCbTHRIjvWQDFDg5fw6eWmhQGJ81BkXCF1FKqSpOUhbkMCPkU7yzMlE8wKt8JQIAw==,iv:VQMYUTFssyN6tyYbqiio+nlqLifULs6gqiwg1p51Z+0=,tag:c0phnxXoACk4vtoakugrxw==,type:str] hetzner_token: ENC[AES256_GCM,data:44uodS8rOH/X95Bj8QBh1rH1Wz/VxaopzOtU3B9La0oZECxpn+uc4cnUrARwv7CsVlQLENlq0BXL9TORugCT0Q==,iv:Ytia3hGmYhSafuo+vhIvfYk0HPf5xHsQFzKX7Vifb44=,tag:dyfvagA9pk9pIUrjhT0pgA==,type:str]
grafana_cloud_access_policy: ENC[AES256_GCM,data:SWLp3yqDOz1zb8GcCPuIcJ/eMK1Yvx7PpuXbx/ThX5DIb0NTjfUx2KdVWx2hFYY70E3TUnur4/2qD/Ys+GGkFlqe/wP/0k940atfTo6R6mNR/Y2+uusUsNUO3voCiXuN2uk5vzA7WNgjxkdUaXVpVaq8bNBzFB0/MiD6m6M7wIm6KL5x9/uhag==,iv:OR4G57v+cOw1LigF8bqaqbxb/sE6xLV+3se/So3Imu0=,tag:UOALoE5IQEnhztJfwCSBuw==,type:str]
grafana_service_account_token: ENC[AES256_GCM,data:Y++SArCCSC+8zIc8KAruEgT0x9Hbuqw5nJl2W3F6CY5+fC/6ooiV6iV3Fa/neQ==,iv:aKmvGcGUwpFMid/osAtICUC07KSJOyh+gIouN+sQx9M=,tag:viXk15D2a1mxoWse69Yjwg==,type:str]
grafana_synthetic_monitoring_access_token: ENC[AES256_GCM,data:iVIAA4HLCqANNxbkpSKesfc27tJTy4tRjS1XoawvcF1c0nz6lbop+ydsMhHU4LiWYK4hBI7rbu6toxuRHtqcjUNsT13PVsf1anDrBrxMt5+dou1wP8lOfG5zunNma1Hjh/BOn+Nl70QLQnUFBET1IJdFTNEV99xA,iv:y40ztRtfjxoHk2Sv4YFdbi33AjhMdlT0kBVyjrnp/Bs=,tag:OJJqdGmzoe9pPrAHKXaCiw==,type:str]
grafana_fleet_management_auth: ENC[AES256_GCM,data:DhhaP9AHlQ9pCAw9lL9M96pM1vV+45NaBY+oSiCfS/LSEKhQOjNzh4TKTwNurJeJ4r9Mpkac896svhbDix46loA2ZVBN407pWOMPcXFIiDcXMt19uaByWdnHGXAEA9XWCynmQicNqtUV6YhGDrok7e0426SeCxbjKFfx8YsPv5wXCqF6BgpWklbAmWhezDx6M2dSUSoTCDZcVXIz1vHrWeK5AIixY2eh1S0R7xesffPp84QD,iv:kmwmGffw6DzP2ESbwJtpfEOR18DA2JIGVFzNujcTURM=,tag:NhPczMMFYsDpbDtLDCcuyA==,type:str]
plex_token: ENC[AES256_GCM,data:Ff3dMZQ/SJW/UdiUZSoymFhB9S8=,iv:yOGAA0PMtk2LKM+Ke3cb83F4b/cO2RLkMkv4j+WNaZk=,tag:CWAoU3wqlNle2OAZbDVSGw==,type:str]
sops: sops:
age: age:
- recipient: age1r8uh2w2qad2z5sgq9q7l73962q2sp8zz9hdnh6sjuvanxl565vmswn8squ - recipient: age1r8uh2w2qad2z5sgq9q7l73962q2sp8zz9hdnh6sjuvanxl565vmswn8squ
enc: | enc: |
-----BEGIN AGE ENCRYPTED FILE----- -----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtUmhhbUpST2xJNVYxTFYx YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVbFc3TFU1d2RMcndYZjYw
bXVic1VTTmhwVHJjay9VSUhMN1hyQjVuVVhNCmd2YVJIY3E4WWZtdFg2ZUhycEpR OGVlSXZWZkxLc0NrSFFEMVpaVTdnY0xxSm0wClJjSEtpbEROL0F6by9EemMxU2xx
aHNxbTc2amYyRGxJVEFJeTVlU1o4QzQKLS0tIHVkUGVwNDVFVk9seEgzSTZiVVhv cnVjNUJTR09DdVhmZ1V0SmRKeUlQdFkKLS0tIGZ3N0Fka0ZhRUpKUEh5ZTJiVytB
MDNISS9UWjdSR3Q2TnBoYTgyNjFlUUEK1vsRrHA6WQDyUO6UJSywBXCnJbgLogwc NDlNWGVGMFNRaHBDZWpXRXB5YnNXbGsKLNpH4vJunfYBet5S1GJPfq+vIQT5iyTv
JeLReyACLqUyDaxtaJwvBA29IguJLLTDdPV4aqZ/uhZxxMB3Yc5hYw== 6bS0IyViMy5ag7O5ACrsHGYTAmRDhYwM9RdEE4F4m5hp3E0Haqhedw==
-----END AGE ENCRYPTED FILE----- -----END AGE ENCRYPTED FILE-----
lastmodified: "2026-05-02T13:12:18Z" lastmodified: "2026-05-04T12:11:23Z"
mac: ENC[AES256_GCM,data:XSm141YbD/KglqujQ2y0vm6U0F/uFuBfBr0G3IxzuYKa6Y/pCPTG3CdzuuUpGsMzZM4PtffH9jVnPAF5MyN7lTH2CKmeRWErJJTkPUQ2Iep+7p28AL46J0sy6YPwh7iZz1NUjvGNCNLWDtIbR/ygL2oibTv9btYBExQVrElAD9I=,iv:6h7ZJW4GQKJEu+zmBnrXnJ7AVIf767UneH7nRCC36gg=,tag:4YKOBolrmaqDdo1v3VTBCg==,type:str] mac: ENC[AES256_GCM,data:0UYeI+h7nwAnAvy3J3E68XXMkGAO7sFbqHtgYXQ9Yz5bD3tR+m4rfU6722QqMH4+j5xKa4KL0SVFTYdQ9RSUHrDpbBcbG000CkZIysmKrLROeg8iQy36XUhf9eqfrIBT2g67piUAUprVREULUySwyuFTz9Z/pFJ57ClidP3hCZw=,iv:Fq02QA+rCu80SGhOar9fDCXWMPlylQeNihWqHGbHBGM=,tag:13RTowa3UBd6Jk/zeOzSWg==,type:str]
unencrypted_suffix: _unencrypted unencrypted_suffix: _unencrypted
version: 3.12.2 version: 3.12.2