From c495b73720b9c60df34c863ed8990f164f7fe5d9 Mon Sep 17 00:00:00 2001 From: "Rasmus \"Pez\" Wejlgaard" Date: Tue, 21 Apr 2026 20:44:37 +0100 Subject: [PATCH] template prometheus config (#67) --- ansible/inventory/host_vars/copenhagen-a.yml | 1 + ansible/inventory/host_vars/copenhagen-c.yml | 1 + ansible/inventory/host_vars/helsinki-a.yml | 1 + ansible/inventory/host_vars/london-a.yml | 1 + ansible/inventory/host_vars/london-b.yml | 1 + ansible/inventory/host_vars/london-c.yml | 1 + ansible/inventory/host_vars/nuremberg-a.yml | 1 + ansible/playbooks/monitoring.yml | 11 +-- ansible/services/prometheus/prometheus.yml | 84 ------------------- ansible/services/prometheus/prometheus.yml.j2 | 79 +++++++++++++++++ 10 files changed, 88 insertions(+), 93 deletions(-) delete mode 100644 ansible/services/prometheus/prometheus.yml create mode 100644 ansible/services/prometheus/prometheus.yml.j2 diff --git a/ansible/inventory/host_vars/copenhagen-a.yml b/ansible/inventory/host_vars/copenhagen-a.yml index 50f4518..ffe084c 100644 --- a/ansible/inventory/host_vars/copenhagen-a.yml +++ b/ansible/inventory/host_vars/copenhagen-a.yml @@ -2,6 +2,7 @@ host_role: gaming host_description: "Gaming servers (Minecraft, WoW/MaNGOS)" host_location: "Copenhagen" +prometheus_location: copenhagen # MaNGOS runs as systemd services, not Docker # MariaDB is the MaNGOS database backend (character, world, auth DBs) diff --git a/ansible/inventory/host_vars/copenhagen-c.yml b/ansible/inventory/host_vars/copenhagen-c.yml index f1911d1..c9d83ad 100644 --- a/ansible/inventory/host_vars/copenhagen-c.yml +++ b/ansible/inventory/host_vars/copenhagen-c.yml @@ -2,3 +2,4 @@ host_role: idle host_description: "Idle/available" host_location: "Copenhagen" +prometheus_location: copenhagen diff --git a/ansible/inventory/host_vars/helsinki-a.yml b/ansible/inventory/host_vars/helsinki-a.yml index 1fee8d1..77d9dab 100644 --- a/ansible/inventory/host_vars/helsinki-a.yml +++ b/ansible/inventory/host_vars/helsinki-a.yml @@ -2,6 +2,7 @@ host_role: proxy host_description: "Reverse proxy (Caddy), main traffic gateway" host_location: "Hetzner Cloud" +prometheus_location: cloud caddy_config_src: "{{ playbook_dir }}/services/caddy/Caddyfile" diff --git a/ansible/inventory/host_vars/london-a.yml b/ansible/inventory/host_vars/london-a.yml index bdb39ed..fcfa20a 100644 --- a/ansible/inventory/host_vars/london-a.yml +++ b/ansible/inventory/host_vars/london-a.yml @@ -3,6 +3,7 @@ node_exporter_bind_tailscale: true host_role: monitoring host_description: "Monitoring stack (Prometheus, Grafana)" host_location: "London" +prometheus_location: london ansible_python_interpreter: /usr/local/bin/python3 grafana_provisioning_dir: /usr/local/etc/grafana/provisioning diff --git a/ansible/inventory/host_vars/london-b.yml b/ansible/inventory/host_vars/london-b.yml index 56a5d68..74eef67 100644 --- a/ansible/inventory/host_vars/london-b.yml +++ b/ansible/inventory/host_vars/london-b.yml @@ -2,6 +2,7 @@ host_role: storage host_description: "Primary storage (ZFS), Docker services" host_location: "London" +prometheus_location: london docker_services: - nextcloud-aio diff --git a/ansible/inventory/host_vars/london-c.yml b/ansible/inventory/host_vars/london-c.yml index 3fb29cc..6340a2c 100644 --- a/ansible/inventory/host_vars/london-c.yml +++ b/ansible/inventory/host_vars/london-c.yml @@ -2,3 +2,4 @@ host_role: idle host_description: "Idle/available" host_location: "London" +prometheus_location: london diff --git a/ansible/inventory/host_vars/nuremberg-a.yml b/ansible/inventory/host_vars/nuremberg-a.yml index b4df27e..fb6fa2e 100644 --- a/ansible/inventory/host_vars/nuremberg-a.yml +++ b/ansible/inventory/host_vars/nuremberg-a.yml @@ -2,6 +2,7 @@ host_role: mail host_description: "Mail server (poste.io)" host_location: "Hetzner Cloud" +prometheus_location: cloud ansible_python_interpreter: /usr/bin/python3 docker_services: diff --git a/ansible/playbooks/monitoring.yml b/ansible/playbooks/monitoring.yml index ffbc146..a11084c 100644 --- a/ansible/playbooks/monitoring.yml +++ b/ansible/playbooks/monitoring.yml @@ -11,19 +11,12 @@ file: "{{ playbook_dir }}/../group_vars/all/secrets.yaml" no_log: true tasks: - - name: Check for Prometheus config - delegate_to: localhost - ansible.builtin.stat: - path: "{{ playbook_dir }}/../services/prometheus/prometheus.yml" - register: prometheus_config - - name: Deploy Prometheus config - ansible.builtin.copy: - src: "{{ playbook_dir }}/../services/prometheus/prometheus.yml" + ansible.builtin.template: + src: "{{ playbook_dir }}/../services/prometheus/prometheus.yml.j2" dest: /usr/local/etc/prometheus.yml mode: '0644' backup: true - when: prometheus_config.stat.exists notify: Restart prometheus - name: Deploy Prometheus alerting rules diff --git a/ansible/services/prometheus/prometheus.yml b/ansible/services/prometheus/prometheus.yml deleted file mode 100644 index c41c570..0000000 --- a/ansible/services/prometheus/prometheus.yml +++ /dev/null @@ -1,84 +0,0 @@ ---- -# Prometheus configuration — extracted from london-a (FreeBSD) -# Config file location on london-a: /usr/local/etc/prometheus.yml -# Prometheus runs as: /usr/local/bin/prometheus --config.file=/usr/local/etc/prometheus.yml -# Data directory: /var/db/prometheus - -global: - scrape_interval: 15s - evaluation_interval: 15s - -# Alerting notifications are handled by Grafana (unified alerting with -# PagerDuty + email contact points), not Alertmanager. No alerting: -# section is needed here. Prometheus still evaluates these rule_files -# so the ALERTS / ALERTS_FOR_STATE metrics are available for queries. -rule_files: - - /usr/local/etc/prometheus/rules/*.rules - -scrape_configs: - - job_name: "prometheus" - static_configs: - - targets: ["localhost:9090"] - - - job_name: "node_exporter" - static_configs: - - targets: ["100.122.219.41:9100"] - labels: - location: london - server: london-a - - targets: ["100.84.65.101:9100"] - labels: - location: london - server: london-b - - targets: ["100.89.206.60:9100"] - labels: - location: copenhagen - server: copenhagen-a - - targets: ["100.115.45.53:9100"] - labels: - location: copenhagen - server: copenhagen-c - - targets: ["100.117.235.28:9100"] - labels: - location: cloud - server: nuremberg-a - - targets: ["100.67.6.27:9100"] - labels: - location: cloud - server: helsinki-a - - - job_name: "smartmontools" - static_configs: - - targets: ["100.84.65.101:9633"] - labels: - location: london - server: london-b - - targets: ["100.89.206.60:9633"] - labels: - location: copenhagen - server: copenhagen-a - - - job_name: "plex" - static_configs: - - targets: ["100.84.65.101:9000"] - labels: - location: london - server: london-b - - - job_name: "systemd_exporter" - static_configs: - - targets: ["100.84.65.101:9558"] - labels: - location: london - server: london-b - - targets: ["100.89.206.60:9558"] - labels: - location: copenhagen - server: copenhagen-a - - - job_name: "caddy" - static_configs: - - targets: ["100.67.6.27:2019"] - labels: - location: cloud - server: helsinki-a diff --git a/ansible/services/prometheus/prometheus.yml.j2 b/ansible/services/prometheus/prometheus.yml.j2 new file mode 100644 index 0000000..f755d00 --- /dev/null +++ b/ansible/services/prometheus/prometheus.yml.j2 @@ -0,0 +1,79 @@ +--- +# Ansible managed — generated from prometheus.yml.j2 +# Config file location on london-a: /usr/local/etc/prometheus.yml +# Prometheus runs as: /usr/local/bin/prometheus --config.file=/usr/local/etc/prometheus.yml +# Data directory: /var/db/prometheus + +global: + scrape_interval: 15s + evaluation_interval: 15s + +# Alerting notifications are handled by Grafana (unified alerting with +# PagerDuty + email contact points), not Alertmanager. No alerting: +# section is needed here. Prometheus still evaluates these rule_files +# so the ALERTS / ALERTS_FOR_STATE metrics are available for queries. +rule_files: + - /usr/local/etc/prometheus/rules/*.rules + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + + - job_name: "node_exporter" + static_configs: +{% for host in groups['all'] | sort %} +{% set h = hostvars[host] %} +{% if h.ansible_host is defined %} + - targets: ["{{ h.ansible_host }}:9100"] + labels: + location: {{ h.prometheus_location }} + server: {{ host }} +{% endif %} +{% endfor %} + + - job_name: "smartmontools" + static_configs: +{% for host in groups['all'] | sort %} +{% set h = hostvars[host] %} +{% if 'smartctl-exporter' in (h.docker_services | default([])) %} + - targets: ["{{ h.ansible_host }}:9633"] + labels: + location: {{ h.prometheus_location }} + server: {{ host }} +{% endif %} +{% endfor %} + + - job_name: "plex" + static_configs: +{% for host in groups['all'] | sort %} +{% set h = hostvars[host] %} +{% if 'plex-exporter' in (h.docker_services | default([])) %} + - targets: ["{{ h.ansible_host }}:9000"] + labels: + location: {{ h.prometheus_location }} + server: {{ host }} +{% endif %} +{% endfor %} + + - job_name: "systemd_exporter" + static_configs: +{% for host in groups['systemd_exporter_hosts'] | sort %} +{% set h = hostvars[host] %} + - targets: ["{{ h.ansible_host }}:9558"] + labels: + location: {{ h.prometheus_location }} + server: {{ host }} +{% endfor %} + + - job_name: "caddy" + static_configs: +{% for host in groups['all'] | sort %} +{% set h = hostvars[host] %} +{% if h.caddy_config_src is defined %} + - targets: ["{{ h.ansible_host }}:2019"] + labels: + location: {{ h.prometheus_location }} + server: {{ host }} +{% endif %} +{% endfor %}