template prometheus config (#67)

ansible/inventory/host_vars/copenhagen-a.yml

@@ -2,6 +2,7 @@
 host_role: gaming
 host_description: "Gaming servers (Minecraft, WoW/MaNGOS)"
 host_location: "Copenhagen"
+prometheus_location: copenhagen
 
 # MaNGOS runs as systemd services, not Docker
 # MariaDB is the MaNGOS database backend (character, world, auth DBs)

ansible/inventory/host_vars/copenhagen-c.yml

@@ -2,3 +2,4 @@
 host_role: idle
 host_description: "Idle/available"
 host_location: "Copenhagen"
+prometheus_location: copenhagen

ansible/inventory/host_vars/helsinki-a.yml

@@ -2,6 +2,7 @@
 host_role: proxy
 host_description: "Reverse proxy (Caddy), main traffic gateway"
 host_location: "Hetzner Cloud"
+prometheus_location: cloud
 
 caddy_config_src: "{{ playbook_dir }}/services/caddy/Caddyfile"
 

ansible/inventory/host_vars/london-a.yml

@@ -3,6 +3,7 @@ node_exporter_bind_tailscale: true
 host_role: monitoring
 host_description: "Monitoring stack (Prometheus, Grafana)"
 host_location: "London"
+prometheus_location: london
 ansible_python_interpreter: /usr/local/bin/python3
 grafana_provisioning_dir: /usr/local/etc/grafana/provisioning
 

ansible/inventory/host_vars/london-b.yml

@@ -2,6 +2,7 @@
 host_role: storage
 host_description: "Primary storage (ZFS), Docker services"
 host_location: "London"
+prometheus_location: london
 
 docker_services:
   - nextcloud-aio

ansible/inventory/host_vars/london-c.yml

@@ -2,3 +2,4 @@
 host_role: idle
 host_description: "Idle/available"
 host_location: "London"
+prometheus_location: london

ansible/inventory/host_vars/nuremberg-a.yml

@@ -2,6 +2,7 @@
 host_role: mail
 host_description: "Mail server (poste.io)"
 host_location: "Hetzner Cloud"
+prometheus_location: cloud
 ansible_python_interpreter: /usr/bin/python3
 
 docker_services:

ansible/playbooks/monitoring.yml

@@ -11,19 +11,12 @@
         file: "{{ playbook_dir }}/../group_vars/all/secrets.yaml"
       no_log: true
   tasks:
-    - name: Check for Prometheus config
-      delegate_to: localhost
-      ansible.builtin.stat:
-        path: "{{ playbook_dir }}/../services/prometheus/prometheus.yml"
-      register: prometheus_config
-
     - name: Deploy Prometheus config
-      ansible.builtin.copy:
-        src: "{{ playbook_dir }}/../services/prometheus/prometheus.yml"
+      ansible.builtin.template:
+        src: "{{ playbook_dir }}/../services/prometheus/prometheus.yml.j2"
         dest: /usr/local/etc/prometheus.yml
         mode: '0644'
         backup: true
-      when: prometheus_config.stat.exists
       notify: Restart prometheus
 
     - name: Deploy Prometheus alerting rules

ansible/services/prometheus/prometheus.yml

@@ -1,84 +0,0 @@
----
-# Prometheus configuration — extracted from london-a (FreeBSD)
-# Config file location on london-a: /usr/local/etc/prometheus.yml
-# Prometheus runs as: /usr/local/bin/prometheus --config.file=/usr/local/etc/prometheus.yml
-# Data directory: /var/db/prometheus
-
-global:
-  scrape_interval: 15s
-  evaluation_interval: 15s
-
-# Alerting notifications are handled by Grafana (unified alerting with
-# PagerDuty + email contact points), not Alertmanager.  No alerting:
-# section is needed here.  Prometheus still evaluates these rule_files
-# so the ALERTS / ALERTS_FOR_STATE metrics are available for queries.
-rule_files:
-  - /usr/local/etc/prometheus/rules/*.rules
-
-scrape_configs:
-  - job_name: "prometheus"
-    static_configs:
-      - targets: ["localhost:9090"]
-
-  - job_name: "node_exporter"
-    static_configs:
-      - targets: ["100.122.219.41:9100"]
-        labels:
-          location: london
-          server: london-a
-      - targets: ["100.84.65.101:9100"]
-        labels:
-          location: london
-          server: london-b
-      - targets: ["100.89.206.60:9100"]
-        labels:
-          location: copenhagen
-          server: copenhagen-a
-      - targets: ["100.115.45.53:9100"]
-        labels:
-          location: copenhagen
-          server: copenhagen-c
-      - targets: ["100.117.235.28:9100"]
-        labels:
-          location: cloud
-          server: nuremberg-a
-      - targets: ["100.67.6.27:9100"]
-        labels:
-          location: cloud
-          server: helsinki-a
-
-  - job_name: "smartmontools"
-    static_configs:
-      - targets: ["100.84.65.101:9633"]
-        labels:
-          location: london
-          server: london-b
-      - targets: ["100.89.206.60:9633"]
-        labels:
-          location: copenhagen
-          server: copenhagen-a
-
-  - job_name: "plex"
-    static_configs:
-      - targets: ["100.84.65.101:9000"]
-        labels:
-          location: london
-          server: london-b
-
-  - job_name: "systemd_exporter"
-    static_configs:
-      - targets: ["100.84.65.101:9558"]
-        labels:
-          location: london
-          server: london-b
-      - targets: ["100.89.206.60:9558"]
-        labels:
-          location: copenhagen
-          server: copenhagen-a
-
-  - job_name: "caddy"
-    static_configs:
-      - targets: ["100.67.6.27:2019"]
-        labels:
-          location: cloud
-          server: helsinki-a

ansible/services/prometheus/prometheus.yml.j2

@@ -0,0 +1,79 @@
+---
+# Ansible managed — generated from prometheus.yml.j2
+# Config file location on london-a: /usr/local/etc/prometheus.yml
+# Prometheus runs as: /usr/local/bin/prometheus --config.file=/usr/local/etc/prometheus.yml
+# Data directory: /var/db/prometheus
+
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+# Alerting notifications are handled by Grafana (unified alerting with
+# PagerDuty + email contact points), not Alertmanager.  No alerting:
+# section is needed here.  Prometheus still evaluates these rule_files
+# so the ALERTS / ALERTS_FOR_STATE metrics are available for queries.
+rule_files:
+  - /usr/local/etc/prometheus/rules/*.rules
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  - job_name: "node_exporter"
+    static_configs:
+{% for host in groups['all'] | sort %}
+{% set h = hostvars[host] %}
+{% if h.ansible_host is defined %}
+      - targets: ["{{ h.ansible_host }}:9100"]
+        labels:
+          location: {{ h.prometheus_location }}
+          server: {{ host }}
+{% endif %}
+{% endfor %}
+
+  - job_name: "smartmontools"
+    static_configs:
+{% for host in groups['all'] | sort %}
+{% set h = hostvars[host] %}
+{% if 'smartctl-exporter' in (h.docker_services | default([])) %}
+      - targets: ["{{ h.ansible_host }}:9633"]
+        labels:
+          location: {{ h.prometheus_location }}
+          server: {{ host }}
+{% endif %}
+{% endfor %}
+
+  - job_name: "plex"
+    static_configs:
+{% for host in groups['all'] | sort %}
+{% set h = hostvars[host] %}
+{% if 'plex-exporter' in (h.docker_services | default([])) %}
+      - targets: ["{{ h.ansible_host }}:9000"]
+        labels:
+          location: {{ h.prometheus_location }}
+          server: {{ host }}
+{% endif %}
+{% endfor %}
+
+  - job_name: "systemd_exporter"
+    static_configs:
+{% for host in groups['systemd_exporter_hosts'] | sort %}
+{% set h = hostvars[host] %}
+      - targets: ["{{ h.ansible_host }}:9558"]
+        labels:
+          location: {{ h.prometheus_location }}
+          server: {{ host }}
+{% endfor %}
+
+  - job_name: "caddy"
+    static_configs:
+{% for host in groups['all'] | sort %}
+{% set h = hostvars[host] %}
+{% if h.caddy_config_src is defined %}
+      - targets: ["{{ h.ansible_host }}:2019"]
+        labels:
+          location: {{ h.prometheus_location }}
+          server: {{ host }}
+{% endif %}
+{% endfor %}