From a33ef8efb586e3312905fd44a9f5e39b9893398e Mon Sep 17 00:00:00 2001
From: Rasmus Wejlgaard <pez@pez.sh>
Date: Fri, 3 Apr 2026 00:48:28 +0000
Subject: [PATCH] fix: remove custom node_exporter, standardise on package
 version

london-b had both a custom node_exporter.service and the
package-managed prometheus-node-exporter.service installed.
Both tried to bind port 9100, causing the package version to fail.

- Add cleanup tasks to remove custom /etc/systemd/system/node_exporter.service
  and /usr/local/bin/node_exporter if present
- Add node_exporter_extra_collectors variable for configurable collectors
- Configure london-b with systemd/processes/sysctl/ethtool/zfs collectors
  matching its previous custom setup

Resolves PESO-109
---
 ansible/inventory/host_vars/london-b.yml      |  7 ++++
 ansible/roles/node_exporter/defaults/main.yml |  4 ++
 ansible/roles/node_exporter/handlers/main.yml |  4 ++
 ansible/roles/node_exporter/tasks/main.yml    | 38 +++++++++++++++++--
 4 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/ansible/inventory/host_vars/london-b.yml b/ansible/inventory/host_vars/london-b.yml
index 30bf2c0..ba5d8f6 100644
--- a/ansible/inventory/host_vars/london-b.yml
+++ b/ansible/inventory/host_vars/london-b.yml
@@ -15,6 +15,13 @@ docker_services:
 zfs_pools:
   - hdd
 
+node_exporter_extra_collectors:
+  - systemd
+  - processes
+  - sysctl
+  - ethtool
+  - zfs
+
 common_ufw_allowed_ports:
   - {port: 32400, proto: tcp, comment: "Plex Media Server"}
   - {port: 6881, proto: tcp, comment: "BitTorrent"}
diff --git a/ansible/roles/node_exporter/defaults/main.yml b/ansible/roles/node_exporter/defaults/main.yml
index a6d19c0..f89b1d6 100644
--- a/ansible/roles/node_exporter/defaults/main.yml
+++ b/ansible/roles/node_exporter/defaults/main.yml
@@ -2,3 +2,7 @@
 # When true, bind node_exporter to the Tailscale IP (ansible_host) only.
 # Use on public-facing hosts to avoid exposing metrics on 0.0.0.0.
 node_exporter_bind_tailscale: false
+
+# Extra collectors to enable beyond the defaults.
+# Each entry is a collector name (e.g. "systemd", "processes").
+node_exporter_extra_collectors: []
diff --git a/ansible/roles/node_exporter/handlers/main.yml b/ansible/roles/node_exporter/handlers/main.yml
index 1c993e9..acffd90 100644
--- a/ansible/roles/node_exporter/handlers/main.yml
+++ b/ansible/roles/node_exporter/handlers/main.yml
@@ -1,4 +1,8 @@
 ---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true
+
 - name: Restart node-exporter (Debian)
   ansible.builtin.service:
     name: prometheus-node-exporter
diff --git a/ansible/roles/node_exporter/tasks/main.yml b/ansible/roles/node_exporter/tasks/main.yml
index 708138d..3ed9936 100644
--- a/ansible/roles/node_exporter/tasks/main.yml
+++ b/ansible/roles/node_exporter/tasks/main.yml
@@ -3,6 +3,29 @@
 # Uses system packages on Linux, pkg on FreeBSD.
 # Optionally binds to Tailscale IP on public-facing hosts.
 
+# ── Cleanup old custom installs ──────────────────────────────
+- name: Stop and disable custom node_exporter service if present
+  ansible.builtin.service:
+    name: node_exporter
+    state: stopped
+    enabled: false
+  failed_when: false
+  when: ansible_facts["os_family"] == "Debian"
+
+- name: Remove custom node_exporter service file
+  ansible.builtin.file:
+    path: /etc/systemd/system/node_exporter.service
+    state: absent
+  when: ansible_facts["os_family"] == "Debian"
+  notify: Reload systemd
+
+- name: Remove custom node_exporter binary
+  ansible.builtin.file:
+    path: /usr/local/bin/node_exporter
+    state: absent
+  when: ansible_facts["os_family"] == "Debian"
+
+# ── Install ──────────────────────────────────────────────────
 - name: Install prometheus-node-exporter (Debian)
   ansible.builtin.apt:
     name: prometheus-node-exporter
@@ -15,14 +38,22 @@
     state: present
   when: ansible_facts["os_family"] == "Alpine"
 
-- name: Configure listen address (Debian)
+# ── Configure (Debian) ──────────────────────────────────────
+- name: Build ARGS for prometheus-node-exporter
+  ansible.builtin.set_fact:
+    _node_exporter_args: >-
+      {{ (node_exporter_extra_collectors | map('regex_replace', '^(.*)$', '--collector.\1') | list)
+         + (['--web.listen-address=' + ansible_host + ':9100'] if node_exporter_bind_tailscale | bool else []) }}
+  when: ansible_facts["os_family"] == "Debian"
+
+- name: Configure prometheus-node-exporter ARGS (Debian)
   ansible.builtin.lineinfile:
     path: /etc/default/prometheus-node-exporter
     regexp: '^ARGS='
-    line: 'ARGS="--web.listen-address={{ ansible_host }}:9100"'
+    line: 'ARGS="{{ _node_exporter_args | join(" ") }}"'
   when:
     - ansible_facts["os_family"] == "Debian"
-    - node_exporter_bind_tailscale | bool
+    - (_node_exporter_args | length > 0)
   notify: Restart node-exporter (Debian)
 
 - name: Enable and start node-exporter (Debian)
@@ -39,6 +70,7 @@
     enabled: true
   when: ansible_facts["os_family"] == "Alpine"
 
+# ── FreeBSD ──────────────────────────────────────────────────
 - name: Install node_exporter (FreeBSD)
   community.general.pkgng:
     name: node_exporter