Add systemd_exporter Ansible role and Prometheus scrape config

- Create systemd_exporter role (download binary, create user, deploy service)
- Add scrape job for london-b:9558 and copenhagen-a:9558
- Add systemd_exporter_hosts inventory group
- Add stage 3b to deploy.yml
- Map role to deploy-on-merge scope

Closes PESO-120

.github/workflows/deploy-on-merge.yml

@@ -65,6 +65,8 @@ jobs:
                 HOSTS="$HOSTS london-b" ;;
               ansible/roles/firewall_alpine/*)
                 HOSTS="$HOSTS nuremberg-a" ;;
+              ansible/roles/systemd_exporter/*)
+                HOSTS="$HOSTS london-b copenhagen-a" ;;
               ansible/roles/systemd_services/*)
                 HOSTS="$HOSTS helsinki-a copenhagen-a" ;;
               ansible/roles/zfs/*)

ansible/deploy.yml

@@ -42,6 +42,15 @@
   roles:
     - role: node_exporter
 
+# ──────────────────────────────────────────────
+# Stage 3b: systemd_exporter — Linux hosts with systemd metrics
+# ──────────────────────────────────────────────
+- name: "Stage 3b: systemd_exporter"
+  hosts: systemd_exporter_hosts
+  tags: [monitoring, systemd_exporter]
+  roles:
+    - role: systemd_exporter
+
 # ──────────────────────────────────────────────
 # Stage 4: Per-host services
 # ──────────────────────────────────────────────

ansible/inventory/hosts.ini

@@ -17,6 +17,10 @@ london-a     ansible_host=100.122.219.41
 london-a
 london-b
 
+[systemd_exporter_hosts]
+london-b
+copenhagen-a
+
 [docker_hosts]
 helsinki-a
 london-b

ansible/roles/systemd_exporter/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+systemd_exporter_version: "0.6.0"
+systemd_exporter_listen_address: "0.0.0.0"
+systemd_exporter_listen_port: 9558
+systemd_exporter_log_level: "info"
+systemd_exporter_user: "systemd-exporter"
+systemd_exporter_group: "systemd-exporter"

ansible/roles/systemd_exporter/handlers/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Restart systemd_exporter
+  ansible.builtin.service:
+    name: systemd_exporter
+    state: restarted

ansible/roles/systemd_exporter/tasks/main.yml

@@ -0,0 +1,80 @@
+---
+# Install and configure systemd_exporter for Prometheus monitoring.
+# Downloads the binary from GitHub releases and deploys a systemd service.
+# Linux only — systemd_exporter has no FreeBSD equivalent.
+
+- name: Create systemd_exporter group
+  ansible.builtin.group:
+    name: "{{ systemd_exporter_group }}"
+    system: true
+    state: present
+
+- name: Create systemd_exporter user
+  ansible.builtin.user:
+    name: "{{ systemd_exporter_user }}"
+    group: "{{ systemd_exporter_group }}"
+    system: true
+    shell: /usr/sbin/nologin
+    create_home: false
+
+- name: Check if systemd_exporter binary exists
+  ansible.builtin.stat:
+    path: /usr/local/bin/systemd_exporter
+  register: _systemd_exporter_bin
+
+- name: Get installed version
+  ansible.builtin.command: /usr/local/bin/systemd_exporter --version
+  register: _systemd_exporter_installed_version
+  changed_when: false
+  failed_when: false
+  when: _systemd_exporter_bin.stat.exists
+
+- name: Download and install systemd_exporter
+  when: >-
+    not _systemd_exporter_bin.stat.exists or
+    systemd_exporter_version not in (_systemd_exporter_installed_version.stdout | default(''))
+  block:
+    - name: Download systemd_exporter tarball
+      ansible.builtin.get_url:
+        url: "https://github.com/prometheus-community/systemd_exporter/releases/download/v{{ systemd_exporter_version }}/systemd_exporter-{{ systemd_exporter_version }}.linux-{{ ansible_architecture | regex_replace('x86_64', 'amd64') | regex_replace('aarch64', 'arm64') }}.tar.gz"
+        dest: /tmp/systemd_exporter.tar.gz
+        mode: '0644'
+
+    - name: Extract systemd_exporter binary
+      ansible.builtin.unarchive:
+        src: /tmp/systemd_exporter.tar.gz
+        dest: /tmp
+        remote_src: true
+
+    - name: Install systemd_exporter binary
+      ansible.builtin.copy:
+        src: "/tmp/systemd_exporter-{{ systemd_exporter_version }}.linux-{{ ansible_architecture | regex_replace('x86_64', 'amd64') | regex_replace('aarch64', 'arm64') }}/systemd_exporter"
+        dest: /usr/local/bin/systemd_exporter
+        mode: '0755'
+        owner: root
+        group: root
+        remote_src: true
+      notify: Restart systemd_exporter
+
+    - name: Clean up tarball
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - /tmp/systemd_exporter.tar.gz
+        - "/tmp/systemd_exporter-{{ systemd_exporter_version }}.linux-{{ ansible_architecture | regex_replace('x86_64', 'amd64') | regex_replace('aarch64', 'arm64') }}"
+
+- name: Deploy systemd_exporter service file
+  ansible.builtin.template:
+    src: systemd_exporter.service.j2
+    dest: /etc/systemd/system/systemd_exporter.service
+    mode: '0644'
+  notify:
+    - Reload systemd
+    - Restart systemd_exporter
+
+- name: Enable and start systemd_exporter
+  ansible.builtin.service:
+    name: systemd_exporter
+    state: started
+    enabled: true

ansible/roles/systemd_exporter/templates/systemd_exporter.service.j2

@@ -0,0 +1,31 @@
+#
+# Ansible managed
+#
+
+[Unit]
+Description=Prometheus SystemD Exporter
+After=network-online.target
+
+[Service]
+Type=simple
+User={{ systemd_exporter_user }}
+Group={{ systemd_exporter_group }}
+ExecStart=/usr/local/bin/systemd_exporter \
+    --log.level={{ systemd_exporter_log_level }} \
+    --web.listen-address={{ systemd_exporter_listen_address }}:{{ systemd_exporter_listen_port }}
+
+SyslogIdentifier=systemd_exporter
+Restart=always
+RestartSec=1
+StartLimitInterval=0
+
+ProtectHome=yes
+NoNewPrivileges=yes
+
+ProtectSystem=strict
+ProtectControlGroups=true
+ProtectKernelModules=true
+ProtectKernelTunables=yes
+
+[Install]
+WantedBy=multi-user.target

ansible/services/prometheus/prometheus.yml

@@ -61,6 +61,17 @@ scrape_configs:
           location: london
           server: london-b
 
+  - job_name: "systemd_exporter"
+    static_configs:
+      - targets: ["100.84.65.101:9558"]
+        labels:
+          location: london
+          server: london-b
+      - targets: ["100.89.206.60:9558"]
+        labels:
+          location: copenhagen
+          server: copenhagen-a
+
   - job_name: "caddy"
     static_configs:
       - targets: ["100.67.6.27:2019"]