.gitignore

@@ -13,7 +13,6 @@ override.tf.json
 
 # Ansible
 *.retry
-.ansible/
 
 # Secrets — never commit plaintext secrets
 *.key

ansible/README.md

@@ -47,9 +47,8 @@ Ad-hoc runs against a single host or playbook go through the manual `deploy.yml`
 The deploy playbook runs in stages, each independently taggable (see `deploy.yml`):
 
 1. **common / baseline** — Baseline packages, SSH hardening, fish shell, dotfiles
-2. **alloy** — Grafana Alloy collector on every host
+2. **docker** — Docker engine on container hosts (`docker_hosts` group)
-3. **docker** — Docker engine on container hosts (`docker_hosts` group)
+3. **services** — Per-host service deployment:
-4. **services** — Per-host service deployment:
    - `helsinki-a`: Caddy + status-page + custom systemd units
    - `docker_hosts`: Docker Compose stacks from `services/`
    - `nuremberg-a`: poste.io mail (Docker)
@@ -57,7 +56,7 @@ The deploy playbook runs in stages, each independently taggable (see `deploy.yml
    - `proxmox_hosts` (`london-a`, `copenhagen-a`): `proxmox_ve` (apt repo, nag patch, CIFS storage on london-a only)
    - `zfs_hosts`: ZFS scrub scheduling
 
-Observability is part of the baseline — every host gets node_exporter and systemd_exporter from `common`, and the Grafana Alloy collector from `alloy`.
+Observability (node_exporter, systemd_exporter, Grafana Alloy) is part of the `common` baseline — every host gets it.
 
 Run a single stage: `ansible-playbook deploy.yml --tags docker`
 
@@ -65,8 +64,7 @@ Run a single stage: `ansible-playbook deploy.yml --tags docker`
 
 | Role | Description |
 |------|-------------|
-| `common` | Base packages, SSH hardening, fish shell, exporters |
+| `common` | Base packages, SSH hardening, fish shell, exporters, Alloy |
-| `alloy` | Grafana Alloy install, Grafana Cloud credentials, Fleet Management enrolment |
 | `dotfiles` | Shell config from `dotfiles/` |
 | `docker` | Docker engine install and setup + monthly log-cleanup cron |
 | `docker_services` | Deploy compose files from `services/` |

ansible/deploy.yml

@@ -22,12 +22,6 @@
     - role: common
     - role: dotfiles
 
-- name: Grafana Alloy
-  hosts: all
-  tags: [common, baseline, alloy]
-  roles:
-    - role: alloy
-
 - name: Docker engine
   hosts: docker_hosts
   tags: [docker]

ansible/group_vars/all/secrets.enc.yaml

@@ -5,7 +5,6 @@ proxmox_ve_smb_user: ENC[AES256_GCM,data:13dtQg==,iv:aUOadm20b9FpmOxXGINv8KkwsfB
 proxmox_ve_smb_password: ENC[AES256_GCM,data:kYegZCZTpEo7,iv:Jm8yMLKQXyaQQPyybhZyAy9bhN4+OjI63f7azOzqF+w=,tag:IIKCI2bsRzer31hF0h4b9g==,type:str]
 proxmox_ve_backup_smb_user: ENC[AES256_GCM,data:sax7tELTAJhZ,iv:OIuvOoZH6bdrN0CgmM3pVIUoIE4syTahyL2pFGaOURA=,tag:Vdx6iSN3cvCqxPCiKuiJRw==,type:str]
 proxmox_ve_backup_smb_password: ENC[AES256_GCM,data:I9wM28Vn+w49C42LD4BgHxLN2aSF9e5+LRZGB5iXfNYSCfOlcp8aJQ==,iv:UG34yCmwDvOzkITd4LJeUu+Et0Tdx/aJXAT6rUbXJVg=,tag:P/O3jLir9i4NnIzcLjcRcA==,type:str]
-alloy_gcloud_rw_api_key: ENC[AES256_GCM,data:0fb/TjxGfXIJiiTWDvGsK83AIhe06vsqwyM4kBXYTP5dPLla+/b3WaX33nM1EkuMl9hG0FmYS5G/4ft3uKdZESQYERgGBichVBuEQjj8eNL5FunSmPLOm+Q26vp0Cj68EyybZ7vEoFoWhbiPEEMjyH3PZglO7/Om5E1c+i5xe4x7Q3CRSTGhNkz9HCXghXVrjpkesg==,iv:KkeMc5ph7AUS/VwBzsnH/N2fiP/rpgNGsjsyr0782xM=,tag:VfE5vQp4CZhuFvQvd8dQKw==,type:str]
 sops:
     age:
         - recipient: age1r8uh2w2qad2z5sgq9q7l73962q2sp8zz9hdnh6sjuvanxl565vmswn8squ
@@ -17,7 +16,7 @@ sops:
             Ti9TNjFta0kyRTNrWGVWbGFGeC9CR3cKkdhytnSS+fCMc1FOSZ5WC3dO16Kw+akB
             O69PKWul3WI7kFmcRk8mzlbFqS941tnPbvVaTYP0+GZTgSX1wlhSeg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-09-18T19:58:02Z"
+    lastmodified: "2026-09-16T16:44:41Z"
-    mac: ENC[AES256_GCM,data:Pb28OOyodaQeyWBbwR+3vDkUzC1eULil2Cq04EqQfh5+mi1xOeUQuEAgJq9u/Lfz8Vb/XdayZ2ArrV4sazLa32PBvJt822V0tIpg8O9vhQjKtVEu8pS2cmWlqyS8mgSMFrBtaaqEVI0ai3Ouypc/5FDtkwXoqmkI6oA1rb5oIkc=,iv:QJAxGwml2bvBg78BDFBM2q4HGxkkHpTgO2bCWW9xqOc=,tag:cQa5IITFeD2HguLHDmCVyQ==,type:str]
+    mac: ENC[AES256_GCM,data:qMjB7lCALsa6dsrWxlhpnJgoWzlbgpF4gktIIZPKoL791CPJL10Y9qqDznGwqU1HRnlf5LdOUOCVPLow8f+Tmt4xbg3t6Rr2PsyL4cAF5N3p2eY3L8aIo0fs41Mkl5zysZbqyWYI3lrBob+Frc4pXNIGCzUlTNhrOw9B03eiWVk=,iv:Nw60y9qoFVQ2A4FdHLEw5cX3BIxdWGGSB3EHOHwhNPA=,tag:IRU3S+sycHP3EunrRRrYBw==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.12.2

ansible/inventory/host_vars/london-b.yml

@@ -20,45 +20,6 @@ snap_services:
 apt_user_services:
   - syncthing@pez
 
-common_apt_repositories:
-  - name: tailscale
-    uris: https://pkgs.tailscale.com/stable/ubuntu
-    suites: "{{ ansible_facts['distribution_release'] }}"
-    components: main
-    signed_by: "https://pkgs.tailscale.com/stable/ubuntu/{{ ansible_facts['distribution_release'] }}.noarmor.gpg"
-  - name: jellyfin
-    uris: https://repo.jellyfin.org/ubuntu
-    suites: "{{ ansible_facts['distribution_release'] }}"
-    components: main
-    signed_by: https://repo.jellyfin.org/jellyfin_team.gpg.key
-  - name: plexmediaserver
-    uris: https://repo.plex.tv/deb
-    suites: public
-    components: main
-    signed_by: https://downloads.plex.tv/plex-keys/PlexSign.v2.key
-
-common_apt_cleanup_files:
-  - /etc/apt/sources.list.d/datadog.list.distUpgrade
-  - /etc/apt/sources.list.d/graphics-drivers-ubuntu-ppa-lunar.list.distUpgrade
-  - /etc/apt/sources.list.d/graphics-drivers-ubuntu-ppa-lunar.sources
-  - /etc/apt/sources.list.d/jcfp-ubuntu-nobetas-lunar.list.distUpgrade
-  - /etc/apt/sources.list.d/jcfp-ubuntu-nobetas-lunar.sources
-  - /etc/apt/sources.list.d/jellyfin.sources.distUpgrade
-  - /etc/apt/sources.list.d/mediaarea.list.distUpgrade
-  - /etc/apt/sources.list.d/mediaarea.sources
-  - /etc/apt/sources.list.d/mono-official-stable.list.distUpgrade
-  - /etc/apt/sources.list.d/nvidia-container-toolkit.list.distUpgrade
-  - /etc/apt/sources.list.d/plexmediaserver.list.distUpgrade
-  - /etc/apt/sources.list.d/sonarr.list.distUpgrade
-  - /etc/apt/sources.list.d/tailscale.list.distUpgrade
-  - /etc/apt/keyrings/jellyfin.gpg
-  - /usr/share/keyrings/plexmediaserver.gpg
-  - /usr/share/keyrings/mono-official-archive-keyring.gpg
-  - /usr/share/keyrings/mono-official-archive-keyring.gpg~
-  - /usr/share/keyrings/sonarr-keyring.gpg
-  - /usr/share/keyrings/sonarr-keyring.gpg~
-  - /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
-
 zfs_pools:
   - hdd
 

ansible/roles/alloy/defaults/main.yml

@@ -1,13 +0,0 @@
----
-alloy_version: "1.19.2-1"
-
-alloy_collector_id: "{{ inventory_hostname }}"
-
-alloy_fleet_management_url: https://fleet-management-prod-023.grafana.net
-alloy_fleet_management_username: "1621083"
-
-alloy_prometheus_url: https://prometheus-prod-55-prod-gb-south-1.grafana.net/api/prom/push
-alloy_prometheus_username: "3166394"
-
-alloy_loki_url: https://logs-prod-035.grafana.net/loki/api/v1/push
-alloy_loki_username: "1578872"

ansible/roles/alloy/handlers/main.yml

@@ -1,6 +0,0 @@
----
-- name: Restart alloy
-  ansible.builtin.systemd:
-    name: alloy
-    state: restarted
-    daemon_reload: true

ansible/roles/alloy/tasks/main.yml

@@ -1,82 +0,0 @@
----
-- name: Install deb822_repository dependency
-  ansible.builtin.apt:
-    name: python3-debian
-    state: present
-
-- name: Configure Grafana apt repository
-  ansible.builtin.deb822_repository:
-    name: grafana
-    types: deb
-    uris: https://apt.grafana.com
-    suites: stable
-    components: main
-    signed_by: https://apt.grafana.com/gpg.key
-  register: alloy_apt_repository
-
-- name: Update apt cache
-  ansible.builtin.apt:
-    update_cache: true
-    cache_valid_time: "{{ 0 if alloy_apt_repository is changed else 3600 }}"
-
-- name: Install Alloy
-  ansible.builtin.apt:
-    name: "alloy={{ alloy_version }}"
-    state: present
-
-- name: Deploy Alloy configuration
-  ansible.builtin.template:
-    src: config.alloy.j2
-    dest: /etc/alloy/config.alloy
-    owner: root
-    group: root
-    mode: '0644'
-  notify: Restart alloy
-
-- name: Create Alloy systemd drop-in directory
-  ansible.builtin.file:
-    path: /etc/systemd/system/alloy.service.d
-    state: directory
-    owner: root
-    group: root
-    mode: '0755'
-
-# StartLimitIntervalSec=0 keeps Alloy retrying through transient upstream
-# failures instead of tripping systemd's start rate-limit and staying dead
-# (PESO-149).
-- name: Install Alloy systemd resilience drop-in
-  ansible.builtin.copy:
-    dest: /etc/systemd/system/alloy.service.d/10-resilience.conf
-    owner: root
-    group: root
-    mode: '0644'
-    content: |
-      # Managed by Ansible (pez-infra) — alloy role.
-      [Unit]
-      StartLimitIntervalSec=0
-
-      [Service]
-      Restart=always
-      RestartSec=30
-  notify: Restart alloy
-
-- name: Install Alloy credentials drop-in
-  ansible.builtin.copy:
-    dest: /etc/systemd/system/alloy.service.d/env.conf
-    owner: root
-    group: root
-    mode: '0600'
-    content: |
-      # Managed by Ansible (pez-infra) — alloy role.
-      [Service]
-      Environment=GCLOUD_RW_API_KEY={{ alloy_gcloud_rw_api_key }}
-      Environment=GCLOUD_FM_COLLECTOR_ID={{ alloy_collector_id }}
-  no_log: true
-  notify: Restart alloy
-
-- name: Enable and start Alloy
-  ansible.builtin.systemd:
-    name: alloy
-    state: started
-    enabled: true
-    daemon_reload: true

ansible/roles/alloy/templates/config.alloy.j2

@@ -1,35 +0,0 @@
-// Managed by Ansible (pez-infra) — alloy role.
-
-remotecfg {
-	url            = "{{ alloy_fleet_management_url }}"
-	id             = "{{ alloy_collector_id }}"
-	poll_frequency = "60s"
-	attributes     = encoding.from_json(coalesce(``, `{}`))
-
-	basic_auth {
-		username = "{{ alloy_fleet_management_username }}"
-		password = sys.env("GCLOUD_RW_API_KEY")
-	}
-}
-
-prometheus.remote_write "metrics_service" {
-	endpoint {
-		url = "{{ alloy_prometheus_url }}"
-
-		basic_auth {
-			username = "{{ alloy_prometheus_username }}"
-			password = sys.env("GCLOUD_RW_API_KEY")
-		}
-	}
-}
-
-loki.write "grafana_cloud_loki" {
-	endpoint {
-		url = "{{ alloy_loki_url }}"
-
-		basic_auth {
-			username = "{{ alloy_loki_username }}"
-			password = sys.env("GCLOUD_RW_API_KEY")
-		}
-	}
-}

ansible/roles/common/defaults/main.yml

@@ -5,6 +5,3 @@ common_ufw_enabled: true
 common_ufw_allowed_ports: []
 #   - {port: 80, proto: tcp, comment: "HTTP"}
 #   - {port: 443, proto: tcp, comment: "HTTPS"}
-
-common_apt_repositories: []
-common_apt_cleanup_files: []

ansible/roles/common/handlers/main.yml

@@ -11,3 +11,9 @@
 - name: Reload systemd daemon
   ansible.builtin.systemd:
     daemon_reload: true
+
+- name: Restart alloy
+  ansible.builtin.systemd:
+    name: alloy
+    state: restarted
+    daemon_reload: true

ansible/roles/common/tasks/main.yml

@@ -20,36 +20,10 @@
     force: true
   when: common_github_cli_source.stat.exists
 
-- name: Install deb822_repository dependency
-  ansible.builtin.apt:
-    name: python3-debian
-    state: present
-  when: common_apt_repositories | length > 0
-
-- name: Configure third-party apt repositories
-  ansible.builtin.deb822_repository:
-    name: "{{ item.name }}"
-    types: deb
-    uris: "{{ item.uris }}"
-    suites: "{{ item.suites }}"
-    components: "{{ item.components }}"
-    signed_by: "{{ item.signed_by }}"
-  loop: "{{ common_apt_repositories }}"
-  loop_control:
-    label: "{{ item.name }}"
-  register: common_apt_repositories_result
-
-- name: Remove stale apt sources and keyrings
-  ansible.builtin.file:
-    path: "{{ item }}"
-    state: absent
-  loop: "{{ common_apt_cleanup_files }}"
-  register: common_apt_cleanup_result
-
 - name: Update apt cache
   ansible.builtin.apt:
     update_cache: true
-    cache_valid_time: "{{ 0 if (common_apt_repositories_result is changed or common_apt_cleanup_result is changed) else 3600 }}"
+    cache_valid_time: 3600
 
 - name: Install baseline packages
   ansible.builtin.apt:
@@ -205,6 +179,35 @@
         state: absent
         purge: true
 
+# --- Grafana Alloy: restart resilience (PESO-149) ---
+# Every host runs alloy.service to ship metrics/logs to Grafana Cloud. On
+# 2026-05-20 a transient TLS failure to Grafana fleet-management tripped
+# systemd's default start rate-limit on copenhagen-c; systemd then gave up and
+# the host went silently unmonitored for ~2.5 weeks. Disable the rate limit so
+# Alloy keeps retrying indefinitely, backing off 30s between attempts, instead
+# of dying permanently on a momentary blip.
+- name: Check for Alloy unit
+  ansible.builtin.stat:
+    path: /lib/systemd/system/alloy.service
+  register: common_alloy_unit
+
+- name: Install Alloy systemd resilience drop-in
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/alloy.service.d/10-resilience.conf
+    mode: '0644'
+    content: |
+      # Managed by Ansible (pez-infra) — common role, PESO-149.
+      # Keep Alloy retrying through transient upstream/TLS failures instead of
+      # hitting systemd's start rate-limit and giving up.
+      [Unit]
+      StartLimitIntervalSec=0
+
+      [Service]
+      Restart=always
+      RestartSec=30
+  when: common_alloy_unit.stat.exists | default(false)
+  notify: Restart alloy
+
 # --- Cleanup: leftover self-hosted Grafana (PESO-149) ---
 # Monitoring moved fully to Grafana Cloud, but the old Grafana OSS package was
 # never removed from copenhagen-c, where grafana-server.service sits enabled

docs/monitoring.md

@@ -26,11 +26,11 @@ Everything in `terraform/grafana/` is the source of truth for the Grafana Cloud
 
 ## Grafana Alloy (per-host collector)
 
-Alloy runs as `alloy.service` on every host in the inventory, installed and configured by the `alloy` Ansible role (Grafana apt repo, `/etc/alloy/config.alloy`, and a `0600` systemd drop-in holding the Grafana Cloud API key from SOPS). Each host is registered as a Grafana Fleet Management collector in `terraform/grafana/fleet_collectors.tf`, tagged with a `location` attribute (`london`, `copenhagen`, `cloud`) so pipelines can target subsets of the fleet.
+Alloy runs as `alloy.service` on every host in the inventory. Each host is registered as a Grafana Fleet Management collector in `terraform/grafana/fleet_collectors.tf`, tagged with a `location` attribute (`london`, `copenhagen`, `cloud`) so pipelines can target subsets of the fleet.
 
 Pipelines (what to scrape, how to relabel, where to ship) live in `terraform/grafana/fleet_pipelines/` and are pushed to Grafana Cloud as a `grafana_fleet_management_pipeline` resource. The Alloy daemons on each host pull their config from Fleet Management.
 
-The `alloy` role drops a `10-resilience.conf` systemd override onto every host (`StartLimitIntervalSec=0`, `Restart=always`, `RestartSec=30`) so a transient upstream/TLS failure can't trip systemd's start rate-limit and permanently kill the collector — it keeps retrying until Grafana Cloud is reachable again. (Added after copenhagen-c sat unmonitored for ~2.5 weeks following one such blip — PESO-149.)
+The `common` role drops a `10-resilience.conf` systemd override onto every host (`StartLimitIntervalSec=0`, `Restart=always`, `RestartSec=30`) so a transient upstream/TLS failure can't trip systemd's start rate-limit and permanently kill the collector — it keeps retrying until Grafana Cloud is reachable again. (Added after copenhagen-c sat unmonitored for ~2.5 weeks following one such blip — PESO-149.)
 
 ### Local exporters scraped by Alloy
 
@@ -47,8 +47,6 @@ The `alloy` role drops a `10-resilience.conf` systemd override onto every host (
 
 Alloy ships systemd journal entries from every host to Grafana Cloud Logs. Log-derived alerts (e.g. SSH brute-force, mail server errors) can be configured directly in Grafana Cloud.
 
-The host's config is bootstrap only — what each collector actually scrapes and ships is decided by the Fleet Management pipelines in `terraform/grafana/fleet_pipelines/`.
-
 ## Synthetic Monitoring
 
 Grafana Cloud's Synthetic Monitoring service runs HTTPS probes from the London region against the public services, every 10 minutes. Configured in `terraform/grafana/synthetic_checks.tf`: