diff --git a/ansible/README.md b/ansible/README.md index c1cf6f2..d357a17 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -47,8 +47,9 @@ Ad-hoc runs against a single host or playbook go through the manual `deploy.yml` The deploy playbook runs in stages, each independently taggable (see `deploy.yml`): 1. **common / baseline** — Baseline packages, SSH hardening, fish shell, dotfiles -2. **docker** — Docker engine on container hosts (`docker_hosts` group) -3. **services** — Per-host service deployment: +2. **alloy** — Grafana Alloy collector on every host +3. **docker** — Docker engine on container hosts (`docker_hosts` group) +4. **services** — Per-host service deployment: - `helsinki-a`: Caddy + status-page + custom systemd units - `docker_hosts`: Docker Compose stacks from `services/` - `nuremberg-a`: poste.io mail (Docker) @@ -56,7 +57,7 @@ The deploy playbook runs in stages, each independently taggable (see `deploy.yml - `proxmox_hosts` (`london-a`, `copenhagen-a`): `proxmox_ve` (apt repo, nag patch, CIFS storage on london-a only) - `zfs_hosts`: ZFS scrub scheduling -Observability (node_exporter, systemd_exporter, Grafana Alloy) is part of the `common` baseline — every host gets it. +Observability is part of the baseline — every host gets node_exporter and systemd_exporter from `common`, and the Grafana Alloy collector from `alloy`. Run a single stage: `ansible-playbook deploy.yml --tags docker` @@ -64,7 +65,8 @@ Run a single stage: `ansible-playbook deploy.yml --tags docker` | Role | Description | |------|-------------| -| `common` | Base packages, SSH hardening, fish shell, exporters, Alloy | +| `common` | Base packages, SSH hardening, fish shell, exporters | +| `alloy` | Grafana Alloy install, Grafana Cloud credentials, Fleet Management enrolment | | `dotfiles` | Shell config from `dotfiles/` | | `docker` | Docker engine install and setup + monthly log-cleanup cron | | `docker_services` | Deploy compose files from `services/` | diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 5420024..5004abb 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -22,6 +22,12 @@ - role: common - role: dotfiles +- name: Grafana Alloy + hosts: all + tags: [common, baseline, alloy] + roles: + - role: alloy + - name: Docker engine hosts: docker_hosts tags: [docker] diff --git a/ansible/group_vars/all/secrets.enc.yaml b/ansible/group_vars/all/secrets.enc.yaml index e200df6..74b897f 100644 --- a/ansible/group_vars/all/secrets.enc.yaml +++ b/ansible/group_vars/all/secrets.enc.yaml @@ -5,6 +5,7 @@ proxmox_ve_smb_user: ENC[AES256_GCM,data:13dtQg==,iv:aUOadm20b9FpmOxXGINv8KkwsfB proxmox_ve_smb_password: ENC[AES256_GCM,data:kYegZCZTpEo7,iv:Jm8yMLKQXyaQQPyybhZyAy9bhN4+OjI63f7azOzqF+w=,tag:IIKCI2bsRzer31hF0h4b9g==,type:str] proxmox_ve_backup_smb_user: ENC[AES256_GCM,data:sax7tELTAJhZ,iv:OIuvOoZH6bdrN0CgmM3pVIUoIE4syTahyL2pFGaOURA=,tag:Vdx6iSN3cvCqxPCiKuiJRw==,type:str] proxmox_ve_backup_smb_password: ENC[AES256_GCM,data:I9wM28Vn+w49C42LD4BgHxLN2aSF9e5+LRZGB5iXfNYSCfOlcp8aJQ==,iv:UG34yCmwDvOzkITd4LJeUu+Et0Tdx/aJXAT6rUbXJVg=,tag:P/O3jLir9i4NnIzcLjcRcA==,type:str] +alloy_gcloud_rw_api_key: ENC[AES256_GCM,data:0fb/TjxGfXIJiiTWDvGsK83AIhe06vsqwyM4kBXYTP5dPLla+/b3WaX33nM1EkuMl9hG0FmYS5G/4ft3uKdZESQYERgGBichVBuEQjj8eNL5FunSmPLOm+Q26vp0Cj68EyybZ7vEoFoWhbiPEEMjyH3PZglO7/Om5E1c+i5xe4x7Q3CRSTGhNkz9HCXghXVrjpkesg==,iv:KkeMc5ph7AUS/VwBzsnH/N2fiP/rpgNGsjsyr0782xM=,tag:VfE5vQp4CZhuFvQvd8dQKw==,type:str] sops: age: - recipient: age1r8uh2w2qad2z5sgq9q7l73962q2sp8zz9hdnh6sjuvanxl565vmswn8squ @@ -16,7 +17,7 @@ sops: Ti9TNjFta0kyRTNrWGVWbGFGeC9CR3cKkdhytnSS+fCMc1FOSZ5WC3dO16Kw+akB O69PKWul3WI7kFmcRk8mzlbFqS941tnPbvVaTYP0+GZTgSX1wlhSeg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-09-16T16:44:41Z" - mac: ENC[AES256_GCM,data:qMjB7lCALsa6dsrWxlhpnJgoWzlbgpF4gktIIZPKoL791CPJL10Y9qqDznGwqU1HRnlf5LdOUOCVPLow8f+Tmt4xbg3t6Rr2PsyL4cAF5N3p2eY3L8aIo0fs41Mkl5zysZbqyWYI3lrBob+Frc4pXNIGCzUlTNhrOw9B03eiWVk=,iv:Nw60y9qoFVQ2A4FdHLEw5cX3BIxdWGGSB3EHOHwhNPA=,tag:IRU3S+sycHP3EunrRRrYBw==,type:str] + lastmodified: "2026-09-18T19:58:02Z" + mac: ENC[AES256_GCM,data:Pb28OOyodaQeyWBbwR+3vDkUzC1eULil2Cq04EqQfh5+mi1xOeUQuEAgJq9u/Lfz8Vb/XdayZ2ArrV4sazLa32PBvJt822V0tIpg8O9vhQjKtVEu8pS2cmWlqyS8mgSMFrBtaaqEVI0ai3Ouypc/5FDtkwXoqmkI6oA1rb5oIkc=,iv:QJAxGwml2bvBg78BDFBM2q4HGxkkHpTgO2bCWW9xqOc=,tag:cQa5IITFeD2HguLHDmCVyQ==,type:str] unencrypted_suffix: _unencrypted version: 3.12.2 diff --git a/ansible/roles/alloy/defaults/main.yml b/ansible/roles/alloy/defaults/main.yml new file mode 100644 index 0000000..d36cca6 --- /dev/null +++ b/ansible/roles/alloy/defaults/main.yml @@ -0,0 +1,13 @@ +--- +alloy_version: "1.19.2-1" + +alloy_collector_id: "{{ inventory_hostname }}" + +alloy_fleet_management_url: https://fleet-management-prod-023.grafana.net +alloy_fleet_management_username: "1621083" + +alloy_prometheus_url: https://prometheus-prod-55-prod-gb-south-1.grafana.net/api/prom/push +alloy_prometheus_username: "3166394" + +alloy_loki_url: https://logs-prod-035.grafana.net/loki/api/v1/push +alloy_loki_username: "1578872" diff --git a/ansible/roles/alloy/handlers/main.yml b/ansible/roles/alloy/handlers/main.yml new file mode 100644 index 0000000..4c88725 --- /dev/null +++ b/ansible/roles/alloy/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: Restart alloy + ansible.builtin.systemd: + name: alloy + state: restarted + daemon_reload: true diff --git a/ansible/roles/alloy/tasks/main.yml b/ansible/roles/alloy/tasks/main.yml new file mode 100644 index 0000000..2261738 --- /dev/null +++ b/ansible/roles/alloy/tasks/main.yml @@ -0,0 +1,82 @@ +--- +- name: Install deb822_repository dependency + ansible.builtin.apt: + name: python3-debian + state: present + +- name: Configure Grafana apt repository + ansible.builtin.deb822_repository: + name: grafana + types: deb + uris: https://apt.grafana.com + suites: stable + components: main + signed_by: https://apt.grafana.com/gpg.key + register: alloy_apt_repository + +- name: Update apt cache + ansible.builtin.apt: + update_cache: true + cache_valid_time: "{{ 0 if alloy_apt_repository is changed else 3600 }}" + +- name: Install Alloy + ansible.builtin.apt: + name: "alloy={{ alloy_version }}" + state: present + +- name: Deploy Alloy configuration + ansible.builtin.template: + src: config.alloy.j2 + dest: /etc/alloy/config.alloy + owner: root + group: root + mode: '0644' + notify: Restart alloy + +- name: Create Alloy systemd drop-in directory + ansible.builtin.file: + path: /etc/systemd/system/alloy.service.d + state: directory + owner: root + group: root + mode: '0755' + +# StartLimitIntervalSec=0 keeps Alloy retrying through transient upstream +# failures instead of tripping systemd's start rate-limit and staying dead +# (PESO-149). +- name: Install Alloy systemd resilience drop-in + ansible.builtin.copy: + dest: /etc/systemd/system/alloy.service.d/10-resilience.conf + owner: root + group: root + mode: '0644' + content: | + # Managed by Ansible (pez-infra) — alloy role. + [Unit] + StartLimitIntervalSec=0 + + [Service] + Restart=always + RestartSec=30 + notify: Restart alloy + +- name: Install Alloy credentials drop-in + ansible.builtin.copy: + dest: /etc/systemd/system/alloy.service.d/env.conf + owner: root + group: root + mode: '0600' + content: | + # Managed by Ansible (pez-infra) — alloy role. + [Service] + Environment=GCLOUD_RW_API_KEY={{ alloy_gcloud_rw_api_key }} + Environment=GCLOUD_FM_COLLECTOR_ID={{ alloy_collector_id }} + no_log: true + notify: Restart alloy + +- name: Enable and start Alloy + ansible.builtin.systemd: + name: alloy + state: started + enabled: true + daemon_reload: true diff --git a/ansible/roles/alloy/templates/config.alloy.j2 b/ansible/roles/alloy/templates/config.alloy.j2 new file mode 100644 index 0000000..6b42525 --- /dev/null +++ b/ansible/roles/alloy/templates/config.alloy.j2 @@ -0,0 +1,35 @@ +// Managed by Ansible (pez-infra) — alloy role. + +remotecfg { + url = "{{ alloy_fleet_management_url }}" + id = "{{ alloy_collector_id }}" + poll_frequency = "60s" + attributes = encoding.from_json(coalesce(``, `{}`)) + + basic_auth { + username = "{{ alloy_fleet_management_username }}" + password = sys.env("GCLOUD_RW_API_KEY") + } +} + +prometheus.remote_write "metrics_service" { + endpoint { + url = "{{ alloy_prometheus_url }}" + + basic_auth { + username = "{{ alloy_prometheus_username }}" + password = sys.env("GCLOUD_RW_API_KEY") + } + } +} + +loki.write "grafana_cloud_loki" { + endpoint { + url = "{{ alloy_loki_url }}" + + basic_auth { + username = "{{ alloy_loki_username }}" + password = sys.env("GCLOUD_RW_API_KEY") + } + } +} diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml index 43bc533..259efaf 100644 --- a/ansible/roles/common/handlers/main.yml +++ b/ansible/roles/common/handlers/main.yml @@ -11,9 +11,3 @@ - name: Reload systemd daemon ansible.builtin.systemd: daemon_reload: true - -- name: Restart alloy - ansible.builtin.systemd: - name: alloy - state: restarted - daemon_reload: true diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index 3268b85..1277ba0 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -205,35 +205,6 @@ state: absent purge: true -# --- Grafana Alloy: restart resilience (PESO-149) --- -# Every host runs alloy.service to ship metrics/logs to Grafana Cloud. On -# 2026-05-20 a transient TLS failure to Grafana fleet-management tripped -# systemd's default start rate-limit on copenhagen-c; systemd then gave up and -# the host went silently unmonitored for ~2.5 weeks. Disable the rate limit so -# Alloy keeps retrying indefinitely, backing off 30s between attempts, instead -# of dying permanently on a momentary blip. -- name: Check for Alloy unit - ansible.builtin.stat: - path: /lib/systemd/system/alloy.service - register: common_alloy_unit - -- name: Install Alloy systemd resilience drop-in - ansible.builtin.copy: - dest: /etc/systemd/system/alloy.service.d/10-resilience.conf - mode: '0644' - content: | - # Managed by Ansible (pez-infra) — common role, PESO-149. - # Keep Alloy retrying through transient upstream/TLS failures instead of - # hitting systemd's start rate-limit and giving up. - [Unit] - StartLimitIntervalSec=0 - - [Service] - Restart=always - RestartSec=30 - when: common_alloy_unit.stat.exists | default(false) - notify: Restart alloy - # --- Cleanup: leftover self-hosted Grafana (PESO-149) --- # Monitoring moved fully to Grafana Cloud, but the old Grafana OSS package was # never removed from copenhagen-c, where grafana-server.service sits enabled diff --git a/docs/monitoring.md b/docs/monitoring.md index ccf8282..fee7eba 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -26,11 +26,11 @@ Everything in `terraform/grafana/` is the source of truth for the Grafana Cloud ## Grafana Alloy (per-host collector) -Alloy runs as `alloy.service` on every host in the inventory. Each host is registered as a Grafana Fleet Management collector in `terraform/grafana/fleet_collectors.tf`, tagged with a `location` attribute (`london`, `copenhagen`, `cloud`) so pipelines can target subsets of the fleet. +Alloy runs as `alloy.service` on every host in the inventory, installed and configured by the `alloy` Ansible role (Grafana apt repo, `/etc/alloy/config.alloy`, and a `0600` systemd drop-in holding the Grafana Cloud API key from SOPS). Each host is registered as a Grafana Fleet Management collector in `terraform/grafana/fleet_collectors.tf`, tagged with a `location` attribute (`london`, `copenhagen`, `cloud`) so pipelines can target subsets of the fleet. Pipelines (what to scrape, how to relabel, where to ship) live in `terraform/grafana/fleet_pipelines/` and are pushed to Grafana Cloud as a `grafana_fleet_management_pipeline` resource. The Alloy daemons on each host pull their config from Fleet Management. -The `common` role drops a `10-resilience.conf` systemd override onto every host (`StartLimitIntervalSec=0`, `Restart=always`, `RestartSec=30`) so a transient upstream/TLS failure can't trip systemd's start rate-limit and permanently kill the collector — it keeps retrying until Grafana Cloud is reachable again. (Added after copenhagen-c sat unmonitored for ~2.5 weeks following one such blip — PESO-149.) +The `alloy` role drops a `10-resilience.conf` systemd override onto every host (`StartLimitIntervalSec=0`, `Restart=always`, `RestartSec=30`) so a transient upstream/TLS failure can't trip systemd's start rate-limit and permanently kill the collector — it keeps retrying until Grafana Cloud is reachable again. (Added after copenhagen-c sat unmonitored for ~2.5 weeks following one such blip — PESO-149.) ### Local exporters scraped by Alloy @@ -47,6 +47,8 @@ The `common` role drops a `10-resilience.conf` systemd override onto every host Alloy ships systemd journal entries from every host to Grafana Cloud Logs. Log-derived alerts (e.g. SSH brute-force, mail server errors) can be configured directly in Grafana Cloud. +The host's config is bootstrap only — what each collector actually scrapes and ships is decided by the Fleet Management pipelines in `terraform/grafana/fleet_pipelines/`. + ## Synthetic Monitoring Grafana Cloud's Synthetic Monitoring service runs HTTPS probes from the London region against the public services, every 10 minutes. Configured in `terraform/grafana/synthetic_checks.tf`: