From 18c098913c7a04d6140c3583b023223028111f13 Mon Sep 17 00:00:00 2001
From: Rasmus Wejlgaard <pez@pez.sh>
Date: Sat, 19 Sep 2026 08:58:35 +0100
Subject: [PATCH] Fix Alloy state ownership after the 1.19 upgrade

The 1.19 package runs alloy.service as User=alloy; 1.14 ran it as root. The
state left behind in /var/lib/alloy is still root-owned, so the service user
can't write its WAL or positions.yml, every remotecfg pipeline fails to build,
and the host falls back to a cached config and stops shipping.

That took node metrics off london-b, helsinki-a, london-c and nuremberg-a from
last night's deploy. I noticed because the ZFS pool state metric vanished off
london-b, which is the only ZFS host and so the only visible symptom.

Chown the state dir to the service user when anything in it isn't already
owned by them, then restart. The detect step keeps it a no-op on hosts that
are already correct, so london-a and copenhagen-a don't get churned.
---
 ansible/roles/alloy/defaults/main.yml |  4 ++++
 ansible/roles/alloy/tasks/main.yml    | 20 ++++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/ansible/roles/alloy/defaults/main.yml b/ansible/roles/alloy/defaults/main.yml
index d36cca6..911e8c6 100644
--- a/ansible/roles/alloy/defaults/main.yml
+++ b/ansible/roles/alloy/defaults/main.yml
@@ -1,6 +1,10 @@
 ---
 alloy_version: "1.19.2-1"
 
+alloy_user: alloy
+alloy_group: alloy
+alloy_state_dir: /var/lib/alloy
+
 alloy_collector_id: "{{ inventory_hostname }}"
 
 alloy_fleet_management_url: https://fleet-management-prod-023.grafana.net
diff --git a/ansible/roles/alloy/tasks/main.yml b/ansible/roles/alloy/tasks/main.yml
index 2261738..074cc49 100644
--- a/ansible/roles/alloy/tasks/main.yml
+++ b/ansible/roles/alloy/tasks/main.yml
@@ -24,6 +24,26 @@
     name: "alloy={{ alloy_version }}"
     state: present
 
+# Alloy ran as root until 1.19 and its state is still root-owned, so the
+# service user can't write its own WAL or positions.yml and every remotecfg
+# pipeline fails to build.
+- name: Find Alloy state not owned by the service user
+  ansible.builtin.command: find {{ alloy_state_dir }} ! -user {{ alloy_user }} -print -quit
+  register: alloy_foreign_state
+  changed_when: alloy_foreign_state.stdout | length > 0
+  check_mode: false
+  failed_when: false
+
+- name: Fix ownership of Alloy state
+  ansible.builtin.file:
+    path: "{{ alloy_state_dir }}"
+    state: directory
+    owner: "{{ alloy_user }}"
+    group: "{{ alloy_group }}"
+    recurse: true
+  when: alloy_foreign_state.stdout | length > 0
+  notify: Restart alloy
+
 - name: Deploy Alloy configuration
   ansible.builtin.template:
     src: config.alloy.j2