mirror of
https://github.com/RWejlgaard/pez-infra.git
synced 2026-11-01 03:21:02 +00:00
Rasmus "Pez" Wejlgaard
72970e7675
The 1.19 package runs alloy.service as User=alloy; 1.14 ran it as root. The state left behind in /var/lib/alloy is still root-owned, so the service user can't write its WAL or positions.yml, every remotecfg pipeline fails to build, and the host falls back to a cached config and stops shipping. That took node metrics off london-b, helsinki-a, london-c and nuremberg-a from last night's deploy. I noticed because the ZFS pool state metric vanished off london-b, which is the only ZFS host and so the only visible symptom. Chown the state dir to the service user when anything in it isn't already owned by them, then restart. The detect step keeps it a no-op on hosts that are already correct, so london-a and copenhagen-a don't get churned.
102 lines
2.8 KiB
YAML
102 lines
2.8 KiB
YAML
---
|
|
- name: Install deb822_repository dependency
|
|
ansible.builtin.apt:
|
|
name: python3-debian
|
|
state: present
|
|
|
|
- name: Configure Grafana apt repository
|
|
ansible.builtin.deb822_repository:
|
|
name: grafana
|
|
types: deb
|
|
uris: https://apt.grafana.com
|
|
suites: stable
|
|
components: main
|
|
signed_by: https://apt.grafana.com/gpg.key
|
|
register: alloy_apt_repository
|
|
|
|
- name: Update apt cache
|
|
ansible.builtin.apt:
|
|
update_cache: true
|
|
cache_valid_time: "{{ 0 if alloy_apt_repository is changed else 3600 }}"
|
|
|
|
- name: Install Alloy
|
|
ansible.builtin.apt:
|
|
name: "alloy={{ alloy_version }}"
|
|
state: present
|
|
|
|
# Alloy ran as root until 1.19 and its state is still root-owned, so the
|
|
# service user can't write its own WAL or positions.yml and every remotecfg
|
|
# pipeline fails to build.
|
|
- name: Find Alloy state not owned by the service user
|
|
ansible.builtin.command: find {{ alloy_state_dir }} ! -user {{ alloy_user }} -print -quit
|
|
register: alloy_foreign_state
|
|
changed_when: alloy_foreign_state.stdout | length > 0
|
|
check_mode: false
|
|
failed_when: false
|
|
|
|
- name: Fix ownership of Alloy state
|
|
ansible.builtin.file:
|
|
path: "{{ alloy_state_dir }}"
|
|
state: directory
|
|
owner: "{{ alloy_user }}"
|
|
group: "{{ alloy_group }}"
|
|
recurse: true
|
|
when: alloy_foreign_state.stdout | length > 0
|
|
notify: Restart alloy
|
|
|
|
- name: Deploy Alloy configuration
|
|
ansible.builtin.template:
|
|
src: config.alloy.j2
|
|
dest: /etc/alloy/config.alloy
|
|
owner: root
|
|
group: root
|
|
mode: '0644'
|
|
notify: Restart alloy
|
|
|
|
- name: Create Alloy systemd drop-in directory
|
|
ansible.builtin.file:
|
|
path: /etc/systemd/system/alloy.service.d
|
|
state: directory
|
|
owner: root
|
|
group: root
|
|
mode: '0755'
|
|
|
|
# StartLimitIntervalSec=0 keeps Alloy retrying through transient upstream
|
|
# failures instead of tripping systemd's start rate-limit and staying dead
|
|
# (PESO-149).
|
|
- name: Install Alloy systemd resilience drop-in
|
|
ansible.builtin.copy:
|
|
dest: /etc/systemd/system/alloy.service.d/10-resilience.conf
|
|
owner: root
|
|
group: root
|
|
mode: '0644'
|
|
content: |
|
|
# Managed by Ansible (pez-infra) — alloy role.
|
|
[Unit]
|
|
StartLimitIntervalSec=0
|
|
|
|
[Service]
|
|
Restart=always
|
|
RestartSec=30
|
|
notify: Restart alloy
|
|
|
|
- name: Install Alloy credentials drop-in
|
|
ansible.builtin.copy:
|
|
dest: /etc/systemd/system/alloy.service.d/env.conf
|
|
owner: root
|
|
group: root
|
|
mode: '0600'
|
|
content: |
|
|
# Managed by Ansible (pez-infra) — alloy role.
|
|
[Service]
|
|
Environment=GCLOUD_RW_API_KEY={{ alloy_gcloud_rw_api_key }}
|
|
Environment=GCLOUD_FM_COLLECTOR_ID={{ alloy_collector_id }}
|
|
no_log: true
|
|
notify: Restart alloy
|
|
|
|
- name: Enable and start Alloy
|
|
ansible.builtin.systemd:
|
|
name: alloy
|
|
state: started
|
|
enabled: true
|
|
daemon_reload: true
|