mirror of
https://github.com/RWejlgaard/pez-infra.git
synced 2026-10-31 19:11:03 +00:00
* Install Grafana Alloy from Ansible (PESO-158) copenhagen-a came back from its Proxmox repave with no Alloy at all, so it's been unmonitored since. Nothing in this repo installs Alloy — every host got it from Grafana's onboarding one-liner, and the common role only bolted on a resilience drop-in if the unit already happened to exist. A rebuild silently loses monitoring. New alloy role owns the whole thing: the Grafana apt repo, the package, the config, the credentials drop-in and the resilience override. Runs on every host, so this can't happen again on the next repave. Moved the resilience drop-in out of common into the role that owns Alloy. The rendered config is byte-identical to what's on the hosts today apart from the managed-by header, so the only effect on the healthy hosts is one restart. * Pin Alloy to 1.19.2-1 and add the Grafana Cloud key to SOPS Adding the apt repo means the fleet is upgradable for the first time, so pin the version explicitly rather than letting copenhagen-a come up five minor versions ahead of everyone else. All six hosts land on 1.19.2-1. Checked the 1.14 -> 1.19 changelog against what we actually run. Nothing breaking touches the bootstrap config or the fleet pipelines; the only entry that lands anywhere near us is 1.18.0 giving otelcol.receiver.otlp upstream default timeouts, which affects the Caddy tracing pipeline on helsinki-a and sets no timeouts explicitly today.
232 lines
6.9 KiB
YAML
232 lines
6.9 KiB
YAML
---
|
|
# Common baseline for all Linux hosts.
|
|
# Installs core packages, configures SSH, sets up the shell environment.
|
|
|
|
# GitHub's gh apt signing key expired on 2026-09-05 and was replaced upstream.
|
|
# Hosts with the repo configured kept the stale keyring, so apt update failed
|
|
# (NO_PUBKEY) and took the whole deploy down. Keep the keyring in sync with
|
|
# upstream wherever the repo exists; get_url only reports a change when the
|
|
# key actually differs.
|
|
- name: Detect GitHub CLI apt source
|
|
ansible.builtin.stat:
|
|
path: /etc/apt/sources.list.d/github-cli.list
|
|
register: common_github_cli_source
|
|
|
|
- name: Refresh GitHub CLI apt keyring
|
|
ansible.builtin.get_url:
|
|
url: https://cli.github.com/packages/githubcli-archive-keyring.gpg
|
|
dest: /usr/share/keyrings/githubcli-archive-keyring.gpg
|
|
mode: '0644'
|
|
force: true
|
|
when: common_github_cli_source.stat.exists
|
|
|
|
- name: Install deb822_repository dependency
|
|
ansible.builtin.apt:
|
|
name: python3-debian
|
|
state: present
|
|
when: common_apt_repositories | length > 0
|
|
|
|
- name: Configure third-party apt repositories
|
|
ansible.builtin.deb822_repository:
|
|
name: "{{ item.name }}"
|
|
types: deb
|
|
uris: "{{ item.uris }}"
|
|
suites: "{{ item.suites }}"
|
|
components: "{{ item.components }}"
|
|
signed_by: "{{ item.signed_by }}"
|
|
loop: "{{ common_apt_repositories }}"
|
|
loop_control:
|
|
label: "{{ item.name }}"
|
|
register: common_apt_repositories_result
|
|
|
|
- name: Remove stale apt sources and keyrings
|
|
ansible.builtin.file:
|
|
path: "{{ item }}"
|
|
state: absent
|
|
loop: "{{ common_apt_cleanup_files }}"
|
|
register: common_apt_cleanup_result
|
|
|
|
- name: Update apt cache
|
|
ansible.builtin.apt:
|
|
update_cache: true
|
|
cache_valid_time: "{{ 0 if (common_apt_repositories_result is changed or common_apt_cleanup_result is changed) else 3600 }}"
|
|
|
|
- name: Install baseline packages
|
|
ansible.builtin.apt:
|
|
name:
|
|
- curl
|
|
- wget
|
|
- git
|
|
- htop
|
|
- tmux
|
|
- vim
|
|
- jq
|
|
- unzip
|
|
- fish
|
|
- rsync
|
|
- fail2ban
|
|
- ufw
|
|
state: present
|
|
|
|
- name: Get fish shell path
|
|
ansible.builtin.command: which fish
|
|
changed_when: false
|
|
register: common_fish_path
|
|
when: inventory_hostname != 'london-a'
|
|
|
|
- name: Set fish as default shell
|
|
ansible.builtin.user:
|
|
name: root
|
|
shell: "{{ common_fish_path.stdout }}"
|
|
when: inventory_hostname != 'london-a'
|
|
|
|
- name: Ensure SSH directory exists
|
|
ansible.builtin.file:
|
|
path: /root/.ssh
|
|
state: directory
|
|
mode: '0700'
|
|
|
|
- name: Harden SSH config
|
|
ansible.builtin.lineinfile:
|
|
path: /etc/ssh/sshd_config
|
|
regexp: "{{ item.regexp }}"
|
|
line: "{{ item.line }}"
|
|
state: present
|
|
loop:
|
|
- {regexp: '^#?PermitRootLogin', line: 'PermitRootLogin prohibit-password'}
|
|
- {regexp: '^#?PasswordAuthentication', line: 'PasswordAuthentication no'}
|
|
- {regexp: '^#?X11Forwarding', line: 'X11Forwarding no'}
|
|
notify: Restart sshd
|
|
|
|
- name: Enable fail2ban
|
|
ansible.builtin.service:
|
|
name: fail2ban
|
|
state: started
|
|
enabled: true
|
|
|
|
# --- UFW firewall ---
|
|
|
|
- name: Set UFW default deny incoming
|
|
community.general.ufw:
|
|
direction: incoming
|
|
default: deny
|
|
when: common_ufw_enabled | bool
|
|
notify: Reload ufw
|
|
|
|
- name: Set UFW default allow outgoing
|
|
community.general.ufw:
|
|
direction: outgoing
|
|
default: allow
|
|
when: common_ufw_enabled | bool
|
|
notify: Reload ufw
|
|
|
|
- name: Allow all traffic on Tailscale interface
|
|
community.general.ufw:
|
|
rule: allow
|
|
direction: in
|
|
interface: tailscale0
|
|
comment: "Tailscale mesh - allow all"
|
|
when: common_ufw_enabled | bool
|
|
notify: Reload ufw
|
|
|
|
- name: Allow SSH (safety net)
|
|
community.general.ufw:
|
|
rule: allow
|
|
port: '22'
|
|
proto: tcp
|
|
comment: "SSH"
|
|
when: common_ufw_enabled | bool
|
|
notify: Reload ufw
|
|
|
|
- name: Allow host-specific ports
|
|
community.general.ufw:
|
|
rule: allow
|
|
port: "{{ item.port | string }}"
|
|
proto: "{{ item.proto | default('tcp') }}"
|
|
from_ip: "{{ item.from_ip | default(omit) }}"
|
|
comment: "{{ item.comment | default(omit) }}"
|
|
loop: "{{ common_ufw_allowed_ports }}"
|
|
when:
|
|
- common_ufw_enabled | bool
|
|
- common_ufw_allowed_ports | length > 0
|
|
notify: Reload ufw
|
|
|
|
# When a port is restricted to a source (from_ip), make sure the older
|
|
# unrestricted "allow from anywhere" variant of the same rule isn't left
|
|
# lingering on the host — UFW keeps it otherwise, which would defeat the
|
|
# source restriction. Deleting an absent rule is a no-op, so this is safe
|
|
# on hosts that never had the broad rule.
|
|
- name: Remove superseded world-open rules for source-restricted ports
|
|
community.general.ufw:
|
|
rule: allow
|
|
port: "{{ item.port | string }}"
|
|
proto: "{{ item.proto | default('tcp') }}"
|
|
delete: true
|
|
loop: "{{ common_ufw_allowed_ports | selectattr('from_ip', 'defined') | list }}"
|
|
when: common_ufw_enabled | bool
|
|
notify: Reload ufw
|
|
|
|
- name: Enable UFW
|
|
community.general.ufw:
|
|
state: enabled
|
|
when: common_ufw_enabled | bool
|
|
|
|
# --- Cleanup: orphaned cloudflared (PESO-138) ---
|
|
# Cloudflare Tunnels were retired in favour of Caddy + Authelia (PESO-134, #56),
|
|
# which removed cloudflared from ansible config. copenhagen-a was unreachable at
|
|
# the time, so its cloudflared.service was never actually stopped and is still
|
|
# running (it has since been repaved as Proxmox VE, so this is now historical).
|
|
# Remove it wherever the unit lingers.
|
|
- name: Detect lingering cloudflared unit
|
|
ansible.builtin.stat:
|
|
path: /etc/systemd/system/cloudflared.service
|
|
register: common_cloudflared_unit
|
|
|
|
- name: Remove orphaned cloudflared
|
|
when:
|
|
- common_cloudflared_unit.stat.exists | default(false)
|
|
block:
|
|
- name: Stop and disable cloudflared
|
|
ansible.builtin.systemd:
|
|
name: cloudflared
|
|
state: stopped
|
|
enabled: false
|
|
failed_when: false
|
|
|
|
- name: Remove cloudflared systemd unit
|
|
ansible.builtin.file:
|
|
path: /etc/systemd/system/cloudflared.service
|
|
state: absent
|
|
notify: Reload systemd daemon
|
|
|
|
- name: Uninstall cloudflared package
|
|
ansible.builtin.apt:
|
|
name: cloudflared
|
|
state: absent
|
|
purge: true
|
|
|
|
# --- Cleanup: leftover self-hosted Grafana (PESO-149) ---
|
|
# Monitoring moved fully to Grafana Cloud, but the old Grafana OSS package was
|
|
# never removed from copenhagen-c, where grafana-server.service sits enabled
|
|
# and failing. Remove it wherever it lingers (no-op on every other host).
|
|
- name: Detect leftover grafana-server unit
|
|
ansible.builtin.stat:
|
|
path: /lib/systemd/system/grafana-server.service
|
|
register: common_grafana_unit
|
|
|
|
- name: Remove leftover self-hosted Grafana
|
|
when: common_grafana_unit.stat.exists | default(false)
|
|
block:
|
|
- name: Stop and disable grafana-server
|
|
ansible.builtin.systemd:
|
|
name: grafana-server
|
|
state: stopped
|
|
enabled: false
|
|
failed_when: false
|
|
|
|
- name: Uninstall grafana package
|
|
ansible.builtin.apt:
|
|
name: grafana
|
|
state: absent
|
|
purge: true
|
|
notify: Reload systemd daemon
|