mirror of
https://github.com/RWejlgaard/pez-infra.git
synced 2026-10-31 19:11:03 +00:00
Rasmus Wejlgaard
40b2d49db6
The lunar -> noble upgrade disabled every third-party repo on london-b, so tailscale, jellyfin and plex have been frozen since. The common role can now manage deb822 repos and clean up stale source/keyring files per host, and london-b uses it for tailscale, jellyfin and plex (now on repo.plex.tv with the v2 key). The dead sonarr v3, mono, nvidia, datadog and PPA leftovers go.
266 lines
8.4 KiB
YAML
266 lines
8.4 KiB
YAML
---
|
|
# Common baseline for all Linux hosts.
|
|
# Installs core packages, configures SSH, sets up the shell environment.
|
|
|
|
# GitHub's gh apt signing key expired on 2026-09-05 and was replaced upstream.
|
|
# Hosts with the repo configured kept the stale keyring, so apt update failed
|
|
# (NO_PUBKEY) and took the whole deploy down. Keep the keyring in sync with
|
|
# upstream wherever the repo exists; get_url only reports a change when the
|
|
# key actually differs.
|
|
- name: Detect GitHub CLI apt source
|
|
ansible.builtin.stat:
|
|
path: /etc/apt/sources.list.d/github-cli.list
|
|
register: common_github_cli_source
|
|
|
|
- name: Refresh GitHub CLI apt keyring
|
|
ansible.builtin.get_url:
|
|
url: https://cli.github.com/packages/githubcli-archive-keyring.gpg
|
|
dest: /usr/share/keyrings/githubcli-archive-keyring.gpg
|
|
mode: '0644'
|
|
force: true
|
|
when: common_github_cli_source.stat.exists
|
|
|
|
# Third-party apt repos are declared per host in common_apt_repositories. A
|
|
# release upgrade disables them silently (london-b's lunar -> noble upgrade
|
|
# left tailscale, jellyfin and plex frozen for a year), so they live in code.
|
|
# A URL in signed_by is fetched into /etc/apt/keyrings by deb822_repository.
|
|
- name: Install deb822_repository dependency
|
|
ansible.builtin.apt:
|
|
name: python3-debian
|
|
state: present
|
|
when: common_apt_repositories | length > 0
|
|
|
|
- name: Configure third-party apt repositories
|
|
ansible.builtin.deb822_repository:
|
|
name: "{{ item.name }}"
|
|
types: deb
|
|
uris: "{{ item.uris }}"
|
|
suites: "{{ item.suites }}"
|
|
components: "{{ item.components }}"
|
|
signed_by: "{{ item.signed_by }}"
|
|
loop: "{{ common_apt_repositories }}"
|
|
loop_control:
|
|
label: "{{ item.name }}"
|
|
register: common_apt_repositories_result
|
|
|
|
- name: Remove stale apt sources and keyrings
|
|
ansible.builtin.file:
|
|
path: "{{ item }}"
|
|
state: absent
|
|
loop: "{{ common_apt_cleanup_files }}"
|
|
register: common_apt_cleanup_result
|
|
|
|
# Refresh straight away when the sources changed, otherwise at most hourly.
|
|
- name: Update apt cache
|
|
ansible.builtin.apt:
|
|
update_cache: true
|
|
cache_valid_time: "{{ 0 if (common_apt_repositories_result is changed or common_apt_cleanup_result is changed) else 3600 }}"
|
|
|
|
- name: Install baseline packages
|
|
ansible.builtin.apt:
|
|
name:
|
|
- curl
|
|
- wget
|
|
- git
|
|
- htop
|
|
- tmux
|
|
- vim
|
|
- jq
|
|
- unzip
|
|
- fish
|
|
- rsync
|
|
- fail2ban
|
|
- ufw
|
|
state: present
|
|
|
|
- name: Get fish shell path
|
|
ansible.builtin.command: which fish
|
|
changed_when: false
|
|
register: common_fish_path
|
|
when: inventory_hostname != 'london-a'
|
|
|
|
- name: Set fish as default shell
|
|
ansible.builtin.user:
|
|
name: root
|
|
shell: "{{ common_fish_path.stdout }}"
|
|
when: inventory_hostname != 'london-a'
|
|
|
|
- name: Ensure SSH directory exists
|
|
ansible.builtin.file:
|
|
path: /root/.ssh
|
|
state: directory
|
|
mode: '0700'
|
|
|
|
- name: Harden SSH config
|
|
ansible.builtin.lineinfile:
|
|
path: /etc/ssh/sshd_config
|
|
regexp: "{{ item.regexp }}"
|
|
line: "{{ item.line }}"
|
|
state: present
|
|
loop:
|
|
- {regexp: '^#?PermitRootLogin', line: 'PermitRootLogin prohibit-password'}
|
|
- {regexp: '^#?PasswordAuthentication', line: 'PasswordAuthentication no'}
|
|
- {regexp: '^#?X11Forwarding', line: 'X11Forwarding no'}
|
|
notify: Restart sshd
|
|
|
|
- name: Enable fail2ban
|
|
ansible.builtin.service:
|
|
name: fail2ban
|
|
state: started
|
|
enabled: true
|
|
|
|
# --- UFW firewall ---
|
|
|
|
- name: Set UFW default deny incoming
|
|
community.general.ufw:
|
|
direction: incoming
|
|
default: deny
|
|
when: common_ufw_enabled | bool
|
|
notify: Reload ufw
|
|
|
|
- name: Set UFW default allow outgoing
|
|
community.general.ufw:
|
|
direction: outgoing
|
|
default: allow
|
|
when: common_ufw_enabled | bool
|
|
notify: Reload ufw
|
|
|
|
- name: Allow all traffic on Tailscale interface
|
|
community.general.ufw:
|
|
rule: allow
|
|
direction: in
|
|
interface: tailscale0
|
|
comment: "Tailscale mesh - allow all"
|
|
when: common_ufw_enabled | bool
|
|
notify: Reload ufw
|
|
|
|
- name: Allow SSH (safety net)
|
|
community.general.ufw:
|
|
rule: allow
|
|
port: '22'
|
|
proto: tcp
|
|
comment: "SSH"
|
|
when: common_ufw_enabled | bool
|
|
notify: Reload ufw
|
|
|
|
- name: Allow host-specific ports
|
|
community.general.ufw:
|
|
rule: allow
|
|
port: "{{ item.port | string }}"
|
|
proto: "{{ item.proto | default('tcp') }}"
|
|
from_ip: "{{ item.from_ip | default(omit) }}"
|
|
comment: "{{ item.comment | default(omit) }}"
|
|
loop: "{{ common_ufw_allowed_ports }}"
|
|
when:
|
|
- common_ufw_enabled | bool
|
|
- common_ufw_allowed_ports | length > 0
|
|
notify: Reload ufw
|
|
|
|
# When a port is restricted to a source (from_ip), make sure the older
|
|
# unrestricted "allow from anywhere" variant of the same rule isn't left
|
|
# lingering on the host — UFW keeps it otherwise, which would defeat the
|
|
# source restriction. Deleting an absent rule is a no-op, so this is safe
|
|
# on hosts that never had the broad rule.
|
|
- name: Remove superseded world-open rules for source-restricted ports
|
|
community.general.ufw:
|
|
rule: allow
|
|
port: "{{ item.port | string }}"
|
|
proto: "{{ item.proto | default('tcp') }}"
|
|
delete: true
|
|
loop: "{{ common_ufw_allowed_ports | selectattr('from_ip', 'defined') | list }}"
|
|
when: common_ufw_enabled | bool
|
|
notify: Reload ufw
|
|
|
|
- name: Enable UFW
|
|
community.general.ufw:
|
|
state: enabled
|
|
when: common_ufw_enabled | bool
|
|
|
|
# --- Cleanup: orphaned cloudflared (PESO-138) ---
|
|
# Cloudflare Tunnels were retired in favour of Caddy + Authelia (PESO-134, #56),
|
|
# which removed cloudflared from ansible config. copenhagen-a was unreachable at
|
|
# the time, so its cloudflared.service was never actually stopped and is still
|
|
# running (it has since been repaved as Proxmox VE, so this is now historical).
|
|
# Remove it wherever the unit lingers.
|
|
- name: Detect lingering cloudflared unit
|
|
ansible.builtin.stat:
|
|
path: /etc/systemd/system/cloudflared.service
|
|
register: common_cloudflared_unit
|
|
|
|
- name: Remove orphaned cloudflared
|
|
when:
|
|
- common_cloudflared_unit.stat.exists | default(false)
|
|
block:
|
|
- name: Stop and disable cloudflared
|
|
ansible.builtin.systemd:
|
|
name: cloudflared
|
|
state: stopped
|
|
enabled: false
|
|
failed_when: false
|
|
|
|
- name: Remove cloudflared systemd unit
|
|
ansible.builtin.file:
|
|
path: /etc/systemd/system/cloudflared.service
|
|
state: absent
|
|
notify: Reload systemd daemon
|
|
|
|
- name: Uninstall cloudflared package
|
|
ansible.builtin.apt:
|
|
name: cloudflared
|
|
state: absent
|
|
purge: true
|
|
|
|
# --- Grafana Alloy: restart resilience (PESO-149) ---
|
|
# Every host runs alloy.service to ship metrics/logs to Grafana Cloud. On
|
|
# 2026-05-20 a transient TLS failure to Grafana fleet-management tripped
|
|
# systemd's default start rate-limit on copenhagen-c; systemd then gave up and
|
|
# the host went silently unmonitored for ~2.5 weeks. Disable the rate limit so
|
|
# Alloy keeps retrying indefinitely, backing off 30s between attempts, instead
|
|
# of dying permanently on a momentary blip.
|
|
- name: Check for Alloy unit
|
|
ansible.builtin.stat:
|
|
path: /lib/systemd/system/alloy.service
|
|
register: common_alloy_unit
|
|
|
|
- name: Install Alloy systemd resilience drop-in
|
|
ansible.builtin.copy:
|
|
dest: /etc/systemd/system/alloy.service.d/10-resilience.conf
|
|
mode: '0644'
|
|
content: |
|
|
# Managed by Ansible (pez-infra) — common role, PESO-149.
|
|
# Keep Alloy retrying through transient upstream/TLS failures instead of
|
|
# hitting systemd's start rate-limit and giving up.
|
|
[Unit]
|
|
StartLimitIntervalSec=0
|
|
|
|
[Service]
|
|
Restart=always
|
|
RestartSec=30
|
|
when: common_alloy_unit.stat.exists | default(false)
|
|
notify: Restart alloy
|
|
|
|
# --- Cleanup: leftover self-hosted Grafana (PESO-149) ---
|
|
# Monitoring moved fully to Grafana Cloud, but the old Grafana OSS package was
|
|
# never removed from copenhagen-c, where grafana-server.service sits enabled
|
|
# and failing. Remove it wherever it lingers (no-op on every other host).
|
|
- name: Detect leftover grafana-server unit
|
|
ansible.builtin.stat:
|
|
path: /lib/systemd/system/grafana-server.service
|
|
register: common_grafana_unit
|
|
|
|
- name: Remove leftover self-hosted Grafana
|
|
when: common_grafana_unit.stat.exists | default(false)
|
|
block:
|
|
- name: Stop and disable grafana-server
|
|
ansible.builtin.systemd:
|
|
name: grafana-server
|
|
state: stopped
|
|
enabled: false
|
|
failed_when: false
|
|
|
|
- name: Uninstall grafana package
|
|
ansible.builtin.apt:
|
|
name: grafana
|
|
state: absent
|
|
purge: true
|
|
notify: Reload systemd daemon
|