From 0a552b10da31d10689999d4a11d0c3810f69f837 Mon Sep 17 00:00:00 2001 From: "Rasmus \"Pez\" Wejlgaard" Date: Wed, 16 Sep 2026 17:52:08 +0100 Subject: [PATCH] Back up the Proxmox VMs to london-b Neither Proxmox host had a single backup job, so the game servers, the k3s control planes and the workspace VMs had no backups at all. copenhagen-a has nowhere local to keep them (`local` is the root LV and already 82% full), so both hosts now back up to a new pve-backups Samba share on london-b, each into its own directory. VMIDs overlap between the two hosts, and retention prunes by VMID, so they can't share one. - media_stack creates a Samba-only pvebackup user on london-b, the share and the per-host directories. The password comes from SOPS and gets re-set if the login stops working. smb.conf is now checked with testparm before it's deployed. - proxmox_ve adds a backup-only london-b-backups CIFS storage, pushes the credentials again if the storage goes inactive, and fails the deploy if it still can't reach london-b. - Jobs are defined in host_vars. Each job keeps a hash of its settings in its comment, so the role only touches jobs that changed and removes ones I delete from host_vars. Jobs made by hand are left alone. london-a backs up the control plane every night. copenhagen-a backs up the control plane and game servers every night and the workspace VMs weekly, since every run is a full archive going over the WAN. --- ansible/group_vars/all/secrets.enc.yaml | 6 +- ansible/inventory/host_vars/copenhagen-a.yml | 13 +++ ansible/inventory/host_vars/london-a.yml | 8 ++ ansible/roles/media_stack/tasks/main.yml | 50 +++++++++ ansible/roles/proxmox_ve/defaults/main.yml | 16 +++ ansible/roles/proxmox_ve/tasks/main.yml | 108 +++++++++++++++++++ ansible/services/samba/config/london-b.conf | 13 +++ docs/hosts/copenhagen-a.md | 17 ++- docs/hosts/london-a.md | 12 +++ docs/hosts/london-b.md | 3 +- 10 files changed, 240 insertions(+), 6 deletions(-) create mode 100644 ansible/roles/proxmox_ve/defaults/main.yml diff --git a/ansible/group_vars/all/secrets.enc.yaml b/ansible/group_vars/all/secrets.enc.yaml index bb23260..e200df6 100644 --- a/ansible/group_vars/all/secrets.enc.yaml +++ b/ansible/group_vars/all/secrets.enc.yaml @@ -3,6 +3,8 @@ plex_exporter_plex_token: ENC[AES256_GCM,data:oT+cmlY9p9zGOMd7lVe1loyxh/E=,iv:kj octopus_energy_api_key: ENC[AES256_GCM,data:wINFSZO/P1K31AQfsehv69eYPUWJtnIU65MDPg9IfI3y75R3xE9n7g==,iv:kIKnxIwl5e+mBzq8s1FQO922oSrNIO4yJCg7BKrH33I=,tag:4IAsh8KW+01YXf83tITy2A==,type:str] proxmox_ve_smb_user: ENC[AES256_GCM,data:13dtQg==,iv:aUOadm20b9FpmOxXGINv8KkwsfBavX755U8SmVCxkCQ=,tag:vV5404IKENKHuBkDwC9GvQ==,type:str] proxmox_ve_smb_password: ENC[AES256_GCM,data:kYegZCZTpEo7,iv:Jm8yMLKQXyaQQPyybhZyAy9bhN4+OjI63f7azOzqF+w=,tag:IIKCI2bsRzer31hF0h4b9g==,type:str] +proxmox_ve_backup_smb_user: ENC[AES256_GCM,data:sax7tELTAJhZ,iv:OIuvOoZH6bdrN0CgmM3pVIUoIE4syTahyL2pFGaOURA=,tag:Vdx6iSN3cvCqxPCiKuiJRw==,type:str] +proxmox_ve_backup_smb_password: ENC[AES256_GCM,data:I9wM28Vn+w49C42LD4BgHxLN2aSF9e5+LRZGB5iXfNYSCfOlcp8aJQ==,iv:UG34yCmwDvOzkITd4LJeUu+Et0Tdx/aJXAT6rUbXJVg=,tag:P/O3jLir9i4NnIzcLjcRcA==,type:str] sops: age: - recipient: age1r8uh2w2qad2z5sgq9q7l73962q2sp8zz9hdnh6sjuvanxl565vmswn8squ @@ -14,7 +16,7 @@ sops: Ti9TNjFta0kyRTNrWGVWbGFGeC9CR3cKkdhytnSS+fCMc1FOSZ5WC3dO16Kw+akB O69PKWul3WI7kFmcRk8mzlbFqS941tnPbvVaTYP0+GZTgSX1wlhSeg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-05-14T19:42:19Z" - mac: ENC[AES256_GCM,data:E5WuwPXt+FwofwAJaCOZj0so4TUBNtQ5J9Swt3Pwd157XBT4PBt65ZHd+gwUOHnyFa7iJlGXN7miMbtx2XZ72cIP2tl0JsO5NOJoLYTFdnK1NbSnrxyESAWnWenONVHILz10Zy3PJqPaN1P9nlME96LBVr/ii/179UJiAEJMxo8=,iv:3P4C1aW94i5FIvtd8yJswN4WRcqF5uuIpBVnFuPEkT4=,tag:Z3WShXsRs4p8xMG0+MJ54Q==,type:str] + lastmodified: "2026-09-16T16:44:41Z" + mac: ENC[AES256_GCM,data:qMjB7lCALsa6dsrWxlhpnJgoWzlbgpF4gktIIZPKoL791CPJL10Y9qqDznGwqU1HRnlf5LdOUOCVPLow8f+Tmt4xbg3t6Rr2PsyL4cAF5N3p2eY3L8aIo0fs41Mkl5zysZbqyWYI3lrBob+Frc4pXNIGCzUlTNhrOw9B03eiWVk=,iv:Nw60y9qoFVQ2A4FdHLEw5cX3BIxdWGGSB3EHOHwhNPA=,tag:IRU3S+sycHP3EunrRRrYBw==,type:str] unencrypted_suffix: _unencrypted version: 3.12.2 diff --git a/ansible/inventory/host_vars/copenhagen-a.yml b/ansible/inventory/host_vars/copenhagen-a.yml index d19c5c2..8963e2d 100644 --- a/ansible/inventory/host_vars/copenhagen-a.yml +++ b/ansible/inventory/host_vars/copenhagen-a.yml @@ -7,3 +7,16 @@ ansible_python_interpreter: /usr/bin/python3 # Don't mount london-b's CIFS share here — it's on the London LAN, not worth # the WAN/Tailscale hop for VM storage from Copenhagen. proxmox_ve_mount_cifs_storage: false + +# Backups cross the WAN to london-b, and every run is a full archive, so only +# the VMs whose data changes daily (the game servers and the k3s control plane) +# run nightly. The workspace VMs are backed up weekly. +proxmox_ve_backup_jobs: + - id: daily + schedule: "03:00" + vmids: [100, 102, 103] + prune_backups: "keep-daily=7,keep-weekly=4" + - id: weekly + schedule: "sun 04:30" + vmids: [101, 104] + prune_backups: "keep-weekly=4" diff --git a/ansible/inventory/host_vars/london-a.yml b/ansible/inventory/host_vars/london-a.yml index 96e2505..43003cb 100644 --- a/ansible/inventory/host_vars/london-a.yml +++ b/ansible/inventory/host_vars/london-a.yml @@ -3,3 +3,11 @@ host_role: proxmox-ve host_description: "Proxmox VE hypervisor" host_location: "London" ansible_python_interpreter: /usr/bin/python3 + +# Nightly backup of the k3s control plane. Karpenter worker VMs are disposable +# and the template (1000) is rebuilt by Karpenter, so neither is backed up. +proxmox_ve_backup_jobs: + - id: daily + schedule: "02:30" + vmids: [101] + prune_backups: "keep-daily=7,keep-weekly=4" diff --git a/ansible/roles/media_stack/tasks/main.yml b/ansible/roles/media_stack/tasks/main.yml index 4168642..a444b94 100644 --- a/ansible/roles/media_stack/tasks/main.yml +++ b/ansible/roles/media_stack/tasks/main.yml @@ -93,6 +93,26 @@ state: started enabled: true +# ── Proxmox backup share ── +# london-a and copenhagen-a write vzdump archives to the [pve-backups] share +# as a dedicated Samba-only user, one directory per Proxmox host. + +- name: Create Proxmox backup user + ansible.builtin.user: + name: "{{ proxmox_ve_backup_smb_user }}" + system: true + shell: /usr/sbin/nologin + create_home: false + +- name: Ensure Proxmox backup directories exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ proxmox_ve_backup_smb_user }}" + group: "{{ proxmox_ve_backup_smb_user }}" + mode: '0750' + loop: "{{ ['/hdd/pve/backups'] + groups['proxmox_hosts'] | map('regex_replace', '^', '/hdd/pve/backups/') | list }}" + # ── Configuration files ── - name: Deploy samba config @@ -101,8 +121,38 @@ dest: /etc/samba/smb.conf mode: '0644' backup: true + validate: testparm -s %s notify: Restart smbd +# smbd has to be serving [pve-backups] before the login check below means +# anything, so don't wait for the end-of-play handler. +- name: Apply samba config before checking the backup login + ansible.builtin.meta: flush_handlers + +# A failed login against the share (valid users only, so no guest fallback) is +# the only reliable way to tell whether the Samba password matches SOPS. +- name: Check Proxmox backup user's Samba login + ansible.builtin.command: smbclient //localhost/pve-backups -U {{ proxmox_ve_backup_smb_user }} -c exit + environment: + PASSWD: "{{ proxmox_ve_backup_smb_password }}" + register: media_stack_pve_backup_login + changed_when: false + failed_when: false + check_mode: false + no_log: true + +- name: Set Proxmox backup user's Samba password + ansible.builtin.shell: | + set -o pipefail + printf '%s\n%s\n' "$SMB_PASSWORD" "$SMB_PASSWORD" | smbpasswd -s -a {{ proxmox_ve_backup_smb_user }} + args: + executable: /bin/bash + environment: + SMB_PASSWORD: "{{ proxmox_ve_backup_smb_password }}" + changed_when: true + when: media_stack_pve_backup_login.rc != 0 + no_log: true + - name: Deploy transmission settings ansible.builtin.copy: src: "{{ playbook_dir }}/services/transmission/config/settings.json" diff --git a/ansible/roles/proxmox_ve/defaults/main.yml b/ansible/roles/proxmox_ve/defaults/main.yml new file mode 100644 index 0000000..be2f5fc --- /dev/null +++ b/ansible/roles/proxmox_ve/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# --- VM backups (vzdump) --- +# Backups go to london-b's [pve-backups] Samba share over Tailscale, into a +# subdirectory named after the Proxmox host. Nothing is configured unless the +# host defines proxmox_ve_backup_jobs. +proxmox_ve_backup_storage: london-b-backups +proxmox_ve_backup_server: 100.84.65.101 +proxmox_ve_backup_share: pve-backups + +# List of scheduled vzdump jobs, e.g. +# - id: daily +# schedule: "03:00" # systemd calendar event, host local time +# vmids: [101] +# prune_backups: "keep-daily=7,keep-weekly=4" +# Jobs this role created that are no longer listed are deleted. +proxmox_ve_backup_jobs: [] diff --git a/ansible/roles/proxmox_ve/tasks/main.yml b/ansible/roles/proxmox_ve/tasks/main.yml index 62598b9..c1dd7a7 100644 --- a/ansible/roles/proxmox_ve/tasks/main.yml +++ b/ansible/roles/proxmox_ve/tasks/main.yml @@ -80,3 +80,111 @@ when: - proxmox_ve_mount_cifs_storage | default(true) - proxmox_ve_hdd_check.rc != 0 + +# --- VM backups (vzdump) --- +# Proxmox has nowhere local to keep backups (copenhagen-a's `local` is the root +# LV, and LVM-thin can't hold archives), so each host backs up to its own +# directory on london-b. Job definitions live in host_vars. + +- name: Configure VM backups + when: proxmox_ve_backup_jobs | length > 0 + block: + - name: Check if backup storage is configured + ansible.builtin.command: pvesh get /storage/{{ proxmox_ve_backup_storage }} + register: proxmox_ve_backup_storage_check + changed_when: false + failed_when: false + check_mode: false + + - name: Add backup storage (london-b pve-backups share) + ansible.builtin.command: >- + pvesm add cifs {{ proxmox_ve_backup_storage }} + --server {{ proxmox_ve_backup_server }} + --share {{ proxmox_ve_backup_share }} + --subdir /{{ inventory_hostname }} + --username {{ proxmox_ve_backup_smb_user }} + --password {{ proxmox_ve_backup_smb_password }} + --content backup + --prune-backups keep-all=1 + no_log: true + changed_when: true + when: proxmox_ve_backup_storage_check.rc != 0 + + - name: Check backup storage is active + ansible.builtin.command: pvesm status --storage {{ proxmox_ve_backup_storage }} + register: proxmox_ve_backup_storage_status + changed_when: false + failed_when: false + check_mode: false + + # The add above only runs once, so re-push the credentials whenever the + # storage can't log in — e.g. after the password is rotated in SOPS. + - name: Refresh backup storage credentials + ansible.builtin.command: >- + pvesm set {{ proxmox_ve_backup_storage }} + --username {{ proxmox_ve_backup_smb_user }} + --password {{ proxmox_ve_backup_smb_password }} + no_log: true + changed_when: true + when: + - proxmox_ve_backup_storage_check.rc == 0 + - "'active' not in proxmox_ve_backup_storage_status.stdout.split()" + + - name: Verify backup storage is active + ansible.builtin.command: pvesm status --storage {{ proxmox_ve_backup_storage }} + register: proxmox_ve_backup_storage_verify + changed_when: false + failed_when: "'active' not in proxmox_ve_backup_storage_verify.stdout.split()" + when: not ansible_check_mode + + - name: List existing backup jobs + ansible.builtin.command: pvesh get /cluster/backup --output-format json + register: proxmox_ve_backup_jobs_current + changed_when: false + check_mode: false + + # Each managed job carries a hash of its desired settings in its comment, + # so drift is detected without having to normalise the API's output. + - name: Work out backup job changes + ansible.builtin.set_fact: + proxmox_ve_backup_jobs_existing: >- + {{ dict(proxmox_ve_backup_jobs_current.stdout | from_json | map(attribute='id') + | zip(proxmox_ve_backup_jobs_current.stdout | from_json | map(attribute='comment', default=''))) }} + proxmox_ve_backup_comment_prefix: "Managed by Ansible (pez-infra)" + + - name: Create or update backup jobs + ansible.builtin.command: >- + pvesh {{ 'set /cluster/backup/' ~ item.id if item.id in proxmox_ve_backup_jobs_existing + else 'create /cluster/backup --id ' ~ item.id }} + --schedule {{ item.schedule | quote }} + --storage {{ proxmox_ve_backup_storage }} + --vmid {{ item.vmids | join(',') }} + --mode snapshot + --compress zstd + --enabled 1 + --prune-backups {{ item.prune_backups | quote }} + --comment {{ (proxmox_ve_backup_comment_prefix ~ ' ' ~ proxmox_ve_backup_job_hash) | quote }} + vars: + # Include everything passed to pvesh, so changing a fixed flag here also + # re-applies the jobs. + proxmox_ve_backup_job_hash: >- + {{ (item | combine({'storage': proxmox_ve_backup_storage, 'mode': 'snapshot', 'compress': 'zstd'}) + | to_json | hash('sha1'))[:12] }} + loop: "{{ proxmox_ve_backup_jobs }}" + loop_control: + label: "{{ item.id }}" + changed_when: true + when: >- + proxmox_ve_backup_jobs_existing.get(item.id) + != proxmox_ve_backup_comment_prefix ~ ' ' ~ proxmox_ve_backup_job_hash + + - name: Remove backup jobs no longer defined + ansible.builtin.command: pvesh delete /cluster/backup/{{ item.key }} + loop: "{{ proxmox_ve_backup_jobs_existing | dict2items }}" + loop_control: + label: "{{ item.key }}" + changed_when: true + when: + - item.value is string + - item.value.startswith(proxmox_ve_backup_comment_prefix) + - item.key not in proxmox_ve_backup_jobs | map(attribute='id') diff --git a/ansible/services/samba/config/london-b.conf b/ansible/services/samba/config/london-b.conf index 3492080..1dfb211 100644 --- a/ansible/services/samba/config/london-b.conf +++ b/ansible/services/samba/config/london-b.conf @@ -24,6 +24,19 @@ public = no writable = yes +# vzdump target for the Proxmox hosts. One subdirectory per host, because +# VMIDs overlap between london-a and copenhagen-a and retention prunes by VMID. +# The user is created by the media_stack role; its name must match +# proxmox_ve_backup_smb_user in group_vars/all/secrets.enc.yaml. +[pve-backups] + comment = Proxmox VE backups + path = /hdd/pve/backups + valid users = pvebackup + browseable = no + writable = yes + create mask = 0640 + directory mask = 0750 + [Movies] comment = Movies path = /hdd/movies diff --git a/docs/hosts/copenhagen-a.md b/docs/hosts/copenhagen-a.md index acea8d2..0e6d2de 100644 --- a/docs/hosts/copenhagen-a.md +++ b/docs/hosts/copenhagen-a.md @@ -32,12 +32,22 @@ Minisforum MS-A2 mini PC. Small, quiet, and draws minimal power relative to its ### Storage -Unlike london-a, this host does **not** mount london-b's CIFS share — it's not worth the WAN/Tailscale hop from Copenhagen for VM storage. Only local storage is configured (`proxmox_ve_mount_cifs_storage: false` in `host_vars/copenhagen-a.yml`). +Unlike london-a, this host does **not** mount london-b's `/pve` CIFS share — it's not worth the WAN/Tailscale hop from Copenhagen for VM storage. It does use london-b for backups (below). Only local storage is configured (`proxmox_ve_mount_cifs_storage: false` in `host_vars/copenhagen-a.yml`). | Storage ID | Type | Backing | |---|---|---| | `local-lvm` | LVM-Thin | Local boot disk | -| `local` | Directory | ISO/template/backup storage | +| `local` | Directory | ISO/template storage | +| `london-b-backups` | CIFS | london-b `pve-backups` share over Tailscale, `/copenhagen-a` subdir — backups only | + +### Backups + +There's no room for backups locally, so they go to london-b over Tailscale. Every vzdump run is a full archive, so only the VMs that change daily run nightly. Jobs are defined in `proxmox_ve_backup_jobs` in `host_vars/copenhagen-a.yml`: + +| Job | Schedule | VMs | Retention | +|---|---|---|---| +| `daily` | 03:00 | 100 (k8s-control-plane), 102 (minecraft), 103 (mangos-zero) | 7 daily, 4 weekly | +| `weekly` | Sun 04:30 | 101 (workspace), 104 (workspace-alpine) | 4 weekly | ### VMs @@ -50,7 +60,8 @@ Part of the `proxmox_hosts` group alongside london-a, sharing the `proxmox_ve` r - Swaps the enterprise apt repo for `pve-no-subscription` so updates work without a paid subscription - Patches `proxmoxlib.js` to suppress the subscription nag dialog - Restricts the web UI to the `tailscale0` interface via UFW -- Skips the london-b CIFS mount (`proxmox_ve_mount_cifs_storage: false`) — local storage only +- Skips the london-b `/pve` CIFS mount (`proxmox_ve_mount_cifs_storage: false`) +- Adds the `london-b-backups` storage and reconciles the vzdump jobs in `proxmox_ve_backup_jobs` ## Networking diff --git a/docs/hosts/london-a.md b/docs/hosts/london-a.md index 20f71a5..2ccb3d8 100644 --- a/docs/hosts/london-a.md +++ b/docs/hosts/london-a.md @@ -37,6 +37,17 @@ Proxmox is connected to a CIFS share on **london-b** (`100.84.65.101 /pve`) for |---|---|---| | `local-lvm` | LVM-Thin | Local boot disk | | `hdd` | CIFS | london-b `/pve` share | +| `london-b-backups` | CIFS | london-b `pve-backups` share, `/london-a` subdir — backups only | + +### Backups + +vzdump jobs are defined in `proxmox_ve_backup_jobs` in `host_vars/london-a.yml` and applied by the `proxmox_ve` role: + +| Job | Schedule | VMs | Retention | +|---|---|---|---| +| `daily` | 02:30 | 101 (k8s-control-plane) | 7 daily, 4 weekly | + +Karpenter worker VMs and the Karpenter template are not backed up. ### VMs @@ -54,6 +65,7 @@ The `proxmox_ve` role: - Patches `proxmoxlib.js` to suppress the subscription nag dialog - Restricts the web UI to the `tailscale0` interface via UFW - Mounts the london-b CIFS storage +- Adds the `london-b-backups` storage and reconciles the vzdump jobs in `proxmox_ve_backup_jobs` ## Networking diff --git a/docs/hosts/london-b.md b/docs/hosts/london-b.md index 02f93ab..891a338 100644 --- a/docs/hosts/london-b.md +++ b/docs/hosts/london-b.md @@ -116,7 +116,8 @@ Snap: Overseerr (`latest/beta` channel). | HDD | /hdd | pez, root (rw) | | Movies | /hdd/movies | public (ro) | | TV Shows | /hdd/tv | public (ro) | -| pve | /hdd/pve | london-a Proxmox (rw) — ISO/template/backup storage | +| pve | /hdd/pve | london-a Proxmox (rw) — ISO/template storage. Not currently defined in smb.conf, see PESO-159 | +| pve-backups | /hdd/pve/backups | `pvebackup` (rw) — vzdump backups from the Proxmox hosts, one directory per host | Media is served directly from the ZFS pool. Docker root (`/hdd/docker`) and PVE storage (`/hdd/pve`) live on the pool too.