fix: slight tweaks (#103)

* fix: slight tweaks

* remove vendor

ansible/deploy.yml

@@ -74,24 +74,12 @@
     - role: systemd_services
     - role: mariadb
 
-# london-a: Cockpit VM host (Debian)
+# london-a: Proxmox VE hypervisor
-- name: "Stage 4f: Cockpit VM host (london-a)"
+- name: "Stage 4f: Proxmox VE (london-a)"
   hosts: london-a
-  tags: [services, cockpit]
+  tags: [services, proxmox]
-  tasks:
+  roles:
-    - name: Install cockpit and cockpit-machines
+    - role: proxmox_ve
-      ansible.builtin.apt:
-        name:
-          - cockpit
-          - cockpit-machines
-        state: present
-        update_cache: true
-
-    - name: Enable and start cockpit
-      ansible.builtin.service:
-        name: cockpit
-        state: started
-        enabled: true
 
 # ──────────────────────────────────────────────
 # Stage 4g: ZFS scrub scheduling — zfs_hosts

ansible/inventory/host_vars/london-a.yml

@@ -1,5 +1,5 @@
 ---
-host_role: vm-host
+host_role: proxmox-ve
-host_description: "VM host (Cockpit + KVM)"
+host_description: "Proxmox VE hypervisor"
 host_location: "London"
 ansible_python_interpreter: /usr/bin/python3

ansible/inventory/hosts.ini

@@ -3,7 +3,7 @@
 
 [linux]
 helsinki-a   ansible_host=100.67.6.27
-london-a     ansible_host=100.90.111.19
+london-a     ansible_host=100.122.180.98
 london-b     ansible_host=100.84.65.101
 london-c     ansible_host=100.123.72.87
 copenhagen-a ansible_host=100.89.206.60

ansible/roles/proxmox_ve/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+- name: Update apt cache
+  ansible.builtin.apt:
+    update_cache: true

ansible/roles/proxmox_ve/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+# Configure Proxmox VE apt sources for the no-subscription tier.
+# Removes the enterprise repo (requires paid subscription) and enables
+# the free no-subscription repo so apt update works without credentials.
+
+- name: Remove PVE enterprise apt source
+  ansible.builtin.file:
+    path: /etc/apt/sources.list.d/pve-enterprise.list
+    state: absent
+
+- name: Remove Ceph enterprise apt source
+  ansible.builtin.file:
+    path: /etc/apt/sources.list.d/ceph.list
+    state: absent
+
+- name: Add PVE no-subscription apt source
+  ansible.builtin.copy:
+    dest: /etc/apt/sources.list.d/pve-no-subscription.list
+    content: "deb http://download.proxmox.com/debian/pve bookworm pve-no-subscription\n"
+    owner: root
+    group: root
+    mode: '0644'
+  notify: Update apt cache
+
+- name: Update apt cache
+  ansible.builtin.apt:
+    update_cache: true

ansible/services/caddy/Caddyfile

@@ -14,13 +14,9 @@
 
 ## LONDON-A SERVICES ##
 
-# Cockpit
+# Proxmox
 london-a.pez.sh {
-        forward_auth localhost:9091 {
+        reverse_proxy 100.122.180.98:8006 {
-		uri /api/authz/forward-auth
-		copy_headers Remote-User Remote-Groups Remote-Name Remote-Email
-	}
-        reverse_proxy 100.90.111.19:9090 {
                 transport http {
                         tls_insecure_skip_verify
                 }

terraform/README.md

@@ -4,19 +4,34 @@ Infrastructure-as-code for cloud and edge services. Uses [OpenTofu](https://open
 
 ## What's managed
 
-- **Cloudflare DNS** — All `pez.sh` records (A, CNAME, MX, TXT)
+- **Hetzner Cloud** — Two servers (`nuremberg-a`, `helsinki-a`), firewalls, and DNS for `pez.sh`
+- **Grafana Cloud** — Stack, dashboards, synthetic monitoring checks, alert rules, Fleet collectors and pipelines
+- **PagerDuty** — Service, escalation policy, and Grafana integration
 
-## CI/CD
+## Secrets
 
-The original GitHub Actions workflow (`apply.yml`) ran plan on push to master, then applied with manual approval via a `prod` environment gate. This workflow lived in the standalone `pez-terraform` repo and would need adapting for the monorepo structure (e.g., path-filtered triggers).
+Secrets are stored encrypted in `secrets.enc.yaml` via [SOPS](https://github.com/getsops/sops) and decrypted at plan/apply time into `secrets.yaml`. The Makefile handles decryption automatically.
+
+Required secret keys: `hetzner_token`, `grafana_cloud_access_policy`, `grafana_synthetic_monitoring_access_token`, `grafana_fleet_management_auth`, `grafana_service_account_token`, `pagerduty_token`, `plex_token`, `backblaze_key_id`.
+
+## State
+
+State is stored in a Backblaze B2 bucket (`pez-infra-tfstate`) using an S3-compatible backend. Credentials are read from `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` environment variables.
+
+## Usage
+
+```sh
+make init   # initialize providers and backend
+make plan   # preview changes
+make apply  # apply changes
+make fmt    # format all .tf files
+```
 
 ## Provider versions
 
 | Provider | Source | Version |
 |----------|--------|---------|
-| Cloudflare | `cloudflare/cloudflare` | `~> 5.18` |
+| Hetzner Cloud | `hetznercloud/hcloud` | `~> 1.45` |
+| Grafana | `grafana/grafana` | `~> 4.35` |
+| PagerDuty | `pagerduty/pagerduty` | `~> 2.2` |
 | OpenTofu | — | `>= 1.6.0` |
-
-## Migrated from
-
-This directory replaces the standalone [`pez-terraform`](https://github.com/RWejlgaard/pez-terraform) repo.

terraform/grafana/synthetic_check_alerts.tf

@@ -1,83 +0,0 @@
-resource "grafana_synthetic_monitoring_check_alerts" "pez_sh" {
-  check_id = grafana_synthetic_monitoring_check.pez_sh.id
-  alerts = [
-    {
-      name        = "ProbeFailedExecutionsTooHigh"
-      threshold   = 3
-      period      = "30m"
-      runbook_url = ""
-    }
-  ]
-}
-
-resource "grafana_synthetic_monitoring_check_alerts" "pez_solutions" {
-  check_id = grafana_synthetic_monitoring_check.pez_solutions.id
-  alerts = [
-    {
-      name        = "ProbeFailedExecutionsTooHigh"
-      threshold   = 3
-      period      = "30m"
-      runbook_url = ""
-    }
-  ]
-}
-
-resource "grafana_synthetic_monitoring_check_alerts" "jellyfin" {
-  check_id = grafana_synthetic_monitoring_check.jellyfin.id
-  alerts = [
-    {
-      name        = "ProbeFailedExecutionsTooHigh"
-      threshold   = 3
-      period      = "30m"
-      runbook_url = ""
-    }
-  ]
-}
-
-resource "grafana_synthetic_monitoring_check_alerts" "plex" {
-  check_id = grafana_synthetic_monitoring_check.plex.id
-  alerts = [
-    {
-      name        = "ProbeFailedExecutionsTooHigh"
-      threshold   = 3
-      period      = "30m"
-      runbook_url = ""
-    }
-  ]
-}
-
-resource "grafana_synthetic_monitoring_check_alerts" "request" {
-  check_id = grafana_synthetic_monitoring_check.request.id
-  alerts = [
-    {
-      name        = "ProbeFailedExecutionsTooHigh"
-      threshold   = 3
-      period      = "30m"
-      runbook_url = ""
-    }
-  ]
-}
-
-resource "grafana_synthetic_monitoring_check_alerts" "jellyfin-requests" {
-  check_id = grafana_synthetic_monitoring_check.jellyfin-requests.id
-  alerts = [
-    {
-      name        = "ProbeFailedExecutionsTooHigh"
-      threshold   = 3
-      period      = "30m"
-      runbook_url = ""
-    }
-  ]
-}
-
-resource "grafana_synthetic_monitoring_check_alerts" "git" {
-  check_id = grafana_synthetic_monitoring_check.git.id
-  alerts = [
-    {
-      name        = "ProbeFailedExecutionsTooHigh"
-      threshold   = 3
-      period      = "30m"
-      runbook_url = ""
-    }
-  ]
-}

terraform/grafana/synthetic_checks.tf

@@ -1,11 +1,31 @@
-resource "grafana_synthetic_monitoring_check" "pez_sh" {
+locals {
-  job     = "pez.sh"
+  probe_london    = 14
-  target  = "https://pez.sh"
+  check_frequency = 600000
-  enabled = true
+  check_timeout   = 3000
-  probes  = [14] # 14 = London, UK
+
+  synthetic_checks = {
+    pez_sh            = { job = "pez.sh", target = "https://pez.sh", headers = [] }
+    pez_solutions     = { job = "pez.solutions", target = "https://pez.solutions", headers = [] }
+    jellyfin          = { job = "jellyfin.pez.sh", target = "https://jellyfin.pez.sh", headers = [] }
+    plex              = { job = "plex.pez.sh", target = "https://plex.pez.sh", headers = ["X-Plex-Token:${var.plex_token}"] }
+    request           = { job = "request.pez.sh", target = "https://request.pez.sh", headers = [] }
+    jellyfin_requests = { job = "jellyfin-requests.pez.sh", target = "https://jellyfin-requests.pez.sh", headers = [] }
+    git               = { job = "git.pez.sh", target = "https://git.pez.sh", headers = [] }
+  }
+}
+
+resource "grafana_synthetic_monitoring_check" "this" {
+  for_each  = local.synthetic_checks
+  job       = each.value.job
+  target    = each.value.target
+  enabled   = true
+  probes    = [local.probe_london]
+  frequency = local.check_frequency
+  timeout   = local.check_timeout
   settings {
     http {
       method              = "GET"
+      headers             = each.value.headers
       compression         = "none"
       fail_if_not_ssl     = true
       ip_version          = "V4"
@@ -13,142 +33,20 @@ resource "grafana_synthetic_monitoring_check" "pez_sh" {
       valid_status_codes  = ["200"]
     }
   }
-  frequency = 600000
-  timeout   = 3000
   lifecycle {
     ignore_changes = [settings]
   }
 }
 
-resource "grafana_synthetic_monitoring_check" "pez_solutions" {
+resource "grafana_synthetic_monitoring_check_alerts" "this" {
-  job     = "pez.solutions"
+  for_each = grafana_synthetic_monitoring_check.this
-  target  = "https://pez.solutions"
+  check_id = each.value.id
-  enabled = true
+  alerts = [
-  probes  = [14] # 14 = London, UK
+    {
-  settings {
+      name        = "ProbeFailedExecutionsTooHigh"
-    http {
+      threshold   = 3
-      method              = "GET"
+      period      = "30m"
-      compression         = "none"
+      runbook_url = ""
-      fail_if_not_ssl     = true
-      ip_version          = "V4"
-      valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
-      valid_status_codes  = ["200"]
     }
-  }
+  ]
-  frequency = 600000
-  timeout   = 3000
-  lifecycle {
-    ignore_changes = [settings]
-  }
-}
-
-resource "grafana_synthetic_monitoring_check" "jellyfin" {
-  job     = "jellyfin.pez.sh"
-  target  = "https://jellyfin.pez.sh"
-  enabled = true
-  probes  = [14] # 14 = London, UK
-  settings {
-    http {
-      method              = "GET"
-      compression         = "none"
-      fail_if_not_ssl     = true
-      ip_version          = "V4"
-      valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
-      valid_status_codes  = ["200"]
-    }
-  }
-  frequency = 600000
-  timeout   = 3000
-  lifecycle {
-    ignore_changes = [settings]
-  }
-}
-
-resource "grafana_synthetic_monitoring_check" "plex" {
-  job     = "plex.pez.sh"
-  target  = "https://plex.pez.sh"
-  enabled = true
-  probes  = [14] # 14 = London, UK
-  settings {
-    http {
-      method              = "GET"
-      headers             = ["X-Plex-Token:${var.plex_token}"]
-      compression         = "none"
-      fail_if_not_ssl     = true
-      ip_version          = "V4"
-      valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
-      valid_status_codes  = ["200"]
-    }
-  }
-  frequency = 600000
-  timeout   = 3000
-  lifecycle {
-    ignore_changes = [settings]
-  }
-}
-
-resource "grafana_synthetic_monitoring_check" "request" {
-  job     = "request.pez.sh"
-  target  = "https://request.pez.sh"
-  enabled = true
-  probes  = [14] # 14 = London, UK
-  settings {
-    http {
-      method              = "GET"
-      compression         = "none"
-      fail_if_not_ssl     = true
-      ip_version          = "V4"
-      valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
-      valid_status_codes  = ["200"]
-    }
-  }
-  frequency = 600000
-  timeout   = 3000
-  lifecycle {
-    ignore_changes = [settings]
-  }
-}
-
-resource "grafana_synthetic_monitoring_check" "jellyfin-requests" {
-  job     = "jellyfin-requests.pez.sh"
-  target  = "https://jellyfin-requests.pez.sh"
-  enabled = true
-  probes  = [14] # 14 = London, UK
-  settings {
-    http {
-      method              = "GET"
-      compression         = "none"
-      fail_if_not_ssl     = true
-      ip_version          = "V4"
-      valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
-      valid_status_codes  = ["200"]
-    }
-  }
-  frequency = 600000
-  timeout   = 3000
-  lifecycle {
-    ignore_changes = [settings]
-  }
-}
-
-resource "grafana_synthetic_monitoring_check" "git" {
-  job     = "git.pez.sh"
-  target  = "https://git.pez.sh"
-  enabled = true
-  probes  = [14] # 14 = London, UK
-  settings {
-    http {
-      method              = "GET"
-      compression         = "none"
-      fail_if_not_ssl     = true
-      ip_version          = "V4"
-      valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"]
-      valid_status_codes  = ["200"]
-    }
-  }
-  frequency = 600000
-  timeout   = 3000
-  lifecycle {
-    ignore_changes = [settings]
-  }
 }

terraform/grafana/vars.tf

@@ -1,4 +1,5 @@
 variable "plex_token" {
-  type      = string
+  type        = string
-  sensitive = true
+  sensitive   = true
+  description = "Plex API token used as a header in the synthetic monitoring check for plex.pez.sh"
 }

terraform/hetzner/dns.tf

@@ -8,6 +8,7 @@ locals {
   nuremberg_a    = hcloud_server.nuremberg-a.ipv4_address
   nuremberg_aaaa = hcloud_server.nuremberg-a.ipv6_address
   copenhagen     = "83.94.248.182"
+  dns_ttl        = 300
 }
 
 resource "hcloud_zone_rrset" "A_helsinki_a" {
@@ -20,7 +21,7 @@ resource "hcloud_zone_rrset" "A_helsinki_a" {
   zone    = hcloud_zone.pezsh.name
   name    = each.value
   type    = "A"
-  ttl     = 300
+  ttl     = local.dns_ttl
   records = [{ value = local.helsinki_a }]
 }
 
@@ -32,7 +33,7 @@ resource "hcloud_zone_rrset" "nuremberg_mail" {
   zone    = hcloud_zone.pezsh.name
   name    = "mail"
   type    = each.key
-  ttl     = 300
+  ttl     = local.dns_ttl
   records = [{ value = each.value }]
 }
 
@@ -41,7 +42,7 @@ resource "hcloud_zone_rrset" "A_copenhagen" {
   zone     = hcloud_zone.pezsh.name
   name     = each.value
   type     = "A"
-  ttl      = 300
+  ttl      = local.dns_ttl
   records  = [{ value = local.copenhagen }]
 }
 
@@ -49,7 +50,7 @@ resource "hcloud_zone_rrset" "CNAME_public" {
   zone    = hcloud_zone.pezsh.name
   name    = "public"
   type    = "CNAME"
-  ttl     = 300
+  ttl     = local.dns_ttl
   records = [{ value = "public.r2.dev." }]
 }
 
@@ -57,7 +58,7 @@ resource "hcloud_zone_rrset" "MX_root" {
   zone = hcloud_zone.pezsh.name
   name = "@"
   type = "MX"
-  ttl  = 300
+  ttl  = local.dns_ttl
   records = [
     { value = "10 mail.pez.sh." },
   ]
@@ -67,7 +68,7 @@ resource "hcloud_zone_rrset" "TXT_dkim" {
   zone = hcloud_zone.pezsh.name
   name = "dkim._domainkey"
   type = "TXT"
-  ttl  = 300
+  ttl  = local.dns_ttl
   records = [{
     value = "\"v=DKIM1;k=rsa;t=s;s=email;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmT/TGkPkfbjleqRYuQoI67/xvM0J5gGmdlzo2jO5qTABz5+nzOS+PefrXkeEZ0IZrpLPKqLyi7K469Ql+HG5wDFDxQRRG7lHJkWJ4tnZgjZWgeszFPhoME74lT6i+j3x29WyxhyzNg0f3NhSwttOe5knmS4zsOb+JK4jShoF9zZkOUCHAZ/vKvY\" \"tJdV+8qpmU8wfgyrzN1OWxjHIjzPP8iMD4g0iCfobbvSvWXHYBveCS7b/Nr3jw3E8twtEAUEGYNGd4h0wKNbNagYUsb5My8tMxQQwZf6imKHgCeYC7buH8TvaJHATReeea4Dzj9UzdPgwdbFLiMB/HXlN0GPhlQIDAQAB\""
   }]
@@ -77,7 +78,7 @@ resource "hcloud_zone_rrset" "TXT_dmarc" {
   zone    = hcloud_zone.pezsh.name
   name    = "_dmarc"
   type    = "TXT"
-  ttl     = 300
+  ttl     = local.dns_ttl
   records = [{ value = "\"v=DMARC1; p=quarantine; rua=mailto:pez@pez.sh; adkim=r; aspf=r\"" }]
 }
 
@@ -85,6 +86,6 @@ resource "hcloud_zone_rrset" "TXT_spf" {
   zone    = hcloud_zone.pezsh.name
   name    = "@"
   type    = "TXT"
-  ttl     = 300
+  ttl     = local.dns_ttl
   records = [{ value = "\"v=spf1 ip4:${local.nuremberg_a} ip6:${local.nuremberg_aaaa} -all\"" }]
 }

terraform/hetzner/outputs.tf

@@ -0,0 +1,12 @@
+output "server_ips" {
+  description = "Public IPv4 addresses of all managed servers"
+  value = {
+    nuremberg_a = hcloud_server.nuremberg-a.ipv4_address
+    helsinki_a  = hcloud_server.helsinki-a.ipv4_address
+  }
+}
+
+output "dns_zone" {
+  description = "The managed DNS zone name"
+  value       = hcloud_zone.pezsh.name
+}

terraform/hetzner/ssh_keys.tf

@@ -1,4 +1,4 @@
 resource "hcloud_ssh_key" "personal" {
-  name = "personal"
+  name       = "personal"
   public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDlU2h+JgVMVsHkkcxed9WbrUCKWfuUrY6yErmGIIREP6X2cua2qE4H+329FSJXQs0Yd0OiNwsXzfW88kl0+aMopQXaccY3q8109KR43RNrRrril9od+PidVvT/fvV8eNYVE9M4gyT1c9t8ZLD85vJf9rILFWbLG4DqqFL3z33W2u//Bl8uVLoY3tSgBmukVt45If9g9mxVfSstLmZj7j75rghS0EbE2kzwgUH397mJGMlJJdFhzRtP+/D09hE+zgFxl45V6dszEu9ggawRRGvEcR1dXDB0g6n3/7h6M+pb8/77ZAxk4AwD6CzZi8k7SlVkzCKZQRPpge+C0xLdm9EAY7byj30XdGgpo80eiCJmVImYm4VmPnjh39IumQWkDgpXkYQ9aj9jUDvcSrEmwTBRJOqmaO7BW0sVbP0BDW3UjCyeUQ8zprmWsUscoB0u9r4bMOLnhNldXljjKcDRdX2JciIILiCEfnn781Q3uxLgOoHEnYto0tSxbLQI/o9WB4M="
 }

terraform/pagerduty/integrations.tf

@@ -1,11 +1,7 @@
-data "pagerduty_vendor" "prometheus" {
-  name = "Prometheus"
-}
-
 resource "pagerduty_service_integration" "grafana_cloud" {
   name    = "Grafana"
   service = pagerduty_service.pez_solutions.id
-  vendor  = data.pagerduty_vendor.prometheus.id
+  vendor  = ""
 }
 
 output "pagerduty_integration_key" {