diff --git a/terraform/README.md b/terraform/README.md index 7ee79a9..327a624 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -4,19 +4,34 @@ Infrastructure-as-code for cloud and edge services. Uses [OpenTofu](https://open ## What's managed -- **Cloudflare DNS** — All `pez.sh` records (A, CNAME, MX, TXT) +- **Hetzner Cloud** — Two servers (`nuremberg-a`, `helsinki-a`), firewalls, and DNS for `pez.sh` +- **Grafana Cloud** — Stack, dashboards, synthetic monitoring checks, alert rules, Fleet collectors and pipelines +- **PagerDuty** — Service, escalation policy, and Grafana integration -## CI/CD +## Secrets -The original GitHub Actions workflow (`apply.yml`) ran plan on push to master, then applied with manual approval via a `prod` environment gate. This workflow lived in the standalone `pez-terraform` repo and would need adapting for the monorepo structure (e.g., path-filtered triggers). +Secrets are stored encrypted in `secrets.enc.yaml` via [SOPS](https://github.com/getsops/sops) and decrypted at plan/apply time into `secrets.yaml`. The Makefile handles decryption automatically. + +Required secret keys: `hetzner_token`, `grafana_cloud_access_policy`, `grafana_synthetic_monitoring_access_token`, `grafana_fleet_management_auth`, `grafana_service_account_token`, `pagerduty_token`, `plex_token`, `backblaze_key_id`. + +## State + +State is stored in a Backblaze B2 bucket (`pez-infra-tfstate`) using an S3-compatible backend. Credentials are read from `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` environment variables. + +## Usage + +```sh +make init # initialize providers and backend +make plan # preview changes +make apply # apply changes +make fmt # format all .tf files +``` ## Provider versions | Provider | Source | Version | |----------|--------|---------| -| Cloudflare | `cloudflare/cloudflare` | `~> 5.18` | +| Hetzner Cloud | `hetznercloud/hcloud` | `~> 1.45` | +| Grafana | `grafana/grafana` | `~> 4.35` | +| PagerDuty | `pagerduty/pagerduty` | `~> 2.2` | | OpenTofu | — | `>= 1.6.0` | - -## Migrated from - -This directory replaces the standalone [`pez-terraform`](https://github.com/RWejlgaard/pez-terraform) repo. diff --git a/terraform/grafana/synthetic_check_alerts.tf b/terraform/grafana/synthetic_check_alerts.tf deleted file mode 100644 index 466549e..0000000 --- a/terraform/grafana/synthetic_check_alerts.tf +++ /dev/null @@ -1,83 +0,0 @@ -resource "grafana_synthetic_monitoring_check_alerts" "pez_sh" { - check_id = grafana_synthetic_monitoring_check.pez_sh.id - alerts = [ - { - name = "ProbeFailedExecutionsTooHigh" - threshold = 3 - period = "30m" - runbook_url = "" - } - ] -} - -resource "grafana_synthetic_monitoring_check_alerts" "pez_solutions" { - check_id = grafana_synthetic_monitoring_check.pez_solutions.id - alerts = [ - { - name = "ProbeFailedExecutionsTooHigh" - threshold = 3 - period = "30m" - runbook_url = "" - } - ] -} - -resource "grafana_synthetic_monitoring_check_alerts" "jellyfin" { - check_id = grafana_synthetic_monitoring_check.jellyfin.id - alerts = [ - { - name = "ProbeFailedExecutionsTooHigh" - threshold = 3 - period = "30m" - runbook_url = "" - } - ] -} - -resource "grafana_synthetic_monitoring_check_alerts" "plex" { - check_id = grafana_synthetic_monitoring_check.plex.id - alerts = [ - { - name = "ProbeFailedExecutionsTooHigh" - threshold = 3 - period = "30m" - runbook_url = "" - } - ] -} - -resource "grafana_synthetic_monitoring_check_alerts" "request" { - check_id = grafana_synthetic_monitoring_check.request.id - alerts = [ - { - name = "ProbeFailedExecutionsTooHigh" - threshold = 3 - period = "30m" - runbook_url = "" - } - ] -} - -resource "grafana_synthetic_monitoring_check_alerts" "jellyfin-requests" { - check_id = grafana_synthetic_monitoring_check.jellyfin-requests.id - alerts = [ - { - name = "ProbeFailedExecutionsTooHigh" - threshold = 3 - period = "30m" - runbook_url = "" - } - ] -} - -resource "grafana_synthetic_monitoring_check_alerts" "git" { - check_id = grafana_synthetic_monitoring_check.git.id - alerts = [ - { - name = "ProbeFailedExecutionsTooHigh" - threshold = 3 - period = "30m" - runbook_url = "" - } - ] -} \ No newline at end of file diff --git a/terraform/grafana/synthetic_checks.tf b/terraform/grafana/synthetic_checks.tf index 0c3ec46..c231330 100644 --- a/terraform/grafana/synthetic_checks.tf +++ b/terraform/grafana/synthetic_checks.tf @@ -1,11 +1,31 @@ -resource "grafana_synthetic_monitoring_check" "pez_sh" { - job = "pez.sh" - target = "https://pez.sh" - enabled = true - probes = [14] # 14 = London, UK +locals { + probe_london = 14 + check_frequency = 600000 + check_timeout = 3000 + + synthetic_checks = { + pez_sh = { job = "pez.sh", target = "https://pez.sh", headers = [] } + pez_solutions = { job = "pez.solutions", target = "https://pez.solutions", headers = [] } + jellyfin = { job = "jellyfin.pez.sh", target = "https://jellyfin.pez.sh", headers = [] } + plex = { job = "plex.pez.sh", target = "https://plex.pez.sh", headers = ["X-Plex-Token:${var.plex_token}"] } + request = { job = "request.pez.sh", target = "https://request.pez.sh", headers = [] } + jellyfin_requests = { job = "jellyfin-requests.pez.sh", target = "https://jellyfin-requests.pez.sh", headers = [] } + git = { job = "git.pez.sh", target = "https://git.pez.sh", headers = [] } + } +} + +resource "grafana_synthetic_monitoring_check" "this" { + for_each = local.synthetic_checks + job = each.value.job + target = each.value.target + enabled = true + probes = [local.probe_london] + frequency = local.check_frequency + timeout = local.check_timeout settings { http { method = "GET" + headers = each.value.headers compression = "none" fail_if_not_ssl = true ip_version = "V4" @@ -13,142 +33,20 @@ resource "grafana_synthetic_monitoring_check" "pez_sh" { valid_status_codes = ["200"] } } - frequency = 600000 - timeout = 3000 lifecycle { ignore_changes = [settings] } } -resource "grafana_synthetic_monitoring_check" "pez_solutions" { - job = "pez.solutions" - target = "https://pez.solutions" - enabled = true - probes = [14] # 14 = London, UK - settings { - http { - method = "GET" - compression = "none" - fail_if_not_ssl = true - ip_version = "V4" - valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"] - valid_status_codes = ["200"] +resource "grafana_synthetic_monitoring_check_alerts" "this" { + for_each = grafana_synthetic_monitoring_check.this + check_id = each.value.id + alerts = [ + { + name = "ProbeFailedExecutionsTooHigh" + threshold = 3 + period = "30m" + runbook_url = "" } - } - frequency = 600000 - timeout = 3000 - lifecycle { - ignore_changes = [settings] - } -} - -resource "grafana_synthetic_monitoring_check" "jellyfin" { - job = "jellyfin.pez.sh" - target = "https://jellyfin.pez.sh" - enabled = true - probes = [14] # 14 = London, UK - settings { - http { - method = "GET" - compression = "none" - fail_if_not_ssl = true - ip_version = "V4" - valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"] - valid_status_codes = ["200"] - } - } - frequency = 600000 - timeout = 3000 - lifecycle { - ignore_changes = [settings] - } -} - -resource "grafana_synthetic_monitoring_check" "plex" { - job = "plex.pez.sh" - target = "https://plex.pez.sh" - enabled = true - probes = [14] # 14 = London, UK - settings { - http { - method = "GET" - headers = ["X-Plex-Token:${var.plex_token}"] - compression = "none" - fail_if_not_ssl = true - ip_version = "V4" - valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"] - valid_status_codes = ["200"] - } - } - frequency = 600000 - timeout = 3000 - lifecycle { - ignore_changes = [settings] - } -} - -resource "grafana_synthetic_monitoring_check" "request" { - job = "request.pez.sh" - target = "https://request.pez.sh" - enabled = true - probes = [14] # 14 = London, UK - settings { - http { - method = "GET" - compression = "none" - fail_if_not_ssl = true - ip_version = "V4" - valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"] - valid_status_codes = ["200"] - } - } - frequency = 600000 - timeout = 3000 - lifecycle { - ignore_changes = [settings] - } -} - -resource "grafana_synthetic_monitoring_check" "jellyfin-requests" { - job = "jellyfin-requests.pez.sh" - target = "https://jellyfin-requests.pez.sh" - enabled = true - probes = [14] # 14 = London, UK - settings { - http { - method = "GET" - compression = "none" - fail_if_not_ssl = true - ip_version = "V4" - valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"] - valid_status_codes = ["200"] - } - } - frequency = 600000 - timeout = 3000 - lifecycle { - ignore_changes = [settings] - } -} - -resource "grafana_synthetic_monitoring_check" "git" { - job = "git.pez.sh" - target = "https://git.pez.sh" - enabled = true - probes = [14] # 14 = London, UK - settings { - http { - method = "GET" - compression = "none" - fail_if_not_ssl = true - ip_version = "V4" - valid_http_versions = ["HTTP/2.0", "HTTP/1.1", "HTTP/1.0"] - valid_status_codes = ["200"] - } - } - frequency = 600000 - timeout = 3000 - lifecycle { - ignore_changes = [settings] - } + ] } diff --git a/terraform/grafana/vars.tf b/terraform/grafana/vars.tf index 3cd8803..a540059 100644 --- a/terraform/grafana/vars.tf +++ b/terraform/grafana/vars.tf @@ -1,4 +1,5 @@ variable "plex_token" { - type = string - sensitive = true + type = string + sensitive = true + description = "Plex API token used as a header in the synthetic monitoring check for plex.pez.sh" } diff --git a/terraform/hetzner/dns.tf b/terraform/hetzner/dns.tf index 33969e0..96286ea 100644 --- a/terraform/hetzner/dns.tf +++ b/terraform/hetzner/dns.tf @@ -8,6 +8,7 @@ locals { nuremberg_a = hcloud_server.nuremberg-a.ipv4_address nuremberg_aaaa = hcloud_server.nuremberg-a.ipv6_address copenhagen = "83.94.248.182" + dns_ttl = 300 } resource "hcloud_zone_rrset" "A_helsinki_a" { @@ -20,7 +21,7 @@ resource "hcloud_zone_rrset" "A_helsinki_a" { zone = hcloud_zone.pezsh.name name = each.value type = "A" - ttl = 300 + ttl = local.dns_ttl records = [{ value = local.helsinki_a }] } @@ -32,7 +33,7 @@ resource "hcloud_zone_rrset" "nuremberg_mail" { zone = hcloud_zone.pezsh.name name = "mail" type = each.key - ttl = 300 + ttl = local.dns_ttl records = [{ value = each.value }] } @@ -41,7 +42,7 @@ resource "hcloud_zone_rrset" "A_copenhagen" { zone = hcloud_zone.pezsh.name name = each.value type = "A" - ttl = 300 + ttl = local.dns_ttl records = [{ value = local.copenhagen }] } @@ -49,7 +50,7 @@ resource "hcloud_zone_rrset" "CNAME_public" { zone = hcloud_zone.pezsh.name name = "public" type = "CNAME" - ttl = 300 + ttl = local.dns_ttl records = [{ value = "public.r2.dev." }] } @@ -57,7 +58,7 @@ resource "hcloud_zone_rrset" "MX_root" { zone = hcloud_zone.pezsh.name name = "@" type = "MX" - ttl = 300 + ttl = local.dns_ttl records = [ { value = "10 mail.pez.sh." }, ] @@ -67,7 +68,7 @@ resource "hcloud_zone_rrset" "TXT_dkim" { zone = hcloud_zone.pezsh.name name = "dkim._domainkey" type = "TXT" - ttl = 300 + ttl = local.dns_ttl records = [{ value = "\"v=DKIM1;k=rsa;t=s;s=email;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmT/TGkPkfbjleqRYuQoI67/xvM0J5gGmdlzo2jO5qTABz5+nzOS+PefrXkeEZ0IZrpLPKqLyi7K469Ql+HG5wDFDxQRRG7lHJkWJ4tnZgjZWgeszFPhoME74lT6i+j3x29WyxhyzNg0f3NhSwttOe5knmS4zsOb+JK4jShoF9zZkOUCHAZ/vKvY\" \"tJdV+8qpmU8wfgyrzN1OWxjHIjzPP8iMD4g0iCfobbvSvWXHYBveCS7b/Nr3jw3E8twtEAUEGYNGd4h0wKNbNagYUsb5My8tMxQQwZf6imKHgCeYC7buH8TvaJHATReeea4Dzj9UzdPgwdbFLiMB/HXlN0GPhlQIDAQAB\"" }] @@ -77,7 +78,7 @@ resource "hcloud_zone_rrset" "TXT_dmarc" { zone = hcloud_zone.pezsh.name name = "_dmarc" type = "TXT" - ttl = 300 + ttl = local.dns_ttl records = [{ value = "\"v=DMARC1; p=quarantine; rua=mailto:pez@pez.sh; adkim=r; aspf=r\"" }] } @@ -85,6 +86,6 @@ resource "hcloud_zone_rrset" "TXT_spf" { zone = hcloud_zone.pezsh.name name = "@" type = "TXT" - ttl = 300 + ttl = local.dns_ttl records = [{ value = "\"v=spf1 ip4:${local.nuremberg_a} ip6:${local.nuremberg_aaaa} -all\"" }] } diff --git a/terraform/hetzner/outputs.tf b/terraform/hetzner/outputs.tf new file mode 100644 index 0000000..59d9526 --- /dev/null +++ b/terraform/hetzner/outputs.tf @@ -0,0 +1,12 @@ +output "server_ips" { + description = "Public IPv4 addresses of all managed servers" + value = { + nuremberg_a = hcloud_server.nuremberg-a.ipv4_address + helsinki_a = hcloud_server.helsinki-a.ipv4_address + } +} + +output "dns_zone" { + description = "The managed DNS zone name" + value = hcloud_zone.pezsh.name +} diff --git a/terraform/hetzner/ssh_keys.tf b/terraform/hetzner/ssh_keys.tf index 426851a..af2f99b 100644 --- a/terraform/hetzner/ssh_keys.tf +++ b/terraform/hetzner/ssh_keys.tf @@ -1,4 +1,4 @@ resource "hcloud_ssh_key" "personal" { - name = "personal" + name = "personal" public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDlU2h+JgVMVsHkkcxed9WbrUCKWfuUrY6yErmGIIREP6X2cua2qE4H+329FSJXQs0Yd0OiNwsXzfW88kl0+aMopQXaccY3q8109KR43RNrRrril9od+PidVvT/fvV8eNYVE9M4gyT1c9t8ZLD85vJf9rILFWbLG4DqqFL3z33W2u//Bl8uVLoY3tSgBmukVt45If9g9mxVfSstLmZj7j75rghS0EbE2kzwgUH397mJGMlJJdFhzRtP+/D09hE+zgFxl45V6dszEu9ggawRRGvEcR1dXDB0g6n3/7h6M+pb8/77ZAxk4AwD6CzZi8k7SlVkzCKZQRPpge+C0xLdm9EAY7byj30XdGgpo80eiCJmVImYm4VmPnjh39IumQWkDgpXkYQ9aj9jUDvcSrEmwTBRJOqmaO7BW0sVbP0BDW3UjCyeUQ8zprmWsUscoB0u9r4bMOLnhNldXljjKcDRdX2JciIILiCEfnn781Q3uxLgOoHEnYto0tSxbLQI/o9WB4M=" }