Clou dflaring out mayday mayday mayday (#89)

* phase 1 - add all the records to both providers to A/B test

* dkim fix

ansible/inventory/host_vars/london-b.yml

@@ -30,6 +30,11 @@ node_exporter_extra_collectors:
   - ethtool
   - zfs
 
+docker_daemon_extra:
+  metrics-addr: "0.0.0.0:9323"
+  data-root: "/hdd/docker"
+  storage-driver: "zfs"
+
 common_ufw_allowed_ports:
   - { port: 32400, proto: tcp, comment: "Plex Media Server" }
   - { port: 6881, proto: tcp, comment: "BitTorrent" }

ansible/roles/alloy/tasks/main.yml

@@ -20,6 +20,15 @@
     state: present
   when: ansible_facts["os_family"] == "Alpine"
 
+- name: Fix alloy storage dir ownership (Alpine)
+  ansible.builtin.file:
+    path: /var/lib/alloy
+    state: directory
+    owner: alloy
+    group: alloy
+    recurse: true
+  when: ansible_facts["os_family"] == "Alpine"
+
 # ── FreeBSD: pkgng ────────────────────────────────────────────────────────────
 
 - name: Install alloy (FreeBSD)
@@ -28,14 +37,13 @@
     state: present
   when: ansible_facts["os_family"] == "FreeBSD"
 
-- name: Create alloy directories (FreeBSD)
+- name: Fix alloy storage dir ownership (FreeBSD)
   ansible.builtin.file:
-    path: "{{ item }}"
+    path: /var/alloy
     state: directory
+    owner: nobody
+    group: nobody
     mode: '0755'
-  loop:
-    - /usr/local/etc/alloy
-    - /var/db/alloy
   when: ansible_facts["os_family"] == "FreeBSD"
 
 # ── Docker socket access ─────────────────────────────────────────────────────
@@ -53,7 +61,7 @@
 - name: Set alloy config path fact
   ansible.builtin.set_fact:
     alloy_config_path: >-
-      {{ '/usr/local/etc/alloy/config.alloy'
+      {{ '/usr/local/etc/alloy.flow'
          if ansible_facts['os_family'] == 'FreeBSD'
          else '/etc/alloy/config.alloy' }}
 
@@ -86,12 +94,6 @@
     value: "YES"
   when: ansible_facts["os_family"] == "FreeBSD"
 
-- name: Set alloy config in rc.conf (FreeBSD)
-  community.general.sysrc:
-    name: alloy_config
-    value: /usr/local/etc/alloy/config.alloy
-  when: ansible_facts["os_family"] == "FreeBSD"
-
 - name: Start alloy (FreeBSD)
   ansible.builtin.service:
     name: alloy

ansible/roles/alloy/templates/alloy.config.alloy.j2

@@ -4,12 +4,22 @@
 // ─── System logs ─────────────────────────────────────────────────────────────
 
 {% if ansible_facts['os_family'] == 'Debian' %}
-local.file_match "system" {
-  path_targets = [
-    {"__path__" = "/var/log/syslog",   "job" = "syslog", "host" = "{{ inventory_hostname }}"},
-    {"__path__" = "/var/log/auth.log", "job" = "auth",   "host" = "{{ inventory_hostname }}"},
-    {"__path__" = "/var/log/kern.log", "job" = "kern",   "host" = "{{ inventory_hostname }}"},
-  ]
+loki.source.journal "system" {
+  forward_to = [loki.write.default.receiver]
+  labels     = {"host" = "{{ inventory_hostname }}"}
+  relabel_rules = loki.relabel.journal.rules
+}
+
+loki.relabel "journal" {
+  forward_to = []
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+    target_label  = "unit"
+  }
+  rule {
+    source_labels = ["__journal_priority_keyword"]
+    target_label  = "level"
+  }
 }
 {% elif ansible_facts['os_family'] == 'Alpine' %}
 local.file_match "system" {
@@ -26,48 +36,11 @@ local.file_match "system" {
 }
 {% endif %}
 
+{% if ansible_facts['os_family'] != 'Debian' %}
 loki.source.file "system" {
   targets    = local.file_match.system.targets
   forward_to = [loki.write.default.receiver]
 }
-
-{% if 'docker_hosts' in group_names %}
-// ─── Docker container logs ────────────────────────────────────────────────────
-
-discovery.docker "containers" {
-  host             = "unix:///var/run/docker.sock"
-  refresh_interval = "15s"
-}
-
-discovery.relabel "docker_containers" {
-  targets = discovery.docker.containers.targets
-
-  rule {
-    source_labels = ["__meta_docker_container_state"]
-    action        = "keep"
-    regex         = "running"
-  }
-  rule {
-    source_labels = ["__meta_docker_container_name"]
-    regex         = "/(.*)"
-    target_label  = "container"
-  }
-  rule {
-    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
-    target_label  = "compose_service"
-  }
-  rule {
-    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
-    target_label  = "compose_project"
-  }
-}
-
-loki.source.docker "containers" {
-  host       = "unix:///var/run/docker.sock"
-  targets    = discovery.relabel.docker_containers.output
-  forward_to = [loki.write.default.receiver]
-  labels     = {"host" = "{{ inventory_hostname }}"}
-}
 {% endif %}
 
 {% if inventory_hostname == 'london-b' %}

ansible/roles/docker/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart docker
+  ansible.builtin.service:
+    name: docker
+    state: restarted
+  listen: Restart docker

ansible/roles/docker/tasks/main.yml

@@ -86,6 +86,26 @@
     state: started
     enabled: true
 
+# ── Loki logging driver ───────────────────────────────────────────────────────
+
+- name: Install Loki Docker logging plugin
+  ansible.builtin.command:
+    cmd: docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
+  register: docker_loki_plugin_install
+  changed_when: "'Installed plugin' in docker_loki_plugin_install.stdout"
+  failed_when:
+    - docker_loki_plugin_install.rc != 0
+    - "'already exists' not in docker_loki_plugin_install.stderr"
+
+- name: Deploy Docker daemon.json
+  ansible.builtin.template:
+    src: daemon.json.j2
+    dest: /etc/docker/daemon.json
+    mode: '0644'
+  notify: Restart docker
+
+# ── Compose project directories ───────────────────────────────────────────────
+
 - name: Create docker compose project directories
   ansible.builtin.file:
     path: "/opt/docker/{{ item }}"

ansible/roles/docker/templates/daemon.json.j2

@@ -0,0 +1,12 @@
+{{ {
+  "log-driver": "loki",
+  "log-opts": {
+    "loki-url": "http://" ~ hostvars['london-a']['ansible_host'] ~ ":3100/loki/api/v1/push",
+    "loki-external-labels": "host=" ~ inventory_hostname ~ ",job=docker",
+    "loki-retries": "5",
+    "loki-batch-size": "400",
+    "loki-timeout": "10s",
+    "mode": "non-blocking",
+    "max-buffer-size": "5m"
+  }
+} | combine(docker_daemon_extra | default({})) | to_nice_json }}

terraform/hetzner_dns.tf

@@ -0,0 +1,291 @@
+resource "hcloud_zone" "pezsh" {
+  name = "pez.sh"
+  mode = "primary"
+}
+
+# =============================================================================
+# A Records
+# =============================================================================
+
+resource "hcloud_zone_rrset" "A_apps" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "apps"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_auth" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "auth"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_bitwarden" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "bitwarden"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_download" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "download"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_git" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "git"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_grafana" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "grafana"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_helsinki-a" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "helsinki-a"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_jellyfin" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "jellyfin"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_jellyfin-requests" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "jellyfin-requests"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_ldap" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "ldap"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_lidarr" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "lidarr"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_mail" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "mail"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.nuremberg-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_minecraft" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "minecraft"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = "83.94.248.182" }]
+}
+
+resource "hcloud_zone_rrset" "A_music" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "music"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_naveen" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "naveen"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_root" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "@"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_plex" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "plex"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_prometheus" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "prometheus"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_prowlarr" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "prowlarr"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_radarr" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "radarr"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_readarr" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "readarr"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_request" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "request"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_rss" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "rss"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_sonarr" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "sonarr"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_soulseek" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "soulseek"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_status" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "status"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = hcloud_server.helsinki-a.ipv4_address }]
+}
+
+resource "hcloud_zone_rrset" "A_wow" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "wow"
+  type    = "A"
+  ttl     = 300
+  records = [{ value = "83.94.248.182" }]
+}
+
+# =============================================================================
+# AAAA Records
+# =============================================================================
+
+resource "hcloud_zone_rrset" "AAAA_mail" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "mail"
+  type    = "AAAA"
+  ttl     = 300
+  records = [{ value = hcloud_server.nuremberg-a.ipv6_address }]
+}
+
+# =============================================================================
+# CNAME Records
+# =============================================================================
+
+resource "hcloud_zone_rrset" "CNAME_public" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "public"
+  type    = "CNAME"
+  ttl     = 300
+  records = [{ value = "public.r2.dev." }]
+}
+
+# =============================================================================
+# MX Records
+# =============================================================================
+
+resource "hcloud_zone_rrset" "MX_root" {
+  zone = hcloud_zone.pezsh.name
+  name = "@"
+  type = "MX"
+  ttl  = 300
+  records = [
+    { value = "10 mail.pez.sh." },
+    { value = "20 mail.pez.sh." },
+  ]
+}
+
+# =============================================================================
+# TXT Records
+# =============================================================================
+
+resource "hcloud_zone_rrset" "TXT_dkim" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "dkim._domainkey"
+  type    = "TXT"
+  ttl     = 300
+  records = [{ value = "\"v=DKIM1;k=rsa;t=s;s=email;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmT/TGkPkfbjleqRYuQoI67/xvM0J5gGmdlzo2jO5qTABz5+nzOS+PefrXkeEZ0IZrpLPKqLyi7K469Ql+HG5wDFDxQRRG7lHJkWJ4tnZgjZWgeszFPhoME74lT6i+j3x29WyxhyzNg0f3NhSwttOe5knmS4zsOb+JK4jShoF9zZkOUCHAZ/vKvY\" \"tJdV+8qpmU8wfgyrzN1OWxjHIjzPP8iMD4g0iCfobbvSvWXHYBveCS7b/Nr3jw3E8twtEAUEGYNGd4h0wKNbNagYUsb5My8tMxQQwZf6imKHgCeYC7buH8TvaJHATReeea4Dzj9UzdPgwdbFLiMB/HXlN0GPhlQIDAQAB\"" }]
+}
+
+resource "hcloud_zone_rrset" "TXT_dmarc" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "_dmarc"
+  type    = "TXT"
+  ttl     = 300
+  records = [{ value = "\"v=DMARC1; p=quarantine; rua=mailto:pez@pez.sh; adkim=r; aspf=r\"" }]
+}
+
+resource "hcloud_zone_rrset" "TXT_root_spf" {
+  zone    = hcloud_zone.pezsh.name
+  name    = "@"
+  type    = "TXT"
+  ttl     = 300
+  records = [{ value = "\"v=spf1 ip4:${hcloud_server.nuremberg-a.ipv4_address} ip6:${hcloud_server.nuremberg-a.ipv6_address} -all\"" }]
+}