fix: use docker driver instead of hacky alloy setup

ansible/inventory/host_vars/london-b.yml

@@ -30,6 +30,11 @@ node_exporter_extra_collectors:
   - ethtool
   - zfs
 
+docker_daemon_extra:
+  metrics-addr: "0.0.0.0:9323"
+  data-root: "/hdd/docker"
+  storage-driver: "zfs"
+
 common_ufw_allowed_ports:
   - { port: 32400, proto: tcp, comment: "Plex Media Server" }
   - { port: 6881, proto: tcp, comment: "BitTorrent" }

ansible/roles/alloy/tasks/main.yml

@@ -20,6 +20,15 @@
     state: present
   when: ansible_facts["os_family"] == "Alpine"
 
+- name: Fix alloy storage dir ownership (Alpine)
+  ansible.builtin.file:
+    path: /var/lib/alloy
+    state: directory
+    owner: alloy
+    group: alloy
+    recurse: true
+  when: ansible_facts["os_family"] == "Alpine"
+
 # ── FreeBSD: pkgng ────────────────────────────────────────────────────────────
 
 - name: Install alloy (FreeBSD)
@@ -28,32 +37,21 @@
     state: present
   when: ansible_facts["os_family"] == "FreeBSD"
 
-- name: Create alloy directories (FreeBSD)
+- name: Fix alloy storage dir ownership (FreeBSD)
   ansible.builtin.file:
-    path: "{{ item }}"
+    path: /var/alloy
     state: directory
+    owner: nobody
+    group: nobody
     mode: '0755'
-  loop:
-    - /usr/local/etc/alloy
-    - /var/db/alloy
   when: ansible_facts["os_family"] == "FreeBSD"
 
-# ── Docker socket access ─────────────────────────────────────────────────────
-
-- name: Add alloy to docker group
-  ansible.builtin.user:
-    name: alloy
-    groups: docker
-    append: true
-  when: "'docker_hosts' in group_names"
-  notify: "Restart alloy ({{ ansible_facts['os_family'] }})"
-
 # ── Config — all OS ───────────────────────────────────────────────────────────
 
 - name: Set alloy config path fact
   ansible.builtin.set_fact:
     alloy_config_path: >-
-      {{ '/usr/local/etc/alloy/config.alloy'
+      {{ '/usr/local/etc/alloy.flow'
          if ansible_facts['os_family'] == 'FreeBSD'
          else '/etc/alloy/config.alloy' }}
 
@@ -86,12 +84,6 @@
     value: "YES"
   when: ansible_facts["os_family"] == "FreeBSD"
 
-- name: Set alloy config in rc.conf (FreeBSD)
-  community.general.sysrc:
-    name: alloy_config
-    value: /usr/local/etc/alloy/config.alloy
-  when: ansible_facts["os_family"] == "FreeBSD"
-
 - name: Start alloy (FreeBSD)
   ansible.builtin.service:
     name: alloy

ansible/roles/alloy/templates/alloy.config.alloy.j2

@@ -4,12 +4,22 @@
 // ─── System logs ─────────────────────────────────────────────────────────────
 
 {% if ansible_facts['os_family'] == 'Debian' %}
-local.file_match "system" {
-  path_targets = [
-    {"__path__" = "/var/log/syslog",   "job" = "syslog", "host" = "{{ inventory_hostname }}"},
-    {"__path__" = "/var/log/auth.log", "job" = "auth",   "host" = "{{ inventory_hostname }}"},
-    {"__path__" = "/var/log/kern.log", "job" = "kern",   "host" = "{{ inventory_hostname }}"},
-  ]
+loki.source.journal "system" {
+  forward_to = [loki.write.default.receiver]
+  labels     = {"host" = "{{ inventory_hostname }}"}
+  relabel_rules = loki.relabel.journal.rules
+}
+
+loki.relabel "journal" {
+  forward_to = []
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+    target_label  = "unit"
+  }
+  rule {
+    source_labels = ["__journal_priority_keyword"]
+    target_label  = "level"
+  }
 }
 {% elif ansible_facts['os_family'] == 'Alpine' %}
 local.file_match "system" {
@@ -26,48 +36,11 @@ local.file_match "system" {
 }
 {% endif %}
 
+{% if ansible_facts['os_family'] != 'Debian' %}
 loki.source.file "system" {
   targets    = local.file_match.system.targets
   forward_to = [loki.write.default.receiver]
 }
-
-{% if 'docker_hosts' in group_names %}
-// ─── Docker container logs ────────────────────────────────────────────────────
-
-discovery.docker "containers" {
-  host             = "unix:///var/run/docker.sock"
-  refresh_interval = "15s"
-}
-
-discovery.relabel "docker_containers" {
-  targets = discovery.docker.containers.targets
-
-  rule {
-    source_labels = ["__meta_docker_container_state"]
-    action        = "keep"
-    regex         = "running"
-  }
-  rule {
-    source_labels = ["__meta_docker_container_name"]
-    regex         = "/(.*)"
-    target_label  = "container"
-  }
-  rule {
-    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
-    target_label  = "compose_service"
-  }
-  rule {
-    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
-    target_label  = "compose_project"
-  }
-}
-
-loki.source.docker "containers" {
-  host       = "unix:///var/run/docker.sock"
-  targets    = discovery.relabel.docker_containers.output
-  forward_to = [loki.write.default.receiver]
-  labels     = {"host" = "{{ inventory_hostname }}"}
-}
 {% endif %}
 
 {% if inventory_hostname == 'london-b' %}

ansible/roles/docker/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart docker
+  ansible.builtin.service:
+    name: docker
+    state: restarted
+  listen: Restart docker

ansible/roles/docker/tasks/main.yml

@@ -86,6 +86,26 @@
     state: started
     enabled: true
 
+# ── Loki logging driver ───────────────────────────────────────────────────────
+
+- name: Install Loki Docker logging plugin
+  ansible.builtin.command:
+    cmd: docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
+  register: loki_plugin_install
+  changed_when: "'Installed plugin' in loki_plugin_install.stdout"
+  failed_when:
+    - loki_plugin_install.rc != 0
+    - "'already exists' not in loki_plugin_install.stderr"
+
+- name: Deploy Docker daemon.json
+  ansible.builtin.template:
+    src: daemon.json.j2
+    dest: /etc/docker/daemon.json
+    mode: '0644'
+  notify: Restart docker
+
+# ── Compose project directories ───────────────────────────────────────────────
+
 - name: Create docker compose project directories
   ansible.builtin.file:
     path: "/opt/docker/{{ item }}"

ansible/roles/docker/templates/daemon.json.j2

@@ -0,0 +1,12 @@
+{{ {
+  "log-driver": "loki",
+  "log-opts": {
+    "loki-url": "http://" ~ hostvars['london-a']['ansible_host'] ~ ":3100/loki/api/v1/push",
+    "loki-external-labels": "host=" ~ inventory_hostname ~ ",job=docker",
+    "loki-retries": "5",
+    "loki-batch-size": "400",
+    "loki-timeout": "10s",
+    "mode": "non-blocking",
+    "max-buffer-size": "5m"
+  }
+} | combine(docker_daemon_extra | default({})) | to_nice_json }}