fix: loki & alloy

ansible/deploy.yml

@@ -51,6 +51,24 @@
   roles:
     - role: systemd_exporter
 
+# ──────────────────────────────────────────────
+# Stage 3c: Alloy — all hosts (log shipping agent)
+# ──────────────────────────────────────────────
+- name: "Stage 3c: Alloy"
+  hosts: alloy_hosts
+  tags: [monitoring, alloy]
+  roles:
+    - role: alloy
+
+# ──────────────────────────────────────────────
+# Stage 3d: Loki — london-a (log aggregation server)
+# ──────────────────────────────────────────────
+- name: "Stage 3d: Loki"
+  hosts: london-a
+  tags: [monitoring, loki]
+  roles:
+    - role: loki
+
 # ──────────────────────────────────────────────
 # Stage 4: Per-host services
 # ──────────────────────────────────────────────

ansible/inventory/host_vars/london-a.yml

@@ -14,10 +14,12 @@ zfs_pools:
 # 0 12 * * sun zpool scrub zroot
 zfs_scrub_schedule: "0 12 * * 0"
 
+loki_push_url: "http://localhost:3100/loki/api/v1/push"
+
 # --- Services enabled in rc.conf ---
 
 # Core services (documented)
-# sshd, ntpd, powerd, zfs, tailscaled, grafana, prometheus, node_exporter
+# sshd, ntpd, powerd, zfs, tailscaled, grafana, prometheus, node_exporter, loki, alloy
 
 # --- Disabled/removed services ---
 # cloudflared — removed 2026-04-03 (PESO-134). Replaced by Caddy + Authelia.

ansible/inventory/hosts.ini

@@ -33,5 +33,14 @@ copenhagen-a
 [monitoring]
 london-a
 
+[alloy_hosts]
+helsinki-a
+london-b
+london-c
+copenhagen-a
+copenhagen-c
+nuremberg-a
+london-a
+
 [all:vars]
 ansible_user=root

ansible/roles/alloy/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+# Used for Alpine binary download only; Debian uses the Grafana apt repo.
+alloy_version: "1.5.1"
+loki_push_url: "http://{{ hostvars['london-a']['ansible_host'] }}:3100/loki/api/v1/push"

ansible/roles/alloy/handlers/main.yml

@@ -0,0 +1,18 @@
+---
+- name: Restart alloy (Debian)
+  ansible.builtin.service:
+    name: alloy
+    state: restarted
+  listen: "Restart alloy (Debian)"
+
+- name: Restart alloy (Alpine)
+  ansible.builtin.service:
+    name: alloy
+    state: restarted
+  listen: "Restart alloy (Alpine)"
+
+- name: Restart alloy (FreeBSD)
+  ansible.builtin.service:
+    name: alloy
+    state: restarted
+  listen: "Restart alloy (FreeBSD)"

ansible/roles/alloy/tasks/main.yml

@@ -0,0 +1,187 @@
+---
+# Install and configure Grafana Alloy log shipping agent.
+# Debian/Ubuntu: Grafana apt repo + alloy package.
+# Alpine: musl binary download from GitHub + OpenRC init script.
+# FreeBSD: pkgng (grafana-alloy).
+
+# ── Debian/Ubuntu: Grafana apt repo ─────────────────────────────────────────
+
+- name: Create apt keyrings directory (Debian)
+  ansible.builtin.file:
+    path: /etc/apt/keyrings
+    state: directory
+    mode: '0755'
+  when: ansible_facts["os_family"] == "Debian"
+
+- name: Set architecture fact
+  ansible.builtin.set_fact:
+    alloy_arch: >-
+      {{ ansible_facts['architecture']
+         | regex_replace('x86_64', 'amd64')
+         | regex_replace('aarch64', 'arm64') }}
+  when: ansible_facts["os_family"] in ["Debian", "Alpine"]
+
+- name: Add Grafana GPG key (Debian)
+  ansible.builtin.get_url:
+    url: https://apt.grafana.com/gpg.key
+    dest: /etc/apt/keyrings/grafana.gpg
+    mode: '0644'
+    force: false
+  when: ansible_facts["os_family"] == "Debian"
+
+- name: Add Grafana apt repository (Debian)
+  ansible.builtin.apt_repository:
+    repo: >-
+      deb [arch={{ alloy_arch }} signed-by=/etc/apt/keyrings/grafana.gpg]
+      https://apt.grafana.com stable main
+    filename: grafana
+    state: present
+    update_cache: true
+  when: ansible_facts["os_family"] == "Debian"
+
+- name: Install alloy (Debian)
+  ansible.builtin.apt:
+    name: alloy
+    state: present
+  when: ansible_facts["os_family"] == "Debian"
+
+# ── Alpine: binary download (musl build) ─────────────────────────────────────
+
+- name: Create alloy directories (Alpine)
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - /etc/alloy
+    - /var/lib/alloy/data
+  when: ansible_facts["os_family"] == "Alpine"
+
+- name: Check if alloy binary exists (Alpine)
+  ansible.builtin.stat:
+    path: /usr/local/bin/alloy
+  register: alloy_bin
+  when: ansible_facts["os_family"] == "Alpine"
+
+- name: Get installed alloy version (Alpine)
+  ansible.builtin.command: /usr/local/bin/alloy --version
+  register: alloy_installed_version
+  changed_when: false
+  failed_when: false
+  when:
+    - ansible_facts["os_family"] == "Alpine"
+    - alloy_bin.stat.exists
+
+- name: Download and install alloy binary (Alpine)
+  when:
+    - ansible_facts["os_family"] == "Alpine"
+    - not alloy_bin.stat.exists or
+      alloy_version not in (alloy_installed_version.stdout | default(''))
+  block:
+    - name: Download alloy musl zip
+      ansible.builtin.get_url:
+        url: >-
+          https://github.com/grafana/alloy/releases/download/v{{ alloy_version
+          }}/alloy-linux-{{ alloy_arch }}-musl.zip
+        dest: /tmp/alloy.zip
+        mode: '0644'
+
+    - name: Extract alloy binary
+      ansible.builtin.unarchive:
+        src: /tmp/alloy.zip
+        dest: /tmp
+        remote_src: true
+
+    - name: Install alloy binary
+      ansible.builtin.copy:
+        src: "/tmp/alloy-linux-{{ alloy_arch }}-musl"
+        dest: /usr/local/bin/alloy
+        mode: '0755'
+        owner: root
+        group: root
+        remote_src: true
+      notify: Restart alloy (Alpine)
+
+    - name: Clean up alloy download
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - /tmp/alloy.zip
+        - "/tmp/alloy-linux-{{ alloy_arch }}-musl"
+
+- name: Deploy alloy OpenRC init script (Alpine)
+  ansible.builtin.template:
+    src: alloy_openrc.j2
+    dest: /etc/init.d/alloy
+    mode: '0755'
+  when: ansible_facts["os_family"] == "Alpine"
+  notify: Restart alloy (Alpine)
+
+# ── FreeBSD: pkgng ────────────────────────────────────────────────────────────
+
+- name: Install alloy (FreeBSD)
+  community.general.pkgng:
+    name: alloy
+    state: present
+  when: ansible_facts["os_family"] == "FreeBSD"
+
+- name: Create alloy directories (FreeBSD)
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - /usr/local/etc/alloy
+    - /var/db/alloy
+  when: ansible_facts["os_family"] == "FreeBSD"
+
+# ── Config — all OS ───────────────────────────────────────────────────────────
+
+- name: Set alloy config path fact
+  ansible.builtin.set_fact:
+    alloy_config_path: >-
+      {{ '/usr/local/etc/alloy/config.alloy'
+         if ansible_facts['os_family'] == 'FreeBSD'
+         else '/etc/alloy/config.alloy' }}
+
+- name: Deploy alloy config
+  ansible.builtin.template:
+    src: alloy.config.alloy.j2
+    dest: "{{ alloy_config_path }}"
+    mode: '0644'
+  notify: "Restart alloy ({{ ansible_facts['os_family'] }})"
+
+# ── Service enable + start ────────────────────────────────────────────────────
+
+- name: Enable and start alloy (Debian)
+  ansible.builtin.service:
+    name: alloy
+    state: started
+    enabled: true
+  when: ansible_facts["os_family"] == "Debian"
+
+- name: Enable and start alloy (Alpine)
+  ansible.builtin.service:
+    name: alloy
+    state: started
+    enabled: true
+  when: ansible_facts["os_family"] == "Alpine"
+
+- name: Enable alloy (FreeBSD)
+  community.general.sysrc:
+    name: alloy_enable
+    value: "YES"
+  when: ansible_facts["os_family"] == "FreeBSD"
+
+- name: Set alloy config in rc.conf (FreeBSD)
+  community.general.sysrc:
+    name: alloy_config
+    value: /usr/local/etc/alloy/config.alloy
+  when: ansible_facts["os_family"] == "FreeBSD"
+
+- name: Start alloy (FreeBSD)
+  ansible.builtin.service:
+    name: alloy
+    state: started
+  when: ansible_facts["os_family"] == "FreeBSD"

ansible/roles/alloy/templates/alloy.config.alloy.j2

@@ -0,0 +1,95 @@
+// Ansible managed — generated from alloy.config.alloy.j2
+// Grafana Alloy log shipping agent — {{ inventory_hostname }}
+
+// ─── System logs ─────────────────────────────────────────────────────────────
+
+{% if ansible_facts['os_family'] == 'Debian' %}
+local.file_match "system" {
+  path_targets = [
+    {"__path__" = "/var/log/syslog",   "job" = "syslog", "host" = "{{ inventory_hostname }}"},
+    {"__path__" = "/var/log/auth.log", "job" = "auth",   "host" = "{{ inventory_hostname }}"},
+    {"__path__" = "/var/log/kern.log", "job" = "kern",   "host" = "{{ inventory_hostname }}"},
+  ]
+}
+{% elif ansible_facts['os_family'] == 'Alpine' %}
+local.file_match "system" {
+  path_targets = [
+    {"__path__" = "/var/log/messages", "job" = "messages", "host" = "{{ inventory_hostname }}"},
+  ]
+}
+{% elif ansible_facts['os_family'] == 'FreeBSD' %}
+local.file_match "system" {
+  path_targets = [
+    {"__path__" = "/var/log/messages",  "job" = "syslog", "host" = "{{ inventory_hostname }}"},
+    {"__path__" = "/var/log/auth.log",  "job" = "auth",   "host" = "{{ inventory_hostname }}"},
+  ]
+}
+{% endif %}
+
+loki.source.file "system" {
+  targets    = local.file_match.system.targets
+  forward_to = [loki.write.default.receiver]
+}
+
+{% if 'docker_hosts' in group_names %}
+// ─── Docker container logs ────────────────────────────────────────────────────
+
+discovery.docker "containers" {
+  host             = "unix:///var/run/docker.sock"
+  refresh_interval = "15s"
+}
+
+discovery.relabel "docker_containers" {
+  targets = discovery.docker.containers.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_state"]
+    action        = "keep"
+    regex         = "running"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "compose_service"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
+    target_label  = "compose_project"
+  }
+}
+
+loki.source.docker "containers" {
+  host       = "unix:///var/run/docker.sock"
+  targets    = discovery.relabel.docker_containers.output
+  forward_to = [loki.write.default.receiver]
+  labels     = {"host" = "{{ inventory_hostname }}"}
+}
+{% endif %}
+
+{% if inventory_hostname == 'london-b' %}
+// ─── london-b app logs ────────────────────────────────────────────────────────
+
+local.file_match "apps" {
+  path_targets = [
+    {"__path__" = "/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Logs/*.log", "job" = "plex",     "host" = "london-b"},
+    {"__path__" = "/var/log/jellyfin/*.log",                                                            "job" = "jellyfin", "host" = "london-b"},
+  ]
+}
+
+loki.source.file "apps" {
+  targets    = local.file_match.apps.targets
+  forward_to = [loki.write.default.receiver]
+}
+{% endif %}
+
+// ─── Loki output ──────────────────────────────────────────────────────────────
+
+loki.write "default" {
+  endpoint {
+    url = "{{ loki_push_url }}"
+  }
+}

ansible/roles/alloy/templates/alloy_openrc.j2

@@ -0,0 +1,14 @@
+#!/sbin/openrc-run
+# Ansible managed
+
+name="alloy"
+description="Grafana Alloy log shipping agent"
+command="/usr/local/bin/alloy"
+command_args="run --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy"
+command_background=true
+pidfile="/run/${RC_SVCNAME}.pid"
+
+depend() {
+    need net
+    use logger
+}

ansible/roles/loki/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+loki_http_listen_port: 3100
+loki_grpc_listen_port: 9096
+loki_data_dir: /var/db/loki
+loki_retention_period: 720h
+loki_ingestion_rate_mb: 4
+loki_ingestion_burst_size_mb: 6

ansible/roles/loki/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Restart loki
+  ansible.builtin.service:
+    name: loki
+    state: restarted

ansible/roles/loki/tasks/main.yml

@@ -0,0 +1,54 @@
+---
+# Install and configure Grafana Loki on FreeBSD (london-a).
+# Co-located with Prometheus and Grafana; all three run as native FreeBSD services.
+# FreeBSD only — Loki is the log aggregation backend for Promtail on all hosts.
+
+- name: Install loki (FreeBSD)
+  community.general.pkgng:
+    name: grafana-loki
+    state: present
+  when: ansible_facts["os_family"] == "FreeBSD"
+
+- name: Ensure Loki data directory exists
+  ansible.builtin.file:
+    path: "{{ loki_data_dir }}"
+    state: directory
+    mode: '0755'
+    owner: loki
+    group: loki
+  when: ansible_facts["os_family"] == "FreeBSD"
+
+- name: Ensure Loki config directory exists
+  ansible.builtin.file:
+    path: /usr/local/etc/loki
+    state: directory
+    mode: '0755'
+  when: ansible_facts["os_family"] == "FreeBSD"
+
+- name: Deploy Loki config
+  ansible.builtin.template:
+    src: loki.yml.j2
+    dest: /usr/local/etc/loki/config.yml
+    mode: '0644'
+    owner: root
+    group: wheel
+  when: ansible_facts["os_family"] == "FreeBSD"
+  notify: Restart loki
+
+- name: Enable loki (FreeBSD)
+  community.general.sysrc:
+    name: loki_enable
+    value: "YES"
+  when: ansible_facts["os_family"] == "FreeBSD"
+
+- name: Set loki config path in rc.conf (FreeBSD)
+  community.general.sysrc:
+    name: loki_config
+    value: /usr/local/etc/loki/config.yml
+  when: ansible_facts["os_family"] == "FreeBSD"
+
+- name: Start loki (FreeBSD)
+  ansible.builtin.service:
+    name: loki
+    state: started
+  when: ansible_facts["os_family"] == "FreeBSD"

ansible/roles/loki/templates/loki.yml.j2

@@ -0,0 +1,51 @@
+# Ansible managed — generated from loki.yml.j2
+# Grafana Loki — london-a (FreeBSD)
+# Single-node, filesystem storage, {{ loki_retention_period }} retention
+
+auth_enabled: false
+
+server:
+  http_listen_port: {{ loki_http_listen_port }}
+  grpc_listen_port: {{ loki_grpc_listen_port }}
+  log_level: info
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: {{ loki_data_dir }}
+  storage:
+    filesystem:
+      chunks_directory: {{ loki_data_dir }}/chunks
+      rules_directory: {{ loki_data_dir }}/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  retention_period: {{ loki_retention_period }}
+  ingestion_rate_mb: {{ loki_ingestion_rate_mb }}
+  ingestion_burst_size_mb: {{ loki_ingestion_burst_size_mb }}
+
+compactor:
+  working_directory: {{ loki_data_dir }}/compactor
+  compaction_interval: 10m
+  retention_enabled: true
+  retention_delete_delay: 2h
+  delete_request_store: filesystem
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100

ansible/roles/media_stack/handlers/main.yml

@@ -3,11 +3,6 @@
   ansible.builtin.systemd:
     daemon_reload: true
 
-- name: Restart promtail
-  ansible.builtin.systemd:
-    name: promtail
-    state: restarted
-
 - name: Restart smbd
   ansible.builtin.systemd:
     name: smbd

ansible/roles/media_stack/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 # media_stack role — deploys the full media stack on london-b
 # Manages: *arr suite, jellyfin, plex, transmission, samba,
-#           ollama, promtail, vsftpd, and cron jobs.
+#           ollama, vsftpd, and cron jobs.
 
 # ── Systemd service units (custom, not package-managed) ──
 
@@ -17,7 +17,6 @@
     - readarr
     - whisparr
     - ollama
-    - promtail
   notify: Reload systemd daemon
 
 - name: Enable and start custom systemd services
@@ -31,7 +30,6 @@
     - lidarr
     - readarr
     - ollama
-    - promtail
 
 # Whisparr is installed but disabled (kept as-is)
 - name: Ensure whisparr unit is present but disabled
@@ -82,13 +80,6 @@
 
 # ── Configuration files ──
 
-- name: Deploy promtail config
-  ansible.builtin.copy:
-    src: "{{ playbook_dir }}/services/promtail/config/london-b.yml"
-    dest: /etc/promtail/config.yml
-    mode: '0644'
-  notify: Restart promtail
-
 - name: Deploy samba config
   ansible.builtin.copy:
     src: "{{ playbook_dir }}/services/samba/config/london-b.conf"

ansible/services/grafana/provisioning/datasources/datasources.json

@@ -14,5 +14,17 @@
     "jsonData": {
       "pdcInjected": false
     }
+  },
+  {
+    "uid": "loki_london_a",
+    "name": "Loki",
+    "type": "loki",
+    "access": "proxy",
+    "url": "http://localhost:3100",
+    "basicAuth": false,
+    "isDefault": false,
+    "jsonData": {
+      "maxLines": 1000
+    }
   }
 ]