From fe31d38028edc0719429f85e21131bf9605bb4a7 Mon Sep 17 00:00:00 2001 From: Rasmus Wejlgaard Date: Tue, 28 Apr 2026 16:34:52 +0100 Subject: [PATCH] fix: loki & alloy --- ansible/deploy.yml | 18 ++ ansible/inventory/host_vars/london-a.yml | 4 +- ansible/inventory/hosts.ini | 9 + ansible/roles/alloy/defaults/main.yml | 4 + ansible/roles/alloy/handlers/main.yml | 18 ++ ansible/roles/alloy/tasks/main.yml | 187 ++++++++++++++++++ .../alloy/templates/alloy.config.alloy.j2 | 95 +++++++++ ansible/roles/alloy/templates/alloy_openrc.j2 | 14 ++ ansible/roles/loki/defaults/main.yml | 7 + ansible/roles/loki/handlers/main.yml | 5 + ansible/roles/loki/tasks/main.yml | 54 +++++ ansible/roles/loki/templates/loki.yml.j2 | 51 +++++ ansible/roles/media_stack/handlers/main.yml | 5 - ansible/roles/media_stack/tasks/main.yml | 11 +- .../provisioning/datasources/datasources.json | 12 ++ 15 files changed, 478 insertions(+), 16 deletions(-) create mode 100644 ansible/roles/alloy/defaults/main.yml create mode 100644 ansible/roles/alloy/handlers/main.yml create mode 100644 ansible/roles/alloy/tasks/main.yml create mode 100644 ansible/roles/alloy/templates/alloy.config.alloy.j2 create mode 100644 ansible/roles/alloy/templates/alloy_openrc.j2 create mode 100644 ansible/roles/loki/defaults/main.yml create mode 100644 ansible/roles/loki/handlers/main.yml create mode 100644 ansible/roles/loki/tasks/main.yml create mode 100644 ansible/roles/loki/templates/loki.yml.j2 diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 6bc56f3..f348246 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -51,6 +51,24 @@ roles: - role: systemd_exporter +# ────────────────────────────────────────────── +# Stage 3c: Alloy — all hosts (log shipping agent) +# ────────────────────────────────────────────── +- name: "Stage 3c: Alloy" + hosts: alloy_hosts + tags: [monitoring, alloy] + roles: + - role: alloy + +# ────────────────────────────────────────────── +# Stage 3d: Loki — london-a (log aggregation server) +# ────────────────────────────────────────────── +- name: "Stage 3d: Loki" + hosts: london-a + tags: [monitoring, loki] + roles: + - role: loki + # ────────────────────────────────────────────── # Stage 4: Per-host services # ────────────────────────────────────────────── diff --git a/ansible/inventory/host_vars/london-a.yml b/ansible/inventory/host_vars/london-a.yml index fcfa20a..093a339 100644 --- a/ansible/inventory/host_vars/london-a.yml +++ b/ansible/inventory/host_vars/london-a.yml @@ -14,10 +14,12 @@ zfs_pools: # 0 12 * * sun zpool scrub zroot zfs_scrub_schedule: "0 12 * * 0" +loki_push_url: "http://localhost:3100/loki/api/v1/push" + # --- Services enabled in rc.conf --- # Core services (documented) -# sshd, ntpd, powerd, zfs, tailscaled, grafana, prometheus, node_exporter +# sshd, ntpd, powerd, zfs, tailscaled, grafana, prometheus, node_exporter, loki, alloy # --- Disabled/removed services --- # cloudflared — removed 2026-04-03 (PESO-134). Replaced by Caddy + Authelia. diff --git a/ansible/inventory/hosts.ini b/ansible/inventory/hosts.ini index 655220b..2ff0d2d 100644 --- a/ansible/inventory/hosts.ini +++ b/ansible/inventory/hosts.ini @@ -33,5 +33,14 @@ copenhagen-a [monitoring] london-a +[alloy_hosts] +helsinki-a +london-b +london-c +copenhagen-a +copenhagen-c +nuremberg-a +london-a + [all:vars] ansible_user=root diff --git a/ansible/roles/alloy/defaults/main.yml b/ansible/roles/alloy/defaults/main.yml new file mode 100644 index 0000000..f4b6e30 --- /dev/null +++ b/ansible/roles/alloy/defaults/main.yml @@ -0,0 +1,4 @@ +--- +# Used for Alpine binary download only; Debian uses the Grafana apt repo. +alloy_version: "1.5.1" +loki_push_url: "http://{{ hostvars['london-a']['ansible_host'] }}:3100/loki/api/v1/push" diff --git a/ansible/roles/alloy/handlers/main.yml b/ansible/roles/alloy/handlers/main.yml new file mode 100644 index 0000000..725a826 --- /dev/null +++ b/ansible/roles/alloy/handlers/main.yml @@ -0,0 +1,18 @@ +--- +- name: Restart alloy (Debian) + ansible.builtin.service: + name: alloy + state: restarted + listen: "Restart alloy (Debian)" + +- name: Restart alloy (Alpine) + ansible.builtin.service: + name: alloy + state: restarted + listen: "Restart alloy (Alpine)" + +- name: Restart alloy (FreeBSD) + ansible.builtin.service: + name: alloy + state: restarted + listen: "Restart alloy (FreeBSD)" diff --git a/ansible/roles/alloy/tasks/main.yml b/ansible/roles/alloy/tasks/main.yml new file mode 100644 index 0000000..f5a3236 --- /dev/null +++ b/ansible/roles/alloy/tasks/main.yml @@ -0,0 +1,187 @@ +--- +# Install and configure Grafana Alloy log shipping agent. +# Debian/Ubuntu: Grafana apt repo + alloy package. +# Alpine: musl binary download from GitHub + OpenRC init script. +# FreeBSD: pkgng (grafana-alloy). + +# ── Debian/Ubuntu: Grafana apt repo ───────────────────────────────────────── + +- name: Create apt keyrings directory (Debian) + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + when: ansible_facts["os_family"] == "Debian" + +- name: Set architecture fact + ansible.builtin.set_fact: + alloy_arch: >- + {{ ansible_facts['architecture'] + | regex_replace('x86_64', 'amd64') + | regex_replace('aarch64', 'arm64') }} + when: ansible_facts["os_family"] in ["Debian", "Alpine"] + +- name: Add Grafana GPG key (Debian) + ansible.builtin.get_url: + url: https://apt.grafana.com/gpg.key + dest: /etc/apt/keyrings/grafana.gpg + mode: '0644' + force: false + when: ansible_facts["os_family"] == "Debian" + +- name: Add Grafana apt repository (Debian) + ansible.builtin.apt_repository: + repo: >- + deb [arch={{ alloy_arch }} signed-by=/etc/apt/keyrings/grafana.gpg] + https://apt.grafana.com stable main + filename: grafana + state: present + update_cache: true + when: ansible_facts["os_family"] == "Debian" + +- name: Install alloy (Debian) + ansible.builtin.apt: + name: alloy + state: present + when: ansible_facts["os_family"] == "Debian" + +# ── Alpine: binary download (musl build) ───────────────────────────────────── + +- name: Create alloy directories (Alpine) + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: '0755' + loop: + - /etc/alloy + - /var/lib/alloy/data + when: ansible_facts["os_family"] == "Alpine" + +- name: Check if alloy binary exists (Alpine) + ansible.builtin.stat: + path: /usr/local/bin/alloy + register: alloy_bin + when: ansible_facts["os_family"] == "Alpine" + +- name: Get installed alloy version (Alpine) + ansible.builtin.command: /usr/local/bin/alloy --version + register: alloy_installed_version + changed_when: false + failed_when: false + when: + - ansible_facts["os_family"] == "Alpine" + - alloy_bin.stat.exists + +- name: Download and install alloy binary (Alpine) + when: + - ansible_facts["os_family"] == "Alpine" + - not alloy_bin.stat.exists or + alloy_version not in (alloy_installed_version.stdout | default('')) + block: + - name: Download alloy musl zip + ansible.builtin.get_url: + url: >- + https://github.com/grafana/alloy/releases/download/v{{ alloy_version + }}/alloy-linux-{{ alloy_arch }}-musl.zip + dest: /tmp/alloy.zip + mode: '0644' + + - name: Extract alloy binary + ansible.builtin.unarchive: + src: /tmp/alloy.zip + dest: /tmp + remote_src: true + + - name: Install alloy binary + ansible.builtin.copy: + src: "/tmp/alloy-linux-{{ alloy_arch }}-musl" + dest: /usr/local/bin/alloy + mode: '0755' + owner: root + group: root + remote_src: true + notify: Restart alloy (Alpine) + + - name: Clean up alloy download + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /tmp/alloy.zip + - "/tmp/alloy-linux-{{ alloy_arch }}-musl" + +- name: Deploy alloy OpenRC init script (Alpine) + ansible.builtin.template: + src: alloy_openrc.j2 + dest: /etc/init.d/alloy + mode: '0755' + when: ansible_facts["os_family"] == "Alpine" + notify: Restart alloy (Alpine) + +# ── FreeBSD: pkgng ──────────────────────────────────────────────────────────── + +- name: Install alloy (FreeBSD) + community.general.pkgng: + name: alloy + state: present + when: ansible_facts["os_family"] == "FreeBSD" + +- name: Create alloy directories (FreeBSD) + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: '0755' + loop: + - /usr/local/etc/alloy + - /var/db/alloy + when: ansible_facts["os_family"] == "FreeBSD" + +# ── Config — all OS ─────────────────────────────────────────────────────────── + +- name: Set alloy config path fact + ansible.builtin.set_fact: + alloy_config_path: >- + {{ '/usr/local/etc/alloy/config.alloy' + if ansible_facts['os_family'] == 'FreeBSD' + else '/etc/alloy/config.alloy' }} + +- name: Deploy alloy config + ansible.builtin.template: + src: alloy.config.alloy.j2 + dest: "{{ alloy_config_path }}" + mode: '0644' + notify: "Restart alloy ({{ ansible_facts['os_family'] }})" + +# ── Service enable + start ──────────────────────────────────────────────────── + +- name: Enable and start alloy (Debian) + ansible.builtin.service: + name: alloy + state: started + enabled: true + when: ansible_facts["os_family"] == "Debian" + +- name: Enable and start alloy (Alpine) + ansible.builtin.service: + name: alloy + state: started + enabled: true + when: ansible_facts["os_family"] == "Alpine" + +- name: Enable alloy (FreeBSD) + community.general.sysrc: + name: alloy_enable + value: "YES" + when: ansible_facts["os_family"] == "FreeBSD" + +- name: Set alloy config in rc.conf (FreeBSD) + community.general.sysrc: + name: alloy_config + value: /usr/local/etc/alloy/config.alloy + when: ansible_facts["os_family"] == "FreeBSD" + +- name: Start alloy (FreeBSD) + ansible.builtin.service: + name: alloy + state: started + when: ansible_facts["os_family"] == "FreeBSD" diff --git a/ansible/roles/alloy/templates/alloy.config.alloy.j2 b/ansible/roles/alloy/templates/alloy.config.alloy.j2 new file mode 100644 index 0000000..2457d19 --- /dev/null +++ b/ansible/roles/alloy/templates/alloy.config.alloy.j2 @@ -0,0 +1,95 @@ +// Ansible managed — generated from alloy.config.alloy.j2 +// Grafana Alloy log shipping agent — {{ inventory_hostname }} + +// ─── System logs ───────────────────────────────────────────────────────────── + +{% if ansible_facts['os_family'] == 'Debian' %} +local.file_match "system" { + path_targets = [ + {"__path__" = "/var/log/syslog", "job" = "syslog", "host" = "{{ inventory_hostname }}"}, + {"__path__" = "/var/log/auth.log", "job" = "auth", "host" = "{{ inventory_hostname }}"}, + {"__path__" = "/var/log/kern.log", "job" = "kern", "host" = "{{ inventory_hostname }}"}, + ] +} +{% elif ansible_facts['os_family'] == 'Alpine' %} +local.file_match "system" { + path_targets = [ + {"__path__" = "/var/log/messages", "job" = "messages", "host" = "{{ inventory_hostname }}"}, + ] +} +{% elif ansible_facts['os_family'] == 'FreeBSD' %} +local.file_match "system" { + path_targets = [ + {"__path__" = "/var/log/messages", "job" = "syslog", "host" = "{{ inventory_hostname }}"}, + {"__path__" = "/var/log/auth.log", "job" = "auth", "host" = "{{ inventory_hostname }}"}, + ] +} +{% endif %} + +loki.source.file "system" { + targets = local.file_match.system.targets + forward_to = [loki.write.default.receiver] +} + +{% if 'docker_hosts' in group_names %} +// ─── Docker container logs ──────────────────────────────────────────────────── + +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" + refresh_interval = "15s" +} + +discovery.relabel "docker_containers" { + targets = discovery.docker.containers.targets + + rule { + source_labels = ["__meta_docker_container_state"] + action = "keep" + regex = "running" + } + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "compose_service" + } + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_project"] + target_label = "compose_project" + } +} + +loki.source.docker "containers" { + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.docker_containers.output + forward_to = [loki.write.default.receiver] + labels = {"host" = "{{ inventory_hostname }}"} +} +{% endif %} + +{% if inventory_hostname == 'london-b' %} +// ─── london-b app logs ──────────────────────────────────────────────────────── + +local.file_match "apps" { + path_targets = [ + {"__path__" = "/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Logs/*.log", "job" = "plex", "host" = "london-b"}, + {"__path__" = "/var/log/jellyfin/*.log", "job" = "jellyfin", "host" = "london-b"}, + ] +} + +loki.source.file "apps" { + targets = local.file_match.apps.targets + forward_to = [loki.write.default.receiver] +} +{% endif %} + +// ─── Loki output ────────────────────────────────────────────────────────────── + +loki.write "default" { + endpoint { + url = "{{ loki_push_url }}" + } +} diff --git a/ansible/roles/alloy/templates/alloy_openrc.j2 b/ansible/roles/alloy/templates/alloy_openrc.j2 new file mode 100644 index 0000000..4496cde --- /dev/null +++ b/ansible/roles/alloy/templates/alloy_openrc.j2 @@ -0,0 +1,14 @@ +#!/sbin/openrc-run +# Ansible managed + +name="alloy" +description="Grafana Alloy log shipping agent" +command="/usr/local/bin/alloy" +command_args="run --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy" +command_background=true +pidfile="/run/${RC_SVCNAME}.pid" + +depend() { + need net + use logger +} diff --git a/ansible/roles/loki/defaults/main.yml b/ansible/roles/loki/defaults/main.yml new file mode 100644 index 0000000..1baf18c --- /dev/null +++ b/ansible/roles/loki/defaults/main.yml @@ -0,0 +1,7 @@ +--- +loki_http_listen_port: 3100 +loki_grpc_listen_port: 9096 +loki_data_dir: /var/db/loki +loki_retention_period: 720h +loki_ingestion_rate_mb: 4 +loki_ingestion_burst_size_mb: 6 diff --git a/ansible/roles/loki/handlers/main.yml b/ansible/roles/loki/handlers/main.yml new file mode 100644 index 0000000..e11cf74 --- /dev/null +++ b/ansible/roles/loki/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart loki + ansible.builtin.service: + name: loki + state: restarted diff --git a/ansible/roles/loki/tasks/main.yml b/ansible/roles/loki/tasks/main.yml new file mode 100644 index 0000000..9f0d75c --- /dev/null +++ b/ansible/roles/loki/tasks/main.yml @@ -0,0 +1,54 @@ +--- +# Install and configure Grafana Loki on FreeBSD (london-a). +# Co-located with Prometheus and Grafana; all three run as native FreeBSD services. +# FreeBSD only — Loki is the log aggregation backend for Promtail on all hosts. + +- name: Install loki (FreeBSD) + community.general.pkgng: + name: grafana-loki + state: present + when: ansible_facts["os_family"] == "FreeBSD" + +- name: Ensure Loki data directory exists + ansible.builtin.file: + path: "{{ loki_data_dir }}" + state: directory + mode: '0755' + owner: loki + group: loki + when: ansible_facts["os_family"] == "FreeBSD" + +- name: Ensure Loki config directory exists + ansible.builtin.file: + path: /usr/local/etc/loki + state: directory + mode: '0755' + when: ansible_facts["os_family"] == "FreeBSD" + +- name: Deploy Loki config + ansible.builtin.template: + src: loki.yml.j2 + dest: /usr/local/etc/loki/config.yml + mode: '0644' + owner: root + group: wheel + when: ansible_facts["os_family"] == "FreeBSD" + notify: Restart loki + +- name: Enable loki (FreeBSD) + community.general.sysrc: + name: loki_enable + value: "YES" + when: ansible_facts["os_family"] == "FreeBSD" + +- name: Set loki config path in rc.conf (FreeBSD) + community.general.sysrc: + name: loki_config + value: /usr/local/etc/loki/config.yml + when: ansible_facts["os_family"] == "FreeBSD" + +- name: Start loki (FreeBSD) + ansible.builtin.service: + name: loki + state: started + when: ansible_facts["os_family"] == "FreeBSD" diff --git a/ansible/roles/loki/templates/loki.yml.j2 b/ansible/roles/loki/templates/loki.yml.j2 new file mode 100644 index 0000000..a369fbe --- /dev/null +++ b/ansible/roles/loki/templates/loki.yml.j2 @@ -0,0 +1,51 @@ +# Ansible managed — generated from loki.yml.j2 +# Grafana Loki — london-a (FreeBSD) +# Single-node, filesystem storage, {{ loki_retention_period }} retention + +auth_enabled: false + +server: + http_listen_port: {{ loki_http_listen_port }} + grpc_listen_port: {{ loki_grpc_listen_port }} + log_level: info + +common: + instance_addr: 127.0.0.1 + path_prefix: {{ loki_data_dir }} + storage: + filesystem: + chunks_directory: {{ loki_data_dir }}/chunks + rules_directory: {{ loki_data_dir }}/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: {{ loki_retention_period }} + ingestion_rate_mb: {{ loki_ingestion_rate_mb }} + ingestion_burst_size_mb: {{ loki_ingestion_burst_size_mb }} + +compactor: + working_directory: {{ loki_data_dir }}/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + delete_request_store: filesystem + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 diff --git a/ansible/roles/media_stack/handlers/main.yml b/ansible/roles/media_stack/handlers/main.yml index 94caced..1f817e6 100644 --- a/ansible/roles/media_stack/handlers/main.yml +++ b/ansible/roles/media_stack/handlers/main.yml @@ -3,11 +3,6 @@ ansible.builtin.systemd: daemon_reload: true -- name: Restart promtail - ansible.builtin.systemd: - name: promtail - state: restarted - - name: Restart smbd ansible.builtin.systemd: name: smbd diff --git a/ansible/roles/media_stack/tasks/main.yml b/ansible/roles/media_stack/tasks/main.yml index f9b25fe..1a2ca57 100644 --- a/ansible/roles/media_stack/tasks/main.yml +++ b/ansible/roles/media_stack/tasks/main.yml @@ -1,7 +1,7 @@ --- # media_stack role — deploys the full media stack on london-b # Manages: *arr suite, jellyfin, plex, transmission, samba, -# ollama, promtail, vsftpd, and cron jobs. +# ollama, vsftpd, and cron jobs. # ── Systemd service units (custom, not package-managed) ── @@ -17,7 +17,6 @@ - readarr - whisparr - ollama - - promtail notify: Reload systemd daemon - name: Enable and start custom systemd services @@ -31,7 +30,6 @@ - lidarr - readarr - ollama - - promtail # Whisparr is installed but disabled (kept as-is) - name: Ensure whisparr unit is present but disabled @@ -82,13 +80,6 @@ # ── Configuration files ── -- name: Deploy promtail config - ansible.builtin.copy: - src: "{{ playbook_dir }}/services/promtail/config/london-b.yml" - dest: /etc/promtail/config.yml - mode: '0644' - notify: Restart promtail - - name: Deploy samba config ansible.builtin.copy: src: "{{ playbook_dir }}/services/samba/config/london-b.conf" diff --git a/ansible/services/grafana/provisioning/datasources/datasources.json b/ansible/services/grafana/provisioning/datasources/datasources.json index f7c5bf4..bed8923 100644 --- a/ansible/services/grafana/provisioning/datasources/datasources.json +++ b/ansible/services/grafana/provisioning/datasources/datasources.json @@ -14,5 +14,17 @@ "jsonData": { "pdcInjected": false } + }, + { + "uid": "loki_london_a", + "name": "Loki", + "type": "loki", + "access": "proxy", + "url": "http://localhost:3100", + "basicAuth": false, + "isDefault": false, + "jsonData": { + "maxLines": 1000 + } } ]