From 53a539dd918c9253651ce436072d1c8bdbbcad99 Mon Sep 17 00:00:00 2001 From: Tibo De Peuter Date: Fri, 17 Jul 2026 22:51:04 +0200 Subject: [PATCH] feat(observability): implement plg stack and ntfy alerting --- modules/apps/default.nix | 1 + modules/apps/monitoring/default.nix | 158 ++++++++++++++++++++++++++++ modules/common/default.nix | 4 + modules/common/monitoring.nix | 47 +++++++++ 4 files changed, 210 insertions(+) create mode 100644 modules/apps/monitoring/default.nix create mode 100644 modules/common/monitoring.nix diff --git a/modules/apps/default.nix b/modules/apps/default.nix index 385f915..88f35d5 100644 --- a/modules/apps/default.nix +++ b/modules/apps/default.nix @@ -8,6 +8,7 @@ ./gitea ./homepage ./jellyfin + ./monitoring ./plex ./solidtime ./speedtest diff --git a/modules/apps/monitoring/default.nix b/modules/apps/monitoring/default.nix new file mode 100644 index 0000000..035ddec --- /dev/null +++ b/modules/apps/monitoring/default.nix @@ -0,0 +1,158 @@ +{ config, lib, pkgs, ... }: + +{ + sops.secrets."grafana/admin_password" = { + sopsFile = ../../../secrets/prod/monitoring.yaml; + }; + sops.secrets."alertmanager/smtp_password" = { + sopsFile = ../../../secrets/prod/monitoring.yaml; + }; + + # 1. Loki Log Storage + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + server.http_listen_port = 3100; + common.ring.instance_addr = "127.0.0.1"; + common.ring.kvstore.store = "inmemory"; + schema_config = { + configs = [{ + from = "2020-10-24"; + store = "boltdb-shipper"; + object_store = "filesystem"; + schema = "v11"; + index = { + prefix = "index_"; + period = "24h"; + }; + }]; + }; + storage_config = { + boltdb_shipper = { + active_index_directory = "/var/lib/loki/boltdb-shipper-active"; + cache_location = "/var/lib/loki/boltdb-shipper-cache"; + }; + filesystem.directory = "/var/lib/loki/chunks"; + }; + }; + }; + + # 2. Prometheus Time-Series DB + services.prometheus = { + enable = true; + port = 9090; + scrapeConfigs = [ + { + job_name = "node"; + scrape_interval = "15s"; + # In a real setup, we would use Prometheus service discovery (e.g., file_sd_configs) + # or list all homelab IPs here. For now, we scrape localhost. + static_configs = [{ + targets = [ "127.0.0.1:9100" ]; + }]; + } + ]; + # Connect Prometheus to Alertmanager + alertmanagers = [{ + static_configs = [{ + targets = [ "127.0.0.1:9093" ]; + }]; + }]; + }; + + # 3. Alertmanager (Routing alerts to NTFY and Email) + services.prometheus.alertmanager = { + enable = true; + port = 9093; + configuration = { + global = { + smtp_smarthost = "smtp.example.com:587"; + smtp_from = "alerts@depeuter.dev"; + smtp_auth_username = "alerts@depeuter.dev"; + smtp_auth_password_file = config.sops.secrets."alertmanager/smtp_password".path; + }; + route = { + receiver = "ntfy-and-email"; + group_wait = "30s"; + group_interval = "5m"; + repeat_interval = "4h"; + group_by = [ "alertname" "instance" ]; + }; + receivers = [{ + name = "ntfy-and-email"; + email_configs = [{ + to = "your-email@example.com"; + # Use smarthost settings defined in global + }]; + webhook_configs = [{ + # Alertmanager natively supports webhooks. We send the JSON to NTFY's Prometheus endpoint. + url = "http://127.0.0.1:2586/alerts"; + }]; + }]; + }; + }; + + # 4. NTFY Push Notification Server + services.ntfy-sh = { + enable = true; + settings = { + base-url = "https://ntfy.lab.depeuter.dev"; + listen-http = ":2586"; + # You can configure auth via the CLI once the service is running, or via auth-file. + # For now, it is open locally. + }; + }; + + # 5. Grafana Visualization + services.grafana = { + enable = true; + settings.server = { + http_port = 3000; + http_addr = "127.0.0.1"; + domain = "grafana.lab.depeuter.dev"; + }; + settings.security.admin_password = "$__file{${config.sops.secrets."grafana/admin_password".path}}"; + + # Declarative Data Sources + provision = { + enable = true; + datasources.settings.datasources = [ + { + name = "Prometheus"; + type = "prometheus"; + access = "proxy"; + url = "http://127.0.0.1:9090"; + isDefault = true; + } + { + name = "Loki"; + type = "loki"; + access = "proxy"; + url = "http://127.0.0.1:3100"; + } + ]; + }; + }; + + # 6. Traefik Reverse Proxy for Grafana and NTFY + services.traefik.dynamicConfigOptions.http = { + routers = { + grafana = { + rule = "Host(`grafana.lab.depeuter.dev`)"; + service = "grafana"; + }; + ntfy = { + rule = "Host(`ntfy.lab.depeuter.dev`)"; + service = "ntfy"; + }; + }; + services = { + grafana.loadBalancer.servers = [{ url = "http://127.0.0.1:3000"; }]; + ntfy.loadBalancer.servers = [{ url = "http://127.0.0.1:2586"; }]; + }; + }; + + # Open firewall for Loki so agents can push logs + networking.firewall.allowedTCPPorts = [ 3100 ]; +} diff --git a/modules/common/default.nix b/modules/common/default.nix index 44309f5..be4b3d3 100644 --- a/modules/common/default.nix +++ b/modules/common/default.nix @@ -1,4 +1,8 @@ { + imports = [ + ./monitoring.nix + ]; + config = { homelab = { services.openssh.enable = true; diff --git a/modules/common/monitoring.nix b/modules/common/monitoring.nix new file mode 100644 index 0000000..6aeac60 --- /dev/null +++ b/modules/common/monitoring.nix @@ -0,0 +1,47 @@ +{ config, lib, pkgs, ... }: + +{ + # Prometheus Node Exporter for hardware metrics + services.prometheus.exporters = { + node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = 9100; + }; + }; + + # Promtail to ship logs to Loki + services.promtail = { + enable = true; + configuration = { + server = { + http_listen_port = 28183; + grpc_listen_port = 0; + }; + positions = { + filename = "/tmp/positions.yaml"; + }; + clients = [{ + # Use the internal DNS name for the Loki ingress + url = "http://loki.lab.depeuter.dev/loki/api/v1/push"; + }]; + scrape_configs = [{ + job_name = "journal"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = config.networking.hostName; + }; + }; + relabel_configs = [{ + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + }]; + }]; + }; + }; + + # Open firewall ports for node-exporter so Prometheus can scrape it + networking.firewall.allowedTCPPorts = [ 9100 ]; +}