{ config, lib, pkgs, ... }: let cfg = config.homelab.apps.monitoring; in { options.homelab.apps.monitoring.enable = lib.mkEnableOption "Homelab Monitoring Stack"; config = lib.mkIf cfg.enable { sops.secrets."grafana/admin_password" = { sopsFile = ../../../../secrets/prod/monitoring.yaml; }; sops.secrets."alertmanager/smtp_password" = { sopsFile = ../../../../secrets/prod/monitoring.yaml; }; # 1. Loki Log Storage services.loki = { enable = true; configuration = { auth_enabled = false; server.http_listen_port = 3100; common.ring.instance_addr = "127.0.0.1"; common.ring.kvstore.store = "inmemory"; schema_config = { configs = [{ from = "2020-10-24"; store = "boltdb-shipper"; object_store = "filesystem"; schema = "v11"; index = { prefix = "index_"; period = "24h"; }; }]; }; storage_config = { boltdb_shipper = { active_index_directory = "/var/lib/loki/boltdb-shipper-active"; cache_location = "/var/lib/loki/boltdb-shipper-cache"; }; filesystem.directory = "/var/lib/loki/chunks"; }; }; }; # 2. Prometheus Time-Series DB services.prometheus = { enable = true; port = 9090; scrapeConfigs = [ { job_name = "node"; scrape_interval = "15s"; # In a real setup, we would use Prometheus service discovery (e.g., file_sd_configs) # or list all homelab IPs here. For now, we scrape localhost. static_configs = [{ targets = [ "127.0.0.1:9100" ]; }]; } ]; # Connect Prometheus to Alertmanager alertmanagers = [{ static_configs = [{ targets = [ "127.0.0.1:9093" ]; }]; }]; }; # 3. Alertmanager (Routing alerts to NTFY and Email) services.prometheus.alertmanager = { enable = true; port = 9093; configuration = { global = { smtp_smarthost = "smtp.example.com:587"; smtp_from = "alerts@depeuter.dev"; smtp_auth_username = "alerts@depeuter.dev"; smtp_auth_password_file = config.sops.secrets."alertmanager/smtp_password".path; }; route = { receiver = "ntfy-and-email"; group_wait = "30s"; group_interval = "5m"; repeat_interval = "4h"; group_by = [ "alertname" "instance" ]; }; receivers = [{ name = "ntfy-and-email"; email_configs = [{ to = "your-email@example.com"; # Use smarthost settings defined in global }]; webhook_configs = [{ # Alertmanager natively supports webhooks. We send the JSON to NTFY's Prometheus endpoint. url = "http://127.0.0.1:2586/alerts"; }]; }]; }; }; # 4. NTFY Push Notification Server services.ntfy-sh = { enable = true; settings = { base-url = "https://ntfy.lab.depeuter.dev"; listen-http = ":2586"; # You can configure auth via the CLI once the service is running, or via auth-file. # For now, it is open locally. }; }; # 5. Grafana Visualization services.grafana = { enable = true; settings.server = { http_port = 3000; http_addr = "127.0.0.1"; domain = "grafana.lab.depeuter.dev"; }; settings.security.admin_password = "$__file{${config.sops.secrets."grafana/admin_password".path}}"; # Declarative Data Sources provision = { enable = true; datasources.settings.datasources = [ { name = "Prometheus"; type = "prometheus"; access = "proxy"; url = "http://127.0.0.1:9090"; isDefault = true; } { name = "Loki"; type = "loki"; access = "proxy"; url = "http://127.0.0.1:3100"; } ]; }; }; # 6. Traefik Reverse Proxy for Grafana and NTFY homelab.apps.traefik.dynamicConfigOptions.http = { routers = { grafana = { rule = "Host(`grafana.lab.depeuter.dev`)"; service = "grafana"; }; ntfy = { rule = "Host(`ntfy.lab.depeuter.dev`)"; service = "ntfy"; }; }; services = { grafana.loadBalancer.servers = [{ url = "http://host.docker.internal:3000"; }]; ntfy.loadBalancer.servers = [{ url = "http://host.docker.internal:2586"; }]; }; }; # Open firewall for Loki so agents can push logs networking.firewall.allowedTCPPorts = [ 3100 ]; }; }