feat(observability): implement plg stack and ntfy alerting

modules/apps/default.nix

@@ -8,6 +8,7 @@
     ./gitea
     ./homepage
     ./jellyfin
+    ./monitoring
     ./plex
     ./solidtime
     ./speedtest

modules/apps/monitoring/default.nix

@@ -0,0 +1,158 @@
+{ config, lib, pkgs, ... }:
+
+{
+  sops.secrets."grafana/admin_password" = {
+    sopsFile = ../../../secrets/prod/monitoring.yaml;
+  };
+  sops.secrets."alertmanager/smtp_password" = {
+    sopsFile = ../../../secrets/prod/monitoring.yaml;
+  };
+
+  # 1. Loki Log Storage
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+      server.http_listen_port = 3100;
+      common.ring.instance_addr = "127.0.0.1";
+      common.ring.kvstore.store = "inmemory";
+      schema_config = {
+        configs = [{
+          from = "2020-10-24";
+          store = "boltdb-shipper";
+          object_store = "filesystem";
+          schema = "v11";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }];
+      };
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+        };
+        filesystem.directory = "/var/lib/loki/chunks";
+      };
+    };
+  };
+
+  # 2. Prometheus Time-Series DB
+  services.prometheus = {
+    enable = true;
+    port = 9090;
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        scrape_interval = "15s";
+        # In a real setup, we would use Prometheus service discovery (e.g., file_sd_configs) 
+        # or list all homelab IPs here. For now, we scrape localhost.
+        static_configs = [{
+          targets = [ "127.0.0.1:9100" ];
+        }];
+      }
+    ];
+    # Connect Prometheus to Alertmanager
+    alertmanagers = [{
+      static_configs = [{
+        targets = [ "127.0.0.1:9093" ];
+      }];
+    }];
+  };
+
+  # 3. Alertmanager (Routing alerts to NTFY and Email)
+  services.prometheus.alertmanager = {
+    enable = true;
+    port = 9093;
+    configuration = {
+      global = {
+        smtp_smarthost = "smtp.example.com:587";
+        smtp_from = "alerts@depeuter.dev";
+        smtp_auth_username = "alerts@depeuter.dev";
+        smtp_auth_password_file = config.sops.secrets."alertmanager/smtp_password".path;
+      };
+      route = {
+        receiver = "ntfy-and-email";
+        group_wait = "30s";
+        group_interval = "5m";
+        repeat_interval = "4h";
+        group_by = [ "alertname" "instance" ];
+      };
+      receivers = [{
+        name = "ntfy-and-email";
+        email_configs = [{
+          to = "your-email@example.com";
+          # Use smarthost settings defined in global
+        }];
+        webhook_configs = [{
+          # Alertmanager natively supports webhooks. We send the JSON to NTFY's Prometheus endpoint.
+          url = "http://127.0.0.1:2586/alerts"; 
+        }];
+      }];
+    };
+  };
+
+  # 4. NTFY Push Notification Server
+  services.ntfy-sh = {
+    enable = true;
+    settings = {
+      base-url = "https://ntfy.lab.depeuter.dev";
+      listen-http = ":2586";
+      # You can configure auth via the CLI once the service is running, or via auth-file.
+      # For now, it is open locally.
+    };
+  };
+
+  # 5. Grafana Visualization
+  services.grafana = {
+    enable = true;
+    settings.server = {
+      http_port = 3000;
+      http_addr = "127.0.0.1";
+      domain = "grafana.lab.depeuter.dev";
+    };
+    settings.security.admin_password = "$__file{${config.sops.secrets."grafana/admin_password".path}}";
+    
+    # Declarative Data Sources
+    provision = {
+      enable = true;
+      datasources.settings.datasources = [
+        {
+          name = "Prometheus";
+          type = "prometheus";
+          access = "proxy";
+          url = "http://127.0.0.1:9090";
+          isDefault = true;
+        }
+        {
+          name = "Loki";
+          type = "loki";
+          access = "proxy";
+          url = "http://127.0.0.1:3100";
+        }
+      ];
+    };
+  };
+
+  # 6. Traefik Reverse Proxy for Grafana and NTFY
+  services.traefik.dynamicConfigOptions.http = {
+    routers = {
+      grafana = {
+        rule = "Host(`grafana.lab.depeuter.dev`)";
+        service = "grafana";
+      };
+      ntfy = {
+        rule = "Host(`ntfy.lab.depeuter.dev`)";
+        service = "ntfy";
+      };
+    };
+    services = {
+      grafana.loadBalancer.servers = [{ url = "http://127.0.0.1:3000"; }];
+      ntfy.loadBalancer.servers = [{ url = "http://127.0.0.1:2586"; }];
+    };
+  };
+
+  # Open firewall for Loki so agents can push logs
+  networking.firewall.allowedTCPPorts = [ 3100 ];
+}

modules/common/default.nix

@@ -1,4 +1,8 @@
 {
+  imports = [
+    ./monitoring.nix
+  ];
+
   config = {
     homelab = {
       services.openssh.enable = true;

modules/common/monitoring.nix

@@ -0,0 +1,47 @@
+{ config, lib, pkgs, ... }:
+
+{
+  # Prometheus Node Exporter for hardware metrics
+  services.prometheus.exporters = {
+    node = {
+      enable = true;
+      enabledCollectors = [ "systemd" ];
+      port = 9100;
+    };
+  };
+
+  # Promtail to ship logs to Loki
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 28183;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+      clients = [{
+        # Use the internal DNS name for the Loki ingress
+        url = "http://loki.lab.depeuter.dev/loki/api/v1/push";
+      }];
+      scrape_configs = [{
+        job_name = "journal";
+        journal = {
+          max_age = "12h";
+          labels = {
+            job = "systemd-journal";
+            host = config.networking.hostName;
+          };
+        };
+        relabel_configs = [{
+          source_labels = [ "__journal__systemd_unit" ];
+          target_label = "unit";
+        }];
+      }];
+    };
+  };
+
+  # Open firewall ports for node-exporter so Prometheus can scrape it
+  networking.firewall.allowedTCPPorts = [ 9100 ];
+}