nix-config/nixos/modules/apps/monitoring/default.nix

{ config, lib, pkgs, ... }:

let
  cfg = config.homelab.apps.monitoring;
in {
  options.homelab.apps.monitoring.enable = lib.mkEnableOption "Homelab Monitoring Stack";

  config = lib.mkIf cfg.enable {
    sops.secrets."grafana/admin_password" = {
    sopsFile = ../../../../secrets/prod/monitoring.yaml;
  };
  sops.secrets."alertmanager/smtp_password" = {
    sopsFile = ../../../../secrets/prod/monitoring.yaml;
  };

  # 1. Loki Log Storage
  services.loki = {
    enable = true;
    configuration = {
      auth_enabled = false;
      server.http_listen_port = 3100;
      common.ring.instance_addr = "127.0.0.1";
      common.ring.kvstore.store = "inmemory";
      schema_config = {
        configs = [{
          from = "2020-10-24";
          store = "boltdb-shipper";
          object_store = "filesystem";
          schema = "v11";
          index = {
            prefix = "index_";
            period = "24h";
          };
        }];
      };
      storage_config = {
        boltdb_shipper = {
          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
          cache_location = "/var/lib/loki/boltdb-shipper-cache";
        };
        filesystem.directory = "/var/lib/loki/chunks";
      };
    };
  };

  # 2. Prometheus Time-Series DB
  services.prometheus = {
    enable = true;
    port = 9090;
    scrapeConfigs = [
      {
        job_name = "node";
        scrape_interval = "15s";
        # In a real setup, we would use Prometheus service discovery (e.g., file_sd_configs) 
        # or list all homelab IPs here. For now, we scrape localhost.
        static_configs = [{
          targets = [ "127.0.0.1:9100" ];
        }];
      }
    ];
    # Connect Prometheus to Alertmanager
    alertmanagers = [{
      static_configs = [{
        targets = [ "127.0.0.1:9093" ];
      }];
    }];
  };

  # 3. Alertmanager (Routing alerts to NTFY and Email)
  services.prometheus.alertmanager = {
    enable = true;
    port = 9093;
    configuration = {
      global = {
        smtp_smarthost = "smtp.example.com:587";
        smtp_from = "alerts@depeuter.dev";
        smtp_auth_username = "alerts@depeuter.dev";
        smtp_auth_password_file = config.sops.secrets."alertmanager/smtp_password".path;
      };
      route = {
        receiver = "ntfy-and-email";
        group_wait = "30s";
        group_interval = "5m";
        repeat_interval = "4h";
        group_by = [ "alertname" "instance" ];
      };
      receivers = [{
        name = "ntfy-and-email";
        email_configs = [{
          to = "your-email@example.com";
          # Use smarthost settings defined in global
        }];
        webhook_configs = [{
          # Alertmanager natively supports webhooks. We send the JSON to NTFY's Prometheus endpoint.
          url = "http://127.0.0.1:2586/alerts"; 
        }];
      }];
    };
  };

  # 4. NTFY Push Notification Server
  services.ntfy-sh = {
    enable = true;
    settings = {
      base-url = "https://ntfy.lab.depeuter.dev";
      listen-http = ":2586";
      # You can configure auth via the CLI once the service is running, or via auth-file.
      # For now, it is open locally.
    };
  };

  # 5. Grafana Visualization
  services.grafana = {
    enable = true;
    settings.server = {
      http_port = 3000;
      http_addr = "127.0.0.1";
      domain = "grafana.lab.depeuter.dev";
    };
    settings.security.admin_password = "$__file{${config.sops.secrets."grafana/admin_password".path}}";
    
    # Declarative Data Sources
    provision = {
      enable = true;
      datasources.settings.datasources = [
        {
          name = "Prometheus";
          type = "prometheus";
          access = "proxy";
          url = "http://127.0.0.1:9090";
          isDefault = true;
        }
        {
          name = "Loki";
          type = "loki";
          access = "proxy";
          url = "http://127.0.0.1:3100";
        }
      ];
    };
  };

  # 6. Traefik Reverse Proxy for Grafana and NTFY
  homelab.apps.traefik.dynamicConfigOptions.http = {
    routers = {
      grafana = {
        rule = "Host(`grafana.lab.depeuter.dev`)";
        service = "grafana";
      };
      ntfy = {
        rule = "Host(`ntfy.lab.depeuter.dev`)";
        service = "ntfy";
      };
    };
    services = {
      grafana.loadBalancer.servers = [{ url = "http://host.docker.internal:3000"; }];
      ntfy.loadBalancer.servers = [{ url = "http://host.docker.internal:2586"; }];
    };
  };

  # Open firewall for Loki so agents can push logs
  networking.firewall.allowedTCPPorts = [ 3100 ];
  };
}