zion/modules/monitoring.nix

{
  config,
  lib,
  pkgs,
  ...
}:

with pkgs;

{
  # Notify when a disk starts going haywire
  services.smartd = {
    enable = true;
    notifications.mail = {
      enable = true;
      sender = "akasroua+smartd@disroot.org";
      recipient = "akasroua@disroot.org";
      mailer = "${msmtp}/bin/msmtp -t --read-envelope-from";
    };
  };

  # Notify about zpool problems
  services.zfs.zed = {
    enableMail = false;
    settings = {
      ZED_EMAIL_ADDR = "akasroua+smartd@disroot.org";
      ZED_EMAIL_PROG = "mail";
      ZED_EMAIL_OPTS = "-s '@SUBJECT@' @ADDRESS@";
      ZED_NOTIFY_VERBOSE = false;
    };
  };

  # Set up msmtp as notifier
  programs.msmtp = {
    enable = true;
    defaults = {
      port = 587;
      tls = true;
    };
    accounts = {
      default = {
        auth = true;
        host = "disroot.org";
        user = "akasroua@disroot.org";
        passwordeval = "${coreutils}/bin/cat ${config.age.secrets.msmtp.path}";
      };
    };
  };

  # Metrics collection
  services.prometheus = {
    enable = true;
    port = 9001;
    retentionTime = "1y";
    exporters = {
      node = {
        enable = true;
        enabledCollectors = [ "systemd" ];
        port = 9002;
      };
      postgres.enable = true;
      smartctl.enable = true;
    };
    scrapeConfigs = [
      {
        job_name = "zion";
        static_configs = [
          {
            targets = [
              "localhost:${toString config.services.prometheus.exporters.node.port}"
              "localhost:${toString config.services.prometheus.exporters.postgres.port}"
              "localhost:${toString config.services.prometheus.exporters.smartctl.port}"
            ];
          }
        ];
      }
    ];
  };

  # Grafana configuration
  services.grafana = {
    enable = true;
    settings.server = {
      domain = "grafana.coolneng.duckdns.org";
      http_port = 9009;
      http_addr = "127.0.0.1";
    };
  };

}