zion/modules/monitoring.nix

{ config, lib, pkgs, ... }:

with pkgs;

{
  # Notify when a disk starts going haywire
  services.smartd = {
    enable = true;
    notifications.mail = {
      enable = true;
      sender = "akasroua+smartd@disroot.org";
      recipient = "akasroua@disroot.org";
      mailer = "${msmtp}/bin/msmtp -t --read-envelope-from";
    };
  };

  # Notify about zpool problems
  services.zfs.zed = {
    enableMail = false;
    settings = {
      ZED_EMAIL_ADDR = "akasroua+smartd@disroot.org";
      ZED_EMAIL_PROG = "mail";
      ZED_EMAIL_OPTS = "-s '@SUBJECT@' @ADDRESS@";
      ZED_NOTIFY_VERBOSE = false;
    };
  };

  # Set up msmtp as notifier
  programs.msmtp = {
    enable = true;
    defaults = {
      port = 587;
      tls = true;
    };
    accounts = {
      default = {
        auth = true;
        host = "disroot.org";
        user = "akasroua@disroot.org";
        passwordeval = "${coreutils}/bin/cat ${config.age.secrets.msmtp.path}";
      };
    };
  };

  # Metrics collection
  services.prometheus = {
    enable = true;
    port = 9001;
    retentionTime = "1y";
    exporters = {
      node = {
        enable = true;
        enabledCollectors = [ "systemd" ];
        port = 9002;
      };
      zfs.enable = true;
      wireguard.enable = true;
      postgres.enable = true;
      smartctl.enable = true;
      dnsmasq.enable = true;
      nginx = {
        enable = true;
        scrapeUri = "http://localhost:8282/nginx_status";
      };
    };
    scrapeConfigs = [{
      job_name = "zion";
      static_configs = [{
        targets = [
          "localhost:${toString config.services.prometheus.exporters.node.port}"
          "localhost:${toString config.services.prometheus.exporters.zfs.port}"
          "localhost:${
            toString config.services.prometheus.exporters.wireguard.port
          }"
          "localhost:${
            toString config.services.prometheus.exporters.postgres.port
          }"
          "localhost:${
            toString config.services.prometheus.exporters.smartctl.port
          }"
          "localhost:${
            toString config.services.prometheus.exporters.dnsmasq.port
          }"
          "localhost:${
            toString config.services.prometheus.exporters.nginx.port
          }"
          "localhost:9641"
        ];
      }];
    }];
  };

  # Grafana configuration
  services.grafana = {
    enable = true;
    settings.server = {
      domain = "grafana.coolneng.duckdns.org";
      http_port = 9009;
      http_addr = "127.0.0.1";
    };
  };

}