path: root/modules/services/monitoring/prometheus.nix

{ config, pkgs, lib, ... }:

let
  cfg = config.my.services.monitoring.prometheus;
  blackboxConfig = {
    modules = {
      https_2xx = {
        prober = "http";
        timeout = "5s";
        http = {
          method = "GET";
          valid_status_codes = [ ];
          fail_if_not_ssl = true;
          no_follow_redirects = false;
          tls_config = { insecure_skip_verify = false; };
          preferred_ip_protocol = "ip4";
        };
      };
      icmp = {
        prober = "icmp";
        icmp = { preferred_ip_protocol = "ip4"; };
        timeout = "5s";
      };
    };
  };
  relabelConfigs = [
    {
      source_labels = [ "__address__" ];
      target_label = "instance";
      replacement = "carmel";
      action = "replace";
      regex = "192.168.6.1:(.*)";
    }
    {
      source_labels = [ "__address__" ];
      target_label = "instance";
      replacement = "tahoe";
      action = "replace";
      regex = "192.168.6.40:(.*)";
    }
  ];
in
{
  options.my.services.monitoring.prometheus = with lib; {
    enable = mkEnableOption "Prometheus monitoring solution";
    listenAddress = mkOption {
      type = types.str;
      default = "0.0.0.0";
      description = lib.mdDoc ''
        Address to listen on.
      '';
    };
    listenPort = mkOption {
      type = types.port;
      default = 9090;
      description = lib.mdDoc ''
        Port to listen on.
      '';
    };
  };

  config = lib.mkIf cfg.enable {
    services.prometheus.exporters.blackbox = {
      enable = true;
      listenAddress = "127.0.0.1";
      port = 9115;
      configFile =
        pkgs.writeText "blackbox.yml" (builtins.toJSON blackboxConfig);
    };

    services.prometheus = {
      enable = true;

      port = cfg.listenPort;
      listenAddress = cfg.listenAddress;

      globalConfig.scrape_interval = "15s";

      extraFlags = [
        # 3 years of retention
        "--storage.tsdb.retention=${toString (365 * 3)}d"
        "--web.enable-admin-api"
      ];

      scrapeConfigs = [
        {
          job_name = "blackbox-ping";
          metrics_path = "/probe";
          params = { module = [ "icmp" ]; };
          static_configs = [{
            targets = [ "8.8.8.8" "1.1.1.1" "git.fcuny.net" "fcuny.net" ];
          }];
          relabel_configs = [
            {
              source_labels = [ "__address__" ];
              target_label = "__param_target";
            }
            {
              source_labels = [ "__param_target" ];
              target_label = "instance";
            }
            {
              target_label = "__address__";
              replacement = "localhost:9115";
            }
          ];
        }
        {
          job_name = "blackbox-http";
          metrics_path = "/probe";
          params = { module = [ "https_2xx" ]; };
          static_configs = [{
            targets = [
              "https://fcuny.net"
              "https://git.fcuny.net"
            ];
          }];
          relabel_configs = [
            {
              source_labels = [ "__address__" ];
              target_label = "__param_target";
            }
            {
              source_labels = [ "__param_target" ];
              target_label = "instance";
            }
            {
              target_label = "__address__";
              replacement = "localhost:9115";
            }
          ];
        }
        {
          job_name = "node";
          static_configs =
            [{ targets = [ "192.168.6.1:9100" "192.168.6.40:9100" ]; }];
          relabel_configs = relabelConfigs;
        }
        {
          job_name = "prometheus";
          static_configs = [{ targets = [ "192.168.6.40:9090" ]; }];
          relabel_configs = relabelConfigs;
        }
        {
          job_name = "dnsmasq";
          static_configs = [{ targets = [ "192.168.6.1:9153" ]; }];
          relabel_configs = relabelConfigs;
        }
        {
          job_name = "unifi-poller";
          static_configs = [{ targets = [ "192.168.6.40:9130" ]; }];
          relabel_configs = relabelConfigs;
        }
      ];
    };
  };
}