path: root/hosts/common/server/prometheus.nix

{ config, pkgs, lib, ... }:

let
  relabelConfigs = [
    {
      source_labels = [ "__address__" ];
      target_label = "instance";
      replacement = "nas";
      action = "replace";
      regex = "192.168.6.10:(.*)";
    }
    {
      source_labels = [ "__address__" ];
      target_label = "instance";
      replacement = "rtr";
      action = "replace";
      regex = "192.168.6.1:(.*)";
    }
    {
      source_labels = [ "__address__" ];
      target_label = "instance";
      replacement = "tahoe";
      action = "replace";
      regex = "192.168.6.20:(.*)";
    }
  ];
in {
  services.prometheus = {
    enable = true;

    globalConfig.scrape_interval = "15s";

    extraFlags = [
      # 3 years of retention
      "--storage.tsdb.retention=${toString (365 * 3)}d"
      "--web.enable-admin-api"
    ];

    scrapeConfigs = [
      {
        job_name = "node";
        static_configs =
          [{ targets = [ "192.168.6.1:9100" "192.168.6.20:9100" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "prometheus";
        static_configs = [{ targets = [ "192.168.6.20:9090" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "traefik";
        static_configs = [{ targets = [ "192.168.6.20:8090" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "gitea";
        static_configs = [{ targets = [ "192.168.6.20:8002" ]; }];
        relabel_configs = relabelConfigs;
      }

      {
        job_name = "dnsd";
        static_configs = [{ targets = [ "192.168.6.1:8053" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "dnsdd";
        static_configs = [{ targets = [ "192.168.6.1:9060" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "dhcpd";
        static_configs = [{ targets = [ "192.168.6.1:8067" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "netd";
        static_configs = [{ targets = [ "192.168.6.1:8055" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "unifi-poller";
        static_configs = [{ targets = [ "192.168.6.20:9130" ]; }];
        relabel_configs = relabelConfigs;
      }
    ];
  };

  age.secrets.restic-repo-systems.file =
    ../../../secrets/restic/repo-systems.age;

  services.restic.backups = {
    prometheus = {
      paths = [ "/var/lib/prometheus2" ];
      repository = "/data/slow/backups/systems";
      passwordFile = config.age.secrets.restic-repo-systems.path;
      initialize = true;
      timerConfig = { OnCalendar = "00:25"; };
      extraBackupArgs = [ "--tag prometheus" ];
      pruneOpts = [ "--keep-daily 7" "--keep-weekly 4" ];
    };
  };
}