refactor prometheus as a module

4 files changed, 190 insertions, 180 deletions

diff --git a/hosts/common/server/prometheus.nix b/hosts/common/server/prometheus.nix
deleted file mode 100644
index 0e4afff..0000000
--- a/hosts/common/server/prometheus.nix
+++ /dev/null
@@ -1,179 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-let
-  blackboxConfig = {
-    modules = {
-      https_2xx = {
-        prober = "http";
-        timeout = "5s";
-        http = {
-          method = "GET";
-          valid_status_codes = [ ];
-          fail_if_not_ssl = true;
-          no_follow_redirects = false;
-          tls_config = { insecure_skip_verify = false; };
-          preferred_ip_protocol = "ip4";
-        };
-      };
-      icmp = {
-        prober = "icmp";
-        icmp = { preferred_ip_protocol = "ip4"; };
-        timeout = "5s";
-      };
-    };
-  };
-  relabelConfigs = [
-    {
-      source_labels = [ "__address__" ];
-      target_label = "instance";
-      replacement = "nas";
-      action = "replace";
-      regex = "192.168.6.10:(.*)";
-    }
-    {
-      source_labels = [ "__address__" ];
-      target_label = "instance";
-      replacement = "rtr";
-      action = "replace";
-      regex = "192.168.6.1:(.*)";
-    }
-    {
-      source_labels = [ "__address__" ];
-      target_label = "instance";
-      replacement = "tahoe";
-      action = "replace";
-      regex = "192.168.6.20:(.*)";
-    }
-  ];
-in {
-  services.prometheus.exporters.blackbox = {
-    enable = true;
-    listenAddress = "127.0.0.1";
-    port = 9115;
-    configFile = pkgs.writeText "blackbox.yml" (builtins.toJSON blackboxConfig);
-  };
-
-  services.prometheus = {
-    enable = true;
-
-    globalConfig.scrape_interval = "15s";
-
-    extraFlags = [
-      # 3 years of retention
-      "--storage.tsdb.retention=${toString (365 * 3)}d"
-      "--web.enable-admin-api"
-    ];
-
-    scrapeConfigs = [
-      {
-        job_name = "blackbox-ping";
-        metrics_path = "/probe";
-        params = { module = [ "icmp" ]; };
-        static_configs =
-          [{ targets = [ "8.8.8.8" "1.1.1.1" "git.fcuny.net" "fcuny.net" ]; }];
-        relabel_configs = [
-          {
-            source_labels = [ "__address__" ];
-            target_label = "__param_target";
-          }
-          {
-            source_labels = [ "__param_target" ];
-            target_label = "instance";
-          }
-          {
-            target_label = "__address__";
-            replacement = "localhost:9115";
-          }
-        ];
-      }
-      {
-        job_name = "blackbox-http";
-        metrics_path = "/probe";
-        params = { module = [ "https_2xx" ]; };
-        static_configs = [{
-          targets = [
-            "https://fcuny.net"
-            "https://git.fcuny.net"
-            "https://notes.fcuny.net"
-          ];
-        }];
-        relabel_configs = [
-          {
-            source_labels = [ "__address__" ];
-            target_label = "__param_target";
-          }
-          {
-            source_labels = [ "__param_target" ];
-            target_label = "instance";
-          }
-          {
-            target_label = "__address__";
-            replacement = "localhost:9115";
-          }
-        ];
-      }
-      {
-        job_name = "node";
-        static_configs =
-          [{ targets = [ "192.168.6.1:9100" "192.168.6.20:9100" ]; }];
-        relabel_configs = relabelConfigs;
-      }
-      {
-        job_name = "prometheus";
-        static_configs = [{ targets = [ "192.168.6.20:9090" ]; }];
-        relabel_configs = relabelConfigs;
-      }
-      {
-        job_name = "traefik";
-        static_configs = [{ targets = [ "192.168.6.20:8090" ]; }];
-        relabel_configs = relabelConfigs;
-      }
-      {
-        job_name = "gitea";
-        static_configs = [{ targets = [ "192.168.6.20:8002" ]; }];
-        relabel_configs = relabelConfigs;
-      }
-
-      {
-        job_name = "dnsd";
-        static_configs = [{ targets = [ "192.168.6.1:8053" ]; }];
-        relabel_configs = relabelConfigs;
-      }
-      {
-        job_name = "dnsdd";
-        static_configs = [{ targets = [ "192.168.6.1:9060" ]; }];
-        relabel_configs = relabelConfigs;
-      }
-      {
-        job_name = "dhcpd";
-        static_configs = [{ targets = [ "192.168.6.1:8067" ]; }];
-        relabel_configs = relabelConfigs;
-      }
-      {
-        job_name = "netd";
-        static_configs = [{ targets = [ "192.168.6.1:8055" ]; }];
-        relabel_configs = relabelConfigs;
-      }
-      {
-        job_name = "unifi-poller";
-        static_configs = [{ targets = [ "192.168.6.20:9130" ]; }];
-        relabel_configs = relabelConfigs;
-      }
-    ];
-  };
-
-  age.secrets.restic-repo-systems.file =
-    ../../../secrets/restic/repo-systems.age;
-
-  services.restic.backups = {
-    prometheus = {
-      paths = [ "/var/lib/prometheus2" ];
-      repository = "/data/slow/backups/systems";
-      passwordFile = config.age.secrets.restic-repo-systems.path;
-      initialize = true;
-      timerConfig = { OnCalendar = "00:25"; };
-      extraBackupArgs = [ "--tag prometheus" ];
-      pruneOpts = [ "--keep-daily 7" "--keep-weekly 4" ];
-    };
-  };
-}
diff --git a/hosts/profiles/nas.nix b/hosts/profiles/nas.nix
index 33fd6b5..bf5f772 100644
--- a/hosts/profiles/nas.nix
+++ b/hosts/profiles/nas.nix
@@ -2,7 +2,6 @@
   imports = [
     # other profiles
     ./server.nix
-    ../common/server/prometheus.nix
     ../common/server/grafana.nix
     ../common/server/traefik.nix
     ../common/server/transmission.nix
@@ -27,6 +26,7 @@
       musicFolder = "/data/fast/music";
     };
     unifi = { enable = true; };
+    prometheus = { enable = true; };
   };
 
   services.restic.backups = {
diff --git a/modules/services/default.nix b/modules/services/default.nix
index 8da870e..97a326a 100644
--- a/modules/services/default.nix
+++ b/modules/services/default.nix
@@ -4,6 +4,7 @@
   imports = [
     ./fwupd
     ./navidrome
+    ./prometheus
     ./samba
     ./ssh-server
     ./tailscale
diff --git a/modules/services/prometheus/default.nix b/modules/services/prometheus/default.nix
new file mode 100644
index 0000000..6eeee16
--- /dev/null
+++ b/modules/services/prometheus/default.nix
@@ -0,0 +1,188 @@
+{ config, pkgs, lib, ... }:
+
+let
+  cfg = config.my.services.prometheus;
+  blackboxConfig = {
+    modules = {
+      https_2xx = {
+        prober = "http";
+        timeout = "5s";
+        http = {
+          method = "GET";
+          valid_status_codes = [ ];
+          fail_if_not_ssl = true;
+          no_follow_redirects = false;
+          tls_config = { insecure_skip_verify = false; };
+          preferred_ip_protocol = "ip4";
+        };
+      };
+      icmp = {
+        prober = "icmp";
+        icmp = { preferred_ip_protocol = "ip4"; };
+        timeout = "5s";
+      };
+    };
+  };
+  relabelConfigs = [
+    {
+      source_labels = [ "__address__" ];
+      target_label = "instance";
+      replacement = "nas";
+      action = "replace";
+      regex = "192.168.6.10:(.*)";
+    }
+    {
+      source_labels = [ "__address__" ];
+      target_label = "instance";
+      replacement = "rtr";
+      action = "replace";
+      regex = "192.168.6.1:(.*)";
+    }
+    {
+      source_labels = [ "__address__" ];
+      target_label = "instance";
+      replacement = "tahoe";
+      action = "replace";
+      regex = "192.168.6.20:(.*)";
+    }
+  ];
+in {
+  options.my.services.navidrome = with lib; {
+    enable = mkEnableOption "Prometheus monitoring solution";
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.prometheus.exporters.blackbox = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 9115;
+      configFile =
+        pkgs.writeText "blackbox.yml" (builtins.toJSON blackboxConfig);
+    };
+
+    services.prometheus = {
+      enable = true;
+
+      globalConfig.scrape_interval = "15s";
+
+      extraFlags = [
+        # 3 years of retention
+        "--storage.tsdb.retention=${toString (365 * 3)}d"
+        "--web.enable-admin-api"
+      ];
+
+      scrapeConfigs = [
+        {
+          job_name = "blackbox-ping";
+          metrics_path = "/probe";
+          params = { module = [ "icmp" ]; };
+          static_configs = [{
+            targets = [ "8.8.8.8" "1.1.1.1" "git.fcuny.net" "fcuny.net" ];
+          }];
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "localhost:9115";
+            }
+          ];
+        }
+        {
+          job_name = "blackbox-http";
+          metrics_path = "/probe";
+          params = { module = [ "https_2xx" ]; };
+          static_configs = [{
+            targets = [
+              "https://fcuny.net"
+              "https://git.fcuny.net"
+              "https://notes.fcuny.net"
+            ];
+          }];
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "localhost:9115";
+            }
+          ];
+        }
+        {
+          job_name = "node";
+          static_configs =
+            [{ targets = [ "192.168.6.1:9100" "192.168.6.20:9100" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "prometheus";
+          static_configs = [{ targets = [ "192.168.6.20:9090" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "traefik";
+          static_configs = [{ targets = [ "192.168.6.20:8090" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "gitea";
+          static_configs = [{ targets = [ "192.168.6.20:8002" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+
+        {
+          job_name = "dnsd";
+          static_configs = [{ targets = [ "192.168.6.1:8053" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "dnsdd";
+          static_configs = [{ targets = [ "192.168.6.1:9060" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "dhcpd";
+          static_configs = [{ targets = [ "192.168.6.1:8067" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "netd";
+          static_configs = [{ targets = [ "192.168.6.1:8055" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "unifi-poller";
+          static_configs = [{ targets = [ "192.168.6.20:9130" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+      ];
+    };
+
+    age.secrets.restic-repo-systems.file =
+      ../../../secrets/restic/repo-systems.age;
+
+    services.restic.backups = {
+      prometheus = {
+        paths = [ "/var/lib/prometheus2" ];
+        repository = "/data/slow/backups/systems";
+        passwordFile = config.age.secrets.restic-repo-systems.path;
+        initialize = true;
+        timerConfig = { OnCalendar = "00:25"; };
+        extraBackupArgs = [ "--tag prometheus" ];
+        pruneOpts = [ "--keep-daily 7" "--keep-weekly 4" ];
+      };
+    };
+  };
+}