From 58159636e820926eaf731aed4c3de657717dabe8 Mon Sep 17 00:00:00 2001
From: Franck Cuny <franck@fcuny.net>
Date: Fri, 19 Dec 2025 08:51:52 -0800
Subject: add monitoring

Run victoria metrics and grafana on rivendell. Grafana is using authelia
for auth. We run some collectors on all the machines, and they publish
to VM through the wireguard interface.
---
 profiles/core-metrics.nix  |  62 ++++++++++++++++++++++++++
 profiles/monitoring.nix    | 105 +++++++++++++++++++++++++++++++++++++++++++++
 profiles/reverse-proxy.nix |   6 +++
 3 files changed, 173 insertions(+)
 create mode 100644 profiles/core-metrics.nix
 create mode 100644 profiles/monitoring.nix

(limited to 'profiles')

diff --git a/profiles/core-metrics.nix b/profiles/core-metrics.nix
new file mode 100644
index 0000000..3f817e5
--- /dev/null
+++ b/profiles/core-metrics.nix
@@ -0,0 +1,62 @@
+{ hostName, ... }:
+let
+  relabel_configs = [
+    {
+      action = "replace";
+      replacement = hostName;
+      target_label = "instance";
+    }
+  ];
+in
+{
+  services.prometheus.exporters = {
+    node.enable = true;
+    systemd.enable = true;
+    process.enable = true;
+  };
+  services.vmagent = {
+    enable = true;
+    remoteWrite.url = "http://10.100.0.60:8428/api/v1/write";
+    prometheusConfig = {
+      global = {
+        external_labels = {
+          "host" = hostName;
+        };
+      };
+      scrape_configs = [
+        {
+          job_name = "node";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:9100" ]; }
+          ];
+          inherit relabel_configs;
+        }
+        {
+          job_name = "systemd";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:9558" ]; }
+          ];
+          inherit relabel_configs;
+        }
+        {
+          job_name = "process";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:9256" ]; }
+          ];
+          inherit relabel_configs;
+        }
+        {
+          job_name = "vmagent";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:8429" ]; }
+          ];
+          inherit relabel_configs;
+        }
+      ];
+    };
+  };
+}
diff --git a/profiles/monitoring.nix b/profiles/monitoring.nix
new file mode 100644
index 0000000..7c62b9e
--- /dev/null
+++ b/profiles/monitoring.nix
@@ -0,0 +1,105 @@
+{ config, ... }:
+{
+
+  age.secrets.grafana-oidc.file = ../secrets/grafana-oidc.age;
+
+  services.victoriametrics.enable = true;
+
+  services.grafana.enable = true;
+  services.grafana.declarativePlugins = [ ];
+  services.grafana.provision.enable = true;
+  services.grafana.provision.datasources.settings = {
+    datasources = [
+      {
+        name = "VictoriaMetrics";
+        type = "prometheus";
+        url = "http://localhost:8428";
+        isDefault = true;
+        jsonData = {
+          httpMethod = "POST";
+          manageAlerts = true;
+        };
+      }
+    ];
+  };
+  services.grafana.settings = {
+    server = {
+      enable_gzip = true;
+      http_port = 3000;
+      http_addr = "10.100.0.60";
+      domain = "dash.fcuny.net";
+      root_url = "https://dash.fcuny.net/";
+    };
+    analytics = {
+      reporting_enabled = false;
+      check_for_updates = false;
+    };
+    users = {
+      allow_signup = false;
+    };
+    "auth.generic_oauth" = {
+      enabled = true;
+      allow_sign_up = true;
+      auto_login = true;
+      name = "Authelia";
+      icon = "signin";
+      client_id = "grafana";
+      # nix run nixpkgs#authelia -- crypto hash generate pbkdf2 --variant sha512 --random --random.length 72 --random.charset rfc3986
+      client_secret = "$__file{/run/credentials/grafana.service/oauth2-client-secret}";
+      scopes = [
+        "openid"
+        "profile"
+        "email"
+        "groups"
+      ];
+      empty_scopes = false;
+      auth_url = "https://auth.fcuny.net/api/oidc/authorization";
+      token_url = "https://auth.fcuny.net/api/oidc/token";
+      api_url = "https://auth.fcuny.net/api/oidc/userinfo";
+      login_attribute_path = "preferred_username";
+      groups_attribute_path = "groups";
+      name_attribute_path = "name";
+      email_attribute_path = "email";
+      use_pkce = true;
+      allow_assign_grafana_admin = true;
+      # Refrain from adding trailing or, see github:grafana/grafana#106686
+      role_attribute_path = builtins.concatStringsSep " || " [
+        "contains(groups, 'grafana-admins') && 'GrafanaAdmin'"
+        "contains(groups, 'grafana-editors') && 'Editor'"
+        "contains(groups, 'grafana-viewers') && 'Viewer'"
+      ];
+      role_attribute_strict = true;
+      skip_org_role_sync = false;
+    };
+  };
+
+  systemd.services.grafana.serviceConfig.LoadCredential = [
+    "oauth2-client-secret:${config.age.secrets.grafana-oidc.path}"
+  ];
+
+  services.authelia.instances.main.settings.identity_providers.oidc.clients = [
+    {
+      id = "grafana";
+      description = "Grafana";
+      client_secret = "$pbkdf2-sha512$310000$yDK1zYFV8y9Zo5iHCv.eQQ$mDpNy3lQ27uqtsbssUaOb8t0rtxD5MBce4sFUqJKE.5y3mVWZir0a1B2q1RaRK/KfgyWxKtNyKRT21Kx7C56Tw";
+      public = false;
+      authorization_policy = "two_factor";
+      require_pkce = true;
+      pkce_challenge_method = "S256";
+      redirect_uris = [ "https://dash.fcuny.net/login/generic_oauth" ];
+      scopes = [
+        "openid"
+        "profile"
+        "email"
+        "groups"
+      ];
+      response_types = [ "code" ];
+      grant_types = [
+        "authorization_code"
+      ];
+      access_token_signed_response_alg = "none";
+      userinfo_signed_response_alg = "none";
+      token_endpoint_auth_method = "client_secret_post";
+    }
+  ];
+}
diff --git a/profiles/reverse-proxy.nix b/profiles/reverse-proxy.nix
index f136ba0..daf2ecb 100644
--- a/profiles/reverse-proxy.nix
+++ b/profiles/reverse-proxy.nix
@@ -68,6 +68,12 @@ in
         forceSSL = true;
         locations."/".proxyPass = "http://${httpHost}:8002";
       };
+      "dash.fcuny.net" = {
+        enableACME = true;
+        acmeRoot = null;
+        forceSSL = true;
+        locations."/".proxyPass = "http://${httpHost}:3000";
+      };
       "fcuny.net" = {
         enableACME = true;
         acmeRoot = null;
-- 
cgit v1.2.3