add monitoring

Run victoria metrics and grafana on rivendell. Grafana is using authelia
for auth. We run some collectors on all the machines, and they publish
to VM through the wireguard interface.

3 files changed, 173 insertions, 0 deletions

diff --git a/profiles/core-metrics.nix b/profiles/core-metrics.nix
new file mode 100644
index 0000000..3f817e5
--- /dev/null
+++ b/profiles/core-metrics.nix
@@ -0,0 +1,62 @@
+{ hostName, ... }:
+let
+  relabel_configs = [
+    {
+      action = "replace";
+      replacement = hostName;
+      target_label = "instance";
+    }
+  ];
+in
+{
+  services.prometheus.exporters = {
+    node.enable = true;
+    systemd.enable = true;
+    process.enable = true;
+  };
+  services.vmagent = {
+    enable = true;
+    remoteWrite.url = "http://10.100.0.60:8428/api/v1/write";
+    prometheusConfig = {
+      global = {
+        external_labels = {
+          "host" = hostName;
+        };
+      };
+      scrape_configs = [
+        {
+          job_name = "node";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:9100" ]; }
+          ];
+          inherit relabel_configs;
+        }
+        {
+          job_name = "systemd";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:9558" ]; }
+          ];
+          inherit relabel_configs;
+        }
+        {
+          job_name = "process";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:9256" ]; }
+          ];
+          inherit relabel_configs;
+        }
+        {
+          job_name = "vmagent";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:8429" ]; }
+          ];
+          inherit relabel_configs;
+        }
+      ];
+    };
+  };
+}
diff --git a/profiles/monitoring.nix b/profiles/monitoring.nix
new file mode 100644
index 0000000..7c62b9e
--- /dev/null
+++ b/profiles/monitoring.nix
@@ -0,0 +1,105 @@
+{ config, ... }:
+{
+
+  age.secrets.grafana-oidc.file = ../secrets/grafana-oidc.age;
+
+  services.victoriametrics.enable = true;
+
+  services.grafana.enable = true;
+  services.grafana.declarativePlugins = [ ];
+  services.grafana.provision.enable = true;
+  services.grafana.provision.datasources.settings = {
+    datasources = [
+      {
+        name = "VictoriaMetrics";
+        type = "prometheus";
+        url = "http://localhost:8428";
+        isDefault = true;
+        jsonData = {
+          httpMethod = "POST";
+          manageAlerts = true;
+        };
+      }
+    ];
+  };
+  services.grafana.settings = {
+    server = {
+      enable_gzip = true;
+      http_port = 3000;
+      http_addr = "10.100.0.60";
+      domain = "dash.fcuny.net";
+      root_url = "https://dash.fcuny.net/";
+    };
+    analytics = {
+      reporting_enabled = false;
+      check_for_updates = false;
+    };
+    users = {
+      allow_signup = false;
+    };
+    "auth.generic_oauth" = {
+      enabled = true;
+      allow_sign_up = true;
+      auto_login = true;
+      name = "Authelia";
+      icon = "signin";
+      client_id = "grafana";
+      # nix run nixpkgs#authelia -- crypto hash generate pbkdf2 --variant sha512 --random --random.length 72 --random.charset rfc3986
+      client_secret = "$__file{/run/credentials/grafana.service/oauth2-client-secret}";
+      scopes = [
+        "openid"
+        "profile"
+        "email"
+        "groups"
+      ];
+      empty_scopes = false;
+      auth_url = "https://auth.fcuny.net/api/oidc/authorization";
+      token_url = "https://auth.fcuny.net/api/oidc/token";
+      api_url = "https://auth.fcuny.net/api/oidc/userinfo";
+      login_attribute_path = "preferred_username";
+      groups_attribute_path = "groups";
+      name_attribute_path = "name";
+      email_attribute_path = "email";
+      use_pkce = true;
+      allow_assign_grafana_admin = true;
+      # Refrain from adding trailing or, see github:grafana/grafana#106686
+      role_attribute_path = builtins.concatStringsSep " || " [
+        "contains(groups, 'grafana-admins') && 'GrafanaAdmin'"
+        "contains(groups, 'grafana-editors') && 'Editor'"
+        "contains(groups, 'grafana-viewers') && 'Viewer'"
+      ];
+      role_attribute_strict = true;
+      skip_org_role_sync = false;
+    };
+  };
+
+  systemd.services.grafana.serviceConfig.LoadCredential = [
+    "oauth2-client-secret:${config.age.secrets.grafana-oidc.path}"
+  ];
+
+  services.authelia.instances.main.settings.identity_providers.oidc.clients = [
+    {
+      id = "grafana";
+      description = "Grafana";
+      client_secret = "$pbkdf2-sha512$310000$yDK1zYFV8y9Zo5iHCv.eQQ$mDpNy3lQ27uqtsbssUaOb8t0rtxD5MBce4sFUqJKE.5y3mVWZir0a1B2q1RaRK/KfgyWxKtNyKRT21Kx7C56Tw";
+      public = false;
+      authorization_policy = "two_factor";
+      require_pkce = true;
+      pkce_challenge_method = "S256";
+      redirect_uris = [ "https://dash.fcuny.net/login/generic_oauth" ];
+      scopes = [
+        "openid"
+        "profile"
+        "email"
+        "groups"
+      ];
+      response_types = [ "code" ];
+      grant_types = [
+        "authorization_code"
+      ];
+      access_token_signed_response_alg = "none";
+      userinfo_signed_response_alg = "none";
+      token_endpoint_auth_method = "client_secret_post";
+    }
+  ];
+}
diff --git a/profiles/reverse-proxy.nix b/profiles/reverse-proxy.nix
index f136ba0..daf2ecb 100644
--- a/profiles/reverse-proxy.nix
+++ b/profiles/reverse-proxy.nix
@@ -68,6 +68,12 @@ in
         forceSSL = true;
         locations."/".proxyPass = "http://${httpHost}:8002";
       };
+      "dash.fcuny.net" = {
+        enableACME = true;
+        acmeRoot = null;
+        forceSSL = true;
+        locations."/".proxyPass = "http://${httpHost}:3000";
+      };
       "fcuny.net" = {
         enableACME = true;
         acmeRoot = null;