add monitoring

Run victoria metrics and grafana on rivendell. Grafana is using authelia
for auth. We run some collectors on all the machines, and they publish
to VM through the wireguard interface.

10 files changed, 195 insertions, 0 deletions

diff --git a/justfile b/justfile
index 7f62818..a63b1bd 100644
--- a/justfile
+++ b/justfile
@@ -51,3 +51,9 @@ sync-agenix-key:
 	mkdir -p ~/.ssh
 	op --account my.1password.com read "op://Private/agenix/private key?ssh-format=openssh" > ~/.ssh/agenix
 	op --account my.1password.com read "op://Private/agenix/public key" > ~/.ssh/agenix.pub
+
+# generate a new OIDC secret
+[group('secrets')]
+oidc-secret:
+	@echo "generate a new OIDC secret..."
+	nix run nixpkgs#authelia -- crypto hash generate pbkdf2 --variant sha512 --random --random.length 72 --random.charset rfc3986
diff --git a/machines/argonath.nix b/machines/argonath.nix
index b99e83f..29d78ae 100644
--- a/machines/argonath.nix
+++ b/machines/argonath.nix
@@ -2,6 +2,7 @@
 {
   imports = [
     ../profiles/acme.nix
+    ../profiles/core-metrics.nix
     ../profiles/cgroups.nix
     ../profiles/defaults.nix
     ../profiles/disk/basic-vm.nix
diff --git a/machines/bree.nix b/machines/bree.nix
index e4cd443..d779ce3 100644
--- a/machines/bree.nix
+++ b/machines/bree.nix
@@ -2,6 +2,7 @@
 {
   imports = [
     ../profiles/cgroups.nix
+    ../profiles/core-metrics.nix
     ../profiles/defaults.nix
     ../profiles/disk/basic-vm.nix
     ../profiles/hardware/synology-vm.nix
diff --git a/machines/rivendell.nix b/machines/rivendell.nix
index 83dcb2e..c48997a 100644
--- a/machines/rivendell.nix
+++ b/machines/rivendell.nix
@@ -3,6 +3,7 @@
   imports = [
     ../profiles/authelia.nix
     ../profiles/cgroups.nix
+    ../profiles/core-metrics.nix
     ../profiles/defaults.nix
     ../profiles/disk/btrfs-on-luks.nix
     ../profiles/git-server.nix
@@ -10,6 +11,7 @@
     ../profiles/home-manager.nix
     ../profiles/makemkv.nix
     ../profiles/miniflux.nix
+    ../profiles/monitoring.nix
     ../profiles/remote-unlock.nix
     ../profiles/restic-backup.nix
     ../profiles/server.nix
diff --git a/profiles/core-metrics.nix b/profiles/core-metrics.nix
new file mode 100644
index 0000000..3f817e5
--- /dev/null
+++ b/profiles/core-metrics.nix
@@ -0,0 +1,62 @@
+{ hostName, ... }:
+let
+  relabel_configs = [
+    {
+      action = "replace";
+      replacement = hostName;
+      target_label = "instance";
+    }
+  ];
+in
+{
+  services.prometheus.exporters = {
+    node.enable = true;
+    systemd.enable = true;
+    process.enable = true;
+  };
+  services.vmagent = {
+    enable = true;
+    remoteWrite.url = "http://10.100.0.60:8428/api/v1/write";
+    prometheusConfig = {
+      global = {
+        external_labels = {
+          "host" = hostName;
+        };
+      };
+      scrape_configs = [
+        {
+          job_name = "node";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:9100" ]; }
+          ];
+          inherit relabel_configs;
+        }
+        {
+          job_name = "systemd";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:9558" ]; }
+          ];
+          inherit relabel_configs;
+        }
+        {
+          job_name = "process";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:9256" ]; }
+          ];
+          inherit relabel_configs;
+        }
+        {
+          job_name = "vmagent";
+          scrape_interval = "10s";
+          static_configs = [
+            { targets = [ "127.0.0.1:8429" ]; }
+          ];
+          inherit relabel_configs;
+        }
+      ];
+    };
+  };
+}
diff --git a/profiles/monitoring.nix b/profiles/monitoring.nix
new file mode 100644
index 0000000..7c62b9e
--- /dev/null
+++ b/profiles/monitoring.nix
@@ -0,0 +1,105 @@
+{ config, ... }:
+{
+
+  age.secrets.grafana-oidc.file = ../secrets/grafana-oidc.age;
+
+  services.victoriametrics.enable = true;
+
+  services.grafana.enable = true;
+  services.grafana.declarativePlugins = [ ];
+  services.grafana.provision.enable = true;
+  services.grafana.provision.datasources.settings = {
+    datasources = [
+      {
+        name = "VictoriaMetrics";
+        type = "prometheus";
+        url = "http://localhost:8428";
+        isDefault = true;
+        jsonData = {
+          httpMethod = "POST";
+          manageAlerts = true;
+        };
+      }
+    ];
+  };
+  services.grafana.settings = {
+    server = {
+      enable_gzip = true;
+      http_port = 3000;
+      http_addr = "10.100.0.60";
+      domain = "dash.fcuny.net";
+      root_url = "https://dash.fcuny.net/";
+    };
+    analytics = {
+      reporting_enabled = false;
+      check_for_updates = false;
+    };
+    users = {
+      allow_signup = false;
+    };
+    "auth.generic_oauth" = {
+      enabled = true;
+      allow_sign_up = true;
+      auto_login = true;
+      name = "Authelia";
+      icon = "signin";
+      client_id = "grafana";
+      # nix run nixpkgs#authelia -- crypto hash generate pbkdf2 --variant sha512 --random --random.length 72 --random.charset rfc3986
+      client_secret = "$__file{/run/credentials/grafana.service/oauth2-client-secret}";
+      scopes = [
+        "openid"
+        "profile"
+        "email"
+        "groups"
+      ];
+      empty_scopes = false;
+      auth_url = "https://auth.fcuny.net/api/oidc/authorization";
+      token_url = "https://auth.fcuny.net/api/oidc/token";
+      api_url = "https://auth.fcuny.net/api/oidc/userinfo";
+      login_attribute_path = "preferred_username";
+      groups_attribute_path = "groups";
+      name_attribute_path = "name";
+      email_attribute_path = "email";
+      use_pkce = true;
+      allow_assign_grafana_admin = true;
+      # Refrain from adding trailing or, see github:grafana/grafana#106686
+      role_attribute_path = builtins.concatStringsSep " || " [
+        "contains(groups, 'grafana-admins') && 'GrafanaAdmin'"
+        "contains(groups, 'grafana-editors') && 'Editor'"
+        "contains(groups, 'grafana-viewers') && 'Viewer'"
+      ];
+      role_attribute_strict = true;
+      skip_org_role_sync = false;
+    };
+  };
+
+  systemd.services.grafana.serviceConfig.LoadCredential = [
+    "oauth2-client-secret:${config.age.secrets.grafana-oidc.path}"
+  ];
+
+  services.authelia.instances.main.settings.identity_providers.oidc.clients = [
+    {
+      id = "grafana";
+      description = "Grafana";
+      client_secret = "$pbkdf2-sha512$310000$yDK1zYFV8y9Zo5iHCv.eQQ$mDpNy3lQ27uqtsbssUaOb8t0rtxD5MBce4sFUqJKE.5y3mVWZir0a1B2q1RaRK/KfgyWxKtNyKRT21Kx7C56Tw";
+      public = false;
+      authorization_policy = "two_factor";
+      require_pkce = true;
+      pkce_challenge_method = "S256";
+      redirect_uris = [ "https://dash.fcuny.net/login/generic_oauth" ];
+      scopes = [
+        "openid"
+        "profile"
+        "email"
+        "groups"
+      ];
+      response_types = [ "code" ];
+      grant_types = [
+        "authorization_code"
+      ];
+      access_token_signed_response_alg = "none";
+      userinfo_signed_response_alg = "none";
+      token_endpoint_auth_method = "client_secret_post";
+    }
+  ];
+}
diff --git a/profiles/reverse-proxy.nix b/profiles/reverse-proxy.nix
index f136ba0..daf2ecb 100644
--- a/profiles/reverse-proxy.nix
+++ b/profiles/reverse-proxy.nix
@@ -68,6 +68,12 @@ in
         forceSSL = true;
         locations."/".proxyPass = "http://${httpHost}:8002";
       };
+      "dash.fcuny.net" = {
+        enableACME = true;
+        acmeRoot = null;
+        forceSSL = true;
+        locations."/".proxyPass = "http://${httpHost}:3000";
+      };
       "fcuny.net" = {
         enableACME = true;
         acmeRoot = null;
diff --git a/secrets/authelia-users.yaml.age b/secrets/authelia-users.yaml.age
index dc8fe62..d21f4e0 100644
--- a/secrets/authelia-users.yaml.age
+++ b/secrets/authelia-users.yaml.age
diff --git a/secrets/grafana-oidc.age b/secrets/grafana-oidc.age
new file mode 100644
index 0000000..deaf0c4
--- /dev/null
+++ b/secrets/grafana-oidc.age
@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 pFjJaA nXdpTOxE+KOi+hkTl8WrFzsXTLlX6JQhY/6+w6ZcZ0k
+6TZjec0mdP37hXGXEev7dN27BqGhvO0EVEJi7XPJsrc
+-> ssh-ed25519 Y5h84Q 1um4Z+C9sRiHVMEJszpc4ygNhONX0tNvAsABlvDmwHA
+IN3pQyGFCRWphTHLAaxrCVci0OaRViHUaZYqZPEA14A
+--- ABsJxwFEMn+GNkH+BqcrSIFfeZJaqSvRTNid1yEDJaA
+���F�꧒�b��R��M��w�ɨ�qo�
�;�\�1���n
�D4 X���Q�LU*��oI�M:��Yy�I�t��ƖJ�E�@ ��i˸\�a%�
\ No newline at end of file
diff --git a/secrets/secrets.nix b/secrets/secrets.nix
index adb15e1..2b645eb 100644
--- a/secrets/secrets.nix
+++ b/secrets/secrets.nix
@@ -42,6 +42,11 @@ in
     hosts.rivendell
   ];
 
+  "grafana-oidc.age".publicKeys = [
+    users.fcuny
+    hosts.rivendell
+  ];
+
   # generated with:
   # openssl rand 64 | openssl base64 -A | tr '+/' '-_' | tr -d '='
   "authelia-storage-key.age".publicKeys = [