From 58159636e820926eaf731aed4c3de657717dabe8 Mon Sep 17 00:00:00 2001 From: Franck Cuny Date: Fri, 19 Dec 2025 08:51:52 -0800 Subject: add monitoring Run victoria metrics and grafana on rivendell. Grafana is using authelia for auth. We run some collectors on all the machines, and they publish to VM through the wireguard interface. --- justfile | 6 +++ machines/argonath.nix | 1 + machines/bree.nix | 1 + machines/rivendell.nix | 2 + profiles/core-metrics.nix | 62 ++++++++++++++++++++++++ profiles/monitoring.nix | 105 ++++++++++++++++++++++++++++++++++++++++ profiles/reverse-proxy.nix | 6 +++ secrets/authelia-users.yaml.age | Bin 556 -> 581 bytes secrets/grafana-oidc.age | 7 +++ secrets/secrets.nix | 5 ++ 10 files changed, 195 insertions(+) create mode 100644 profiles/core-metrics.nix create mode 100644 profiles/monitoring.nix create mode 100644 secrets/grafana-oidc.age diff --git a/justfile b/justfile index 7f62818..a63b1bd 100644 --- a/justfile +++ b/justfile @@ -51,3 +51,9 @@ sync-agenix-key: mkdir -p ~/.ssh op --account my.1password.com read "op://Private/agenix/private key?ssh-format=openssh" > ~/.ssh/agenix op --account my.1password.com read "op://Private/agenix/public key" > ~/.ssh/agenix.pub + +# generate a new OIDC secret +[group('secrets')] +oidc-secret: + @echo "generate a new OIDC secret..." + nix run nixpkgs#authelia -- crypto hash generate pbkdf2 --variant sha512 --random --random.length 72 --random.charset rfc3986 diff --git a/machines/argonath.nix b/machines/argonath.nix index b99e83f..29d78ae 100644 --- a/machines/argonath.nix +++ b/machines/argonath.nix @@ -2,6 +2,7 @@ { imports = [ ../profiles/acme.nix + ../profiles/core-metrics.nix ../profiles/cgroups.nix ../profiles/defaults.nix ../profiles/disk/basic-vm.nix diff --git a/machines/bree.nix b/machines/bree.nix index e4cd443..d779ce3 100644 --- a/machines/bree.nix +++ b/machines/bree.nix @@ -2,6 +2,7 @@ { imports = [ ../profiles/cgroups.nix + ../profiles/core-metrics.nix ../profiles/defaults.nix ../profiles/disk/basic-vm.nix ../profiles/hardware/synology-vm.nix diff --git a/machines/rivendell.nix b/machines/rivendell.nix index 83dcb2e..c48997a 100644 --- a/machines/rivendell.nix +++ b/machines/rivendell.nix @@ -3,6 +3,7 @@ imports = [ ../profiles/authelia.nix ../profiles/cgroups.nix + ../profiles/core-metrics.nix ../profiles/defaults.nix ../profiles/disk/btrfs-on-luks.nix ../profiles/git-server.nix @@ -10,6 +11,7 @@ ../profiles/home-manager.nix ../profiles/makemkv.nix ../profiles/miniflux.nix + ../profiles/monitoring.nix ../profiles/remote-unlock.nix ../profiles/restic-backup.nix ../profiles/server.nix diff --git a/profiles/core-metrics.nix b/profiles/core-metrics.nix new file mode 100644 index 0000000..3f817e5 --- /dev/null +++ b/profiles/core-metrics.nix @@ -0,0 +1,62 @@ +{ hostName, ... }: +let + relabel_configs = [ + { + action = "replace"; + replacement = hostName; + target_label = "instance"; + } + ]; +in +{ + services.prometheus.exporters = { + node.enable = true; + systemd.enable = true; + process.enable = true; + }; + services.vmagent = { + enable = true; + remoteWrite.url = "http://10.100.0.60:8428/api/v1/write"; + prometheusConfig = { + global = { + external_labels = { + "host" = hostName; + }; + }; + scrape_configs = [ + { + job_name = "node"; + scrape_interval = "10s"; + static_configs = [ + { targets = [ "127.0.0.1:9100" ]; } + ]; + inherit relabel_configs; + } + { + job_name = "systemd"; + scrape_interval = "10s"; + static_configs = [ + { targets = [ "127.0.0.1:9558" ]; } + ]; + inherit relabel_configs; + } + { + job_name = "process"; + scrape_interval = "10s"; + static_configs = [ + { targets = [ "127.0.0.1:9256" ]; } + ]; + inherit relabel_configs; + } + { + job_name = "vmagent"; + scrape_interval = "10s"; + static_configs = [ + { targets = [ "127.0.0.1:8429" ]; } + ]; + inherit relabel_configs; + } + ]; + }; + }; +} diff --git a/profiles/monitoring.nix b/profiles/monitoring.nix new file mode 100644 index 0000000..7c62b9e --- /dev/null +++ b/profiles/monitoring.nix @@ -0,0 +1,105 @@ +{ config, ... }: +{ + + age.secrets.grafana-oidc.file = ../secrets/grafana-oidc.age; + + services.victoriametrics.enable = true; + + services.grafana.enable = true; + services.grafana.declarativePlugins = [ ]; + services.grafana.provision.enable = true; + services.grafana.provision.datasources.settings = { + datasources = [ + { + name = "VictoriaMetrics"; + type = "prometheus"; + url = "http://localhost:8428"; + isDefault = true; + jsonData = { + httpMethod = "POST"; + manageAlerts = true; + }; + } + ]; + }; + services.grafana.settings = { + server = { + enable_gzip = true; + http_port = 3000; + http_addr = "10.100.0.60"; + domain = "dash.fcuny.net"; + root_url = "https://dash.fcuny.net/"; + }; + analytics = { + reporting_enabled = false; + check_for_updates = false; + }; + users = { + allow_signup = false; + }; + "auth.generic_oauth" = { + enabled = true; + allow_sign_up = true; + auto_login = true; + name = "Authelia"; + icon = "signin"; + client_id = "grafana"; + # nix run nixpkgs#authelia -- crypto hash generate pbkdf2 --variant sha512 --random --random.length 72 --random.charset rfc3986 + client_secret = "$__file{/run/credentials/grafana.service/oauth2-client-secret}"; + scopes = [ + "openid" + "profile" + "email" + "groups" + ]; + empty_scopes = false; + auth_url = "https://auth.fcuny.net/api/oidc/authorization"; + token_url = "https://auth.fcuny.net/api/oidc/token"; + api_url = "https://auth.fcuny.net/api/oidc/userinfo"; + login_attribute_path = "preferred_username"; + groups_attribute_path = "groups"; + name_attribute_path = "name"; + email_attribute_path = "email"; + use_pkce = true; + allow_assign_grafana_admin = true; + # Refrain from adding trailing or, see github:grafana/grafana#106686 + role_attribute_path = builtins.concatStringsSep " || " [ + "contains(groups, 'grafana-admins') && 'GrafanaAdmin'" + "contains(groups, 'grafana-editors') && 'Editor'" + "contains(groups, 'grafana-viewers') && 'Viewer'" + ]; + role_attribute_strict = true; + skip_org_role_sync = false; + }; + }; + + systemd.services.grafana.serviceConfig.LoadCredential = [ + "oauth2-client-secret:${config.age.secrets.grafana-oidc.path}" + ]; + + services.authelia.instances.main.settings.identity_providers.oidc.clients = [ + { + id = "grafana"; + description = "Grafana"; + client_secret = "$pbkdf2-sha512$310000$yDK1zYFV8y9Zo5iHCv.eQQ$mDpNy3lQ27uqtsbssUaOb8t0rtxD5MBce4sFUqJKE.5y3mVWZir0a1B2q1RaRK/KfgyWxKtNyKRT21Kx7C56Tw"; + public = false; + authorization_policy = "two_factor"; + require_pkce = true; + pkce_challenge_method = "S256"; + redirect_uris = [ "https://dash.fcuny.net/login/generic_oauth" ]; + scopes = [ + "openid" + "profile" + "email" + "groups" + ]; + response_types = [ "code" ]; + grant_types = [ + "authorization_code" + ]; + access_token_signed_response_alg = "none"; + userinfo_signed_response_alg = "none"; + token_endpoint_auth_method = "client_secret_post"; + } + ]; +} diff --git a/profiles/reverse-proxy.nix b/profiles/reverse-proxy.nix index f136ba0..daf2ecb 100644 --- a/profiles/reverse-proxy.nix +++ b/profiles/reverse-proxy.nix @@ -68,6 +68,12 @@ in forceSSL = true; locations."/".proxyPass = "http://${httpHost}:8002"; }; + "dash.fcuny.net" = { + enableACME = true; + acmeRoot = null; + forceSSL = true; + locations."/".proxyPass = "http://${httpHost}:3000"; + }; "fcuny.net" = { enableACME = true; acmeRoot = null; diff --git a/secrets/authelia-users.yaml.age b/secrets/authelia-users.yaml.age index dc8fe62..d21f4e0 100644 Binary files a/secrets/authelia-users.yaml.age and b/secrets/authelia-users.yaml.age differ diff --git a/secrets/grafana-oidc.age b/secrets/grafana-oidc.age new file mode 100644 index 0000000..deaf0c4 --- /dev/null +++ b/secrets/grafana-oidc.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> ssh-ed25519 pFjJaA nXdpTOxE+KOi+hkTl8WrFzsXTLlX6JQhY/6+w6ZcZ0k +6TZjec0mdP37hXGXEev7dN27BqGhvO0EVEJi7XPJsrc +-> ssh-ed25519 Y5h84Q 1um4Z+C9sRiHVMEJszpc4ygNhONX0tNvAsABlvDmwHA +IN3pQyGFCRWphTHLAaxrCVci0OaRViHUaZYqZPEA14A +--- ABsJxwFEMn+GNkH+BqcrSIFfeZJaqSvRTNid1yEDJaA +F꧒bRMwɨqo ;\1nD4 XQLU*oIM:YyItƖJE@ i˸\a% \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index adb15e1..2b645eb 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -42,6 +42,11 @@ in hosts.rivendell ]; + "grafana-oidc.age".publicKeys = [ + users.fcuny + hosts.rivendell + ]; + # generated with: # openssl rand 64 | openssl base64 -A | tr '+/' '-_' | tr -d '=' "authelia-storage-key.age".publicKeys = [ -- cgit v1.2.3