From 16436901243098dbb394d87e0fe53043c691034e Mon Sep 17 00:00:00 2001 From: "Finn@MiBook" Date: Fri, 7 Aug 2026 10:25:04 +0200 Subject: [PATCH] Add monitoring profile: Grafana + Prometheus + Loki + Alloy Native NixOS modules under my.profiles.monitoring, enabled on jupiter. Grafana provisions datasources/dashboards/alert rules from the grafana-content flake input (rechberg dashboards repo). Prometheus scrapes host + stack (90d), Loki+Alloy ship the systemd journal (90d). Grafana LAN-only :3000, anonymous viewer, admin password + secret_key via /etc/grafana file providers. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01CZJkwCnSbykq9rRTTeHc6b --- flake.lock | 17 ++++ flake.nix | 6 ++ machines/jupiter/environments.nix | 1 + modules/environments/default.nix | 1 + modules/environments/monitoring/alloy.nix | 51 ++++++++++++ modules/environments/monitoring/default.nix | 16 ++++ modules/environments/monitoring/grafana.nix | 83 +++++++++++++++++++ modules/environments/monitoring/loki.nix | 55 ++++++++++++ .../environments/monitoring/prometheus.nix | 52 ++++++++++++ 9 files changed, 282 insertions(+) create mode 100644 modules/environments/monitoring/alloy.nix create mode 100644 modules/environments/monitoring/default.nix create mode 100644 modules/environments/monitoring/grafana.nix create mode 100644 modules/environments/monitoring/loki.nix create mode 100644 modules/environments/monitoring/prometheus.nix diff --git a/flake.lock b/flake.lock index 92acc7f..c79a2c1 100644 --- a/flake.lock +++ b/flake.lock @@ -79,6 +79,22 @@ "type": "github" } }, + "grafana-content": { + "flake": false, + "locked": { + "lastModified": 1785940786, + "narHash": "sha256-flH5pwYtB3QWnZ/OYJKoj9twb1fxYrowr9FCh8L1GMY=", + "ref": "refs/heads/master", + "rev": "4017b678ffa3bde4136a3468f18f1eb3bba52374", + "revCount": 1, + "type": "git", + "url": "ssh://git@rechberg.online:222/finn.markwitz/dashboards.git" + }, + "original": { + "type": "git", + "url": "ssh://git@rechberg.online:222/finn.markwitz/dashboards.git" + } + }, "nix": { "inputs": { "flake-compat": "flake-compat", @@ -260,6 +276,7 @@ "root": { "inputs": { "flake-parts": "flake-parts", + "grafana-content": "grafana-content", "nix": "nix", "nixos-generators": "nixos-generators", "nixos-hardware": "nixos-hardware", diff --git a/flake.nix b/flake.nix index 362e14f..0df0de0 100644 --- a/flake.nix +++ b/flake.nix @@ -12,6 +12,12 @@ inputs.nixpkgs.follows = "nixpkgs"; }; + # Grafana content (dashboards, datasources, alert rules) — provisioned from the store. + grafana-content = { + url = "git+ssh://git@rechberg.online:222/finn.markwitz/dashboards.git"; + flake = false; + }; + }; outputs = diff --git a/machines/jupiter/environments.nix b/machines/jupiter/environments.nix index e36c825..c9e69b7 100644 --- a/machines/jupiter/environments.nix +++ b/machines/jupiter/environments.nix @@ -22,6 +22,7 @@ in jellyfin.enable = true; jellyseerr.enable = true; immich.enable = true; + monitoring.enable = true; development.enable = true; home-assistant.enable = true; diff --git a/modules/environments/default.nix b/modules/environments/default.nix index 9158f6a..9eb7257 100644 --- a/modules/environments/default.nix +++ b/modules/environments/default.nix @@ -15,6 +15,7 @@ ./docker ./homepage ./kde-desktop + ./monitoring ./readarr ./sonarr ./jellyfin diff --git a/modules/environments/monitoring/alloy.nix b/modules/environments/monitoring/alloy.nix new file mode 100644 index 0000000..0923299 --- /dev/null +++ b/modules/environments/monitoring/alloy.nix @@ -0,0 +1,51 @@ +# Grafana Alloy: ships the full systemd journal to Loki. +# The nixpkgs module reads /etc/alloy/*.alloy and already runs with the +# systemd-journal supplementary group. Alloy exposes /metrics + UI on :12345. +{ + config, + lib, + ... +}: +let + cfg = config.my.profiles.monitoring; +in +{ + config = lib.mkIf cfg.enable { + services.alloy.enable = true; + + environment.etc."alloy/config.alloy".text = '' + loki.source.journal "journal" { + max_age = "24h" + forward_to = [loki.write.local.receiver] + relabel_rules = loki.relabel.journal.rules + labels = { host = "jupiter" } + } + + loki.relabel "journal" { + forward_to = [] + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "priority" + } + rule { + source_labels = ["__journal__transport"] + target_label = "transport" + } + rule { + source_labels = ["__journal__boot_id"] + target_label = "boot_id" + } + } + + loki.write "local" { + endpoint { + url = "http://127.0.0.1:3100/loki/api/v1/push" + } + } + ''; + }; +} diff --git a/modules/environments/monitoring/default.nix b/modules/environments/monitoring/default.nix new file mode 100644 index 0000000..54dd5c2 --- /dev/null +++ b/modules/environments/monitoring/default.nix @@ -0,0 +1,16 @@ +# Observability stack: Grafana + Prometheus + Loki + Alloy. +# Content (dashboards, datasources, alert rules) lives in a separate repo, +# consumed as the `grafana-content` flake input and provisioned from the store. +{ lib, ... }: +{ + imports = [ + ./grafana.nix + ./prometheus.nix + ./loki.nix + ./alloy.nix + ]; + + options.my.profiles.monitoring = { + enable = lib.mkEnableOption "Grafana + Prometheus + Loki + Alloy observability stack"; + }; +} diff --git a/modules/environments/monitoring/grafana.nix b/modules/environments/monitoring/grafana.nix new file mode 100644 index 0000000..ad4277c --- /dev/null +++ b/modules/environments/monitoring/grafana.nix @@ -0,0 +1,83 @@ +# Grafana service + provisioning from the grafana-content flake input. +# Provisioning shape verified against nixpkgs 26.05 grafana module: +# - datasources.path takes a directory (lndir'd into the provisioning dir) +# - dashboard provider is generated here so the store path can be injected +# - each alerting resource takes its own YAML *file* (they share one target dir, +# so pointing them at a directory would collide) +{ + config, + lib, + inputs, + ... +}: +let + cfg = config.my.profiles.monitoring; + content = inputs.grafana-content; +in +{ + config = lib.mkIf cfg.enable { + services.grafana = { + enable = true; + + settings = { + server = { + http_addr = "0.0.0.0"; + http_port = 3000; + domain = "jupiter.solar.internal"; + root_url = "http://jupiter.solar.internal:3000/"; + }; + security.admin_password = "$__file{/etc/grafana/admin_password}"; + # 26.05 removed the built-in default; encrypts secrets in Grafana's DB. + security.secret_key = "$__file{/etc/grafana/secret_key}"; + "auth.anonymous" = { + enabled = true; + org_role = "Viewer"; + }; + metrics.enabled = true; # /metrics for Prometheus self-scrape + unified_alerting.enabled = true; + alerting.enabled = false; # disable legacy alerting + }; + + provision = { + enable = true; + + # static datasources dir → all *.yaml linked in + datasources.path = "${content}/provisioning/datasources"; + + # dashboard provider generated here with the store path injected + dashboards.settings.providers = [ + { + name = "content"; + type = "file"; + disableDeletion = true; + allowUiUpdates = false; + options = { + path = "${content}/dashboards"; + foldersFromFilesStructure = true; + }; + } + ]; + + # per-resource alerting files (share one target dir → must be files) + alerting = { + rules.path = "${content}/provisioning/alerting/rules.yaml"; + contactPoints.path = "${content}/provisioning/alerting/contactpoints.yaml"; + policies.path = "${content}/provisioning/alerting/policies.yaml"; + }; + }; + }; + + networking.firewall.allowedTCPPorts = [ 3000 ]; + + # Register a homepage-dashboard tile (matches the other profiles). + my.homepage.services = [ + { + group = "Monitoring"; + name = "Grafana"; + description = "Metrics & logs dashboards"; + href = "http://jupiter.solar.internal:3000"; + icon = "grafana.png"; + } + ]; + }; +} diff --git a/modules/environments/monitoring/loki.nix b/modules/environments/monitoring/loki.nix new file mode 100644 index 0000000..5847df4 --- /dev/null +++ b/modules/environments/monitoring/loki.nix @@ -0,0 +1,55 @@ +# Loki single-binary, filesystem storage, tsdb schema, 90d retention via compactor. +# Localhost only — Grafana is the sole consumer. +{ + config, + lib, + ... +}: +let + cfg = config.my.profiles.monitoring; +in +{ + config = lib.mkIf cfg.enable { + services.loki = { + enable = true; + configuration = { + server = { + http_listen_address = "127.0.0.1"; + http_listen_port = 3100; + }; + auth_enabled = false; + + common = { + instance_addr = "127.0.0.1"; + path_prefix = "/var/lib/loki"; + storage.filesystem = { + chunks_directory = "/var/lib/loki/chunks"; + rules_directory = "/var/lib/loki/rules"; + }; + replication_factor = 1; + ring.kvstore.store = "inmemory"; + }; + + schema_config.configs = [ + { + from = "2024-01-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + + limits_config.retention_period = "2160h"; # 90d + compactor = { + working_directory = "/var/lib/loki/compactor"; + retention_enabled = true; + delete_request_store = "filesystem"; + }; + }; + }; + }; +} diff --git a/modules/environments/monitoring/prometheus.nix b/modules/environments/monitoring/prometheus.nix new file mode 100644 index 0000000..a646486 --- /dev/null +++ b/modules/environments/monitoring/prometheus.nix @@ -0,0 +1,52 @@ +# Prometheus + node_exporter. Prometheus UI reachable on the LAN (:9090); +# node_exporter bound to localhost. Scrapes the host plus the stack itself. +{ + config, + lib, + ... +}: +let + cfg = config.my.profiles.monitoring; +in +{ + config = lib.mkIf cfg.enable { + services.prometheus = { + enable = true; + listenAddress = "0.0.0.0"; + port = 9090; + retentionTime = "90d"; + + exporters.node = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9100; + # default collectors + }; + + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ { targets = [ "127.0.0.1:9100" ]; } ]; + } + { + job_name = "prometheus"; + static_configs = [ { targets = [ "127.0.0.1:9090" ]; } ]; + } + { + job_name = "grafana"; + static_configs = [ { targets = [ "127.0.0.1:3000" ]; } ]; + } + { + job_name = "loki"; + static_configs = [ { targets = [ "127.0.0.1:3100" ]; } ]; + } + { + job_name = "alloy"; + static_configs = [ { targets = [ "127.0.0.1:12345" ]; } ]; + } + ]; + }; + + networking.firewall.allowedTCPPorts = [ 9090 ]; + }; +} -- 2.52.0