Add monitoring profile: Grafana + Prometheus + Loki + Alloy

Native NixOS modules under my.profiles.monitoring, enabled on jupiter.
Grafana provisions datasources/dashboards/alert rules from the
grafana-content flake input (rechberg dashboards repo). Prometheus
scrapes host + stack (90d), Loki+Alloy ship the systemd journal (90d).
Grafana LAN-only :3000, anonymous viewer, admin password + secret_key
via /etc/grafana file providers.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CZJkwCnSbykq9rRTTeHc6b

flake.lock

@@ -79,6 +79,22 @@
         "type": "github"
       }
     },
+    "grafana-content": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1785940786,
+        "narHash": "sha256-flH5pwYtB3QWnZ/OYJKoj9twb1fxYrowr9FCh8L1GMY=",
+        "ref": "refs/heads/master",
+        "rev": "4017b678ffa3bde4136a3468f18f1eb3bba52374",
+        "revCount": 1,
+        "type": "git",
+        "url": "ssh://git@rechberg.online:222/finn.markwitz/dashboards.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "ssh://git@rechberg.online:222/finn.markwitz/dashboards.git"
+      }
+    },
     "nix": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -260,6 +276,7 @@
     "root": {
       "inputs": {
         "flake-parts": "flake-parts",
+        "grafana-content": "grafana-content",
         "nix": "nix",
         "nixos-generators": "nixos-generators",
         "nixos-hardware": "nixos-hardware",

flake.nix

@@ -12,6 +12,12 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
+    # Grafana content (dashboards, datasources, alert rules) — provisioned from the store.
+    grafana-content = {
+      url = "git+ssh://git@rechberg.online:222/finn.markwitz/dashboards.git";
+      flake = false;
+    };
+
   };
 
   outputs =

machines/jupiter/environments.nix

@@ -22,6 +22,7 @@ in
     jellyfin.enable = true;
     jellyseerr.enable = true;
     immich.enable = true;
+    monitoring.enable = true;
     development.enable = true;
     home-assistant.enable = true;
 

modules/environments/default.nix

@@ -15,6 +15,7 @@
     ./docker
     ./homepage
     ./kde-desktop
+    ./monitoring
     ./readarr
     ./sonarr
     ./jellyfin

modules/environments/monitoring/alloy.nix

@@ -0,0 +1,51 @@
+# Grafana Alloy: ships the full systemd journal to Loki.
+# The nixpkgs module reads /etc/alloy/*.alloy and already runs with the
+# systemd-journal supplementary group. Alloy exposes /metrics + UI on :12345.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.alloy.enable = true;
+
+    environment.etc."alloy/config.alloy".text = ''
+      loki.source.journal "journal" {
+        max_age       = "24h"
+        forward_to    = [loki.write.local.receiver]
+        relabel_rules = loki.relabel.journal.rules
+        labels        = { host = "jupiter" }
+      }
+
+      loki.relabel "journal" {
+        forward_to = []
+        rule {
+          source_labels = ["__journal__systemd_unit"]
+          target_label  = "unit"
+        }
+        rule {
+          source_labels = ["__journal_priority_keyword"]
+          target_label  = "priority"
+        }
+        rule {
+          source_labels = ["__journal__transport"]
+          target_label  = "transport"
+        }
+        rule {
+          source_labels = ["__journal__boot_id"]
+          target_label  = "boot_id"
+        }
+      }
+
+      loki.write "local" {
+        endpoint {
+          url = "http://127.0.0.1:3100/loki/api/v1/push"
+        }
+      }
+    '';
+  };
+}

modules/environments/monitoring/default.nix

@@ -0,0 +1,16 @@
+# Observability stack: Grafana + Prometheus + Loki + Alloy.
+# Content (dashboards, datasources, alert rules) lives in a separate repo,
+# consumed as the `grafana-content` flake input and provisioned from the store.
+{ lib, ... }:
+{
+  imports = [
+    ./grafana.nix
+    ./prometheus.nix
+    ./loki.nix
+    ./alloy.nix
+  ];
+
+  options.my.profiles.monitoring = {
+    enable = lib.mkEnableOption "Grafana + Prometheus + Loki + Alloy observability stack";
+  };
+}

modules/environments/monitoring/grafana.nix

@@ -0,0 +1,83 @@
+# Grafana service + provisioning from the grafana-content flake input.
+# Provisioning shape verified against nixpkgs 26.05 grafana module:
+#  - datasources.path takes a directory (lndir'd into the provisioning dir)
+#  - dashboard provider is generated here so the store path can be injected
+#  - each alerting resource takes its own YAML *file* (they share one target dir,
+#    so pointing them at a directory would collide)
+{
+  config,
+  lib,
+  inputs,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+  content = inputs.grafana-content;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.grafana = {
+      enable = true;
+
+      settings = {
+        server = {
+          http_addr = "0.0.0.0";
+          http_port = 3000;
+          domain = "jupiter.solar.internal";
+          root_url = "http://jupiter.solar.internal:3000/";
+        };
+        security.admin_password = "$__file{/etc/grafana/admin_password}";
+        # 26.05 removed the built-in default; encrypts secrets in Grafana's DB.
+        security.secret_key = "$__file{/etc/grafana/secret_key}";
+        "auth.anonymous" = {
+          enabled = true;
+          org_role = "Viewer";
+        };
+        metrics.enabled = true; # /metrics for Prometheus self-scrape
+        unified_alerting.enabled = true;
+        alerting.enabled = false; # disable legacy alerting
+      };
+
+      provision = {
+        enable = true;
+
+        # static datasources dir → all *.yaml linked in
+        datasources.path = "${content}/provisioning/datasources";
+
+        # dashboard provider generated here with the store path injected
+        dashboards.settings.providers = [
+          {
+            name = "content";
+            type = "file";
+            disableDeletion = true;
+            allowUiUpdates = false;
+            options = {
+              path = "${content}/dashboards";
+              foldersFromFilesStructure = true;
+            };
+          }
+        ];
+
+        # per-resource alerting files (share one target dir → must be files)
+        alerting = {
+          rules.path = "${content}/provisioning/alerting/rules.yaml";
+          contactPoints.path = "${content}/provisioning/alerting/contactpoints.yaml";
+          policies.path = "${content}/provisioning/alerting/policies.yaml";
+        };
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [ 3000 ];
+
+    # Register a homepage-dashboard tile (matches the other profiles).
+    my.homepage.services = [
+      {
+        group = "Monitoring";
+        name = "Grafana";
+        description = "Metrics & logs dashboards";
+        href = "http://jupiter.solar.internal:3000";
+        icon = "grafana.png";
+      }
+    ];
+  };
+}

modules/environments/monitoring/loki.nix

@@ -0,0 +1,55 @@
+# Loki single-binary, filesystem storage, tsdb schema, 90d retention via compactor.
+# Localhost only — Grafana is the sole consumer.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.loki = {
+      enable = true;
+      configuration = {
+        server = {
+          http_listen_address = "127.0.0.1";
+          http_listen_port = 3100;
+        };
+        auth_enabled = false;
+
+        common = {
+          instance_addr = "127.0.0.1";
+          path_prefix = "/var/lib/loki";
+          storage.filesystem = {
+            chunks_directory = "/var/lib/loki/chunks";
+            rules_directory = "/var/lib/loki/rules";
+          };
+          replication_factor = 1;
+          ring.kvstore.store = "inmemory";
+        };
+
+        schema_config.configs = [
+          {
+            from = "2024-01-01";
+            store = "tsdb";
+            object_store = "filesystem";
+            schema = "v13";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+
+        limits_config.retention_period = "2160h"; # 90d
+        compactor = {
+          working_directory = "/var/lib/loki/compactor";
+          retention_enabled = true;
+          delete_request_store = "filesystem";
+        };
+      };
+    };
+  };
+}

modules/environments/monitoring/prometheus.nix

@@ -0,0 +1,52 @@
+# Prometheus + node_exporter. Prometheus UI reachable on the LAN (:9090);
+# node_exporter bound to localhost. Scrapes the host plus the stack itself.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.prometheus = {
+      enable = true;
+      listenAddress = "0.0.0.0";
+      port = 9090;
+      retentionTime = "90d";
+
+      exporters.node = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        port = 9100;
+        # default collectors
+      };
+
+      scrapeConfigs = [
+        {
+          job_name = "node";
+          static_configs = [ { targets = [ "127.0.0.1:9100" ]; } ];
+        }
+        {
+          job_name = "prometheus";
+          static_configs = [ { targets = [ "127.0.0.1:9090" ]; } ];
+        }
+        {
+          job_name = "grafana";
+          static_configs = [ { targets = [ "127.0.0.1:3000" ]; } ];
+        }
+        {
+          job_name = "loki";
+          static_configs = [ { targets = [ "127.0.0.1:3100" ]; } ];
+        }
+        {
+          job_name = "alloy";
+          static_configs = [ { targets = [ "127.0.0.1:12345" ]; } ];
+        }
+      ];
+    };
+
+    networking.firewall.allowedTCPPorts = [ 9090 ];
+  };
+}