Add monitoring profile: Grafana + Prometheus + Loki + Alloy

Native NixOS modules under my.profiles.monitoring, enabled on jupiter. Grafana provisions datasources/dashboards/alert rules from the grafana-content flake input (rechberg dashboards repo). Prometheus scrapes host + stack (90d), Loki+Alloy ship the systemd journal (90d). Grafana LAN-only :3000, anonymous viewer, admin password + secret_key via /etc/grafana file providers. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CZJkwCnSbykq9rRTTeHc6b
2026-08-07 10:25:04 +02:00
12 changed files with 305 additions and 31 deletions
@@ -21,11 +21,11 @@
        "nixpkgs-lib": "nixpkgs-lib"
      },
      "locked": {
-        "lastModified": 1785627969,
-        "narHash": "sha256-4dtXQk/NMePegK/nWp5NSeuZKLATItOq61lpEvmXqGw=",
+        "lastModified": 1782949081,
+        "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=",
        "owner": "hercules-ci",
        "repo": "flake-parts",
-        "rev": "427bf4bd9435fdf21321c8cc628c24efc14c0f7a",
+        "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e",
        "type": "github"
      },
      "original": {
@@ -79,6 +79,22 @@
        "type": "github"
      }
    },
+    "grafana-content": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1785940786,
+        "narHash": "sha256-flH5pwYtB3QWnZ/OYJKoj9twb1fxYrowr9FCh8L1GMY=",
+        "ref": "refs/heads/master",
+        "rev": "4017b678ffa3bde4136a3468f18f1eb3bba52374",
+        "revCount": 1,
+        "type": "git",
+        "url": "ssh://git@rechberg.online:222/finn.markwitz/dashboards.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "ssh://git@rechberg.online:222/finn.markwitz/dashboards.git"
+      }
+    },
    "nix": {
      "inputs": {
        "flake-compat": "flake-compat",
@@ -89,11 +105,11 @@
        "nixpkgs-regression": "nixpkgs-regression"
      },
      "locked": {
-        "lastModified": 1786447342,
-        "narHash": "sha256-jA79fFQUEsLcpmjWRrlOgVVFVKUQSD4zOZIRKEpEx88=",
+        "lastModified": 1784762557,
+        "narHash": "sha256-R/r6jRnANV50c8F5Fz5+1Q1moab0IGWRk+cg5ME2nMY=",
        "owner": "NixOS",
        "repo": "nix",
-        "rev": "da110e01d913bb32bdc06a301c3797e8c42b8ab7",
+        "rev": "d10c84cd0cc0efdcb29cf2611caf5fbcd10fa071",
        "type": "github"
      },
      "original": {
@@ -143,11 +159,11 @@
        "nixpkgs": "nixpkgs_2"
      },
      "locked": {
-        "lastModified": 1786437054,
-        "narHash": "sha256-I++HzBBAgQ17UaLVU6aSm1/7LDo6c9xr8rAbpByWywE=",
+        "lastModified": 1784723954,
+        "narHash": "sha256-1CfD8ZUjCkTgjsneLZ/lxCHhgDfqxxE7/GX0MmsgiqA=",
        "owner": "NixOS",
        "repo": "nixos-hardware",
-        "rev": "6ed13b1d888d5cb07dbb0723eb1df86bbacd0b9c",
+        "rev": "a017f5b72210026af5b3ac5949f08d94380a6fbd",
        "type": "github"
      },
      "original": {
@@ -187,11 +203,11 @@
    },
    "nixpkgs-lib": {
      "locked": {
-        "lastModified": 1785031560,
-        "narHash": "sha256-OmshNvn2vupOFpYinLUu+1Dnpu4n7Q5N3ggGVNHpkUI=",
+        "lastModified": 1782614948,
+        "narHash": "sha256-ePjCwr1sNm9NYUqywL7QfK3JnlS015msC+eBu2zKlp8=",
        "owner": "nix-community",
        "repo": "nixpkgs.lib",
-        "rev": "0e79af5e3d4dcfcd676ab5ba3f95d2e3352e078c",
+        "rev": "db3f255737b94216eb71cce308e2912cf6bc2d7c",
        "type": "github"
      },
      "original": {
@@ -218,11 +234,11 @@
    },
    "nixpkgs-unstable": {
      "locked": {
-        "lastModified": 1786247143,
-        "narHash": "sha256-qemNveuexlWhK0Qc6Y8o+gtSVTIj5xljzTAXQZxTHWA=",
-        "rev": "279b4a8275f032c566576b3f181fa0f27197f588",
+        "lastModified": 1784796856,
+        "narHash": "sha256-vwxWgF+Gj276WznzGb1LxGsK/39HaQwgQXiU3EkC844=",
+        "rev": "e2587caef70cea85dd97d7daab492899902dbf5d",
        "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1050399.279b4a8275f0/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1040357.e2587caef70c/nixexprs.tar.xz"
      },
      "original": {
        "id": "nixpkgs",
@@ -245,11 +261,11 @@
    },
    "nixpkgs_3": {
      "locked": {
-        "lastModified": 1786313170,
-        "narHash": "sha256-F5163SsL9xqCLcx8CS2Y5MeLA5z6mfgXGDiCBEqSnbo=",
-        "rev": "fcb8fcd6bf2d0adecae5bd491afaaaf8311b758d",
+        "lastModified": 1784707089,
+        "narHash": "sha256-DUedXhD2Rg8q4Xyd07Sb90eZGy4gg6W+Vl/WbLNwAZo=",
+        "rev": "b3fe9581c9061c749abef42b6d4ee7b7c05c33fa",
        "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.7376.fcb8fcd6bf2d/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.5845.b3fe9581c906/nixexprs.tar.xz"
      },
      "original": {
        "id": "nixpkgs",
@@ -260,6 +276,7 @@
    "root": {
      "inputs": {
        "flake-parts": "flake-parts",
+        "grafana-content": "grafana-content",
        "nix": "nix",
        "nixos-generators": "nixos-generators",
        "nixos-hardware": "nixos-hardware",
@@ -12,6 +12,12 @@
      inputs.nixpkgs.follows = "nixpkgs";
    };

+    # Grafana content (dashboards, datasources, alert rules) — provisioned from the store.
+    grafana-content = {
+      url = "git+ssh://git@rechberg.online:222/finn.markwitz/dashboards.git";
+      flake = false;
+    };
+
  };

  outputs =
@@ -22,6 +22,7 @@ in
    jellyfin.enable = true;
    jellyseerr.enable = true;
    immich.enable = true;
+    monitoring.enable = true;
    development.enable = true;
    home-assistant.enable = true;

@@ -4,7 +4,7 @@ _: {
  ];

  networking = {
-    domain = "jupiter";
+    domain = "jupiter.solar.internal";
    search = [ "jupiter.solar.internal" ];
  };
 }
@@ -15,6 +15,7 @@
    ./docker
    ./homepage
    ./kde-desktop
+    ./monitoring
    ./readarr
    ./sonarr
    ./jellyfin
@@ -91,12 +91,10 @@ in
  };

  config = lib.mkIf cfg.enable {
-    networking.firewall.allowedTCPPorts = [ dashboardPort ];
-
    services.homepage-dashboard = {
      enable = true;
      listenPort = dashboardPort;
-      allowedHosts = "${dashboardHost}:${toString dashboardPort},192.168.178.65:${toString dashboardPort},localhost:${toString dashboardPort},127.0.0.1:${toString dashboardPort},jupiter:${toString dashboardPort},jupiter.solar.internal:${toString dashboardPort}";
+      allowedHosts = "${dashboardHost}:${toString dashboardPort},localhost:${toString dashboardPort},127.0.0.1:${toString dashboardPort},jupiter.solar.internal:${toString dashboardPort}";
      bookmarks = import ./bookmarks.nix;
      services = homepageServices ++ manualServices;
      widgets = config.my.homepage.widgets ++ manualWidgets;
@@ -24,13 +24,7 @@ in
      mediaLocation = "/var/lib/immich";
      machine-learning.enable = true;
      accelerationDevices = [ "/dev/dri/renderD128" ];
-      # Setting `settings` puts Immich in config-file mode: the admin settings
-      # UI becomes read-only and system config is managed declaratively here.
-      settings = {
-        server.externalDomain = "http://${hostName}:${toString port}";
-        # Intel Quick Sync hardware transcoding (jupiter's iGPU).
-        ffmpeg.accel = "qsv";
-      };
+      settings.server.externalDomain = "http://${hostName}:${toString port}";
    };

    # The native module does not add GPU groups; required for VAAPI/QSV transcoding.
@@ -0,0 +1,51 @@
+# Grafana Alloy: ships the full systemd journal to Loki.
+# The nixpkgs module reads /etc/alloy/*.alloy and already runs with the
+# systemd-journal supplementary group. Alloy exposes /metrics + UI on :12345.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.alloy.enable = true;
+
+    environment.etc."alloy/config.alloy".text = ''
+      loki.source.journal "journal" {
+        max_age       = "24h"
+        forward_to    = [loki.write.local.receiver]
+        relabel_rules = loki.relabel.journal.rules
+        labels        = { host = "jupiter" }
+      }
+
+      loki.relabel "journal" {
+        forward_to = []
+        rule {
+          source_labels = ["__journal__systemd_unit"]
+          target_label  = "unit"
+        }
+        rule {
+          source_labels = ["__journal_priority_keyword"]
+          target_label  = "priority"
+        }
+        rule {
+          source_labels = ["__journal__transport"]
+          target_label  = "transport"
+        }
+        rule {
+          source_labels = ["__journal__boot_id"]
+          target_label  = "boot_id"
+        }
+      }
+
+      loki.write "local" {
+        endpoint {
+          url = "http://127.0.0.1:3100/loki/api/v1/push"
+        }
+      }
+    '';
+  };
+}
@@ -0,0 +1,16 @@
+# Observability stack: Grafana + Prometheus + Loki + Alloy.
+# Content (dashboards, datasources, alert rules) lives in a separate repo,
+# consumed as the `grafana-content` flake input and provisioned from the store.
+{ lib, ... }:
+{
+  imports = [
+    ./grafana.nix
+    ./prometheus.nix
+    ./loki.nix
+    ./alloy.nix
+  ];
+
+  options.my.profiles.monitoring = {
+    enable = lib.mkEnableOption "Grafana + Prometheus + Loki + Alloy observability stack";
+  };
+}
@@ -0,0 +1,83 @@
+# Grafana service + provisioning from the grafana-content flake input.
+# Provisioning shape verified against nixpkgs 26.05 grafana module:
+#  - datasources.path takes a directory (lndir'd into the provisioning dir)
+#  - dashboard provider is generated here so the store path can be injected
+#  - each alerting resource takes its own YAML *file* (they share one target dir,
+#    so pointing them at a directory would collide)
+{
+  config,
+  lib,
+  inputs,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+  content = inputs.grafana-content;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.grafana = {
+      enable = true;
+
+      settings = {
+        server = {
+          http_addr = "0.0.0.0";
+          http_port = 3000;
+          domain = "jupiter.solar.internal";
+          root_url = "http://jupiter.solar.internal:3000/";
+        };
+        security.admin_password = "$__file{/etc/grafana/admin_password}";
+        # 26.05 removed the built-in default; encrypts secrets in Grafana's DB.
+        security.secret_key = "$__file{/etc/grafana/secret_key}";
+        "auth.anonymous" = {
+          enabled = true;
+          org_role = "Viewer";
+        };
+        metrics.enabled = true; # /metrics for Prometheus self-scrape
+        unified_alerting.enabled = true;
+        alerting.enabled = false; # disable legacy alerting
+      };
+
+      provision = {
+        enable = true;
+
+        # static datasources dir → all *.yaml linked in
+        datasources.path = "${content}/provisioning/datasources";
+
+        # dashboard provider generated here with the store path injected
+        dashboards.settings.providers = [
+          {
+            name = "content";
+            type = "file";
+            disableDeletion = true;
+            allowUiUpdates = false;
+            options = {
+              path = "${content}/dashboards";
+              foldersFromFilesStructure = true;
+            };
+          }
+        ];
+
+        # per-resource alerting files (share one target dir → must be files)
+        alerting = {
+          rules.path = "${content}/provisioning/alerting/rules.yaml";
+          contactPoints.path = "${content}/provisioning/alerting/contactpoints.yaml";
+          policies.path = "${content}/provisioning/alerting/policies.yaml";
+        };
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [ 3000 ];
+
+    # Register a homepage-dashboard tile (matches the other profiles).
+    my.homepage.services = [
+      {
+        group = "Monitoring";
+        name = "Grafana";
+        description = "Metrics & logs dashboards";
+        href = "http://jupiter.solar.internal:3000";
+        icon = "grafana.png";
+      }
+    ];
+  };
+}
@@ -0,0 +1,55 @@
+# Loki single-binary, filesystem storage, tsdb schema, 90d retention via compactor.
+# Localhost only — Grafana is the sole consumer.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.loki = {
+      enable = true;
+      configuration = {
+        server = {
+          http_listen_address = "127.0.0.1";
+          http_listen_port = 3100;
+        };
+        auth_enabled = false;
+
+        common = {
+          instance_addr = "127.0.0.1";
+          path_prefix = "/var/lib/loki";
+          storage.filesystem = {
+            chunks_directory = "/var/lib/loki/chunks";
+            rules_directory = "/var/lib/loki/rules";
+          };
+          replication_factor = 1;
+          ring.kvstore.store = "inmemory";
+        };
+
+        schema_config.configs = [
+          {
+            from = "2024-01-01";
+            store = "tsdb";
+            object_store = "filesystem";
+            schema = "v13";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+
+        limits_config.retention_period = "2160h"; # 90d
+        compactor = {
+          working_directory = "/var/lib/loki/compactor";
+          retention_enabled = true;
+          delete_request_store = "filesystem";
+        };
+      };
+    };
+  };
+}
@@ -0,0 +1,52 @@
+# Prometheus + node_exporter. Prometheus UI reachable on the LAN (:9090);
+# node_exporter bound to localhost. Scrapes the host plus the stack itself.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.prometheus = {
+      enable = true;
+      listenAddress = "0.0.0.0";
+      port = 9090;
+      retentionTime = "90d";
+
+      exporters.node = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        port = 9100;
+        # default collectors
+      };
+
+      scrapeConfigs = [
+        {
+          job_name = "node";
+          static_configs = [ { targets = [ "127.0.0.1:9100" ]; } ];
+        }
+        {
+          job_name = "prometheus";
+          static_configs = [ { targets = [ "127.0.0.1:9090" ]; } ];
+        }
+        {
+          job_name = "grafana";
+          static_configs = [ { targets = [ "127.0.0.1:3000" ]; } ];
+        }
+        {
+          job_name = "loki";
+          static_configs = [ { targets = [ "127.0.0.1:3100" ]; } ];
+        }
+        {
+          job_name = "alloy";
+          static_configs = [ { targets = [ "127.0.0.1:12345" ]; } ];
+        }
+      ];
+    };
+
+    networking.firewall.allowedTCPPorts = [ 9090 ];
+  };
+}