Add monitoring profile: Grafana + Prometheus + Loki + Alloy

Native NixOS modules under my.profiles.monitoring, enabled on jupiter.
Grafana provisions datasources/dashboards/alert rules from the
grafana-content flake input (rechberg dashboards repo). Prometheus
scrapes host + stack (90d), Loki+Alloy ship the systemd journal (90d).
Grafana LAN-only :3000, anonymous viewer, admin password + secret_key
via /etc/grafana file providers.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CZJkwCnSbykq9rRTTeHc6b

flake.lock

@@ -21,11 +21,11 @@
         "nixpkgs-lib": "nixpkgs-lib"
       },
       "locked": {
-        "lastModified": 1785627969,
+        "lastModified": 1782949081,
-        "narHash": "sha256-4dtXQk/NMePegK/nWp5NSeuZKLATItOq61lpEvmXqGw=",
+        "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "427bf4bd9435fdf21321c8cc628c24efc14c0f7a",
+        "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e",
         "type": "github"
       },
       "original": {
@@ -79,6 +79,22 @@
         "type": "github"
       }
     },
+    "grafana-content": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1785940786,
+        "narHash": "sha256-flH5pwYtB3QWnZ/OYJKoj9twb1fxYrowr9FCh8L1GMY=",
+        "ref": "refs/heads/master",
+        "rev": "4017b678ffa3bde4136a3468f18f1eb3bba52374",
+        "revCount": 1,
+        "type": "git",
+        "url": "ssh://git@rechberg.online:222/finn.markwitz/dashboards.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "ssh://git@rechberg.online:222/finn.markwitz/dashboards.git"
+      }
+    },
     "nix": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -89,11 +105,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1787274306,
+        "lastModified": 1784762557,
-        "narHash": "sha256-Qg9f9td5iphUWSQS6zmvyZWO1F+D7j8Z3U6dGyUTg08=",
+        "narHash": "sha256-R/r6jRnANV50c8F5Fz5+1Q1moab0IGWRk+cg5ME2nMY=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "649e823fb24ed118d72e613be35fa8ea1b64afe7",
+        "rev": "d10c84cd0cc0efdcb29cf2611caf5fbcd10fa071",
         "type": "github"
       },
       "original": {
@@ -143,11 +159,11 @@
         "nixpkgs": "nixpkgs_2"
       },
       "locked": {
-        "lastModified": 1787144466,
+        "lastModified": 1784723954,
-        "narHash": "sha256-HHfv2/HkNSKbbSyU9iD/g8lbP6r4tl33sSw1W4rXCk0=",
+        "narHash": "sha256-1CfD8ZUjCkTgjsneLZ/lxCHhgDfqxxE7/GX0MmsgiqA=",
         "owner": "NixOS",
         "repo": "nixos-hardware",
-        "rev": "0471accf8d0a8210b31d947497d179ecc99e0021",
+        "rev": "a017f5b72210026af5b3ac5949f08d94380a6fbd",
         "type": "github"
       },
       "original": {
@@ -187,11 +203,11 @@
     },
     "nixpkgs-lib": {
       "locked": {
-        "lastModified": 1785031560,
+        "lastModified": 1782614948,
-        "narHash": "sha256-OmshNvn2vupOFpYinLUu+1Dnpu4n7Q5N3ggGVNHpkUI=",
+        "narHash": "sha256-ePjCwr1sNm9NYUqywL7QfK3JnlS015msC+eBu2zKlp8=",
         "owner": "nix-community",
         "repo": "nixpkgs.lib",
-        "rev": "0e79af5e3d4dcfcd676ab5ba3f95d2e3352e078c",
+        "rev": "db3f255737b94216eb71cce308e2912cf6bc2d7c",
         "type": "github"
       },
       "original": {
@@ -218,11 +234,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1787135253,
+        "lastModified": 1784796856,
-        "narHash": "sha256-M5/r2v++FjVhdsxXYMb4BDJ5YLAdCWFt3aZotcshocA=",
+        "narHash": "sha256-vwxWgF+Gj276WznzGb1LxGsK/39HaQwgQXiU3EkC844=",
-        "rev": "ffb3c9b700e759be2ef13237c9d8f953b32a1e46",
+        "rev": "e2587caef70cea85dd97d7daab492899902dbf5d",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1058091.ffb3c9b700e7/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1040357.e2587caef70c/nixexprs.tar.xz"
       },
       "original": {
         "id": "nixpkgs",
@@ -245,11 +261,11 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1787101114,
+        "lastModified": 1784707089,
-        "narHash": "sha256-BA7sSNjLDuPGSOYBGpr6WQjke1MQ8AZpJ8GlYZM/mOc=",
+        "narHash": "sha256-DUedXhD2Rg8q4Xyd07Sb90eZGy4gg6W+Vl/WbLNwAZo=",
-        "rev": "b18a4b905f8d028dc4476412e6d6891728695379",
+        "rev": "b3fe9581c9061c749abef42b6d4ee7b7c05c33fa",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.8045.b18a4b905f8d/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.5845.b3fe9581c906/nixexprs.tar.xz"
       },
       "original": {
         "id": "nixpkgs",
@@ -260,6 +276,7 @@
     "root": {
       "inputs": {
         "flake-parts": "flake-parts",
+        "grafana-content": "grafana-content",
         "nix": "nix",
         "nixos-generators": "nixos-generators",
         "nixos-hardware": "nixos-hardware",

flake.nix

@@ -12,6 +12,12 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
+    # Grafana content (dashboards, datasources, alert rules) — provisioned from the store.
+    grafana-content = {
+      url = "git+ssh://git@rechberg.online:222/finn.markwitz/dashboards.git";
+      flake = false;
+    };
+
   };
 
   outputs =

machines/jupiter/environments.nix

@@ -22,7 +22,7 @@ in
     jellyfin.enable = true;
     jellyseerr.enable = true;
     immich.enable = true;
-    newsreader.enable = true;
+    monitoring.enable = true;
     development.enable = true;
     home-assistant.enable = true;
 

modules/environments/default.nix

@@ -15,11 +15,11 @@
     ./docker
     ./homepage
     ./kde-desktop
+    ./monitoring
     ./readarr
     ./sonarr
     ./jellyfin
     ./jellyseerr
     ./immich
-    ./newsreader
   ];
 }

modules/environments/immich/default.nix

@@ -24,13 +24,7 @@ in
       mediaLocation = "/var/lib/immich";
       machine-learning.enable = true;
       accelerationDevices = [ "/dev/dri/renderD128" ];
-      # Setting `settings` puts Immich in config-file mode: the admin settings
+      settings.server.externalDomain = "http://${hostName}:${toString port}";
-      # UI becomes read-only and system config is managed declaratively here.
-      settings = {
-        server.externalDomain = "http://${hostName}:${toString port}";
-        # Intel Quick Sync hardware transcoding (jupiter's iGPU).
-        ffmpeg.accel = "qsv";
-      };
     };
 
     # The native module does not add GPU groups; required for VAAPI/QSV transcoding.

modules/environments/monitoring/alloy.nix

@@ -0,0 +1,51 @@
+# Grafana Alloy: ships the full systemd journal to Loki.
+# The nixpkgs module reads /etc/alloy/*.alloy and already runs with the
+# systemd-journal supplementary group. Alloy exposes /metrics + UI on :12345.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.alloy.enable = true;
+
+    environment.etc."alloy/config.alloy".text = ''
+      loki.source.journal "journal" {
+        max_age       = "24h"
+        forward_to    = [loki.write.local.receiver]
+        relabel_rules = loki.relabel.journal.rules
+        labels        = { host = "jupiter" }
+      }
+
+      loki.relabel "journal" {
+        forward_to = []
+        rule {
+          source_labels = ["__journal__systemd_unit"]
+          target_label  = "unit"
+        }
+        rule {
+          source_labels = ["__journal_priority_keyword"]
+          target_label  = "priority"
+        }
+        rule {
+          source_labels = ["__journal__transport"]
+          target_label  = "transport"
+        }
+        rule {
+          source_labels = ["__journal__boot_id"]
+          target_label  = "boot_id"
+        }
+      }
+
+      loki.write "local" {
+        endpoint {
+          url = "http://127.0.0.1:3100/loki/api/v1/push"
+        }
+      }
+    '';
+  };
+}

modules/environments/monitoring/default.nix

@@ -0,0 +1,16 @@
+# Observability stack: Grafana + Prometheus + Loki + Alloy.
+# Content (dashboards, datasources, alert rules) lives in a separate repo,
+# consumed as the `grafana-content` flake input and provisioned from the store.
+{ lib, ... }:
+{
+  imports = [
+    ./grafana.nix
+    ./prometheus.nix
+    ./loki.nix
+    ./alloy.nix
+  ];
+
+  options.my.profiles.monitoring = {
+    enable = lib.mkEnableOption "Grafana + Prometheus + Loki + Alloy observability stack";
+  };
+}

modules/environments/monitoring/grafana.nix

@@ -0,0 +1,83 @@
+# Grafana service + provisioning from the grafana-content flake input.
+# Provisioning shape verified against nixpkgs 26.05 grafana module:
+#  - datasources.path takes a directory (lndir'd into the provisioning dir)
+#  - dashboard provider is generated here so the store path can be injected
+#  - each alerting resource takes its own YAML *file* (they share one target dir,
+#    so pointing them at a directory would collide)
+{
+  config,
+  lib,
+  inputs,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+  content = inputs.grafana-content;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.grafana = {
+      enable = true;
+
+      settings = {
+        server = {
+          http_addr = "0.0.0.0";
+          http_port = 3000;
+          domain = "jupiter.solar.internal";
+          root_url = "http://jupiter.solar.internal:3000/";
+        };
+        security.admin_password = "$__file{/etc/grafana/admin_password}";
+        # 26.05 removed the built-in default; encrypts secrets in Grafana's DB.
+        security.secret_key = "$__file{/etc/grafana/secret_key}";
+        "auth.anonymous" = {
+          enabled = true;
+          org_role = "Viewer";
+        };
+        metrics.enabled = true; # /metrics for Prometheus self-scrape
+        unified_alerting.enabled = true;
+        alerting.enabled = false; # disable legacy alerting
+      };
+
+      provision = {
+        enable = true;
+
+        # static datasources dir → all *.yaml linked in
+        datasources.path = "${content}/provisioning/datasources";
+
+        # dashboard provider generated here with the store path injected
+        dashboards.settings.providers = [
+          {
+            name = "content";
+            type = "file";
+            disableDeletion = true;
+            allowUiUpdates = false;
+            options = {
+              path = "${content}/dashboards";
+              foldersFromFilesStructure = true;
+            };
+          }
+        ];
+
+        # per-resource alerting files (share one target dir → must be files)
+        alerting = {
+          rules.path = "${content}/provisioning/alerting/rules.yaml";
+          contactPoints.path = "${content}/provisioning/alerting/contactpoints.yaml";
+          policies.path = "${content}/provisioning/alerting/policies.yaml";
+        };
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [ 3000 ];
+
+    # Register a homepage-dashboard tile (matches the other profiles).
+    my.homepage.services = [
+      {
+        group = "Monitoring";
+        name = "Grafana";
+        description = "Metrics & logs dashboards";
+        href = "http://jupiter.solar.internal:3000";
+        icon = "grafana.png";
+      }
+    ];
+  };
+}

modules/environments/monitoring/loki.nix

@@ -0,0 +1,55 @@
+# Loki single-binary, filesystem storage, tsdb schema, 90d retention via compactor.
+# Localhost only — Grafana is the sole consumer.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.loki = {
+      enable = true;
+      configuration = {
+        server = {
+          http_listen_address = "127.0.0.1";
+          http_listen_port = 3100;
+        };
+        auth_enabled = false;
+
+        common = {
+          instance_addr = "127.0.0.1";
+          path_prefix = "/var/lib/loki";
+          storage.filesystem = {
+            chunks_directory = "/var/lib/loki/chunks";
+            rules_directory = "/var/lib/loki/rules";
+          };
+          replication_factor = 1;
+          ring.kvstore.store = "inmemory";
+        };
+
+        schema_config.configs = [
+          {
+            from = "2024-01-01";
+            store = "tsdb";
+            object_store = "filesystem";
+            schema = "v13";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+
+        limits_config.retention_period = "2160h"; # 90d
+        compactor = {
+          working_directory = "/var/lib/loki/compactor";
+          retention_enabled = true;
+          delete_request_store = "filesystem";
+        };
+      };
+    };
+  };
+}

modules/environments/monitoring/prometheus.nix

@@ -0,0 +1,52 @@
+# Prometheus + node_exporter. Prometheus UI reachable on the LAN (:9090);
+# node_exporter bound to localhost. Scrapes the host plus the stack itself.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.my.profiles.monitoring;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.prometheus = {
+      enable = true;
+      listenAddress = "0.0.0.0";
+      port = 9090;
+      retentionTime = "90d";
+
+      exporters.node = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        port = 9100;
+        # default collectors
+      };
+
+      scrapeConfigs = [
+        {
+          job_name = "node";
+          static_configs = [ { targets = [ "127.0.0.1:9100" ]; } ];
+        }
+        {
+          job_name = "prometheus";
+          static_configs = [ { targets = [ "127.0.0.1:9090" ]; } ];
+        }
+        {
+          job_name = "grafana";
+          static_configs = [ { targets = [ "127.0.0.1:3000" ]; } ];
+        }
+        {
+          job_name = "loki";
+          static_configs = [ { targets = [ "127.0.0.1:3100" ]; } ];
+        }
+        {
+          job_name = "alloy";
+          static_configs = [ { targets = [ "127.0.0.1:12345" ]; } ];
+        }
+      ];
+    };
+
+    networking.firewall.allowedTCPPorts = [ 9090 ];
+  };
+}

modules/environments/newsreader/default.nix

@@ -1,102 +0,0 @@
-# X (Twitter) news reader: RSSHub feed bridge + Miniflux reader
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.my.profiles.newsreader;
-  hostName = config.networking.hostName;
-
-  # RSSHub only ever talks to Miniflux on the same host, so it stays on
-  # loopback and out of the firewall.
-  rsshubPort = 1200;
-in
-{
-  options.my.profiles.newsreader = with lib; {
-    enable = mkEnableOption "RSSHub + Miniflux news reader";
-
-    port = mkOption {
-      type = types.port;
-      default = 8085; # 8080 is taken by aria on jupiter
-      description = "Port Miniflux listens on.";
-    };
-
-    rsshubSecretFile = mkOption {
-      type = types.path;
-      default = "/var/lib/secrets/rsshub.env";
-      description = ''
-        EnvironmentFile holding RSSHub's X session, in the form
-
-        ```
-        TWITTER_AUTH_TOKEN=<auth_token cookie>,<optional second cookie>
-        ```
-
-        X removed guest access, so the bridge needs a logged-in session: copy
-        the `auth_token` cookie from a burner account and close the tab without
-        logging out, since logging out invalidates it. Listing several cookies
-        gives RSSHub rotation headroom when one gets suspended.
-
-        Create this file by hand, root-owned and chmod 600 — it must not end up
-        in the Nix store.
-      '';
-    };
-
-    minifluxSecretFile = mkOption {
-      type = types.path;
-      default = "/var/lib/secrets/miniflux.env";
-      description = ''
-        EnvironmentFile holding the Miniflux admin account:
-
-        ```
-        ADMIN_USERNAME=finn
-        ADMIN_PASSWORD=<at least 6 characters>
-        ```
-      '';
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    # Turns X accounts, lists and keyword searches into RSS. Feed URLs look
-    # like http://127.0.0.1:1200/twitter/user/<handle>, /twitter/list/<id> or
-    # /twitter/keyword/<query>.
-    services.rsshub = {
-      enable = true;
-      redis.enable = true;
-      secretFiles = [ cfg.rsshubSecretFile ];
-      settings = {
-        PORT = rsshubPort;
-        LISTEN_INADDR_ANY = false;
-        # X throttles aggressively and answers with an empty 200 rather than an
-        # error, so cache for an hour and keep retries low.
-        CACHE_EXPIRE = "3600";
-        REQUEST_RETRY = "3";
-      };
-    };
-
-    services.miniflux = {
-      enable = true;
-      adminCredentialsFile = cfg.minifluxSecretFile;
-      config = {
-        LISTEN_ADDR = "0.0.0.0:${toString cfg.port}";
-        BASE_URL = "http://${hostName}:${toString cfg.port}/";
-        CREATE_ADMIN = 1;
-        # Minutes. Matched to RSSHub's cache; polling harder just burns the
-        # X session for nothing.
-        POLLING_FREQUENCY = 60;
-      };
-    };
-
-    networking.firewall.allowedTCPPorts = [ cfg.port ];
-
-    my.homepage.services = [
-      {
-        group = "Services";
-        name = "Miniflux";
-        description = "RSS reader";
-        href = "http://${hostName}:${toString cfg.port}";
-        icon = "miniflux.png";
-      }
-    ];
-  };
-}