obs,vicm: fix values for storageclass

.github/dependabot.yml

@@ -13,6 +13,7 @@ updates:
       - "/docker/home/zigbee2mqtt"
       - "/docker/infrastructure/mail/roundcube"
       - "/docker/infrastructure/network/traefik"
+      - "/docker/infrastructure/observability"
       - "/docker/infrastructure/squid"
       - "/docker/infrastructure/sshportal"
       - "/docker/personal/fireshare"

.swarmcd/stacks.yaml

@@ -38,6 +38,11 @@ traefik:
   branch: main
   compose_file: docker/infrastructure/network/traefik/docker-compose.yml
 
+observability:
+  repo: homeprod
+  branch: main
+  compose_file: docker/infrastructure/observability/docker-compose.yml
+
 squid:
   repo: homeprod
   branch: main

docker/infrastructure/observability/alloy.river

@@ -0,0 +1,71 @@
+// Grafana Alloy config for the docker host (replaces Promtail, EOL since
+// 2026-03-02). Tails Docker container logs via docker_sd discovery, attaches
+// container_name/compose_service/stream labels, and pushes them to the
+// in-cluster Loki at loki-internal.lan (LAN-only Traefik Ingress on port 80).
+// River config produced by `alloy convert -f promtail` from an equivalent
+// Promtail YAML (so component names and argument shapes are guaranteed
+// correct for Alloy v1.18).
+
+discovery.docker "docker" {
+  host             = "unix:///var/run/docker.sock"
+  refresh_interval = "5s"
+}
+
+loki.process "docker" {
+  forward_to = [loki.write.default.receiver]
+
+  // Parse docker json-file log lines
+  // ({"log":"...","stream":"stdout","time":"..."}).
+  stage.json {
+    expressions = {
+      log    = "log",
+      stream = "stream",
+    }
+  }
+  stage.labels {
+    values = {
+      stream = null,
+    }
+  }
+}
+
+discovery.relabel "docker" {
+  targets = []
+
+  // Strip leading / from container name.
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container_name"
+  }
+  // Compose service name (if label is set).
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "compose_service"
+  }
+  // Log stream (stdout/stderr).
+  rule {
+    source_labels = ["__meta_docker_container_log_stream"]
+    target_label  = "stream"
+  }
+  // Mark the source so dashboards/alerts can distinguish the docker host.
+  rule {
+    target_label = "source"
+    replacement  = "docker-host"
+  }
+}
+
+loki.source.docker "docker" {
+  host             = "unix:///var/run/docker.sock"
+  targets          = discovery.docker.docker.targets
+  forward_to       = [loki.process.docker.receiver]
+  relabel_rules    = discovery.relabel.docker.rules
+  refresh_interval = "5s"
+}
+
+loki.write "default" {
+  endpoint {
+    url = "http://loki-internal.lan/loki/api/v1/push"
+  }
+  external_labels = {}
+}

docker/infrastructure/observability/docker-compose.yml

@@ -0,0 +1,82 @@
+# Docker-host collectors for the central observability stack on Kubernetes.
+# Managed by swarm-cd (see .swarmcd/stacks.yaml).
+#
+# node-exporter + cAdvisor run locally (scraped by vmagent below).
+# vmagent remote_writes metrics to vm-internal.lan (LAN-only Traefik Ingress) ->
+#   vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428
+# alloy tails /var/lib/docker/containers + /var/log and pushes logs to
+#   loki-internal.lan -> loki.observability.svc.cluster.local:3100
+
+services:
+  node-exporter:
+    image: prom/node-exporter:v1.9.1
+    container_name: obs-node-exporter
+    network_mode: host
+    pid: host
+    command:
+      - "--path.rootfs=/host"
+    volumes:
+      - /:/host:ro,rslave
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.49.1
+    container_name: obs-cadvisor
+    ports:
+      - "8080"
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /dev/disk/:/dev/disk:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  vmagent:
+    image: victoriametrics/vmagent:v1.131.0
+    container_name: obs-vmagent
+    # vmagent listens on 8429 (its own metrics), scrapes node-exporter:9100
+    # and cadvisor:8080, remote_writes to vm-internal.lan
+    command:
+      - "-promscrape.config=/etc/vmagent/vmagent.yml"
+      - "-remoteWrite.url=http://vm-internal.lan/api/v1/write"
+    volumes:
+      - type: bind
+        source: /root/homeprod/docker/infrastructure/observability/vmagent.yml
+        target: /etc/vmagent/vmagent.yml
+        read_only: true
+    depends_on:
+      - node-exporter
+      - cadvisor
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  alloy:
+    image: grafana/alloy:v1.18.0
+    container_name: obs-alloy
+    command:
+      - "run"
+      - "/etc/alloy/config.river"
+      - "--server.http.listen.address=0.0.0.0"
+      - "--server.http.listen.port=12345"
+    volumes:
+      - type: bind
+        source: /root/homeprod/docker/infrastructure/observability/alloy.river
+        target: /etc/alloy/config.river
+        read_only: true
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/log:/var/log:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+networks:
+  default:
+    name: proxy
+    external: true

docker/infrastructure/observability/vmagent.yml

@@ -0,0 +1,22 @@
+# vmagent scrape config for the docker host.
+# Scrapes the two local collectors, forwards everything to the central
+# VictoriaMetrics via the LAN-only Caddy route vm-internal.lan (Caddy proxies
+# it to vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428).
+
+scrape_configs:
+  - job_name: docker-host-node
+    static_configs:
+      - targets:
+          - localhost:9100
+        labels:
+          # Distinguishes the docker host from K8s nodes in dashboards/alerts.
+          instance_source: docker-host
+          host: docker-r740
+
+  - job_name: docker-host-cadvisor
+    static_configs:
+      - targets:
+          - localhost:8080
+        labels:
+          instance_source: docker-host
+          host: docker-r740

kubernetes/infrastructure/authentik/values.yaml

@@ -1,14 +1,14 @@
 authentik:
-    secret_key: ENC[AES256_GCM,data:jSWI2kCndcIZVhU0CqH/l1Gsoeec1GkE2G3ZVoDHUGtvfhsNJOGdr4i8EGDX+2vER4bT0Fm1QboS+jzsCQpAwhLy1R1+8Bt5PsAzTb1QitU=,iv:01q0kIYKUNPigQ54ufVJ/RC96Oa+EmK3zSuzU7aOgFA=,tag:0rdAEbBmioDgKgBjhObzvQ==,type:str]
+    secret_key: ENC[AES256_GCM,data:P5e7cSOa0Uw0NEmWuDzIv1QxIkbvLE9vjzULmS8yS8b62fS5jHuFWUF/8VkW+pXiJo51mZ9b3QWDBdD732hpSrR7hUpK19WdZvFfXMKsqVc=,iv:AuuvJtXdlQ2SzeiFdxv96cEVyeMTNW/fG8bPS4ItFtY=,tag:AJC/T+eXGwOdQ/U7rebIdQ==,type:str]
     postgresql:
-        password: ENC[AES256_GCM,data:EVq2/Kh++C4Lbw==,iv:7RI/kDQbRQT4IlYfGB+6Me7eCrcOwa5JjvYUp/TFqqc=,tag:VtbyKGOvChfarkH7W6jl3A==,type:str]
+        password: ENC[AES256_GCM,data:wmidNe0ODeH8FQ==,iv:zh38F/lIKdFBBIQufjLORtfusPWwDnsDIh61b6ltbQE=,tag:SSMCicRKoPBXwPLFoh+uoQ==,type:str]
     email:
         from: webbot@vhaudiquet.fr
         host: mail.vhaudiquet.fr
         port: 587
-        username: ENC[AES256_GCM,data:K4OGfVrjdFOOQC6yycSvDYzed14=,iv:Gb+VCDbD8k13iz8GZ6NJO7C6GdT3pHp2P0pwaV4DsS4=,tag:PuVgudDPMQaPn7vLwFskzA==,type:str]
+        username: ENC[AES256_GCM,data:VV9x5UWZ2F/p/l4pOVdDB3ZeK98=,iv:H7YXijpo2oJEO96LJOERL4lyQtV7QJq86H33wDpKObU=,tag:q+KiivjpV3kXY0AYGKhzFQ==,type:str]
         use_tls: true
-        password: ENC[AES256_GCM,data:48RFR9gl50OSTwTYXVA=,iv:naCBGgUtoJsvtj2IB2L0fZFvCG3xq0GWxcOKy6fXQqY=,tag:5zYlTsd1vW4IvWahPT0oog==,type:str]
+        password: ENC[AES256_GCM,data:pco+hLWairs2UFNFTSY=,iv:OMrxrzKDo91UIMgEhPdN4dTpRxl5A1LKHYk5h4eFD/A=,tag:ecjAP9lv6FPxPppyurJ5cA==,type:str]
 postgresql:
     enabled: true
     volumePermissions:
@@ -17,18 +17,18 @@ postgresql:
             repository: bitnamilegacy/os-shell
             tag: 12-debian-12-r51
     auth:
-        password: ENC[AES256_GCM,data:IP+dvufKG63iYQ==,iv:IHgt2kZURk5HswfR2X7axcHRTAUD5U7EHD98HISOs5A=,tag:9jyMuImdzhLK4W+VJALHfg==,type:str]
+        password: ENC[AES256_GCM,data:cRaZ23HWjZ41ig==,iv:HABmXhmHvMhjUJ989xJOFPWW+EnLTCot1is80vMTBUE=,tag:r+/iRkQIlXvALv/JQbGFRg==,type:str]
     image:
         repository: bitnamilegacy/postgresql
         tag: 15.9.0
     primary:
         args: []
         tolerations:
-            - key: ENC[AES256_GCM,data:ygEHLXnG+j0n,iv:4v3hfYLoBpoIJKsO7yeFNdje+AGQLYs7lz+rk0gLBY8=,tag:0li5/WTWYb40QWFDDcZm9Q==,type:str]
+            - key: ENC[AES256_GCM,data:M14eAjMj2w2+,iv:5vIosyFtwVgqs07CT2UcL1KL2hqg+rvKS5PWRvWRlPQ=,tag:OJ4RwlLAhGDN4JGPX8gypw==,type:str]
               operator: Equal
-              value: ENC[AES256_GCM,data:GNwXSZCmLFQ=,iv:hZ7NbCZ8cxZEU1JG+BB3ny9WiBZBCTygBHdUEmBhkR8=,tag:5CS8fgBbAuP+2d2MfB2dyA==,type:str]
+              value: ENC[AES256_GCM,data:KKZ1Kd5yRa8=,iv:adSUcfmjnDAc9QZ14NDfFz4oATpcBhtn8iHTF2xXpcw=,tag:G//QEbSKXiGP7nD5oAfNSA==,type:str]
               effect: NoSchedule
-            - key: ENC[AES256_GCM,data:9b+GqCYVJdES0yju9wHGGWQJ+3E1ic8UVrFdvL5dzyBN0Tc1xw==,iv:ykCqmML463Uvz9CpVzZ8TvE7Jd6tvBYHaajCwEvRwLQ=,tag:rjQOiBpclJl8c/LiIsVPMw==,type:str]
+            - key: ENC[AES256_GCM,data:KtJS0g+/3y6VZb9VNKv1IWMwIi8FBURPGT6W6qvBAV1CHW4UMg==,iv:1I8o3SNljujN4FqByFC9uHpImJhHjj4Us0QEpHpDHZ4=,tag:FIpRW9I4OSVFVx4ejmau5w==,type:str]
               operator: Exists
               effect: NoSchedule
         affinity:
@@ -37,7 +37,7 @@ postgresql:
                     - weight: 100
                       preference:
                         matchExpressions:
-                            - key: ENC[AES256_GCM,data:x2Ht92rjwcr2u79BYs/uVNiNbcI=,iv:iUQWjZLQ8B7Jk1DcovsDE29YXk7S6PQQNyAr2Ncxov8=,tag:WgpcfsYz498jboBHe6lV9Q==,type:str]
+                            - key: ENC[AES256_GCM,data:IkWdKkS/XlGw0u7I+Q2wYhXL+CU=,iv:2k9d0Jo4Vw5gA4tGeIq1Dw0IiZKQykfV8YAt2lYk7GE=,tag:Xnz58VP61Ew/J8hvIClOVg==,type:str]
                               operator: NotIn
                               values:
                                 - "true"
@@ -62,49 +62,49 @@ server:
         enabled: true
         hosts:
             - authentik.vhaudiquet.fr
-# Failover tolerations + affinity — allow scheduling on p330 during r740 outage
-# Applied to both server and worker deployments
-tolerations:
-    - key: ENC[AES256_GCM,data:tsGiwDVDE81o,iv:bi6QCnWxfMQLWxK9wu/BQ83QWxHLeEK78FTOOeQjpjw=,tag:mUqoFXmMUif5er9gWI2GMQ==,type:str]
-      operator: Equal
-      value: ENC[AES256_GCM,data:+E3R4/1ql4U=,iv:Cl5foY7erSqCqa/0Pm21zZ6xTkKqoisXvhFhfTvVZ8Q=,tag:Xyr7ZjbRwRtgMQ17J8i4rQ==,type:str]
-      effect: NoSchedule
-    - key: ENC[AES256_GCM,data:gEBAJArMSyvmwrn5W64Ym/FEqFxP1dcJhOe/r+0KDZGQ5Ow7JQ==,iv:PkeYsg3kiYcgtkoHeY9k7WNevbm/3wVyDvPLttsfYF8=,tag:OUH7xTYKrUwWoQZLgcbjIQ==,type:str]
-      operator: Exists
-      effect: NoSchedule
-affinity:
-    nodeAffinity:
-        preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 100
-              preference:
-                matchExpressions:
-                    - key: ENC[AES256_GCM,data:yjIISLuamGGnRMCR7aVfdKNhxHs=,iv:AEqH9gL/rJUQnF6r3yiqu4gnpJmg1oQrMP6MiE3SKDg=,tag:SeUVkc1dMXU/GFxyXQu67A==,type:str]
-                      operator: NotIn
-                      values:
-                        - "true"
+# Failover tolerations + affinity — allow scheduling on p330 during r740 outage.
+# The authentik chart reads global.tolerations (applied to server + worker)
+# and global.affinity.nodeAffinity (preset system with type + matchExpressions).
+global:
+    tolerations:
+        - key: ENC[AES256_GCM,data:yy3LVikckQ4y,iv:PlWRKZVoR6g/RtQdGBoIKbnwjX887xZY7ZhciTa0nps=,tag:yRMuJW3yjBV2C3LDWVdEHg==,type:str]
+          operator: Equal
+          value: ENC[AES256_GCM,data:IqKHhgroaK0=,iv:bvg7kLaVQEsH31xHH1rCW29dWlWDlDsAMAmR1cX2nS4=,tag:B77eeEjVnRwk/feuPyi2Kw==,type:str]
+          effect: NoSchedule
+        - key: ENC[AES256_GCM,data:QYGmJBgxhqC2L3UvGYyJKKWDfzSFq91w1ifwNn9rofPnBqL0Ug==,iv:+tI3xw51LqinZnEOi7Lgao9liOD8KT8rOGNtn10qkFg=,tag:fJhk0PifHEyFlX4j75Sklg==,type:str]
+          operator: Exists
+          effect: NoSchedule
+    affinity:
+        nodeAffinity:
+            type: soft
+            matchExpressions:
+                - key: ENC[AES256_GCM,data:qRInCqtomABFKcUFHzDQe2qmzJY=,iv:9cMrKqqpIMtNCUqkIWaCr5YGNoK6zsID3xAKp34q1kM=,tag:Zf1ZjGwCgnaOqTq8Hr++Fg==,type:str]
+                  operator: NotIn
+                  values:
+                    - "true"
 # (postgresql tolerations are set in the postgresql block above)
 sops:
-    lastmodified: "2026-07-25T12:44:32Z"
-    mac: ENC[AES256_GCM,data:pm8/xGCybCIMBXhP3/7XQDwaFzuv9Ay7L4yaq8RXB35Dkj1hzUe+vGnlZwzxa6cvOh0i2h29Y63Wl1LJQ33+9zjjbzd8+ezF9//GOm4rA2m34JKXMUadAKa7YZz+5N6DGG+48ZfmoV9CFVo/JG/ApHIQMmHNNySb9IsgV9ef9Fo=,iv:4rwii6OKxskFtMrnJJ5LPlEwPPz2DudyKGIdjizbJrk=,tag:JvpjQjp4nK7Zbr71sHZJWA==,type:str]
+    lastmodified: "2026-07-25T12:48:03Z"
+    mac: ENC[AES256_GCM,data:noPba/BIxyoUo3sFfwXuMaZwwSMNADgWcwGaIoQchdzu3KodtWRHk5xJ9wy4VGelOuy3gS79j9ErDnsn8I1v05Bqqlrig99Sch5Jlu1UQpeUQtUjq9S7d5BQ0f3QdeU4iP2v28JovI3gBNHEIlZiueV6xYUrpeoKzS9ukrBtJsM=,iv:XRmbI5hX0OCh4I+7MmnI9LyBLJhf2CsfHgvsNaEUWgc=,tag:Ym5JeVRqTWHWl8qSQAFh0w==,type:str]
     pgp:
-        - created_at: "2026-07-25T12:44:32Z"
+        - created_at: "2026-07-25T12:48:03Z"
           enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMA7uy4qQr71wiARAAqvgReqd8cH8t7ApfkQk8OsKTTHCYSkw5nEE+5N/psOsU
-            RVW7TYHRAFKd48JdgLCaWAfmVEfjDSs+nAdZhWGJ9Wamj7HITTMZH/TbHeKtiQ+u
-            g18AsSdKCMmZ9TnEerzIWWFmulxaH/GSva0lJeMO+TCU+RZYdJwlDT5ZVGywrRQz
-            LvLdWV4fRKQm7S7nrFEFIIvYdq0XDe39fea91Zj7V/g/SD9lkw/BBnAuv91J+DiX
-            yUhhWP4Lo2ecMtqbYL16q4BJDd96smLQ1Yp+XDbzvgrGBF1suEfqaGYeNhwO8XS/
-            Z58RmG4H2unXJMgsrDaOn1hEc8Ap42ihuqCrOZU0gB5EW4qTfWFN9kw6fic+5rVe
-            Rnrpk976+YA7507YZ3S4SabY1WgGe9NcvAvvM+ZuT4aO3E4098Sfbx9p5uG7PuxB
-            XSsSn6gqI+CmgWbdbiF7ZJg4ByCtTX7lOedO014WziTAELljzOxX/Or7EeYjuxGp
-            xF7yZajQ626SEx90nZjMDHjRaryo77m9g78Ji4/wMtHNTif18tVRP/hY+EhEucN4
-            Nte/Dg2o4iBCT6yAwHWKnLfs4e8BwZAt6qPpLtEQbhurZDZD2bxaAAK/qC7DWS7u
-            kheSqi4cax5TRV4mP7XqN2m0UfX3bQN/8SGiPyubDvXGbshxMUEe24ppU8DmD1TS
-            XAHnGzxIrQ18psN3PZBU/i9d5eiZ4qIGKY7sN9qbD144pWy1kOU7AFuvGAMxPJUs
-            kbpapaydCJwuUA8aV706ijA5/zvpvJJqS+zf1EAU24znMfZJ13VZBYp1d6aT
-            =KtCa
+            hQIMA7uy4qQr71wiAQ//Sa5nJk6vXXFdf4C5XwsCeRmkSYKSdxJZ2auRzRi+2znv
+            IqicZh0YENwEX++2NQNtt/jaPFCqWUMEVfncvYceutHzZxDZihkbIL/jKs0h/Hjr
+            gL/JNRPQbVD4HXLTQ5dOuAfixJXcN9DfYQGyBg1vOtvpKxR4cD4JirA3KyYjtSWf
+            cYw5fMSalPJ0D3B5pObFZoVJhu8emO3FegSu/tqnZseJuT4mqZqdhpuej17FtkQB
+            il0tNwBdy9pDFXMxiZoGFtVb5Bii7ZU19MPleBBTFO9nNLXE8rCZ/1tJBtR+XUXE
+            acglYoZHt3aHAqe1+8iLWAOmE1KkElUMU6/G5yZCpLNNwwgx1jXqGsvITMQQVptJ
+            RMMKfdNGZSC4mcVXT+Ux/uwvw09DO9Mrbf0tjiTTjLProNxUmY4pPMShwCmGsfTa
+            si/bZbCui/Jryy02dYWuzA4bpReY+ySMwUGZ9C/0Gn/SBEAdeGQKDhJwgmE6BM1n
+            hMZOh9YPcGSc0n4a9q/CzFqcQ7T8m1uHgqciFG/zW8A+l8bC1H5uxsUwcYPALLxY
+            adAZ6XMPZAZ1L/zp3NO/SpdHp7hQVqx+TZOWvmlZ4XiDrICp4AVK8oZAXKw/nrAu
+            k4ZYVsrBKrkLBIMNmM1mxuhi248ELB7V+kN9zDxQZh87J+F2WXk8D4dMwBfRfYzS
+            XAGiurXdFcCH3i8O5mmE8QY+ZgdKzyAj7hVw8BvXlOIne8pPe3YuZsp68zwdz8MQ
+            Gr5oDPaZWhEwFEg+4H+ub5xSIZewY7Ib7FM8bzzfd23vODtZ+0sBoRH+Tq4R
+            =m933
             -----END PGP MESSAGE-----
           fp: DC6910268E657FF70BA7EC289974494E76938DDC
     encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$

kubernetes/infrastructure/observability/alloy-config.yaml

@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alloy-config
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: alloy
+    app.kubernetes.io/component: log-collector
+data:
+  config.river: |
+    // -----------------------------------------------------------------------------
+    // Grafana Alloy — log collection only (metrics cluster+host collection is
+    // handled by vmagent/node-exporter/cAdvisor elsewhere).
+    // Forwards pod logs to the in-cluster Loki single-binary.
+    // River config produced by `alloy convert -f promtail` from an equivalent
+    // Promtail YAML (so component names and argument shapes are guaranteed
+    // correct for Alloy v1.18).
+    // -----------------------------------------------------------------------------
+
+    discovery.kubernetes "kubernetes_pods" {
+      role = "pod"
+
+      selectors {
+        role  = "pod"
+        field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)
+      }
+    }
+
+    loki.process "kubernetes_pods" {
+      forward_to = [loki.write.default.receiver]
+
+      // CRI-style log lines on the host: {"log":"...","stream":"stdout","time":"..."}
+      stage.cri { }
+    }
+
+    discovery.relabel "kubernetes_pods" {
+      targets = discovery.kubernetes.kubernetes_pods.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_container_name"]
+        target_label  = "container"
+      }
+    }
+
+    loki.source.file "kubernetes_pods" {
+      targets    = discovery.relabel.kubernetes_pods.output
+      forward_to = [loki.process.kubernetes_pods.receiver]
+
+      file_match {
+        enabled = true
+      }
+      legacy_positions_file = "/tmp/positions.yaml"
+    }
+
+    loki.write "default" {
+      endpoint {
+        url = "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push"
+      }
+      external_labels = {}
+    }

kubernetes/infrastructure/observability/alloy-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: alloy
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: grafana
+        namespace: observability
+      chart: alloy
+      version: "1.11.0"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: alloy-values

kubernetes/infrastructure/observability/alloy-values.yaml

@@ -0,0 +1,57 @@
+# Grafana Alloy
+# Runs as a DaemonSet on every node (tolerates the failover taint).
+# Tails pod logs from /var/log/pods, attaches K8s labels (pod, namespace,
+# service, container, image), and pushes them to Loki at
+# http://loki.observability.svc.cluster.local:3100 (in-cluster, no auth).
+# Use the externally provisioned ConfigMap (alloy-config) carrying config.river.
+alloy:
+    configMap:
+        create: false
+        name: alloy-config
+        key: ENC[AES256_GCM,data:aevkZsfe4c4H34BJ,iv:4vrNZWwYWaxJUKGbNlnZfk0NM182nBQZuwvqUPslKNA=,tag:lh1iILM3uQAt0An6kN6EHg==,type:str]
+    # Mount host log paths so the DaemonSet can tail pod/container logs.
+    mounts:
+        varlog: true
+        dockercontainers: false
+    stabilityLevel: generally-available
+    tolerations:
+        - key: ENC[AES256_GCM,data:zOMWiA9dG+8o,iv:yHp9aBczV8nVRGlJmsBaKs9WdmE3+ii8gWOaj5Xok6M=,tag:8c5ZqBT+vvC0WCW8mi0c9g==,type:str]
+          value: ENC[AES256_GCM,data:2Q04VR9xLbQ=,iv:6+tQXx3IGUIcrse+poG4X/4pS5dN2dKvhb79sCy+nfA=,tag:jb/TZmGpvdsWnKoC5NdCow==,type:str]
+          operator: Equal
+          effect: NoSchedule
+        - key: ENC[AES256_GCM,data:aVq8GU1ZDg5m9XDEI3PoVbbykU36SnKpEgrohvZ0IBmH47qDKg==,iv:eMcHRO5mGp9hhY4KnuBmdPTjd/9lqtesjA/RRSu4QMw=,tag:2jaGuogkLcIuye2meJQMDQ==,type:str]
+          operator: Exists
+          effect: NoSchedule
+    resources:
+        requests:
+            cpu: 50m
+            memory: 64Mi
+        limits:
+            cpu: 200m
+            memory: 256Mi
+sops:
+    lastmodified: "2026-07-28T13:46:53Z"
+    mac: ENC[AES256_GCM,data:u6J/EIP5aFtOi8ZtT8BPn978wJAEL1E3r0WcmG5WIsWKJBZMMQHUYRmHgkS6n0/UMtczWoCdXlxObS73ZQMSSkdtagQHE/HPh69dDdGkjxyM4mlP+YMwcnIQkxpJBAQTERXA+SVoWCZb7ztKbiJS+zPjOCfA95ixQINmBCdUgEc=,iv:WOXngWu3uNWYp18/xWghGJyONBrq3SSnPDwbjHBDozU=,tag:1fasTfuLBGXBCo9JPytsJw==,type:str]
+    pgp:
+        - created_at: "2026-07-28T13:46:53Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA7uy4qQr71wiAQ/8DSD0jEn3YE7XHTm4DWdfLHPs1IVg9hyiWSHe5/xH0siD
+            zWonfjcuytw8EHDzv6pQMNYmIpyAmJ92Al4ROJmBV8gZ59+U9goHmw5HgaQAupBJ
+            /gy8iFBteGOHJT+XJPoVb/bu449f7Vle4L3DrOAl6mwxE0WT/7ERoQBeSxL+2OTZ
+            iDPdq24Zy+crDHpoM0GcSnFRLX5UpmL2QXGB8PHyctwSQoDW+sbAY2ME6buOw8HI
+            AremKTRtFm6Pu1Ksdh6ZicaHpLSvLyByPPt2zYhlBQsG3s/SjJDlnakUZHhm7T4o
+            oHzSpKqV7NQbE09ZMyGJrEkl+rfCPeu6jHplW2EX3xjqLWNUF8GHcsDWljogZxXF
+            irBPkyKASEDdkHZgiZobU6BmvQlxILj104xNtUYTTChcKu3s0d5BrkIOGe+Y7rJd
+            fNvMfkmO8/URBgtsXy2xPKo0Jzu/0pZsPEp2lxNeyElwxQqo1CFwhKZPkeQ2eLpu
+            4X7/CMN8KJr6jLX8HhX9AivlCj+2CoT8SB5GAPw4p00eD0GIMWSNyOXjyW/qQZi3
+            82EyX9KIpHjALoOI4/3YgFpUO12AEsjirYKWkUM+IoDp5l+YX59QdM7zUVVyzsTG
+            2EXEdQqYVA7DIcMJb0tHqxDGMGxNayaUsikV7e/kcRVUyDSH+xOfaSUrDTqqv+XS
+            XAFro6848HF5uMIP4pfM+dxzkc9rFF4il0CuGWlscbz5q+Aw9LImGbpNzrWk3Lpi
+            beuwWKFTZ8zab5TH9UF/xqsQeg/nyzQnb3fFPk+owvY7fH6XiYbN5WjKjXGW
+            =kBpp
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/infrastructure/observability/grafana-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: observability
+spec:
+  interval: 1m
+  url: https://grafana.github.io/helm-charts

kubernetes/infrastructure/observability/kustomization.yaml

@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: observability
+resources:
+  - namespace.yaml
+  - vmstack-repository.yaml
+  - grafana-repository.yaml
+  - loki-repository.yaml
+  - vmstack-release.yaml
+  - loki-release.yaml
+  - alloy-release.yaml
+  - alloy-config.yaml
+  - vm-internal-ingress.yaml
+  - loki-internal-ingress.yaml
+secretGenerator:
+  - name: vmstack-values
+    files:
+      - values.yaml=vmstack-values.yaml
+  - name: loki-values
+    files:
+      - values.yaml=loki-values.yaml
+  - name: alloy-values
+    files:
+      - values.yaml=alloy-values.yaml
+configurations:
+  - kustomizeconfig.yaml

kubernetes/infrastructure/observability/kustomizeconfig.yaml

@@ -0,0 +1,6 @@
+nameReference:
+  - kind: Secret
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease

kubernetes/infrastructure/observability/loki-internal-ingress.yaml

@@ -0,0 +1,23 @@
+# Ingress exposing the Loki push API (loki:3100) through Traefik as
+# loki-internal.lan — used by the docker-host promtail to push container logs
+# into the central Loki.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: loki-internal
+  namespace: observability
+  annotations:
+    external-dns.alpha.kubernetes.io/enabled: "true"
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: loki-internal.lan
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: loki
+                port:
+                  number: 3100

kubernetes/infrastructure/observability/loki-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: loki
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: grafana-community
+        namespace: observability
+      chart: loki
+      version: "18.5.4"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: loki-values

kubernetes/infrastructure/observability/loki-repository.yaml

@@ -0,0 +1,9 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana-community
+  namespace: observability
+spec:
+  interval: 1m
+  type: oci
+  url: oci://ghcr.io/grafana-community/helm-charts

kubernetes/infrastructure/observability/loki-values.yaml

@@ -0,0 +1,88 @@
+# Loki Helm values
+# Uses Monolithic deployment mode with filesystem storage on a Longhorn PVC.
+loki:
+    deploymentMode: Monolithic
+    # Filesystem storage — chunks + rules on the PVC. The chart v18 requires
+    # an explicit loki.storage block; without it the workload template errors
+    # with "Please define loki.storage.bucketNames.chunks".
+    storage:
+        type: filesystem
+        filesystem:
+            chunks_directory: /var/loki/chunks
+            rules_directory: /var/loki/rules
+    # Required schema config. tsdb store + filesystem object store, schema v13.
+    # See https://grafana.com/docs/loki/latest/operations/storage/schema/.
+    schemaConfig:
+        configs:
+            - from: 2024-04-01T00:00:00Z
+              store: tsdb
+              object_store: filesystem
+              schema: v13
+              index:
+                prefix: index_
+                period: 24h
+    # For monolithic mode, point tsdb_shipper at the local store (no index gateway).
+    storage_config:
+        tsdb_shipper:
+            active_index_directory: /var/loki/index
+            cache_location: /var/loki/index-cache
+        filesystem:
+            chunks_directory: /var/loki/chunks
+            rules_directory: /var/loki/rules
+    monolithic:
+        persistence:
+            storageClassName: longhorn
+            accessModes:
+                - ReadWriteOnce
+            size: 10Gi
+        resources:
+            requests:
+                cpu: 100m
+                memory: 256Mi
+            limits:
+                cpu: "1"
+                memory: 1Gi
+# 7d log retention (user can tune later).
+limitsConfig:
+    retention_period: 7d
+# Disable distributed components + gateway + minio (monolithic only).
+test:
+    enabled: false
+backend:
+    replicas: 0
+read:
+    replicas: 0
+write:
+    replicas: 0
+singleBinary:
+    replicas: 0
+gateway:
+    enabled: false
+minio:
+    enabled: false
+sops:
+    lastmodified: "2026-07-28T16:09:30Z"
+    mac: ENC[AES256_GCM,data:phQ6t8rBVG5rdOR3H4GEWJHiCU1rj0MOzjp2jn2nneOUbbclyn4L4ga7YHOuoce0HEEMWniWxchT2eB0BZM3gtOd3yyBq3z3vTQKI12TsTzQSXxm+AgM4DL855BhPrFJCNZYeXb6IAGVkcRK/N7ihuGWiZORBsPaIcYseW9kjf4=,iv:wNIkcPRr1d79odGsT+yJJ+zqlUVTtqj8c/YkbhqOQjA=,tag:LSXyM6QiCM9qoit++nUkBw==,type:str]
+    pgp:
+        - created_at: "2026-07-28T16:09:30Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQILA7uy4qQr71wiAQ/3c4Q36xa3tNPub4HCEUmJwK7Nmhdrg+sihZYR/TOHuDuV
+            sY3GX+HwaRaMh4rOiJpCDa9E3qJY+qh9Wf+rgq+vaL9hmfNBlC+2q2xGv2u6FdiN
+            xXE501qBbZjyNm/9fVAnCxbTBZP2mkRgdzMHfK2dnhIcL0/5LK7NAu+PsMQ0J8uq
+            J5JYCqQTp/bfoRi8O4YXJD38u5d0UyEIy7HImKSn21za6CUErHEgdOTGUzw+UTH9
+            kYHQykiBVML6bpp3YkqIquBm1tDzNuPvKSynnswJS5DQnGwsWVmESA3Rgfes2doe
+            4d2f4Bso8+1+BRgbPkmp2c3/GdNFeUjjj6ViRar1bX2Bmt/ODLjoxAriRzS5ChRe
+            MBJkKcW1hWotQE1IZgYC8A2I4dvDAbjeTwWmyJW7T1NV0er2rDd8TC9aQfhOPSeh
+            mXdF18PXSmonmAdCj8Fu6Zfd6kcMQUQdI5L3BLRAQNs6+0r/tNjPH8vgYQtRWtxO
+            acguClDwTrMwGM33HiBAYpoZ4hpCk43PRDNf0o2JkDy8tr3uTeBkmHPIA80UsIi8
+            /YmrLzzaF0TaN7OvDLbw/5HUR1R2Wt2Mc3GnTdJIxMogqcw+jO86cNm2v7FJX5v1
+            wG4ocHbrbfH1lcte3ZW8E5z3evuUeXVgUTL7dRHdNWmfL3HUnBlX+H+z7qWoNdJc
+            AUhXpEvLTZ0LkA0ZMi749sUcL9cLsXkKJhAPbiF/3ytc7tLc40QsTY53uuqGPU/8
+            LVVX7jUjqBrmeUENYj3zSN3kxcI1OX/qzHdTQ0LFSPPUrCEPtE8hWJrLaUE=
+            =Nr2M
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/infrastructure/observability/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: observability
+  labels:
+    pod-security.kubernetes.io/enforce: privileged

kubernetes/infrastructure/observability/vm-internal-ingress.yaml

@@ -0,0 +1,26 @@
+# Ingress exposing the VictoriaMetrics vminsert endpoint (vmsingle:8428)
+# through Traefik as vm-internal.lan — used by the docker-host vmagent to
+# remote_write metrics into the central VictoriaMetrics.
+# external-dns auto-creates the vm-internal.lan A record pointing at the
+# Traefik LoadBalancer (see kubernetes/system/external-dns).
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: vm-internal
+  namespace: observability
+  annotations:
+    external-dns.alpha.kubernetes.io/enabled: "true"
+    # No chart ingress is available for vmsingle, so declare it here.
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: vm-internal.lan
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: vmsingle-vm-victoria-metrics-k8s-stack
+                port:
+                  number: 8428

kubernetes/infrastructure/observability/vmstack-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: vm-victoria-metrics-k8s-stack
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: vm
+        namespace: observability
+      chart: victoria-metrics-k8s-stack
+      version: "0.87.0"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: vmstack-values

kubernetes/infrastructure/observability/vmstack-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: vm
+  namespace: observability
+spec:
+  interval: 1m
+  url: https://victoriametrics.github.io/helm-charts/

kubernetes/infrastructure/observability/vmstack-values.yaml

@@ -0,0 +1,199 @@
+# victoria-metrics-k8s-stack Helm values 
+# The chart installs the VictoriaMetrics operator + CRDs and creates VMSingle,
+# VMAgent, VMAlert, VMAlertmanager CRs.
+# ---------------------------------------------------------------------------
+# VictoriaMetrics single-binary (the metrics database)
+# ---------------------------------------------------------------------------
+vmsingle:
+    enabled: true
+    spec:
+        retentionPeriod: 7d
+        storage:
+            storageClassName: longhorn
+            accessModes:
+                - ReadWriteOnce
+            resources:
+                requests:
+                    storage: 20Gi
+        resources:
+            requests:
+                cpu: 100m
+                memory: 256Mi
+            limits:
+                cpu: "1"
+                memory: 1Gi
+# ---------------------------------------------------------------------------
+# vmagent — scrapes node-exporter, kube-state-metrics, kubelet, etc.
+# The docker host runs its own vmagent that remote_writes through
+# vm-internal.lan (the Traefik Ingress → this vmsingle).
+# ---------------------------------------------------------------------------
+vmagent:
+    enabled: true
+    spec:
+        selectAllByDefault: true
+        scrapeInterval: 20s
+        tolerations:
+            - key: ENC[AES256_GCM,data:SyutB0y9QtqQ,iv:EKRXNGtUovQYsESHFQTGygP09O2UWtLuBEQQcYZm3Oc=,tag:zh7CYOZleSgbbQhYZvcT+w==,type:str]
+              value: ENC[AES256_GCM,data:So+8xmHYSxs=,iv:7sfUBAaMZd7imjVx14lusVn58cJQ4HpTkaPy5Eal2dY=,tag:hRBjT6wuEfG3Z+ZiPJhwQQ==,type:str]
+              operator: Equal
+              effect: NoSchedule
+            - key: ENC[AES256_GCM,data:lj9OurynU8pRt4/MhgyevjmV0b7Zq+WG/9+zRRvPeA+RYZsV+A==,iv:LjuhErDT0bDlLcpo/AlNykz8Xu89B+nx1ja+HkPNXwo=,tag:5xM6XHrCdEejE7ituQ8W4Q==,type:str]
+              operator: Exists
+              effect: NoSchedule
+# ---------------------------------------------------------------------------
+# kube-state-metrics + node-exporter (metrics sources)
+# ---------------------------------------------------------------------------
+kube-state-metrics:
+    enabled: true
+prometheus-node-exporter:
+    enabled: true
+# ---------------------------------------------------------------------------
+# Grafana (UI) — served behind ingress at grafana.lan.
+# ---------------------------------------------------------------------------
+grafana:
+    enabled: true
+    persistence:
+        enabled: true
+        storageClassName: longhorn
+        accessModes:
+            - ReadWriteOnce
+        size: 5Gi
+    adminUser: admin
+    # SOPS encrypts this when the file is processed.
+    adminPassword: ENC[AES256_GCM,data:vpPvJA/PktMnsg/IovYfsD4lDBndUw==,iv:b4eLMEcUMJj4BwyeYw1kw2sZuieyr2xL9et+R7ss0BU=,tag:u0RcTp6mkQNAGkblG4/xww==,type:str]
+    # Provision a Loki datasource alongside the chart's default VictoriaMetrics
+    # datasource so metric/log correlation works in one UI.
+    datasources:
+        datasources.yaml:
+            apiVersion: ENC[AES256_GCM,data:Uw==,iv:KYTMBV53yMbojMaqNinYpm/Uj5ylV86jcdJTdxuy+G0=,tag:IbIpiSBQFw7YSTQg0WRGqg==,type:int]
+            datasources:
+                - name: ENC[AES256_GCM,data:BPfR8Q==,iv:MkMhH3IJfBZzbIcqQw7y6bfJB/4Ew4feZNugjovk5O0=,tag:jaa43tzIXkE8kWZp2R/kcg==,type:str]
+                  type: ENC[AES256_GCM,data:WWncww==,iv:hPqJE0KCidSBnf11wxFH+Je+YKztWmeToBqi39Qf4ZQ=,tag:c9enJ2VgQqDJVbasvgG+Dw==,type:str]
+                  url: ENC[AES256_GCM,data:YkS66tpWXpW62Dtful0KNCwoZ3rG3nQtDBUsNQUwdQ+eqtG38JTrAudQqnsRD51q,iv:Mu3Sc6T2xypdot9dA1DHc8nD7UI8372nqlveLITR82E=,tag:1VmmIg+xR9qjWcvPKVYsJQ==,type:str]
+                  access: ENC[AES256_GCM,data:af3GpjE=,iv:MAYKfklF+wCC/XN+4snebjPAnW4KpYzaMMGYds7rG18=,tag:X+oFiDAiCb1f7cfREqFT7A==,type:str]
+                  isDefault: ENC[AES256_GCM,data:Zgu4Hn8=,iv:nnVB/in75nPAQos0Dfl6abBrHwT8tCt5X7lcCYBZbLc=,tag:u1UvkSj6SjcMm06NzKMGDA==,type:bool]
+                  jsonData:
+                    maxLines: ENC[AES256_GCM,data:88JJqA==,iv:lKsZa024I6223UEu0SHuZ3N9LZrzw0WBTfC+L2/3ljI=,tag:YvqieXK36xMmC/sSJ11NIg==,type:int]
+    ingress:
+        enabled: true
+        ingressClassName: traefik
+        annotations:
+            external-dns.alpha.kubernetes.io/enabled: "true"
+        hosts:
+            - grafana.lan
+        path: /
+        pathType: Prefix
+        tls: []
+# ---------------------------------------------------------------------------
+# vmalert — evaluates VMRule CRs against VictoriaMetrics, forwards firing
+# alerts to Alertmanager. selectAllByDefault picks up all VMRules in the
+# namespace (including our vmalert-rules.yaml VMRule CR).
+# ---------------------------------------------------------------------------
+vmalert:
+    enabled: true
+    spec:
+        selectAllByDefault: true
+        evaluationInterval: 20s
+        notifiers:
+            - url: http://vm-victoria-metrics-k8s-stack-alertmanager.observability.svc.cluster.local:9093
+        resources:
+            requests:
+                cpu: 50m
+                memory: 64Mi
+            limits:
+                cpu: 200m
+                memory: 128Mi
+# ---------------------------------------------------------------------------
+# Alertmanager — 1 replica; inline config with a single n8n webhook receiver.
+# n8n runs on the docker host and fans out to email/Telegram/whatever.
+# ---------------------------------------------------------------------------
+alertmanager:
+    enabled: true
+    spec:
+        replicaCount: 1
+        port: "9093"
+        selectAllByDefault: true
+        storage:
+            volumeClaimTemplate:
+                spec:
+                    storageClassName: longhorn
+                    accessModes:
+                        - ReadWriteOnce
+                    resources:
+                        requests:
+                            storage: 2Gi
+        resources:
+            requests:
+                cpu: 50m
+                memory: 64Mi
+            limits:
+                cpu: 200m
+                memory: 128Mi
+    config:
+        route:
+            receiver: n8n-webhook
+            group_wait: 30s
+            group_interval: 5m
+            repeat_interval: 4h
+        receivers:
+            - name: n8n-webhook
+              webhook_configs:
+                - url: http://n8n.lan/webhook/observability-alert
+                  send_resolved: true
+extraRules:
+    node-alerts:
+        groups:
+            - name: node
+              rules:
+                - alert: HighNodeCPU
+                  expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: High CPU on {{ $labels.instance }}
+                    description: CPU usage above 80% for 10 minutes.
+                - alert: HighNodeRAM
+                  expr: 100 - ((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100) > 85
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: High RAM on {{ $labels.instance }}
+                    description: RAM usage above 85% for 10 minutes.
+                - alert: LowDiskSpace
+                  expr: |
+                    100 - ((node_filesystem_avail_bytes{mountpoint!~"/run.*|/var/lib/docker.*"} /
+                            node_filesystem_size_bytes{mountpoint!~"/run.*|/var/lib/docker.*"}) * 100) > 85
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: Low disk space on {{ $labels.instance }} {{ $labels.mountpoint }}
+                    description: Disk usage above 85% for 10 minutes.
+sops:
+    lastmodified: "2026-07-28T16:39:09Z"
+    mac: ENC[AES256_GCM,data:Ymxmmkui0Nuv35Yq6v4BXM4xGhvjAufexW7fIm9fUIdTTXZ3e10PitNCZXKiaDwwCJfdEkNZri7jPAYzhTzKMIdcTKnOMImsMjdiJZwRwwbKcYVay0A6KSLJ8eFjDRRYhlCXbsAIRlQHpFh4UkssKdu40mIiudU0qIVm04ppbno=,iv:R8kpITPYSbQ1yZw5IK0fArssxUtIjNmvzknOrLJQcuk=,tag:/CRcr1VZAIKIPbcag05uTw==,type:str]
+    pgp:
+        - created_at: "2026-07-28T16:39:08Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA7uy4qQr71wiARAAmOxyx5eH5Jykp/GDL79REgf61iGujtPiUVy8CT3o7O5k
+            tTGma6rRXK5wZV/iGN3wOQehF19Oy2iYwMfxilRwpB69RFwyKPKdHDDWMWNXv5Th
+            wJnNjVjWZ0xiRautWPn/TjDOJZ6j6p7lQ3IcQ3vzuap1RVRIXv081DxJGR9JbeYl
+            khmrl1+zecwxS87JK+fLko9NIBb5JgFEtEf1RvLyFak8nZu5cNCu+PCykALvUaO7
+            V5NAaF1aV9cdz8G9TZWmHZsIlDDE9/oro2aLR7cr8uxnKQhWerttqvyDYNwYzNKW
+            qgsUplTxTmyuc9DdnLDfJ55Ncuv9qS5/4/p5GpsHrteD6oiPNjXaxR8cBVFLvGwp
+            kQb/VZgOGAnkYlMCUG8APiwRhbpAlyy51hKBqVh+a9iRlzZEWHKlxcvIrcFAQaHt
+            sDXyg36F0GCoHiMAqHKG0kRcbTV7Sq0JZVoLU/qxvaDfqvyCZO7X/U/YsSWNbWL0
+            emyBfeSvBX9h62NJx2aw8xKyiL+aeVfzSKAaXHMd/r4gKxBw/kVHAljKwRzbL85j
+            UHeLUvqVkXs1mtIqGgUnpCcAVuV83omgcOwF44FitG/f9GCx+UWTvScwaw5ywuG3
+            J/4fOaA3K0Z4e/mOejARd4jLSPktSRpA01k9kjYY8UBn7TzQ0LrMLj5dxh8BpWrS
+            XgECZapi+q6IjIses7+JqkWiP25hcglhqK3LAiUapA0c5UmAa5Vf7jfnkFm8eUdZ
+            bA7ZnhX8nxK9n7JdSxrZhrbPWMO8hF/XKi7KTZMvgXFlb4/qP96y7oeqzKYcnvE=
+            =pmSq
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/kustomization.yaml

@@ -24,6 +24,7 @@ resources:
   # Infrastructure
   - infrastructure/authentik
   - infrastructure/mail/stalwart
+  - infrastructure/observability
   # Personal
   - personal/linkwarden
   - personal/notesnook

kubernetes/system/longhorn/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
   - namespace.yaml
   - repository.yaml
   - release.yaml
+  - recurring-jobs.yaml
 secretGenerator:
   - name: longhorn-values
     files:

kubernetes/system/longhorn/recurring-jobs.yaml

@@ -0,0 +1,34 @@
+# Recurring backup and snapshot schedules for Longhorn.
+apiVersion: longhorn.io/v1beta2
+kind: RecurringJob
+metadata:
+  name: daily-snapshot
+  namespace: longhorn-system
+spec:
+  # Fast local recovery from accidental deletes.
+  name: daily-snapshot
+  task: snapshot
+  cron: "0 2 * * *"
+  retain: 7
+  concurrency: 2
+  labels:
+    type: snapshot
+  groups:
+    - default
+---
+apiVersion: longhorn.io/v1beta2
+kind: RecurringJob
+metadata:
+  name: daily-backup
+  namespace: longhorn-system
+spec:
+  # Daily NFS backup for DR if a node is lost.
+  name: daily-backup
+  task: backup
+  cron: "0 3 * * *"
+  retain: 7
+  concurrency: 2
+  labels:
+    type: backup
+  groups:
+    - default

kubernetes/system/longhorn/values.yaml

@@ -1,10 +1,7 @@
 # Longhorn Helm Values
-# Single-node R740 cluster, backup target on TrueNAS NFS
 defaultSettings:
-    # Single node — only 1 replica needed
-    defaultReplicaCount: 1
-    # NFS backup target on TrueNAS
-    backupTarget: nfs://truenas.lan:/mnt/fast_app_data/longhorn-backup
+    # 2 replicas so volumes survive single-node loss
+    defaultReplicaCount: 2
     # Allow full disk usage on single-node
     storageOverProvisioningPercentage: 100
     storageMinimalAvailablePercentage: 10
@@ -26,7 +23,7 @@ defaultSettings:
     taintToleration: dedicated=failover:NoSchedule;node-role.kubernetes.io/control-plane:NoSchedule
 persistence:
     defaultClass: true
-    defaultClassReplicaCount: 1
+    defaultClassReplicaCount: 2
     # Retain volumes on PVC delete — safer during migration
     reclaimPolicy: Retain
     # Mount options for ext4
@@ -37,23 +34,23 @@ persistence:
 # R740, but keeps things correct if that ever changes).
 longhornManager:
     tolerations:
-        - key: ENC[AES256_GCM,data:ZvL4q5WzJQbz/+jBCaH4Gpp5PdCqwrqcIDCXwPq1oVintCYXxg==,iv:SASkDRml7AG+wJTWe/CRaAm7kv7tJ/IRXfpWzeewHuo=,tag:ToVKgaQeLVav7Qt2bdlpnw==,type:str]
+        - key: ENC[AES256_GCM,data:DDNTXJiwP2LQiOmqgf/L6tLIZiFgqh42dLDXj8xomGv7rlUP9A==,iv:eu6vnl3EiLu1DWWDWlxkQNcN6/uEhSkpufw7L9Ikr14=,tag:DJOMo8LDGmzkqvIEkfCTHg==,type:str]
           operator: Exists
           effect: NoSchedule
         # Tolerate the failover taint so longhorn-manager runs on the P330
-        - key: ENC[AES256_GCM,data:B+Y/ckOHz6xq,iv:27M/gUOPZ09BNZP2OwyjNsK8Ld0opec9l2FjiRiN0Wg=,tag:3UGdvzOsXgSAs7KwBJ5oKQ==,type:str]
-          value: ENC[AES256_GCM,data:YT/EL6wwlYI=,iv:H8CCtKbzmWRahFxtZguzghVLK3/qvHFaE8IQPALL0Oo=,tag:HIMsMtxutYh3HVASeIrxtw==,type:str]
+        - key: ENC[AES256_GCM,data:rbWq0sruC6zC,iv:Eq5izQ4LXZ90Bhf1BJ7V9/Xo7HVMULeDHZ6NgItZfGQ=,tag:ixJt7YD4cBy+SrG2WVn0ng==,type:str]
+          value: ENC[AES256_GCM,data:wxGNkS97rg8=,iv:bBGGqcwibhXE0zkH+ZJEVgr20ogu7JwQxxDXqJcMdsQ=,tag:VLBQh8vRl1SiYB9S/vwLeg==,type:str]
           operator: Equal
           effect: NoSchedule
 # longhorn-driver (CSI provisioner/attacher deployer) — tolerate the same
 # taints so CSI operations work from the failover node.
 longhornDriver:
     tolerations:
-        - key: ENC[AES256_GCM,data:ks2denq5PQL6UjGUe5JFahXJqLDY8jQgAPjQkn3fhryTCsd1Tg==,iv:gR8QtaRW5+4GptIa92hh2v+pk41ve1FmCMC+qlitdOs=,tag:NDGn8o5btPbfqDWg1nS6wQ==,type:str]
+        - key: ENC[AES256_GCM,data:1EBSG4hL4heCnLFJYzTNaQWtFU3v6xwTlG+Y9/H4p5d0TZdFog==,iv:9ILcNXXzxzG5CYA4Ny7bKHBI581B1qJC3OBUObScxbQ=,tag:4B4JaFO0tfyVA6w7ypCY6g==,type:str]
           operator: Exists
           effect: NoSchedule
-        - key: ENC[AES256_GCM,data:nRPej7F1wIQr,iv:fRF9YaiqpPCAEGSXCHJublN7OYoikb3xK1RacxMKDcw=,tag:+leZjB1PGhWvxhrEBHxImg==,type:str]
-          value: ENC[AES256_GCM,data:QdMlIx5PYgU=,iv:TJ+i5qwY6G2j+87HdmOZwDWPzJOd9bDeJUmkC3yl9oA=,tag:niNS/IwVVaw+83e4jJrgag==,type:str]
+        - key: ENC[AES256_GCM,data:ZSC5qg+ChOkI,iv:rIq8j7L7F641kWDzcSV9lrYxhHggT1FTZ8WF53vxvV0=,tag:Hj4RlF3usdIQi0G6Ecmdbg==,type:str]
+          value: ENC[AES256_GCM,data:YfiuCv2EKec=,iv:+6i5CT1ToCKy1gFtVVIFv5FLQaJaKHx4vQk8b0IBy3U=,tag:zHgzNuijbwI1QbvuLnZNvw==,type:str]
           operator: Equal
           effect: NoSchedule
 # longhorn-ui — tolerate the same taints so the UI is reachable even if it
@@ -61,63 +58,46 @@ longhornDriver:
 longhornUI:
     replicas: 1
     tolerations:
-        - key: ENC[AES256_GCM,data:mYYdyvTvfbP57Tr7zEUgAYZmtSmTougPSbou75YNqD93LvQEuw==,iv:vcKiaYE2ikIAornOIs7LB4iIp6XczB8VmW1FgeUO2ho=,tag:i1yOrLaUX1+AzviTLYQFUw==,type:str]
+        - key: ENC[AES256_GCM,data:n4UdwssGYhHQgDTpCpu+WgIPsdNc8uAaZtnURZt0/C3VfeLR4w==,iv:mqy0GqHN0KxtkETS7psu61Mr1oGmtPDhg2vQhVDas6c=,tag:uUd+NY90XK3i0aigc5ndCg==,type:str]
           operator: Exists
           effect: NoSchedule
-        - key: ENC[AES256_GCM,data:rqykd/I0zsFq,iv:lixb2ezSpU2GjXsi8li4z0d1TjNpCRiPXrSMiOASH64=,tag:rMg3tE4fw7jFpidBRZBkIQ==,type:str]
-          value: ENC[AES256_GCM,data:S7G6SgmWjA0=,iv:Wwo3U/bFkch+dNlLZLDNCiaUVID8bIVxhPtizIUt/rE=,tag:7R8s/D6IX4SfZgT/PCkbVw==,type:str]
+        - key: ENC[AES256_GCM,data:isSkUNqax+AU,iv:jWgZyZL9VMmzYqoLCUkIiUuy1iBeOm2RE5CjZGCcXxQ=,tag:714V0YGb7jqebC4IRX/phw==,type:str]
+          value: ENC[AES256_GCM,data:Qbtu1wgV1oI=,iv:RihD43IA1+C3m9OkXACGgd5lY9lDWCyzIcw/MCoSMvg=,tag:WzQtC4rT+uovS6s36DJWUw==,type:str]
           operator: Equal
           effect: NoSchedule
-# NOTE: There is no `longhorn-instance-manager` Helm key. The instance-manager,
-# engine and replica pods are system-managed and created dynamically by
-# longhorn-manager; their tolerations come from `defaultSettings.taintToleration`
-# above. Do not add a `longhorn-instance-manager` block here — it would be
-# silently ignored.
 # Disable ingress for now
 ingress:
     enabled: false
-# Recurring backup and snapshot schedules
-# - Daily snapshots: fast local recovery from accidental deletes
-# - Daily NFS backups: DR if node is lost
-recurringJobs:
-    enable: true
-    jobs:
-        - name: daily-snapshot
-          task: snapshot
-          cron: 0 2 * * *
-          retain: 7
-          concurrency: 2
-          labels:
-            type: snapshot
-        - name: daily-backup
-          task: backup
-          cron: 0 3 * * *
-          retain: 7
-          concurrency: 2
-          labels:
-            type: backup
+# Default backup store (NFS target on TrueNAS).
+defaultBackupStore:
+    # NFS endpoint exposing the TrueNAS dataset as a backup target.
+    backupTarget: nfs://truenas.lan:/mnt/fast_app_data/longhorn-backup
+    # No credentials required for an anonymously-readable NFS export.
+    backupTargetCredentialSecret: ""
+    # Poll the backup target every 5 minutes for new/removed backups.
+    pollInterval: 300
 sops:
-    lastmodified: "2026-07-22T09:25:21Z"
-    mac: ENC[AES256_GCM,data:H61nccieTaqp4w6cySS8EG6VZvplmUdSSo2MFi+mGzZohJFjTfdHxIaJgfKuN7/+/dMoacgozl73CmwSnelzgKMm9CZNf0ucWXRPG/8EiM5IeDXBhRChP+Y8BHX6h7FNS6la6Jzdqy8/jKUPw5sdDQaHT6KnQ37ie+D3JjZAwic=,iv:qb/Cr0upi9qQfYVDAwFyLqnVHSlktkVVbsmhTaFsZu0=,tag:q0jPEGtNl4BmBJpdghgRJg==,type:str]
+    lastmodified: "2026-07-26T08:44:08Z"
+    mac: ENC[AES256_GCM,data:4shHlVWcVol/Z8b+5+tz6shT8x6gCZz2xhYv1VT7dHlgcYWWgq2V5mbGUtsVmWFzkGPC5rYvU3dUEiEkl0zb6Q1rP/fD6xQmlyIgH/ioHrfCvjorY5Rv8t4OFH/TyBK9hMwC+iFdAUJOdE9vYQoDmUMTGz6XwWiDY3PxRv1X9D4=,iv:DDDiStImg5qH2QP9lcSyW5+b+1P1mk6F0TWkt9p2syo=,tag:zJnZ455gsXPkE4Lfup2Rcg==,type:str]
     pgp:
-        - created_at: "2026-07-22T09:25:21Z"
+        - created_at: "2026-07-26T08:44:08Z"
           enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMA7uy4qQr71wiARAAwf/RSzNAtZ+BY13bYXQstWAs+7HYZLm1fNKcqNKevSHT
-            +LKy+YkfXgDk67PZNz2H5v3N3GJi/cDEo1MhfXeutYLioZ7WFVaeXsvd3hxisbzi
-            noliuQtu9b63l5UwZPbWvfFDKLJqVKG2JTfXSp+G0Nv11VYCYKftRNd/KH+xEP89
-            hGWGuL+AhbPW9dFimiAICLk/sGyVXNq1SAKu3rL9o/mtRDh+igSg5YohhYhiA7TC
-            PMp/7SRY6swfZ0xfyw9PKlbR5Gm7i5Wtml6MP5myZ90cjzMAkjOL8NavxGogvfib
-            ajjdN5AqdoSU5Kr+rKx8I3Eq9V0Fs/oXfmLeHYYdNoqNOqpqtupXKnMVEdHyUdeq
-            shBuDUE4dvZWyYKfZKzQz2WASN1GDiVm89IvhQ2MP1nEhRE92UyixKHQmLgNolpm
-            LTkuGzWU3dNzHO1c3JLgjF/lWve5KV01Wa/HrAZFH5rSG3qLN8rkmhB+q4C2jjkN
-            4aXyPY8/vRLbIwQgSnxBKYzD6yNFNj76W8kmdEe7c1sMcJZZdjC2pqmEzlfoP9sl
-            vzLkHQeFe2IGYMVna0KVCuEk+5pTvnU7KaoMjfbGFPvDwv3heRUzSTBAF3furx03
-            QHi9bOVCrhrt3QLb7cQSrhGMwtV+XiphfQVAJqKSS3la0XjIoJzm297hNgXv0OTS
-            XAGMF0oFo/1bytZZY0+BSe10y4g+G8q47cqWq0/bnvG4fP95/U3/tH8njIFeNg0U
-            yqm9O+1fQ6I2b6hRlGyv6RjovRc32J+RVPgEkK5a5dlvSTQf97tw9Bo3tKc6
-            =SxRv
+            hQIMA7uy4qQr71wiAQ/9FsBtkggrKwHfh5FFRUFkyqCU596d/R7ZEvsMP7zzCH9H
+            VAAYJxl8D6/KY5EavpeZH4s2ZnlzZjlb/qS2CKpSzXuN7ADPnK10cBxxAOQLp6KF
+            pqdkv6x5MyWbfv87IzM8cI0D/jlG2/iZPfBZy/YU4Rg8cefIl82vNgNADW2O68bK
+            w1VS75AYKmUfXZ2vqz+wX46YcjdXkrvRwzM9lzfIm2Q2dKMqBKSYzxMICZ/hWGRj
+            t33KbZLKC1Kg0NmYOY4GCt8hi6Ck4TjKvzxktsEpXYFDhWzXe9nr+axtrfypLrAn
+            fcLCQV1FSovMK+P4JxB/dd0u+F3RR500F3tBJPQ1xs1Rc1f8Cd2Ynr/3LWtTstkq
+            JyOfLFAj4/LG8o7ub/FSvLZZI052iRLZqlIIe0cFTqCz31yVEJOnLJWmME36hDEm
+            rfd3X9IvixfwWVTa3K0TxZta7u0Q0DsqHIt3eLdeTJC3wY4J+A/vUvPFv137YEqp
+            O+/ALTbwK7ViSQIupjr4kEcklBtiu9myNi3u8JNg67/XHwahCh0tfGk4jCevve5S
+            dDk4Ud7469LuDwYSZw0RdwuU89jQivvsp8/mZpQIoSVzl//iEQ2EOAff1gMfiPoj
+            1MrwGtrrMAo+2Uoxacd7LfFn/l1zRNiGw1rX21NzswmzIU84US4Grv9LGealQ8TS
+            XgH+datMRHIAPKcYY0u/B/PfyoKArlPUzQsIQOm02+s1zuvK/dArKxgObKTAr/u4
+            uNuVH3ObUVMb6VX4iwlxfS+VKpjotTbgY2TJfV46kGC3JJ0w1LdjEyJbpYlIz/k=
+            =HWEp
             -----END PGP MESSAGE-----
           fp: DC6910268E657FF70BA7EC289974494E76938DDC
     encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$