obs,vicm: fix values for storageclass

.github/dependabot.yml

@@ -13,6 +13,7 @@ updates:
       - "/docker/home/zigbee2mqtt"
       - "/docker/infrastructure/mail/roundcube"
       - "/docker/infrastructure/network/traefik"
+      - "/docker/infrastructure/observability"
       - "/docker/infrastructure/squid"
       - "/docker/infrastructure/sshportal"
       - "/docker/personal/fireshare"

.swarmcd/stacks.yaml

@@ -38,6 +38,11 @@ traefik:
   branch: main
   compose_file: docker/infrastructure/network/traefik/docker-compose.yml
 
+observability:
+  repo: homeprod
+  branch: main
+  compose_file: docker/infrastructure/observability/docker-compose.yml
+
 squid:
   repo: homeprod
   branch: main

docker/infrastructure/observability/alloy.river

@@ -0,0 +1,71 @@
+// Grafana Alloy config for the docker host (replaces Promtail, EOL since
+// 2026-03-02). Tails Docker container logs via docker_sd discovery, attaches
+// container_name/compose_service/stream labels, and pushes them to the
+// in-cluster Loki at loki-internal.lan (LAN-only Traefik Ingress on port 80).
+// River config produced by `alloy convert -f promtail` from an equivalent
+// Promtail YAML (so component names and argument shapes are guaranteed
+// correct for Alloy v1.18).
+
+discovery.docker "docker" {
+  host             = "unix:///var/run/docker.sock"
+  refresh_interval = "5s"
+}
+
+loki.process "docker" {
+  forward_to = [loki.write.default.receiver]
+
+  // Parse docker json-file log lines
+  // ({"log":"...","stream":"stdout","time":"..."}).
+  stage.json {
+    expressions = {
+      log    = "log",
+      stream = "stream",
+    }
+  }
+  stage.labels {
+    values = {
+      stream = null,
+    }
+  }
+}
+
+discovery.relabel "docker" {
+  targets = []
+
+  // Strip leading / from container name.
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container_name"
+  }
+  // Compose service name (if label is set).
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "compose_service"
+  }
+  // Log stream (stdout/stderr).
+  rule {
+    source_labels = ["__meta_docker_container_log_stream"]
+    target_label  = "stream"
+  }
+  // Mark the source so dashboards/alerts can distinguish the docker host.
+  rule {
+    target_label = "source"
+    replacement  = "docker-host"
+  }
+}
+
+loki.source.docker "docker" {
+  host             = "unix:///var/run/docker.sock"
+  targets          = discovery.docker.docker.targets
+  forward_to       = [loki.process.docker.receiver]
+  relabel_rules    = discovery.relabel.docker.rules
+  refresh_interval = "5s"
+}
+
+loki.write "default" {
+  endpoint {
+    url = "http://loki-internal.lan/loki/api/v1/push"
+  }
+  external_labels = {}
+}

docker/infrastructure/observability/docker-compose.yml

@@ -0,0 +1,82 @@
+# Docker-host collectors for the central observability stack on Kubernetes.
+# Managed by swarm-cd (see .swarmcd/stacks.yaml).
+#
+# node-exporter + cAdvisor run locally (scraped by vmagent below).
+# vmagent remote_writes metrics to vm-internal.lan (LAN-only Traefik Ingress) ->
+#   vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428
+# alloy tails /var/lib/docker/containers + /var/log and pushes logs to
+#   loki-internal.lan -> loki.observability.svc.cluster.local:3100
+
+services:
+  node-exporter:
+    image: prom/node-exporter:v1.9.1
+    container_name: obs-node-exporter
+    network_mode: host
+    pid: host
+    command:
+      - "--path.rootfs=/host"
+    volumes:
+      - /:/host:ro,rslave
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.49.1
+    container_name: obs-cadvisor
+    ports:
+      - "8080"
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /dev/disk/:/dev/disk:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  vmagent:
+    image: victoriametrics/vmagent:v1.131.0
+    container_name: obs-vmagent
+    # vmagent listens on 8429 (its own metrics), scrapes node-exporter:9100
+    # and cadvisor:8080, remote_writes to vm-internal.lan
+    command:
+      - "-promscrape.config=/etc/vmagent/vmagent.yml"
+      - "-remoteWrite.url=http://vm-internal.lan/api/v1/write"
+    volumes:
+      - type: bind
+        source: /root/homeprod/docker/infrastructure/observability/vmagent.yml
+        target: /etc/vmagent/vmagent.yml
+        read_only: true
+    depends_on:
+      - node-exporter
+      - cadvisor
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  alloy:
+    image: grafana/alloy:v1.18.0
+    container_name: obs-alloy
+    command:
+      - "run"
+      - "/etc/alloy/config.river"
+      - "--server.http.listen.address=0.0.0.0"
+      - "--server.http.listen.port=12345"
+    volumes:
+      - type: bind
+        source: /root/homeprod/docker/infrastructure/observability/alloy.river
+        target: /etc/alloy/config.river
+        read_only: true
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/log:/var/log:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+networks:
+  default:
+    name: proxy
+    external: true

docker/infrastructure/observability/vmagent.yml

@@ -0,0 +1,22 @@
+# vmagent scrape config for the docker host.
+# Scrapes the two local collectors, forwards everything to the central
+# VictoriaMetrics via the LAN-only Caddy route vm-internal.lan (Caddy proxies
+# it to vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428).
+
+scrape_configs:
+  - job_name: docker-host-node
+    static_configs:
+      - targets:
+          - localhost:9100
+        labels:
+          # Distinguishes the docker host from K8s nodes in dashboards/alerts.
+          instance_source: docker-host
+          host: docker-r740
+
+  - job_name: docker-host-cadvisor
+    static_configs:
+      - targets:
+          - localhost:8080
+        labels:
+          instance_source: docker-host
+          host: docker-r740

kubernetes/infrastructure/observability/alloy-config.yaml

@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alloy-config
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: alloy
+    app.kubernetes.io/component: log-collector
+data:
+  config.river: |
+    // -----------------------------------------------------------------------------
+    // Grafana Alloy — log collection only (metrics cluster+host collection is
+    // handled by vmagent/node-exporter/cAdvisor elsewhere).
+    // Forwards pod logs to the in-cluster Loki single-binary.
+    // River config produced by `alloy convert -f promtail` from an equivalent
+    // Promtail YAML (so component names and argument shapes are guaranteed
+    // correct for Alloy v1.18).
+    // -----------------------------------------------------------------------------
+
+    discovery.kubernetes "kubernetes_pods" {
+      role = "pod"
+
+      selectors {
+        role  = "pod"
+        field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)
+      }
+    }
+
+    loki.process "kubernetes_pods" {
+      forward_to = [loki.write.default.receiver]
+
+      // CRI-style log lines on the host: {"log":"...","stream":"stdout","time":"..."}
+      stage.cri { }
+    }
+
+    discovery.relabel "kubernetes_pods" {
+      targets = discovery.kubernetes.kubernetes_pods.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_container_name"]
+        target_label  = "container"
+      }
+    }
+
+    loki.source.file "kubernetes_pods" {
+      targets    = discovery.relabel.kubernetes_pods.output
+      forward_to = [loki.process.kubernetes_pods.receiver]
+
+      file_match {
+        enabled = true
+      }
+      legacy_positions_file = "/tmp/positions.yaml"
+    }
+
+    loki.write "default" {
+      endpoint {
+        url = "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push"
+      }
+      external_labels = {}
+    }

kubernetes/infrastructure/observability/alloy-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: alloy
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: grafana
+        namespace: observability
+      chart: alloy
+      version: "1.11.0"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: alloy-values

kubernetes/infrastructure/observability/alloy-values.yaml

@@ -0,0 +1,57 @@
+# Grafana Alloy
+# Runs as a DaemonSet on every node (tolerates the failover taint).
+# Tails pod logs from /var/log/pods, attaches K8s labels (pod, namespace,
+# service, container, image), and pushes them to Loki at
+# http://loki.observability.svc.cluster.local:3100 (in-cluster, no auth).
+# Use the externally provisioned ConfigMap (alloy-config) carrying config.river.
+alloy:
+    configMap:
+        create: false
+        name: alloy-config
+        key: ENC[AES256_GCM,data:aevkZsfe4c4H34BJ,iv:4vrNZWwYWaxJUKGbNlnZfk0NM182nBQZuwvqUPslKNA=,tag:lh1iILM3uQAt0An6kN6EHg==,type:str]
+    # Mount host log paths so the DaemonSet can tail pod/container logs.
+    mounts:
+        varlog: true
+        dockercontainers: false
+    stabilityLevel: generally-available
+    tolerations:
+        - key: ENC[AES256_GCM,data:zOMWiA9dG+8o,iv:yHp9aBczV8nVRGlJmsBaKs9WdmE3+ii8gWOaj5Xok6M=,tag:8c5ZqBT+vvC0WCW8mi0c9g==,type:str]
+          value: ENC[AES256_GCM,data:2Q04VR9xLbQ=,iv:6+tQXx3IGUIcrse+poG4X/4pS5dN2dKvhb79sCy+nfA=,tag:jb/TZmGpvdsWnKoC5NdCow==,type:str]
+          operator: Equal
+          effect: NoSchedule
+        - key: ENC[AES256_GCM,data:aVq8GU1ZDg5m9XDEI3PoVbbykU36SnKpEgrohvZ0IBmH47qDKg==,iv:eMcHRO5mGp9hhY4KnuBmdPTjd/9lqtesjA/RRSu4QMw=,tag:2jaGuogkLcIuye2meJQMDQ==,type:str]
+          operator: Exists
+          effect: NoSchedule
+    resources:
+        requests:
+            cpu: 50m
+            memory: 64Mi
+        limits:
+            cpu: 200m
+            memory: 256Mi
+sops:
+    lastmodified: "2026-07-28T13:46:53Z"
+    mac: ENC[AES256_GCM,data:u6J/EIP5aFtOi8ZtT8BPn978wJAEL1E3r0WcmG5WIsWKJBZMMQHUYRmHgkS6n0/UMtczWoCdXlxObS73ZQMSSkdtagQHE/HPh69dDdGkjxyM4mlP+YMwcnIQkxpJBAQTERXA+SVoWCZb7ztKbiJS+zPjOCfA95ixQINmBCdUgEc=,iv:WOXngWu3uNWYp18/xWghGJyONBrq3SSnPDwbjHBDozU=,tag:1fasTfuLBGXBCo9JPytsJw==,type:str]
+    pgp:
+        - created_at: "2026-07-28T13:46:53Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA7uy4qQr71wiAQ/8DSD0jEn3YE7XHTm4DWdfLHPs1IVg9hyiWSHe5/xH0siD
+            zWonfjcuytw8EHDzv6pQMNYmIpyAmJ92Al4ROJmBV8gZ59+U9goHmw5HgaQAupBJ
+            /gy8iFBteGOHJT+XJPoVb/bu449f7Vle4L3DrOAl6mwxE0WT/7ERoQBeSxL+2OTZ
+            iDPdq24Zy+crDHpoM0GcSnFRLX5UpmL2QXGB8PHyctwSQoDW+sbAY2ME6buOw8HI
+            AremKTRtFm6Pu1Ksdh6ZicaHpLSvLyByPPt2zYhlBQsG3s/SjJDlnakUZHhm7T4o
+            oHzSpKqV7NQbE09ZMyGJrEkl+rfCPeu6jHplW2EX3xjqLWNUF8GHcsDWljogZxXF
+            irBPkyKASEDdkHZgiZobU6BmvQlxILj104xNtUYTTChcKu3s0d5BrkIOGe+Y7rJd
+            fNvMfkmO8/URBgtsXy2xPKo0Jzu/0pZsPEp2lxNeyElwxQqo1CFwhKZPkeQ2eLpu
+            4X7/CMN8KJr6jLX8HhX9AivlCj+2CoT8SB5GAPw4p00eD0GIMWSNyOXjyW/qQZi3
+            82EyX9KIpHjALoOI4/3YgFpUO12AEsjirYKWkUM+IoDp5l+YX59QdM7zUVVyzsTG
+            2EXEdQqYVA7DIcMJb0tHqxDGMGxNayaUsikV7e/kcRVUyDSH+xOfaSUrDTqqv+XS
+            XAFro6848HF5uMIP4pfM+dxzkc9rFF4il0CuGWlscbz5q+Aw9LImGbpNzrWk3Lpi
+            beuwWKFTZ8zab5TH9UF/xqsQeg/nyzQnb3fFPk+owvY7fH6XiYbN5WjKjXGW
+            =kBpp
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/infrastructure/observability/grafana-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: observability
+spec:
+  interval: 1m
+  url: https://grafana.github.io/helm-charts

kubernetes/infrastructure/observability/kustomization.yaml

@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: observability
+resources:
+  - namespace.yaml
+  - vmstack-repository.yaml
+  - grafana-repository.yaml
+  - loki-repository.yaml
+  - vmstack-release.yaml
+  - loki-release.yaml
+  - alloy-release.yaml
+  - alloy-config.yaml
+  - vm-internal-ingress.yaml
+  - loki-internal-ingress.yaml
+secretGenerator:
+  - name: vmstack-values
+    files:
+      - values.yaml=vmstack-values.yaml
+  - name: loki-values
+    files:
+      - values.yaml=loki-values.yaml
+  - name: alloy-values
+    files:
+      - values.yaml=alloy-values.yaml
+configurations:
+  - kustomizeconfig.yaml

kubernetes/infrastructure/observability/kustomizeconfig.yaml

@@ -0,0 +1,6 @@
+nameReference:
+  - kind: Secret
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease

kubernetes/infrastructure/observability/loki-internal-ingress.yaml

@@ -0,0 +1,23 @@
+# Ingress exposing the Loki push API (loki:3100) through Traefik as
+# loki-internal.lan — used by the docker-host promtail to push container logs
+# into the central Loki.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: loki-internal
+  namespace: observability
+  annotations:
+    external-dns.alpha.kubernetes.io/enabled: "true"
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: loki-internal.lan
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: loki
+                port:
+                  number: 3100

kubernetes/infrastructure/observability/loki-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: loki
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: grafana-community
+        namespace: observability
+      chart: loki
+      version: "18.5.4"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: loki-values

kubernetes/infrastructure/observability/loki-repository.yaml

@@ -0,0 +1,9 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana-community
+  namespace: observability
+spec:
+  interval: 1m
+  type: oci
+  url: oci://ghcr.io/grafana-community/helm-charts

kubernetes/infrastructure/observability/loki-values.yaml

@@ -0,0 +1,88 @@
+# Loki Helm values
+# Uses Monolithic deployment mode with filesystem storage on a Longhorn PVC.
+loki:
+    deploymentMode: Monolithic
+    # Filesystem storage — chunks + rules on the PVC. The chart v18 requires
+    # an explicit loki.storage block; without it the workload template errors
+    # with "Please define loki.storage.bucketNames.chunks".
+    storage:
+        type: filesystem
+        filesystem:
+            chunks_directory: /var/loki/chunks
+            rules_directory: /var/loki/rules
+    # Required schema config. tsdb store + filesystem object store, schema v13.
+    # See https://grafana.com/docs/loki/latest/operations/storage/schema/.
+    schemaConfig:
+        configs:
+            - from: 2024-04-01T00:00:00Z
+              store: tsdb
+              object_store: filesystem
+              schema: v13
+              index:
+                prefix: index_
+                period: 24h
+    # For monolithic mode, point tsdb_shipper at the local store (no index gateway).
+    storage_config:
+        tsdb_shipper:
+            active_index_directory: /var/loki/index
+            cache_location: /var/loki/index-cache
+        filesystem:
+            chunks_directory: /var/loki/chunks
+            rules_directory: /var/loki/rules
+    monolithic:
+        persistence:
+            storageClassName: longhorn
+            accessModes:
+                - ReadWriteOnce
+            size: 10Gi
+        resources:
+            requests:
+                cpu: 100m
+                memory: 256Mi
+            limits:
+                cpu: "1"
+                memory: 1Gi
+# 7d log retention (user can tune later).
+limitsConfig:
+    retention_period: 7d
+# Disable distributed components + gateway + minio (monolithic only).
+test:
+    enabled: false
+backend:
+    replicas: 0
+read:
+    replicas: 0
+write:
+    replicas: 0
+singleBinary:
+    replicas: 0
+gateway:
+    enabled: false
+minio:
+    enabled: false
+sops:
+    lastmodified: "2026-07-28T16:09:30Z"
+    mac: ENC[AES256_GCM,data:phQ6t8rBVG5rdOR3H4GEWJHiCU1rj0MOzjp2jn2nneOUbbclyn4L4ga7YHOuoce0HEEMWniWxchT2eB0BZM3gtOd3yyBq3z3vTQKI12TsTzQSXxm+AgM4DL855BhPrFJCNZYeXb6IAGVkcRK/N7ihuGWiZORBsPaIcYseW9kjf4=,iv:wNIkcPRr1d79odGsT+yJJ+zqlUVTtqj8c/YkbhqOQjA=,tag:LSXyM6QiCM9qoit++nUkBw==,type:str]
+    pgp:
+        - created_at: "2026-07-28T16:09:30Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQILA7uy4qQr71wiAQ/3c4Q36xa3tNPub4HCEUmJwK7Nmhdrg+sihZYR/TOHuDuV
+            sY3GX+HwaRaMh4rOiJpCDa9E3qJY+qh9Wf+rgq+vaL9hmfNBlC+2q2xGv2u6FdiN
+            xXE501qBbZjyNm/9fVAnCxbTBZP2mkRgdzMHfK2dnhIcL0/5LK7NAu+PsMQ0J8uq
+            J5JYCqQTp/bfoRi8O4YXJD38u5d0UyEIy7HImKSn21za6CUErHEgdOTGUzw+UTH9
+            kYHQykiBVML6bpp3YkqIquBm1tDzNuPvKSynnswJS5DQnGwsWVmESA3Rgfes2doe
+            4d2f4Bso8+1+BRgbPkmp2c3/GdNFeUjjj6ViRar1bX2Bmt/ODLjoxAriRzS5ChRe
+            MBJkKcW1hWotQE1IZgYC8A2I4dvDAbjeTwWmyJW7T1NV0er2rDd8TC9aQfhOPSeh
+            mXdF18PXSmonmAdCj8Fu6Zfd6kcMQUQdI5L3BLRAQNs6+0r/tNjPH8vgYQtRWtxO
+            acguClDwTrMwGM33HiBAYpoZ4hpCk43PRDNf0o2JkDy8tr3uTeBkmHPIA80UsIi8
+            /YmrLzzaF0TaN7OvDLbw/5HUR1R2Wt2Mc3GnTdJIxMogqcw+jO86cNm2v7FJX5v1
+            wG4ocHbrbfH1lcte3ZW8E5z3evuUeXVgUTL7dRHdNWmfL3HUnBlX+H+z7qWoNdJc
+            AUhXpEvLTZ0LkA0ZMi749sUcL9cLsXkKJhAPbiF/3ytc7tLc40QsTY53uuqGPU/8
+            LVVX7jUjqBrmeUENYj3zSN3kxcI1OX/qzHdTQ0LFSPPUrCEPtE8hWJrLaUE=
+            =Nr2M
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/infrastructure/observability/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: observability
+  labels:
+    pod-security.kubernetes.io/enforce: privileged

kubernetes/infrastructure/observability/vm-internal-ingress.yaml

@@ -0,0 +1,26 @@
+# Ingress exposing the VictoriaMetrics vminsert endpoint (vmsingle:8428)
+# through Traefik as vm-internal.lan — used by the docker-host vmagent to
+# remote_write metrics into the central VictoriaMetrics.
+# external-dns auto-creates the vm-internal.lan A record pointing at the
+# Traefik LoadBalancer (see kubernetes/system/external-dns).
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: vm-internal
+  namespace: observability
+  annotations:
+    external-dns.alpha.kubernetes.io/enabled: "true"
+    # No chart ingress is available for vmsingle, so declare it here.
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: vm-internal.lan
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: vmsingle-vm-victoria-metrics-k8s-stack
+                port:
+                  number: 8428

kubernetes/infrastructure/observability/vmstack-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: vm-victoria-metrics-k8s-stack
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: vm
+        namespace: observability
+      chart: victoria-metrics-k8s-stack
+      version: "0.87.0"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: vmstack-values

kubernetes/infrastructure/observability/vmstack-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: vm
+  namespace: observability
+spec:
+  interval: 1m
+  url: https://victoriametrics.github.io/helm-charts/

kubernetes/infrastructure/observability/vmstack-values.yaml

@@ -0,0 +1,199 @@
+# victoria-metrics-k8s-stack Helm values 
+# The chart installs the VictoriaMetrics operator + CRDs and creates VMSingle,
+# VMAgent, VMAlert, VMAlertmanager CRs.
+# ---------------------------------------------------------------------------
+# VictoriaMetrics single-binary (the metrics database)
+# ---------------------------------------------------------------------------
+vmsingle:
+    enabled: true
+    spec:
+        retentionPeriod: 7d
+        storage:
+            storageClassName: longhorn
+            accessModes:
+                - ReadWriteOnce
+            resources:
+                requests:
+                    storage: 20Gi
+        resources:
+            requests:
+                cpu: 100m
+                memory: 256Mi
+            limits:
+                cpu: "1"
+                memory: 1Gi
+# ---------------------------------------------------------------------------
+# vmagent — scrapes node-exporter, kube-state-metrics, kubelet, etc.
+# The docker host runs its own vmagent that remote_writes through
+# vm-internal.lan (the Traefik Ingress → this vmsingle).
+# ---------------------------------------------------------------------------
+vmagent:
+    enabled: true
+    spec:
+        selectAllByDefault: true
+        scrapeInterval: 20s
+        tolerations:
+            - key: ENC[AES256_GCM,data:SyutB0y9QtqQ,iv:EKRXNGtUovQYsESHFQTGygP09O2UWtLuBEQQcYZm3Oc=,tag:zh7CYOZleSgbbQhYZvcT+w==,type:str]
+              value: ENC[AES256_GCM,data:So+8xmHYSxs=,iv:7sfUBAaMZd7imjVx14lusVn58cJQ4HpTkaPy5Eal2dY=,tag:hRBjT6wuEfG3Z+ZiPJhwQQ==,type:str]
+              operator: Equal
+              effect: NoSchedule
+            - key: ENC[AES256_GCM,data:lj9OurynU8pRt4/MhgyevjmV0b7Zq+WG/9+zRRvPeA+RYZsV+A==,iv:LjuhErDT0bDlLcpo/AlNykz8Xu89B+nx1ja+HkPNXwo=,tag:5xM6XHrCdEejE7ituQ8W4Q==,type:str]
+              operator: Exists
+              effect: NoSchedule
+# ---------------------------------------------------------------------------
+# kube-state-metrics + node-exporter (metrics sources)
+# ---------------------------------------------------------------------------
+kube-state-metrics:
+    enabled: true
+prometheus-node-exporter:
+    enabled: true
+# ---------------------------------------------------------------------------
+# Grafana (UI) — served behind ingress at grafana.lan.
+# ---------------------------------------------------------------------------
+grafana:
+    enabled: true
+    persistence:
+        enabled: true
+        storageClassName: longhorn
+        accessModes:
+            - ReadWriteOnce
+        size: 5Gi
+    adminUser: admin
+    # SOPS encrypts this when the file is processed.
+    adminPassword: ENC[AES256_GCM,data:vpPvJA/PktMnsg/IovYfsD4lDBndUw==,iv:b4eLMEcUMJj4BwyeYw1kw2sZuieyr2xL9et+R7ss0BU=,tag:u0RcTp6mkQNAGkblG4/xww==,type:str]
+    # Provision a Loki datasource alongside the chart's default VictoriaMetrics
+    # datasource so metric/log correlation works in one UI.
+    datasources:
+        datasources.yaml:
+            apiVersion: ENC[AES256_GCM,data:Uw==,iv:KYTMBV53yMbojMaqNinYpm/Uj5ylV86jcdJTdxuy+G0=,tag:IbIpiSBQFw7YSTQg0WRGqg==,type:int]
+            datasources:
+                - name: ENC[AES256_GCM,data:BPfR8Q==,iv:MkMhH3IJfBZzbIcqQw7y6bfJB/4Ew4feZNugjovk5O0=,tag:jaa43tzIXkE8kWZp2R/kcg==,type:str]
+                  type: ENC[AES256_GCM,data:WWncww==,iv:hPqJE0KCidSBnf11wxFH+Je+YKztWmeToBqi39Qf4ZQ=,tag:c9enJ2VgQqDJVbasvgG+Dw==,type:str]
+                  url: ENC[AES256_GCM,data:YkS66tpWXpW62Dtful0KNCwoZ3rG3nQtDBUsNQUwdQ+eqtG38JTrAudQqnsRD51q,iv:Mu3Sc6T2xypdot9dA1DHc8nD7UI8372nqlveLITR82E=,tag:1VmmIg+xR9qjWcvPKVYsJQ==,type:str]
+                  access: ENC[AES256_GCM,data:af3GpjE=,iv:MAYKfklF+wCC/XN+4snebjPAnW4KpYzaMMGYds7rG18=,tag:X+oFiDAiCb1f7cfREqFT7A==,type:str]
+                  isDefault: ENC[AES256_GCM,data:Zgu4Hn8=,iv:nnVB/in75nPAQos0Dfl6abBrHwT8tCt5X7lcCYBZbLc=,tag:u1UvkSj6SjcMm06NzKMGDA==,type:bool]
+                  jsonData:
+                    maxLines: ENC[AES256_GCM,data:88JJqA==,iv:lKsZa024I6223UEu0SHuZ3N9LZrzw0WBTfC+L2/3ljI=,tag:YvqieXK36xMmC/sSJ11NIg==,type:int]
+    ingress:
+        enabled: true
+        ingressClassName: traefik
+        annotations:
+            external-dns.alpha.kubernetes.io/enabled: "true"
+        hosts:
+            - grafana.lan
+        path: /
+        pathType: Prefix
+        tls: []
+# ---------------------------------------------------------------------------
+# vmalert — evaluates VMRule CRs against VictoriaMetrics, forwards firing
+# alerts to Alertmanager. selectAllByDefault picks up all VMRules in the
+# namespace (including our vmalert-rules.yaml VMRule CR).
+# ---------------------------------------------------------------------------
+vmalert:
+    enabled: true
+    spec:
+        selectAllByDefault: true
+        evaluationInterval: 20s
+        notifiers:
+            - url: http://vm-victoria-metrics-k8s-stack-alertmanager.observability.svc.cluster.local:9093
+        resources:
+            requests:
+                cpu: 50m
+                memory: 64Mi
+            limits:
+                cpu: 200m
+                memory: 128Mi
+# ---------------------------------------------------------------------------
+# Alertmanager — 1 replica; inline config with a single n8n webhook receiver.
+# n8n runs on the docker host and fans out to email/Telegram/whatever.
+# ---------------------------------------------------------------------------
+alertmanager:
+    enabled: true
+    spec:
+        replicaCount: 1
+        port: "9093"
+        selectAllByDefault: true
+        storage:
+            volumeClaimTemplate:
+                spec:
+                    storageClassName: longhorn
+                    accessModes:
+                        - ReadWriteOnce
+                    resources:
+                        requests:
+                            storage: 2Gi
+        resources:
+            requests:
+                cpu: 50m
+                memory: 64Mi
+            limits:
+                cpu: 200m
+                memory: 128Mi
+    config:
+        route:
+            receiver: n8n-webhook
+            group_wait: 30s
+            group_interval: 5m
+            repeat_interval: 4h
+        receivers:
+            - name: n8n-webhook
+              webhook_configs:
+                - url: http://n8n.lan/webhook/observability-alert
+                  send_resolved: true
+extraRules:
+    node-alerts:
+        groups:
+            - name: node
+              rules:
+                - alert: HighNodeCPU
+                  expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: High CPU on {{ $labels.instance }}
+                    description: CPU usage above 80% for 10 minutes.
+                - alert: HighNodeRAM
+                  expr: 100 - ((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100) > 85
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: High RAM on {{ $labels.instance }}
+                    description: RAM usage above 85% for 10 minutes.
+                - alert: LowDiskSpace
+                  expr: |
+                    100 - ((node_filesystem_avail_bytes{mountpoint!~"/run.*|/var/lib/docker.*"} /
+                            node_filesystem_size_bytes{mountpoint!~"/run.*|/var/lib/docker.*"}) * 100) > 85
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: Low disk space on {{ $labels.instance }} {{ $labels.mountpoint }}
+                    description: Disk usage above 85% for 10 minutes.
+sops:
+    lastmodified: "2026-07-28T16:39:09Z"
+    mac: ENC[AES256_GCM,data:Ymxmmkui0Nuv35Yq6v4BXM4xGhvjAufexW7fIm9fUIdTTXZ3e10PitNCZXKiaDwwCJfdEkNZri7jPAYzhTzKMIdcTKnOMImsMjdiJZwRwwbKcYVay0A6KSLJ8eFjDRRYhlCXbsAIRlQHpFh4UkssKdu40mIiudU0qIVm04ppbno=,iv:R8kpITPYSbQ1yZw5IK0fArssxUtIjNmvzknOrLJQcuk=,tag:/CRcr1VZAIKIPbcag05uTw==,type:str]
+    pgp:
+        - created_at: "2026-07-28T16:39:08Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA7uy4qQr71wiARAAmOxyx5eH5Jykp/GDL79REgf61iGujtPiUVy8CT3o7O5k
+            tTGma6rRXK5wZV/iGN3wOQehF19Oy2iYwMfxilRwpB69RFwyKPKdHDDWMWNXv5Th
+            wJnNjVjWZ0xiRautWPn/TjDOJZ6j6p7lQ3IcQ3vzuap1RVRIXv081DxJGR9JbeYl
+            khmrl1+zecwxS87JK+fLko9NIBb5JgFEtEf1RvLyFak8nZu5cNCu+PCykALvUaO7
+            V5NAaF1aV9cdz8G9TZWmHZsIlDDE9/oro2aLR7cr8uxnKQhWerttqvyDYNwYzNKW
+            qgsUplTxTmyuc9DdnLDfJ55Ncuv9qS5/4/p5GpsHrteD6oiPNjXaxR8cBVFLvGwp
+            kQb/VZgOGAnkYlMCUG8APiwRhbpAlyy51hKBqVh+a9iRlzZEWHKlxcvIrcFAQaHt
+            sDXyg36F0GCoHiMAqHKG0kRcbTV7Sq0JZVoLU/qxvaDfqvyCZO7X/U/YsSWNbWL0
+            emyBfeSvBX9h62NJx2aw8xKyiL+aeVfzSKAaXHMd/r4gKxBw/kVHAljKwRzbL85j
+            UHeLUvqVkXs1mtIqGgUnpCcAVuV83omgcOwF44FitG/f9GCx+UWTvScwaw5ywuG3
+            J/4fOaA3K0Z4e/mOejARd4jLSPktSRpA01k9kjYY8UBn7TzQ0LrMLj5dxh8BpWrS
+            XgECZapi+q6IjIses7+JqkWiP25hcglhqK3LAiUapA0c5UmAa5Vf7jfnkFm8eUdZ
+            bA7ZnhX8nxK9n7JdSxrZhrbPWMO8hF/XKi7KTZMvgXFlb4/qP96y7oeqzKYcnvE=
+            =pmSq
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/kustomization.yaml

@@ -24,6 +24,7 @@ resources:
   # Infrastructure
   - infrastructure/authentik
   - infrastructure/mail/stalwart
+  - infrastructure/observability
   # Personal
   - personal/linkwarden
   - personal/notesnook