observability: deploy observability stack

.github/dependabot.yml

@@ -13,6 +13,7 @@ updates:
       - "/docker/home/zigbee2mqtt"
       - "/docker/infrastructure/mail/roundcube"
       - "/docker/infrastructure/network/traefik"
+      - "/docker/infrastructure/observability"
       - "/docker/infrastructure/squid"
       - "/docker/infrastructure/sshportal"
       - "/docker/personal/fireshare"

.swarmcd/stacks.yaml

@@ -38,6 +38,11 @@ traefik:
   branch: main
   compose_file: docker/infrastructure/network/traefik/docker-compose.yml
 
+observability:
+  repo: homeprod
+  branch: main
+  compose_file: docker/infrastructure/observability/docker-compose.yml
+
 squid:
   repo: homeprod
   branch: main

docker/infrastructure/observability/alloy.river

@@ -0,0 +1,83 @@
+// Grafana Alloy config for the docker host
+// Tails Docker container logs via the docker_sd discovery +
+// file streaming, applies container_name/image labels, and pushes them to
+// the in-cluster Loki at loki-internal.lan (LAN-only Traefik Ingress).
+
+logging {
+  level  = "info"
+  format = "logfmt"
+}
+
+// Discover running Docker containers via the host socket.
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+}
+
+// Relabel rules: strip the leading slash from __meta_docker_container_name and
+// expose useful labels (container name, compose service, image).
+discovery.relabel "containers" {
+  targets = discovery.docker.containers.targets
+
+  // Strip leading / from container name.
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container_name"
+    replacement   = "$1"
+  }
+
+  // Compose service name (if label is set).
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "compose_service"
+  }
+
+  // Log stream (stdout/stderr).
+  rule {
+    source_labels = ["__meta_docker_container_log_stream"]
+    target_label  = "stream"
+  }
+
+  // Mark the source so dashboards/alerts can distinguish the docker host.
+  rule {
+    target_label = "source"
+    replacement  = "docker-host"
+  }
+}
+
+// Tail the container log files based on the docker_sd discovery. Each target
+// exposes a `__path__`idders pointing at /var/lib/docker/containers/<id>/<id>-json.log
+// (the default docker json-file driver path; we mount that host path read-only).
+local.file "container_logs" {
+  targets       = discovery.relabel.containers.output
+  tail_from_end = true
+}
+
+// Parse docker json-file logs ({"log":"...","stream":"stdout","time":"..."}).
+loki.process "containers" {
+  forward_to = [loki.write.default.receiver]
+
+  stage.json {
+    expressions = {
+      log    = "log",
+      stream = "stream",
+      time   = "time",
+    }
+  }
+  stage.labels {
+    values = {
+      stream = "stream",
+    }
+  }
+  stage.timestamp {
+    source = "time"
+    format = "RFC3339Nano"
+  }
+}
+
+// Push to Loki in-cluster via the LAN-only Traefik Ingress (port 80).
+loki.write "default" {
+  endpoint {
+    url = "http://loki-internal.lan/loki/api/v1/push"
+  }
+}

docker/infrastructure/observability/docker-compose.yml

@@ -0,0 +1,82 @@
+# Docker-host collectors for the central observability stack on Kubernetes.
+# Managed by swarm-cd (see .swarmcd/stacks.yaml).
+#
+# node-exporter + cAdvisor run locally (scraped by vmagent below).
+# vmagent remote_writes metrics to vm-internal.lan (LAN-only Traefik Ingress) ->
+#   vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428
+# alloy tails /var/lib/docker/containers + /var/log and pushes logs to
+#   loki-internal.lan -> loki.observability.svc.cluster.local:3100
+
+services:
+  node-exporter:
+    image: prom/node-exporter:v1.9.1
+    container_name: obs-node-exporter
+    network_mode: host
+    pid: host
+    command:
+      - "--path.rootfs=/host"
+    volumes:
+      - /:/host:ro,rslave
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.49.1
+    container_name: obs-cadvisor
+    ports:
+      - "8080"
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /dev/disk/:/dev/disk:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  vmagent:
+    image: victoriametrics/vmagent:v1.131.0
+    container_name: obs-vmagent
+    # vmagent listens on 8429 (its own metrics), scrapes node-exporter:9100
+    # and cadvisor:8080, remote_writes to vm-internal.lan
+    command:
+      - "-promscrape.config=/etc/vmagent/vmagent.yml"
+      - "-remoteWrite.url=http://vm-internal.lan/api/v1/write"
+    volumes:
+      - type: bind
+        source: /root/homeprod/docker/infrastructure/observability/vmagent.yml
+        target: /etc/vmagent/vmagent.yml
+        read_only: true
+    depends_on:
+      - node-exporter
+      - cadvisor
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  alloy:
+    image: grafana/alloy:v1.18.0
+    container_name: obs-alloy
+    command:
+      - "run"
+      - "/etc/alloy/config.river"
+      - "--server.http.listen.address=0.0.0.0"
+      - "--server.http.listen.port=12345"
+    volumes:
+      - type: bind
+        source: /root/homeprod/docker/infrastructure/observability/alloy.river
+        target: /etc/alloy/config.river
+        read_only: true
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/log:/var/log:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+networks:
+  default:
+    name: proxy
+    external: true

docker/infrastructure/observability/vmagent.yml

@@ -0,0 +1,22 @@
+# vmagent scrape config for the docker host.
+# Scrapes the two local collectors, forwards everything to the central
+# VictoriaMetrics via the LAN-only Caddy route vm-internal.lan (Caddy proxies
+# it to vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428).
+
+scrape_configs:
+  - job_name: docker-host-node
+    static_configs:
+      - targets:
+          - localhost:9100
+        labels:
+          # Distinguishes the docker host from K8s nodes in dashboards/alerts.
+          instance_source: docker-host
+          host: docker-r740
+
+  - job_name: docker-host-cadvisor
+    static_configs:
+      - targets:
+          - localhost:8080
+        labels:
+          instance_source: docker-host
+          host: docker-r740

kubernetes/infrastructure/observability/alertmanager-config.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: alertmanager-config
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: alertmanager
+    app.kubernetes.io/component: notifiers
+type: Opaque
+stringData:
+  alertmanager.yaml: |
+    global:
+      resolve_timeout: 5m
+
+    route:
+      receiver: default
+      group_by: ["alertname", "severity", "instance"]
+      group_wait: 30s
+      group_interval: 5m
+      repeat_interval: 4h
+
+    receivers:
+      - name: default
+        webhook_configs:
+          # n8n workflow webhook
+          - url: "http://n8n.lan/webhook/observability-alert"
+            send_resolved: true

kubernetes/infrastructure/observability/alloy-config.yaml

@@ -0,0 +1,93 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alloy-config
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: alloy
+    app.kubernetes.io/component: log-collector
+data:
+  config.river: |
+    // -----------------------------------------------------------------------------
+    // Grafana Alloy — log collection only (metrics cluster+host collection is
+    // handled by vmagent/node-exporter/cAdvisor elsewhere).
+    // Forwards pod logs to the in-cluster Loki single-binary.
+    // -----------------------------------------------------------------------------
+
+    logging {
+      level = "info"
+      format = "logfmt"
+    }
+
+    // Tail every pod's log files written under /var/log/pods/* on the host.
+    local.file_match "pods" {
+      path_targets = [{
+        __path__ = "/var/log/pods/*/*/*.log",
+      }]
+    }
+
+    // Discover Kubernetes metadata for the tailed files (pod_name, namespace,
+    // container_name, uid).
+    discovery.kubernetes "pods" {
+      role = "pod"
+    }
+
+    // Stream-stage relabel: read the filename to extract namespace/pod/container
+    // metadata in line with Loki/Promtail's expectations.
+    lrelabel "pods" {
+      targets = local.file_match.pods.targets
+      rule {
+        source_labels = ["__path__"]
+        regex        = "/var/log/pods/(?P<namespace>[^/]+)/(?P<pod>[^/]+)/(?P<container>[^/]+)/(?P<uid>[^/]+)/.*"
+        target_label  = "namespace"
+        replacement   = "$namespace"
+      }
+      rule {
+        source_labels = ["__path__"]
+        regex        = "/var/log/pods/([^/]+)/([^/]+)/([^/]+)/[^/]+/.*"
+        target_label  = "pod"
+        replacement   = "$pod"
+      }
+      rule {
+        source_labels = ["__path__"]
+        regex        = "/var/log/pods/([^/]+)/([^/]+)/([^/]+)/[^/]+/.*"
+        target_label  = "container"
+        replacement   = "$container"
+      }
+    }
+
+    // Tail the actual log files.
+    local.file "pods_logs" {
+      targets    = lrelabel.pods.output
+      read_from  = "beginning"
+      tail_from_end = false
+    }
+
+    // Parse CRI-style lines ({"log":"...","stream":"stdout","time":"..."}).
+    loki.process "pods" {
+      forward_to = [loki.write.default.receiver]
+
+      stage.json {
+        expressions = {
+          log    = "log",
+          stream = "stream",
+          time   = "time",
+        }
+      }
+      stage.labels {
+        values = {
+          stream = "stream",
+        }
+      }
+      stage.timestamp {
+        source = "time"
+        format = "RFC3339Nano"
+      }
+    }
+
+    // Push to Loki in-cluster (no auth on .lan).
+    loki.write "default" {
+      endpoint {
+        url = "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push"
+      }
+    }

kubernetes/infrastructure/observability/alloy-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: alloy
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: grafana
+        namespace: observability
+      chart: alloy
+      version: "1.11.0"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: alloy-values

kubernetes/infrastructure/observability/alloy-values.yaml

@@ -0,0 +1,57 @@
+# Grafana Alloy
+# Runs as a DaemonSet on every node (tolerates the failover taint).
+# Tails pod logs from /var/log/pods, attaches K8s labels (pod, namespace,
+# service, container, image), and pushes them to Loki at
+# http://loki.observability.svc.cluster.local:3100 (in-cluster, no auth).
+# Use the externally provisioned ConfigMap (alloy-config) carrying config.river.
+alloy:
+    configMap:
+        create: false
+        name: alloy-config
+        key: ENC[AES256_GCM,data:aevkZsfe4c4H34BJ,iv:4vrNZWwYWaxJUKGbNlnZfk0NM182nBQZuwvqUPslKNA=,tag:lh1iILM3uQAt0An6kN6EHg==,type:str]
+    # Mount host log paths so the DaemonSet can tail pod/container logs.
+    mounts:
+        varlog: true
+        dockercontainers: false
+    stabilityLevel: generally-available
+    tolerations:
+        - key: ENC[AES256_GCM,data:zOMWiA9dG+8o,iv:yHp9aBczV8nVRGlJmsBaKs9WdmE3+ii8gWOaj5Xok6M=,tag:8c5ZqBT+vvC0WCW8mi0c9g==,type:str]
+          value: ENC[AES256_GCM,data:2Q04VR9xLbQ=,iv:6+tQXx3IGUIcrse+poG4X/4pS5dN2dKvhb79sCy+nfA=,tag:jb/TZmGpvdsWnKoC5NdCow==,type:str]
+          operator: Equal
+          effect: NoSchedule
+        - key: ENC[AES256_GCM,data:aVq8GU1ZDg5m9XDEI3PoVbbykU36SnKpEgrohvZ0IBmH47qDKg==,iv:eMcHRO5mGp9hhY4KnuBmdPTjd/9lqtesjA/RRSu4QMw=,tag:2jaGuogkLcIuye2meJQMDQ==,type:str]
+          operator: Exists
+          effect: NoSchedule
+    resources:
+        requests:
+            cpu: 50m
+            memory: 64Mi
+        limits:
+            cpu: 200m
+            memory: 256Mi
+sops:
+    lastmodified: "2026-07-28T13:46:53Z"
+    mac: ENC[AES256_GCM,data:u6J/EIP5aFtOi8ZtT8BPn978wJAEL1E3r0WcmG5WIsWKJBZMMQHUYRmHgkS6n0/UMtczWoCdXlxObS73ZQMSSkdtagQHE/HPh69dDdGkjxyM4mlP+YMwcnIQkxpJBAQTERXA+SVoWCZb7ztKbiJS+zPjOCfA95ixQINmBCdUgEc=,iv:WOXngWu3uNWYp18/xWghGJyONBrq3SSnPDwbjHBDozU=,tag:1fasTfuLBGXBCo9JPytsJw==,type:str]
+    pgp:
+        - created_at: "2026-07-28T13:46:53Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA7uy4qQr71wiAQ/8DSD0jEn3YE7XHTm4DWdfLHPs1IVg9hyiWSHe5/xH0siD
+            zWonfjcuytw8EHDzv6pQMNYmIpyAmJ92Al4ROJmBV8gZ59+U9goHmw5HgaQAupBJ
+            /gy8iFBteGOHJT+XJPoVb/bu449f7Vle4L3DrOAl6mwxE0WT/7ERoQBeSxL+2OTZ
+            iDPdq24Zy+crDHpoM0GcSnFRLX5UpmL2QXGB8PHyctwSQoDW+sbAY2ME6buOw8HI
+            AremKTRtFm6Pu1Ksdh6ZicaHpLSvLyByPPt2zYhlBQsG3s/SjJDlnakUZHhm7T4o
+            oHzSpKqV7NQbE09ZMyGJrEkl+rfCPeu6jHplW2EX3xjqLWNUF8GHcsDWljogZxXF
+            irBPkyKASEDdkHZgiZobU6BmvQlxILj104xNtUYTTChcKu3s0d5BrkIOGe+Y7rJd
+            fNvMfkmO8/URBgtsXy2xPKo0Jzu/0pZsPEp2lxNeyElwxQqo1CFwhKZPkeQ2eLpu
+            4X7/CMN8KJr6jLX8HhX9AivlCj+2CoT8SB5GAPw4p00eD0GIMWSNyOXjyW/qQZi3
+            82EyX9KIpHjALoOI4/3YgFpUO12AEsjirYKWkUM+IoDp5l+YX59QdM7zUVVyzsTG
+            2EXEdQqYVA7DIcMJb0tHqxDGMGxNayaUsikV7e/kcRVUyDSH+xOfaSUrDTqqv+XS
+            XAFro6848HF5uMIP4pfM+dxzkc9rFF4il0CuGWlscbz5q+Aw9LImGbpNzrWk3Lpi
+            beuwWKFTZ8zab5TH9UF/xqsQeg/nyzQnb3fFPk+owvY7fH6XiYbN5WjKjXGW
+            =kBpp
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/infrastructure/observability/grafana-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: observability
+spec:
+  interval: 1m
+  url: https://grafana.github.io/helm-charts

kubernetes/infrastructure/observability/kustomization.yaml

@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: observability
+resources:
+  - namespace.yaml
+  - vmstack-repository.yaml
+  - grafana-repository.yaml
+  - loki-repository.yaml
+  - vmstack-release.yaml
+  - loki-release.yaml
+  - alloy-release.yaml
+  - vmalert-rules.yaml
+  - alloy-config.yaml
+  - alertmanager-config.yaml
+  - vm-internal-ingress.yaml
+  - loki-internal-ingress.yaml
+secretGenerator:
+  - name: vmstack-values
+    files:
+      - values.yaml=vmstack-values.yaml
+  - name: loki-values
+    files:
+      - values.yaml=loki-values.yaml
+  - name: alloy-values
+    files:
+      - values.yaml=alloy-values.yaml
+configurations:
+  - kustomizeconfig.yaml

kubernetes/infrastructure/observability/kustomizeconfig.yaml

@@ -0,0 +1,6 @@
+nameReference:
+  - kind: Secret
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease

kubernetes/infrastructure/observability/loki-internal-ingress.yaml

@@ -0,0 +1,23 @@
+# Ingress exposing the Loki push API (loki:3100) through Traefik as
+# loki-internal.lan — used by the docker-host promtail to push container logs
+# into the central Loki.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: loki-internal
+  namespace: observability
+  annotations:
+    external-dns.alpha.kubernetes.io/enabled: "true"
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: loki-internal.lan
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: loki
+                port:
+                  number: 3100

kubernetes/infrastructure/observability/loki-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: loki
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: grafana-community
+        namespace: observability
+      chart: loki
+      version: "18.5.4"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: loki-values

kubernetes/infrastructure/observability/loki-repository.yaml

@@ -0,0 +1,9 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana-community
+  namespace: observability
+spec:
+  interval: 1m
+  type: oci
+  url: oci://ghcr.io/grafana-community/helm-charts

kubernetes/infrastructure/observability/loki-values.yaml

@@ -0,0 +1,65 @@
+# Loki single-binary Helm values
+# SOPS-encrypted before commit (per .sops.yaml rules).
+# Uses the new chart-v18 "Monolithic" deployment mode (SingleBinary mode is
+# deprecated in Loki 4 / chart >=18.x). Bundles storage on a Longhorn PVC —
+# no minio, no gateway. Resource-light; tuned for a 2-node home cluster.
+loki:
+    deploymentMode: Monolithic
+    monolithic:
+        persistence:
+            storageClassName: longhorn
+            accessModes:
+                - ReadWriteOnce
+            size: 10Gi
+        resources:
+            requests:
+                cpu: 100m
+                memory: 256Mi
+            limits:
+                cpu: "1"
+                memory: 1Gi
+# 7d log retention (user can tune later).
+limitsConfig:
+    retention_period: 7d
+# Disable subcharts we don't need in monolithic mode.
+test:
+    enabled: false
+# Single binary only — disable distributed components.
+backend:
+    replicas: 0
+read:
+    replicas: 0
+write:
+    replicas: 0
+singleBinary:
+    replicas: 0
+gateway:
+    enabled: false
+minio:
+    enabled: false
+sops:
+    lastmodified: "2026-07-28T13:46:53Z"
+    mac: ENC[AES256_GCM,data:bk1sxO2qIY/FT3+9dTCfpxKBbb9NlyEfTFuUihURUA/AmMJHV4XOx4EaBK6YFbSOVCLny5/aDbUDfF5DMudZTvKMNyU5sWq4X3+6BBZaL3vHSw2+UTA8pdpkNYpLl4uDq5aDFvAN2v/f/XwOpMIEte1L+yi7Rl96hq3Is9TYxjg=,iv:0Ayu7eiEVRAml+SycHWgmmP+rf4fZxhXAALor6d01rI=,tag:+aVf2mehHqak3K5xOHSJCA==,type:str]
+    pgp:
+        - created_at: "2026-07-28T13:46:53Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA7uy4qQr71wiAQ//cYdDJn/PZWg3nx7eNeHjByT8kpYiTgi28Hbx0MmaR5Dd
+            koqKVn47tSO8UkoIYkJ2uoImBJMmIqSLg5XjIhleUmSrZnrCLm0zhdgljMnkphME
+            N9NYHc2gd65inztRDoZi12zxeDdPLcSo1tKPBvIQi33tWODbT6E8TJ4YGKBEv1+X
+            NggSqgADxmYevrVYrzJtHCf7o369CUjHUedSkGPDEk7ZNKimb2kEdi4qKV+QFiX7
+            y4iyqgeEepTjmloXSF6diC1F4cak96MqsNeIpiJLU5L1vWGGvb5J9znijpgqL6eX
+            aompNyrpUrwrF2dbsfb32o/AW4FjdJsC+7RfLPWs9qBneoUtU7LPisZpN+xQ9kgt
+            cx0CVY966y/0WTPm3coIhzO90fzL1kAn7+Bl5W5WhMBFRGDq7K0oqYcgdQmcr45o
+            yegnYxV4uO6Vgf9NV9zEqCvq+UqRZUwJhQSdmaPkvWQDhx91y7lca0WP90xwYMhN
+            jUvhHVcpZ7b++sbxNnvbhmnBBkAlwesQHVc85WSvok4kb3KpM1wju/j0KmOP2/lb
+            YazrzeXu47diSttopRU+JphtUQyuOdTqQ+k5x/1gt7qd8R4IbfKAFMKWPC/KLBnH
+            d4UIwa+ln/lWhhuR+/PxaPZrXGlm5SidM3r2SlH+1iGJfRzq1USaeEn9Vfp6OVvS
+            XgFZngqHN4Wv9zdT71EUGeixMz142maTlcPBM1OnHgiIkf3gLoInizU1VrZz/f8s
+            OOB4rHLwFjqj//Lze896q0kaWtnvZOuKdvTRsE7ep8VqAG2pVKwzcpINqnyTZss=
+            =ww9K
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/infrastructure/observability/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: observability
+  labels:
+    pod-security.kubernetes.io/enforce: privileged

kubernetes/infrastructure/observability/vm-internal-ingress.yaml

@@ -0,0 +1,26 @@
+# Ingress exposing the VictoriaMetrics vminsert endpoint (vmsingle:8428)
+# through Traefik as vm-internal.lan — used by the docker-host vmagent to
+# remote_write metrics into the central VictoriaMetrics.
+# external-dns auto-creates the vm-internal.lan A record pointing at the
+# Traefik LoadBalancer (see kubernetes/system/external-dns).
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: vm-internal
+  namespace: observability
+  annotations:
+    external-dns.alpha.kubernetes.io/enabled: "true"
+    # No chart ingress is available for vmsingle, so declare it here.
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: vm-internal.lan
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: vmsingle-vm-victoria-metrics-k8s-stack
+                port:
+                  number: 8428

kubernetes/infrastructure/observability/vmalert-rules.yaml

@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vmalert-rules
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: vmalert
+    app.kubernetes.io/component: alerting-rules
+data:
+  node.rules: |
+    groups:
+      - name: node
+        rules:
+          - alert: HighNodeCPU
+            expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "High CPU on {{ $labels.instance }}"
+              description: "CPU usage above 90% for 5m on {{ $labels.instance }}."
+          - alert: HighNodeRAM
+            expr: 1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 0.9
+            for: 10m
+            labels:
+              severity: warning
+            annotations:
+              summary: "RAM >90% on {{ $labels.instance }}"
+              description: "Available RAM below 10% for 10m on {{ $labels.instance }}."
+          - alert: LowDiskSpace
+            expr: 1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) > 0.85
+            for: 10m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Disk >85% full on {{ $labels.instance }} {{ $labels.mountpoint }}"
+              description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} is more than 85% full for 10m."

kubernetes/infrastructure/observability/vmstack-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: vm-victoria-metrics-k8s-stack
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: vm
+        namespace: observability
+      chart: victoria-metrics-k8s-stack
+      version: "0.43.2"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: vmstack-values

kubernetes/infrastructure/observability/vmstack-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: vm
+  namespace: observability
+spec:
+  interval: 1m
+  url: https://victoriametrics.github.io/helm-charts/

kubernetes/infrastructure/observability/vmstack-values.yaml

@@ -0,0 +1,151 @@
+# victoria-metrics-k8s-stack Helm values
+# SOPS-encrypted before commit (per .sops.yaml rules).
+# All values for this chart live here; vmagent, grafana, vmalert, alertmanager,
+# kube-state-metrics and node-exporter are sub-charts toggled on/off below.
+# ---------------------------------------------------------------------------
+# VictoriaMetrics single-binary (the metrics database)
+# ---------------------------------------------------------------------------
+vmsingle:
+    enabled: true
+    spec:
+        # Keep 7d of metrics; user can tune later.
+        retentionPeriod: 7d
+        storage:
+            storageClassName: longhorn
+            accessModes:
+                - ReadWriteOnce
+            resources:
+                requests:
+                    storage: 20Gi
+        resources:
+            requests:
+                cpu: 100m
+                memory: 256Mi
+            limits:
+                cpu: "1"
+                memory: 1Gi
+# vmagent scrapes the cluster (node-exporter, kube-state-metrics, cAdvisor).
+# That same vmagent is reused for the inner cluster; the docker host runs its
+# own vmagent that remote_writes through vm-internal.lan.
+vmagent:
+    enabled: true
+    # Tolerate the failover taint so the scrape DaemonSet covers all nodes.
+    tolerations:
+        - key: ENC[AES256_GCM,data:pERZJq2cU/V7,iv:eMeR91aKxAJmKpKLEL6pA9rcIHOrd+Mg+RmA3gt1xGw=,tag:l2igrTTHLclHPEO3bCq/dg==,type:str]
+          value: ENC[AES256_GCM,data:m0pJQf4RLt4=,iv:Au3k95QkD2wv3OP/yDyNQKDN5ZrDIuNAwhXaMCEEn+g=,tag:mEcqTG4e0aYwoHr4TtmgTg==,type:str]
+          operator: Equal
+          effect: NoSchedule
+        - key: ENC[AES256_GCM,data:lDW9tKXBee7p5HjH/FWmzpwtnrazCuohAQm47B5htn++6QUYIA==,iv:yJJdF/XaFXghwBOwC2lByMRM+M5MNP51ho3qTfAKqq0=,tag:WFNE1PMRFss+M/I6j/8Nbg==,type:str]
+          operator: Exists
+          effect: NoSchedule
+# ---------------------------------------------------------------------------
+# kube-state-metrics + node-exporter (metrics sources)
+# ---------------------------------------------------------------------------
+kubeStateMetrics:
+    enabled: true
+nodeExporter:
+    enabled: true
+# ---------------------------------------------------------------------------
+# Grafana (UI) — served behind ingress at grafana.lan.
+# ---------------------------------------------------------------------------
+grafana:
+    enabled: true
+    persistence:
+        enabled: true
+        storageClassName: longhorn
+        accessModes:
+            - ReadWriteOnce
+        size: 5Gi
+    adminUser: admin
+    # SOPS encrypts this when the file is processed.
+    adminPassword: ENC[AES256_GCM,data:VXtozCJ6XZ7JqWuIrDTmZkfWDeglVXl9JZ9T0sY2OUDY9MM=,iv:PO4leV1zpTFZTB2/hWipitGec4zrHfW/gOkJBrGk24k=,tag:9GinyiqoA4qrbyeKJ33Iuw==,type:str]
+    # Provision both the chart's default VictoriaMetrics datasource AND a Loki
+    # datasource so metric/log correlation works in one UI.
+    additionalDataSources:
+        - name: Loki
+          type: loki
+          url: http://loki.observability.svc.cluster.local:3100
+          access: proxy
+          isDefault: false
+    ingress:
+        enabled: true
+        ingressClassName: traefik
+        annotations:
+            external-dns.alpha.kubernetes.io/enabled: "true"
+        hosts:
+            - grafana.lan
+        path: /
+        pathType: Prefix
+        tls: []
+# ---------------------------------------------------------------------------
+# vmalert — evaluates the node.rules ConfigMap (see vmalert-rules.yaml) against
+# VictoriaMetrics, forwards firing alerts to Alertmanager below.
+# ---------------------------------------------------------------------------
+vmalert:
+    enabled: true
+    spec:
+        rule:
+            configMap: vmalert-rules
+            configMapKeyRef:
+                key: ENC[AES256_GCM,data:7obbOvuMe8Cpdg==,iv:oIzOCzFFA0cptWQMPiIOi5b76dR3zp96Y8JrdKOkE8A=,tag:ArteBgOhdz/7N4wNpWE83g==,type:str]
+                name: vmalert-rules
+        notifier:
+            alertmanagerUrl: http://vm-victoria-metrics-k8s-stack-alertmanager.observability.svc.cluster.local:9093
+        resources:
+            requests:
+                cpu: 50m
+                memory: 64Mi
+            limits:
+                cpu: 200m
+                memory: 128Mi
+# ---------------------------------------------------------------------------
+# Alertmanager — 1 replica; reads its config from the alertmanager-config
+# Secret (see alertmanager-config.yaml), which has a single n8n webhook receiver.
+# ---------------------------------------------------------------------------
+alertmanager:
+    enabled: true
+    replicaCount: 1
+    storage:
+        storageClassName: longhorn
+        accessModes:
+            - ReadWriteOnce
+        size: 2Gi
+    configFromSecretRef: ENC[AES256_GCM,data:C6NDZCkX76QLaZLw/vY3B01+AA==,iv:RSpe3Zv1mPfo0nj73tWrf5IBd+Sfqo1NUrERvmVPBgs=,tag:4ZGHA2Kyj9zpGn6eylNbXw==,type:str]
+    configKey: alertmanager.yaml
+    resources:
+        requests:
+            cpu: 50m
+            memory: 64Mi
+        limits:
+            cpu: 200m
+            memory: 128Mi
+# ---------------------------------------------------------------------------
+# Storage class default
+# ---------------------------------------------------------------------------
+defaultStorageClass: longhorn
+sops:
+    lastmodified: "2026-07-28T13:46:53Z"
+    mac: ENC[AES256_GCM,data:ZyCYO9b3OMJo8Br9tzbK+bN3rZvlkk/scOEUSHwkAABeQxL7JDbcpjHL8WauSY57wLnl3sK9JQnUHS+/ghOiZu/QDuodbhpoRrE8jpdjo1+qZaqJBqjbG1NIYU/aLQnHr4bnIzh8oX0PYnsu2uSqkg1c0c1KCJmrvHtXlFwtWOs=,iv:zh8yVzExwT29JF+4jLcZgmuSPM6R/d/2IWTcYfK3Qn4=,tag:LYTYY6rJVEKJRsOp6G/T0g==,type:str]
+    pgp:
+        - created_at: "2026-07-28T13:46:53Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA7uy4qQr71wiAQ/+KP62YdvAUJjxvFoWH1skMELYx9Lj/5TRXnrHOtqQrpXC
+            UzKNgS3XFmgeUcpA3wXvhTEPhICtdC9KDvxkF0jPTATqM2WFPcF7V2vjhTjuAMtU
+            /g4RR44j6U2WRZJFjs0tEpVGVrw+HXiaACdgqy1P6OmnmgvTSBvZvj3AsBpHOHeM
+            teaE11couXeYnrZ9NJeCvhbcvJfYAkdpqyCLJ9UWEro3QFm4z6tvs6D4rq+nAhBb
+            MntlvArijXunkwbHLZVPh8SyfxgGooy9PBSEX+ymyuxPYmjTlarYgO7oBQJUWjRk
+            BqkKns9KKNPyaUP/37tXJA8Gvgog/Z9JKDT28lq5noe6CAeHbBbUUo4hZZA14zcD
+            /Lmkc03KdOKf/eLnEFGtoWjQJ38FD5+uAh5Zkixm1WW0wV/PdqevJrG8mRjn8FuP
+            6GU+nWo+77BxDyKvA1O13iOtTnGZtg/Z+lXVUbpcM8F7QCGrabhLiWVe51iWjYfU
+            /Ub7HZHRDXmiz7Q6Kem5Wleoy3ME/lxFblx9v0LbO3Dc0lj/85EVNm9os8tt5PD0
+            cvGE/cGsZfUfgwbjEVpPnRN0SzlQKfEr9KVQP4DGNAqDOzEl2AD4Dg3l29oAgSda
+            ZRXdYbEN/ZIAGUg1wNVfZJf2KoJA+wm8c4uk7KAAI0+bIeaongovU4vnylVCHIbS
+            XgGDxEIDZGOueHms+xEMIW6QmdazFdkaLumY0XrEInHMwqhKM0NOyZL2zd/liZtE
+            wb8xm/V+eOWhjnE2RynNBCwMI6VeVYUPkin2MKQcq+nr7SVpWQsm2+Jbj5YtWCk=
+            =yKiR
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/kustomization.yaml

@@ -24,6 +24,7 @@ resources:
   # Infrastructure
   - infrastructure/authentik
   - infrastructure/mail/stalwart
+  - infrastructure/observability
   # Personal
   - personal/linkwarden
   - personal/notesnook