obs,loki: remove redundant values

.github/dependabot.yml

@@ -13,6 +13,7 @@ updates:
       - "/docker/home/zigbee2mqtt"
       - "/docker/infrastructure/mail/roundcube"
       - "/docker/infrastructure/network/traefik"
+      - "/docker/infrastructure/observability"
       - "/docker/infrastructure/squid"
       - "/docker/infrastructure/sshportal"
       - "/docker/personal/fireshare"

.swarmcd/stacks.yaml

@@ -38,6 +38,11 @@ traefik:
   branch: main
   compose_file: docker/infrastructure/network/traefik/docker-compose.yml
 
+observability:
+  repo: homeprod
+  branch: main
+  compose_file: docker/infrastructure/observability/docker-compose.yml
+
 squid:
   repo: homeprod
   branch: main

docker/infrastructure/observability/alloy.river

@@ -0,0 +1,71 @@
+// Grafana Alloy config for the docker host (replaces Promtail, EOL since
+// 2026-03-02). Tails Docker container logs via docker_sd discovery, attaches
+// container_name/compose_service/stream labels, and pushes them to the
+// in-cluster Loki at loki-internal.lan (LAN-only Traefik Ingress on port 80).
+// River config produced by `alloy convert -f promtail` from an equivalent
+// Promtail YAML (so component names and argument shapes are guaranteed
+// correct for Alloy v1.18).
+
+discovery.docker "docker" {
+  host             = "unix:///var/run/docker.sock"
+  refresh_interval = "5s"
+}
+
+loki.process "docker" {
+  forward_to = [loki.write.default.receiver]
+
+  // Parse docker json-file log lines
+  // ({"log":"...","stream":"stdout","time":"..."}).
+  stage.json {
+    expressions = {
+      log    = "log",
+      stream = "stream",
+    }
+  }
+  stage.labels {
+    values = {
+      stream = null,
+    }
+  }
+}
+
+discovery.relabel "docker" {
+  targets = []
+
+  // Strip leading / from container name.
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container_name"
+  }
+  // Compose service name (if label is set).
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "compose_service"
+  }
+  // Log stream (stdout/stderr).
+  rule {
+    source_labels = ["__meta_docker_container_log_stream"]
+    target_label  = "stream"
+  }
+  // Mark the source so dashboards/alerts can distinguish the docker host.
+  rule {
+    target_label = "source"
+    replacement  = "docker-host"
+  }
+}
+
+loki.source.docker "docker" {
+  host             = "unix:///var/run/docker.sock"
+  targets          = discovery.docker.docker.targets
+  forward_to       = [loki.process.docker.receiver]
+  relabel_rules    = discovery.relabel.docker.rules
+  refresh_interval = "5s"
+}
+
+loki.write "default" {
+  endpoint {
+    url = "http://loki-internal.lan/loki/api/v1/push"
+  }
+  external_labels = {}
+}

docker/infrastructure/observability/docker-compose.yml

@@ -0,0 +1,81 @@
+# Docker-host collectors for the central observability stack on Kubernetes.
+# Managed by swarm-cd (see .swarmcd/stacks.yaml).
+#
+# node-exporter + cAdvisor run locally (scraped by vmagent below).
+# vmagent remote_writes metrics to vm-internal.lan (LAN-only Traefik Ingress) ->
+#   vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428
+# alloy tails /var/lib/docker/containers + /var/log and pushes logs to
+#   loki-internal.lan -> loki.observability.svc.cluster.local:3100
+
+services:
+  node-exporter:
+    image: prom/node-exporter:v1.9.1
+    container_name: obs-node-exporter
+    network_mode: host
+    pid: host
+    command:
+      - "--path.rootfs=/host"
+    volumes:
+      - /:/host:ro,rslave
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.49.1
+    container_name: obs-cadvisor
+    ports:
+      - "8080"
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /dev/disk/:/dev/disk:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  vmagent:
+    image: victoriametrics/vmagent:v1.131.0
+    container_name: obs-vmagent
+    # vmagent listens on 8429 (its own metrics), scrapes node-exporter:9100
+    # and cadvisor:8080, remote_writes to vm-internal.lan
+    command:
+      - "-promscrape.config=/etc/vmagent/vmagent.yml"
+      - "-remoteWrite.url=http://vm-internal.lan/api/v1/write"
+    volumes:
+      - type: bind
+        source: /root/homeprod/docker/infrastructure/observability/vmagent.yml
+        target: /etc/vmagent/vmagent.yml
+        read_only: true
+    depends_on:
+      - node-exporter
+      - cadvisor
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  alloy:
+    image: grafana/alloy:v1.18.0
+    container_name: obs-alloy
+    command:
+      - "run"
+      - "/etc/alloy/config.river"
+      - "--server.http.listen-addr=0.0.0.0:12345"
+    volumes:
+      - type: bind
+        source: /root/homeprod/docker/infrastructure/observability/alloy.river
+        target: /etc/alloy/config.river
+        read_only: true
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/log:/var/log:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+networks:
+  default:
+    name: proxy
+    external: true

docker/infrastructure/observability/vmagent.yml

@@ -0,0 +1,22 @@
+# vmagent scrape config for the docker host.
+# Scrapes the two local collectors, forwards everything to the central
+# VictoriaMetrics via the LAN-only Caddy route vm-internal.lan (Caddy proxies
+# it to vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428).
+
+scrape_configs:
+  - job_name: docker-host-node
+    static_configs:
+      - targets:
+          - localhost:9100
+        labels:
+          # Distinguishes the docker host from K8s nodes in dashboards/alerts.
+          instance_source: docker-host
+          host: docker-r740
+
+  - job_name: docker-host-cadvisor
+    static_configs:
+      - targets:
+          - localhost:8080
+        labels:
+          instance_source: docker-host
+          host: docker-r740

infra/rpi4/kube/main.tf

@@ -0,0 +1,214 @@
+# Talos control-plane node for the Raspberry Pi 4 — joins the r740 "kube" cluster
+# as a third etcd member to restore quorum (2-of-3 majority). Unlike the p330
+# failover node, this node is tainted "quorum" so no user workloads are ever
+# scheduled on it; only essential DaemonSets (Cilium, etc.) that tolerate the
+# taint land here for cluster networking.
+#
+# Secret handling: the cluster machine secrets are provided via
+# var.machine_secrets_file (a local, gitignored JSON file in the provider's
+# machine_secrets format). They are consumed by EPHEMERAL resources and
+# WRITE-ONLY attributes so they never land in Terraform state. See
+# variables.tf and scripts/extract-talos-secrets.sh for how to produce the
+# file from the live r740 node.
+terraform {
+  required_providers {
+    talos = {
+      source  = "siderolabs/talos"
+      version = "0.11.0"
+    }
+    null = {
+      source  = "hashicorp/null"
+      version = "3.2.3"
+    }
+  }
+}
+
+locals {
+  # Load the machine secrets from the gitignored JSON file. This local is only
+  # ever referenced by ephemeral resources / write-only attributes, so the
+  # values are never persisted to state.
+  machine_secrets = jsondecode(file(var.machine_secrets_file))
+
+  # Network config: static if node_subnet is provided, otherwise Talos DHCPs.
+  # The rpi4 uses DHCP (node_subnet = null), so only nameservers are patched in.
+  static_network = var.node_subnet == null ? {} : {
+    interfaces = [{
+      interface = var.network_interface
+      addresses = [var.node_subnet]
+      routes    = var.node_gateway == null ? [] : [{ gateway = var.node_gateway }]
+    }]
+  }
+
+  network_patch = {
+    nameservers = var.nameservers
+  }
+  network_patch_merged = merge(local.network_patch, local.static_network)
+
+  machine_patch = {
+    install = {
+      image = var.installer_image
+      disk  = var.install_disk
+    }
+    network = local.network_patch_merged
+    # NOTE: no Longhorn iSCSI/ext4 kernel modules here. This is a quorum-only
+    # node: the quorum taint keeps user workloads (and Longhorn replicas) off
+    # it, so the storage stack is not needed. Essential DaemonSets such as
+    # Cilium still run here for cluster networking and tolerate the taint.
+    sysctls = {
+      "fs.inotify.max_user_instances" = "1024"
+      "fs.inotify.max_user_watches"   = "1048576"
+    }
+    kubelet = {
+      # Register the node already tainted so the scheduler never admits user
+      # workloads even before the null_resource below runs. NoSchedule is
+      # sufficient: essential DaemonSets (Cilium, etc.) tolerate it, but no
+      # user pods are admitted.
+      extraArgs = {
+        "register-with-taints" = "${var.quorum_taint_key}=${var.quorum_taint_value}:${var.quorum_taint_effect}"
+      }
+    }
+  }
+}
+
+# --- Ephemeral resources: secrets never stored in state ---------------------
+#
+# talos_machine_configuration generates the control-plane join config from the
+# provided machine_secrets. The output (machine_configuration) is an ephemeral
+# value — it can only flow into write-only attributes or provisioners, never
+# into a persisted resource attribute.
+ephemeral "talos_machine_configuration" "rpi4" {
+  cluster_name     = var.cluster_name
+  machine_type     = "controlplane"
+  cluster_endpoint = var.cluster_endpoint
+  machine_secrets  = local.machine_secrets
+  config_patches = [
+    yamlencode({
+      machine = local.machine_patch
+    }),
+    # Pin the Kubernetes node name via a HostnameConfig document (Talos v1.13+).
+    # The old machine.network.hostname field conflicts with the default
+    # HostnameConfig document ("static hostname is already set"), so we use the
+    # document-based config with auto: off + an explicit hostname instead.
+    yamlencode({
+      apiVersion = "v1alpha1"
+      kind       = "HostnameConfig"
+      hostname   = var.rpi4_node_name
+      auto       = "off"
+    })
+  ]
+}
+
+# talos_client_configuration generates a Talos client config (talosconfig) from
+# the machine_secrets, scoped to the rpi4 node. Also ephemeral — used only to
+# drive the write-only client_configuration_wo on the apply resource.
+ephemeral "talos_client_configuration" "rpi4" {
+  cluster_name    = var.cluster_name
+  machine_secrets = local.machine_secrets
+  nodes           = [var.rpi4_host]
+}
+
+# --- Apply the config to the node (write-only attrs → no secrets in state) --
+#
+# machine_configuration_input_wo and client_configuration_wo are write-only:
+# Terraform uses them during apply but does NOT persist them to state. Only a
+# hash of the machine config (machine_configuration_hash) is stored, for drift
+# detection. Because the config patch contains a `machine.install` block, when
+# Talos receives this config on a node booted from the SD card (maintenance)
+# image it installs itself to install.disk and reboots into the installed
+# system. As a controlplane node it then joins the existing etcd cluster as a
+# new member and runs the control-plane components. With r740 + p330 + rpi4 the
+# etcd cluster reaches 3 members → 2-of-3 quorum.
+resource "talos_machine_configuration_apply" "rpi4" {
+  node                           = var.rpi4_host
+  client_configuration_wo        = ephemeral.talos_client_configuration.rpi4.client_configuration
+  machine_configuration_input_wo = ephemeral.talos_machine_configuration.rpi4.machine_configuration
+}
+
+# --- Write the rendered config to disk for manual use ----------------------
+#
+# local_file.content cannot accept an ephemeral value (it would persist to
+# state), so we use a null_resource local-exec provisioner instead —
+# provisioners do not persist their arguments to state. This writes rpi4.yaml
+# so the config can also be applied manually with
+# `talosctl apply-config --nodes <rpi4_host> --file rpi4.yaml` if needed.
+resource "null_resource" "rpi4_machine_config_file" {
+  triggers = {
+    # Re-run only when the (non-secret) inputs that shape the config change.
+    node            = var.rpi4_node_name
+    install_disk    = var.install_disk
+    installer_image = var.installer_image
+    taint           = "${var.quorum_taint_key}=${var.quorum_taint_value}:${var.quorum_taint_effect}"
+  }
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      set -euo pipefail
+      cat > "${path.module}/rpi4.yaml" <<'YAMLEOF'
+${ephemeral.talos_machine_configuration.rpi4.machine_configuration}
+YAMLEOF
+      echo "Wrote ${path.module}/rpi4.yaml"
+    EOT
+  }
+
+  depends_on = [talos_machine_configuration_apply.rpi4]
+}
+
+# --- Wait for the node, then label + taint ---------------------------------
+#
+# Wait for the node to register with Kubernetes (kubelet creates the Node
+# object after Talos installs and reboots), then label it and (re)apply the
+# quorum taint. This is idempotent: kubectl exits 0 if the label/taint already
+# exists. The taint is also set via kubelet `register-with-taints`, so this
+# null_resource is a safety net for manual edits / drift. The kubeconfig path
+# is only used inside the provisioner (not persisted to state).
+resource "null_resource" "rpi4_node_label_and_taint" {
+  triggers = {
+    node       = var.rpi4_node_name
+    key        = var.quorum_taint_key
+    value      = var.quorum_taint_value
+    effect     = var.quorum_taint_effect
+    kubeconfig = var.kubeconfig_path
+  }
+
+  provisioner "local-exec" {
+    # Wait for the node to show up, then label + taint. The wait loop is bounded
+    # by kubectl --timeout; tune it via TF_LOG / re-run if the node is slow to
+    # join (a controlplane node must first complete the etcd join handshake).
+    command = <<-EOT
+      set -euo pipefail
+      KUBECONFIG="${var.kubeconfig_path}"
+      export KUBECONFIG
+      NODE="${var.rpi4_node_name}"
+
+      echo "Waiting for node $NODE to be registered (kubelet creates the Node object once Talos has installed, rebooted and joined etcd)..."
+      # kubectl wait --for=condition=Ready fails instantly with NotFound if the
+      # node object doesn't exist yet, so poll for existence first.
+      # /bin/sh (dash) has no $SECONDS, so count iterations with a bounded loop.
+      tries=240   # 240 * 5s = 20 minutes max
+      until kubectl get node "$NODE" >/dev/null 2>&1; do
+        tries=$((tries - 1))
+        if [ "$tries" -le 0 ]; then
+          echo "Timed out waiting for node $NODE to register." >&2
+          exit 1
+        fi
+        sleep 5
+      done
+      echo "Node $NODE registered. Waiting for it to become Ready..."
+
+      # Now wait for Ready (a controlplane node needs etcd joined + apiserver up).
+      kubectl wait --for=condition=Ready "node/$NODE" --timeout=20m || \
+        kubectl wait --for=jsonpath='{.status.conditions[?(@.reason=="KubeletReady")].status}'=True "node/$NODE" --timeout=20m
+
+      # Quorum marker + taint (applied to the controlplane node).
+      kubectl label --overwrite node "$NODE" homeprod.io/quorum=true
+
+      # Apply the taint idempotently (kubectl taint --overwrite is a no-op if it exists).
+      kubectl taint --overwrite node "$NODE" \
+        "${var.quorum_taint_key}=${var.quorum_taint_value}:${var.quorum_taint_effect}"
+
+      echo "Node $NODE ready, labeled and tainted for quorum-only scheduling."
+    EOT
+  }
+
+  depends_on = [talos_machine_configuration_apply.rpi4]
+}

infra/rpi4/kube/variables.tf

@@ -0,0 +1,146 @@
+# Variables for the Raspberry Pi 4 Talos control-plane node that joins the r740
+# "kube" cluster as a third etcd member to restore quorum (2-of-3 majority).
+#
+# Secret handling: the cluster machine secrets (cluster id/secret, etcd/k8s
+# certs, bootstrap token) are NOT read from terraform state (the r740 state is
+# stale) and are NOT generated here (that would create a new, incompatible
+# cluster). Instead they are provided via `machine_secrets_file` — a local,
+# gitignored JSON file in the Talos provider's machine_secrets format. The
+# file is produced once from the live r740 node (see
+# scripts/extract-talos-secrets.sh) and stored in a real secret manager
+# (Bitwarden); you paste it back to disk when running this module. Ephemeral
+# resources + write-only attributes ensure the secrets never land in Terraform
+# state.
+
+variable "rpi4_host" {
+  description = "Reachable IP/hostname of the rpi4 Talos node (for Talos API access). With DHCP this is the leased IP (e.g. 10.1.2.135)."
+  type        = string
+}
+
+variable "rpi4_node_name" {
+  description = "Kubernetes/Talos node name for the rpi4 (e.g. rpi4). Pinned via machine.network.hostname so the node registers with this name regardless of DHCP."
+  type        = string
+  default     = "rpi4"
+}
+
+# --- Cluster identity (no terraform_remote_state — state is stale) ----------
+
+variable "cluster_name" {
+  description = "Name of the existing Talos cluster the rpi4 joins. Must match the cluster the r740 bootstrapped (kube-r740)."
+  type        = string
+  default     = "kube-r740"
+}
+
+variable "cluster_endpoint" {
+  description = "Endpoint (host:port) of the Talos/Kubernetes API on the cluster. Must match the r740 bootstrap endpoint."
+  type        = string
+  default     = "https://kube-r740.lan:6443"
+}
+
+# --- Secrets (provided manually, never in state) ---------------------------
+
+variable "machine_secrets_file" {
+  description = <<EOT
+Path to a local, gitignored JSON file containing the cluster machine secrets in
+the Talos provider's machine_secrets format (cluster.id, cluster.secret, certs,
+secrets.bootstrap_token, secrets.secretbox_encryption_secret, trustdinfo.token).
+Generate it once from the live r740 node with
+scripts/extract-talos-secrets.sh, store the contents in Bitwarden, and paste it
+back to this file when running this module. The file MUST be gitignored — it
+contains the cluster root of trust.
+EOT
+  type        = string
+  default     = "secrets.json"
+}
+
+# --- Install / network -----------------------------------------------------
+
+variable "installer_image" {
+  description = <<EOT
+Talos installer image to use on the rpi4 (bare metal, ARM64).
+Must be an ARM64 Image Factory build (schematic generated at
+https://factory.talos.dev) for the Raspberry Pi 4 platform. Unlike the x86
+control-plane nodes, this quorum node does NOT need the iSCSI/Longhorn
+extensions because no user workloads or Longhorn replicas are scheduled on it
+(the quorum taint keeps it empty); only essential DaemonSets (Cilium, etc.)
+land here.
+EOT
+  type        = string
+  default     = "factory.talos.dev/installer/ee21ef4a5ef808a9b7484cc0dda0f25075021691c8c09a276591eedb638ea1f9:v1.13.6"
+}
+
+variable "install_disk" {
+  description = "Block device path to install Talos on. For the rpi4 booting from the SD card this is /dev/mmcblk0."
+  type        = string
+  default     = "/dev/mmcblk0"
+}
+
+variable "node_subnet" {
+  description = <<EOT
+Static IPv4 address in CIDR notation for the rpi4 node (e.g. 10.1.2.135/24).
+Set to null (default) to use DHCP. The rpi4 uses DHCP, so a static address is
+not required; the node registers with Kubernetes under rpi4_node_name regardless
+of the leased IP.
+EOT
+  type        = string
+  default     = null
+}
+
+variable "node_gateway" {
+  description = "IPv4 gateway for the rpi4 node. Ignored when node_subnet is null (DHCP)."
+  type        = string
+  default     = null
+}
+
+variable "network_interface" {
+  description = <<EOT
+Primary network interface name on the rpi4. The built-in Ethernet port is eth0.
+EOT
+  type        = string
+  default     = "eth0"
+}
+
+variable "nameservers" {
+  description = "DNS nameservers configured on the node (must work independently of kube)."
+  type        = list(string)
+  default     = ["10.1.2.148", "1.1.1.1"]
+}
+
+# --- Quorum taint ----------------------------------------------------------
+
+variable "quorum_taint_key" {
+  description = "Taint key applied to the node to reserve it as a quorum-only member (no user workloads)."
+  type        = string
+  default     = "dedicated"
+}
+
+variable "quorum_taint_value" {
+  description = "Taint value applied to the node."
+  type        = string
+  default     = "quorum"
+}
+
+variable "quorum_taint_effect" {
+  description = "Taint effect applied to the node. NoSchedule is sufficient: essential DaemonSets (Cilium, etc.) tolerate it for networking, but no user workloads are admitted."
+  type        = string
+  default     = "NoSchedule"
+
+  validation {
+    condition     = contains(["NoSchedule", "PreferNoSchedule", "NoExecute"], var.quorum_taint_effect)
+    error_message = "quorum_taint_effect must be NoSchedule, PreferNoSchedule or NoExecute."
+  }
+}
+
+# --- Kubeconfig for the label/taint null_resource --------------------------
+
+variable "kubeconfig_path" {
+  description = <<EOT
+Path to a kubeconfig for the cluster, used by the null_resource that waits for
+the node and applies the quorum label/taint. This is NOT stored in state — it is
+only referenced inside a local-exec provisioner. Point it at the r740 kube
+module's kubeconfig (../../r740/kube/kubeconfig) or any valid kubeconfig for the
+cluster.
+EOT
+  type        = string
+  default     = "../../r740/kube/kubeconfig"
+}

kubernetes/infrastructure/observability/alloy-config.yaml

@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alloy-config
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: alloy
+    app.kubernetes.io/component: log-collector
+data:
+  config.river: |
+    // -----------------------------------------------------------------------------
+    // Grafana Alloy — log collection only (metrics cluster+host collection is
+    // handled by vmagent/node-exporter/cAdvisor elsewhere).
+    // Forwards pod logs to the in-cluster Loki single-binary.
+    // River config produced by `alloy convert -f promtail` from an equivalent
+    // Promtail YAML (so component names and argument shapes are guaranteed
+    // correct for Alloy v1.18).
+    // -----------------------------------------------------------------------------
+
+    discovery.kubernetes "kubernetes_pods" {
+      role = "pod"
+
+      selectors {
+        role  = "pod"
+        field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)
+      }
+    }
+
+    loki.process "kubernetes_pods" {
+      forward_to = [loki.write.default.receiver]
+
+      // CRI-style log lines on the host: {"log":"...","stream":"stdout","time":"..."}
+      stage.cri { }
+    }
+
+    discovery.relabel "kubernetes_pods" {
+      targets = discovery.kubernetes.kubernetes_pods.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label  = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label  = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_container_name"]
+        target_label  = "container"
+      }
+    }
+
+    loki.source.file "kubernetes_pods" {
+      targets    = discovery.relabel.kubernetes_pods.output
+      forward_to = [loki.process.kubernetes_pods.receiver]
+
+      file_match {
+        enabled = true
+      }
+      legacy_positions_file = "/tmp/positions.yaml"
+    }
+
+    loki.write "default" {
+      endpoint {
+        url = "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push"
+      }
+      external_labels = {}
+    }

kubernetes/infrastructure/observability/alloy-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: alloy
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: grafana
+        namespace: observability
+      chart: alloy
+      version: "1.11.0"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: alloy-values

kubernetes/infrastructure/observability/alloy-values.yaml

@@ -0,0 +1,57 @@
+# Grafana Alloy
+# Runs as a DaemonSet on every node (tolerates the failover taint).
+# Tails pod logs from /var/log/pods, attaches K8s labels (pod, namespace,
+# service, container, image), and pushes them to Loki at
+# http://loki.observability.svc.cluster.local:3100 (in-cluster, no auth).
+# Use the externally provisioned ConfigMap (alloy-config) carrying config.river.
+alloy:
+    configMap:
+        create: false
+        name: alloy-config
+        key: ENC[AES256_GCM,data:aevkZsfe4c4H34BJ,iv:4vrNZWwYWaxJUKGbNlnZfk0NM182nBQZuwvqUPslKNA=,tag:lh1iILM3uQAt0An6kN6EHg==,type:str]
+    # Mount host log paths so the DaemonSet can tail pod/container logs.
+    mounts:
+        varlog: true
+        dockercontainers: false
+    stabilityLevel: generally-available
+    tolerations:
+        - key: ENC[AES256_GCM,data:zOMWiA9dG+8o,iv:yHp9aBczV8nVRGlJmsBaKs9WdmE3+ii8gWOaj5Xok6M=,tag:8c5ZqBT+vvC0WCW8mi0c9g==,type:str]
+          value: ENC[AES256_GCM,data:2Q04VR9xLbQ=,iv:6+tQXx3IGUIcrse+poG4X/4pS5dN2dKvhb79sCy+nfA=,tag:jb/TZmGpvdsWnKoC5NdCow==,type:str]
+          operator: Equal
+          effect: NoSchedule
+        - key: ENC[AES256_GCM,data:aVq8GU1ZDg5m9XDEI3PoVbbykU36SnKpEgrohvZ0IBmH47qDKg==,iv:eMcHRO5mGp9hhY4KnuBmdPTjd/9lqtesjA/RRSu4QMw=,tag:2jaGuogkLcIuye2meJQMDQ==,type:str]
+          operator: Exists
+          effect: NoSchedule
+    resources:
+        requests:
+            cpu: 50m
+            memory: 64Mi
+        limits:
+            cpu: 200m
+            memory: 256Mi
+sops:
+    lastmodified: "2026-07-28T13:46:53Z"
+    mac: ENC[AES256_GCM,data:u6J/EIP5aFtOi8ZtT8BPn978wJAEL1E3r0WcmG5WIsWKJBZMMQHUYRmHgkS6n0/UMtczWoCdXlxObS73ZQMSSkdtagQHE/HPh69dDdGkjxyM4mlP+YMwcnIQkxpJBAQTERXA+SVoWCZb7ztKbiJS+zPjOCfA95ixQINmBCdUgEc=,iv:WOXngWu3uNWYp18/xWghGJyONBrq3SSnPDwbjHBDozU=,tag:1fasTfuLBGXBCo9JPytsJw==,type:str]
+    pgp:
+        - created_at: "2026-07-28T13:46:53Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA7uy4qQr71wiAQ/8DSD0jEn3YE7XHTm4DWdfLHPs1IVg9hyiWSHe5/xH0siD
+            zWonfjcuytw8EHDzv6pQMNYmIpyAmJ92Al4ROJmBV8gZ59+U9goHmw5HgaQAupBJ
+            /gy8iFBteGOHJT+XJPoVb/bu449f7Vle4L3DrOAl6mwxE0WT/7ERoQBeSxL+2OTZ
+            iDPdq24Zy+crDHpoM0GcSnFRLX5UpmL2QXGB8PHyctwSQoDW+sbAY2ME6buOw8HI
+            AremKTRtFm6Pu1Ksdh6ZicaHpLSvLyByPPt2zYhlBQsG3s/SjJDlnakUZHhm7T4o
+            oHzSpKqV7NQbE09ZMyGJrEkl+rfCPeu6jHplW2EX3xjqLWNUF8GHcsDWljogZxXF
+            irBPkyKASEDdkHZgiZobU6BmvQlxILj104xNtUYTTChcKu3s0d5BrkIOGe+Y7rJd
+            fNvMfkmO8/URBgtsXy2xPKo0Jzu/0pZsPEp2lxNeyElwxQqo1CFwhKZPkeQ2eLpu
+            4X7/CMN8KJr6jLX8HhX9AivlCj+2CoT8SB5GAPw4p00eD0GIMWSNyOXjyW/qQZi3
+            82EyX9KIpHjALoOI4/3YgFpUO12AEsjirYKWkUM+IoDp5l+YX59QdM7zUVVyzsTG
+            2EXEdQqYVA7DIcMJb0tHqxDGMGxNayaUsikV7e/kcRVUyDSH+xOfaSUrDTqqv+XS
+            XAFro6848HF5uMIP4pfM+dxzkc9rFF4il0CuGWlscbz5q+Aw9LImGbpNzrWk3Lpi
+            beuwWKFTZ8zab5TH9UF/xqsQeg/nyzQnb3fFPk+owvY7fH6XiYbN5WjKjXGW
+            =kBpp
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/infrastructure/observability/grafana-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: observability
+spec:
+  interval: 1m
+  url: https://grafana.github.io/helm-charts

kubernetes/infrastructure/observability/kustomization.yaml

@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: observability
+resources:
+  - namespace.yaml
+  - vmstack-repository.yaml
+  - grafana-repository.yaml
+  - loki-repository.yaml
+  - vmstack-release.yaml
+  - loki-release.yaml
+  - alloy-release.yaml
+  - alloy-config.yaml
+  - vm-internal-ingress.yaml
+  - loki-internal-ingress.yaml
+secretGenerator:
+  - name: vmstack-values
+    files:
+      - values.yaml=vmstack-values.yaml
+  - name: loki-values
+    files:
+      - values.yaml=loki-values.yaml
+  - name: alloy-values
+    files:
+      - values.yaml=alloy-values.yaml
+configurations:
+  - kustomizeconfig.yaml

kubernetes/infrastructure/observability/kustomizeconfig.yaml

@@ -0,0 +1,6 @@
+nameReference:
+  - kind: Secret
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease

kubernetes/infrastructure/observability/loki-internal-ingress.yaml

@@ -0,0 +1,23 @@
+# Ingress exposing the Loki push API (loki:3100) through Traefik as
+# loki-internal.lan — used by the docker-host promtail to push container logs
+# into the central Loki.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: loki-internal
+  namespace: observability
+  annotations:
+    external-dns.alpha.kubernetes.io/enabled: "true"
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: loki-internal.lan
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: loki
+                port:
+                  number: 3100

kubernetes/infrastructure/observability/loki-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: loki
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: grafana-community
+        namespace: observability
+      chart: loki
+      version: "18.5.4"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: loki-values

kubernetes/infrastructure/observability/loki-repository.yaml

@@ -0,0 +1,9 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana-community
+  namespace: observability
+spec:
+  interval: 1m
+  type: oci
+  url: oci://ghcr.io/grafana-community/helm-charts

kubernetes/infrastructure/observability/loki-values.yaml

@@ -0,0 +1,83 @@
+# Loki Helm values
+# Uses Monolithic deployment mode with filesystem storage on a Longhorn PVC.
+loki:
+    deploymentMode: Monolithic
+    # Filesystem storage — chunks + rules on the PVC.
+    storage:
+        type: filesystem
+        filesystem:
+            chunks_directory: /var/loki/chunks
+            rules_directory: /var/loki/rules
+    # Required schema config. tsdb store + filesystem object store, schema v13.
+    # See https://grafana.com/docs/loki/latest/operations/storage/schema/.
+    schemaConfig:
+        configs:
+            - from: "2024-04-01"
+              store: tsdb
+              object_store: filesystem
+              schema: v13
+              index:
+                prefix: index_
+                period: 24h
+    # For monolithic mode, point tsdb_shipper at the local store (no index gateway).
+    storage_config:
+        tsdb_shipper:
+            active_index_directory: /var/loki/index
+            cache_location: /var/loki/index-cache
+    monolithic:
+        persistence:
+            storageClassName: longhorn
+            accessModes:
+                - ReadWriteOnce
+            size: 10Gi
+        resources:
+            requests:
+                cpu: 100m
+                memory: 256Mi
+            limits:
+                cpu: "1"
+                memory: 1Gi
+# 7d log retention (user can tune later).
+limitsConfig:
+    retention_period: 7d
+# Disable distributed components + gateway + minio (monolithic only).
+test:
+    enabled: false
+backend:
+    replicas: 0
+read:
+    replicas: 0
+write:
+    replicas: 0
+singleBinary:
+    replicas: 0
+gateway:
+    enabled: false
+minio:
+    enabled: false
+sops:
+    lastmodified: "2026-08-01T20:19:42Z"
+    mac: ENC[AES256_GCM,data:KYZQGIimeeOlt2lltCrKttLXzk2b7DOT8KNOhpeVqF/VNNcVxP7Srz6nCR5J7dI6wFSePspof5gNJEZlzP3EGJFQaGBf34hOGhHBUtlcscX8vj2JYmmJ85B+pylHjQNPHaXqLP4tAwIFfhcJi368K+GEAu8tztkxJKk7dtOOyrg=,iv:V24R3eSzW0edwPnPSSwEQqqSfiNrK6+ZyQ368xpJKhg=,tag:LmaMhhGwRIIG+SxTUJf5gQ==,type:str]
+    pgp:
+        - created_at: "2026-08-01T20:19:42Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA7uy4qQr71wiAQ/+PDeQtUHMysWOkTjqU8vNAl1IQzDrwHskeBmLXLI99kdc
+            hU0afTjzUbomFdMRGvf0QZrcFbnD5OHskw5KRleDbYxOp4aXjzX0RsqSe9elTFAj
+            ml4XG0UrEV4Ygi4mTNOGzmINaIjobcn0tQd6nZFPUUA//udBcHsczs9x9Rrf+YTv
+            XrUdNAhveRlowvbvfNrDIh1OsOgyWyHJEJsKiLbt2b7e47PKhhRQzET+bNwhXzTb
+            GMY/w3rVYPddqx1wi5HPtYlhL3emYlmMYLLi1ITr1fKvAsgGKSJpaGS6yu+YkyiB
+            AJHou+2aCUy1TVk8U/Ne9TSrhsV1AJSIbtr8+iyKAtSsSuodam0Pyzn59uqL4nNd
+            1S4Nkvm1UDYv97hmSQdAzTGs+Aql6HyB/Wyi+rkMZQ0fl7+hdX5/eEDYZzijD0Nu
+            Mqk1iFbTn/rXir71VRWNttycIM1PfGAOUmLoyOR+TrL5Ecy0Sx9yx2DS9DecnqtS
+            39YI6UNqiEYdvW8Y+XELSFI/B5cjw7C1A5S9tecUxoOIBGZZSeRQa2WDSWb1dYky
+            Rzvncoeemgjkp8CHQhSqrylB+I4s0Y1HMUWR/q0wxew9juNEIP8ZUkfKcueuCYlm
+            5ui/RaeYatMFo8hg+DrxEtQIkbBZ/ILss/QzyHZZG3RMq0i9s/PIn5yipVhOgorS
+            XAFEeOrVaKpN06LfNP7/HTI+D/5HJL++Twuyz9fwC2Eb5saIIB6gpywiICH8HcD5
+            MRmmhl18x2YU+JdmkAcWNfH5+JHVnx8o+D/y9JqeCf6sle/9LyToazJv58Tc
+            =2u7h
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/infrastructure/observability/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: observability
+  labels:
+    pod-security.kubernetes.io/enforce: privileged

kubernetes/infrastructure/observability/vm-internal-ingress.yaml

@@ -0,0 +1,26 @@
+# Ingress exposing the VictoriaMetrics vminsert endpoint (vmsingle:8428)
+# through Traefik as vm-internal.lan — used by the docker-host vmagent to
+# remote_write metrics into the central VictoriaMetrics.
+# external-dns auto-creates the vm-internal.lan A record pointing at the
+# Traefik LoadBalancer (see kubernetes/system/external-dns).
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: vm-internal
+  namespace: observability
+  annotations:
+    external-dns.alpha.kubernetes.io/enabled: "true"
+    # No chart ingress is available for vmsingle, so declare it here.
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: vm-internal.lan
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: vmsingle-vm-victoria-metrics-k8s-stack
+                port:
+                  number: 8428

kubernetes/infrastructure/observability/vmstack-release.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: vm-victoria-metrics-k8s-stack
+  namespace: observability
+spec:
+  interval: 1m
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: vm
+        namespace: observability
+      chart: victoria-metrics-k8s-stack
+      version: "0.87.0"
+      interval: 1m
+  valuesFrom:
+    - kind: Secret
+      name: vmstack-values

kubernetes/infrastructure/observability/vmstack-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: vm
+  namespace: observability
+spec:
+  interval: 1m
+  url: https://victoriametrics.github.io/helm-charts/

kubernetes/infrastructure/observability/vmstack-values.yaml

@@ -0,0 +1,199 @@
+# victoria-metrics-k8s-stack Helm values 
+# The chart installs the VictoriaMetrics operator + CRDs and creates VMSingle,
+# VMAgent, VMAlert, VMAlertmanager CRs.
+# ---------------------------------------------------------------------------
+# VictoriaMetrics single-binary (the metrics database)
+# ---------------------------------------------------------------------------
+vmsingle:
+    enabled: true
+    spec:
+        retentionPeriod: 7d
+        storage:
+            storageClassName: longhorn
+            accessModes:
+                - ReadWriteOnce
+            resources:
+                requests:
+                    storage: 20Gi
+        resources:
+            requests:
+                cpu: 100m
+                memory: 256Mi
+            limits:
+                cpu: "1"
+                memory: 1Gi
+# ---------------------------------------------------------------------------
+# vmagent — scrapes node-exporter, kube-state-metrics, kubelet, etc.
+# The docker host runs its own vmagent that remote_writes through
+# vm-internal.lan (the Traefik Ingress → this vmsingle).
+# ---------------------------------------------------------------------------
+vmagent:
+    enabled: true
+    spec:
+        selectAllByDefault: true
+        scrapeInterval: 20s
+        tolerations:
+            - key: ENC[AES256_GCM,data:SyutB0y9QtqQ,iv:EKRXNGtUovQYsESHFQTGygP09O2UWtLuBEQQcYZm3Oc=,tag:zh7CYOZleSgbbQhYZvcT+w==,type:str]
+              value: ENC[AES256_GCM,data:So+8xmHYSxs=,iv:7sfUBAaMZd7imjVx14lusVn58cJQ4HpTkaPy5Eal2dY=,tag:hRBjT6wuEfG3Z+ZiPJhwQQ==,type:str]
+              operator: Equal
+              effect: NoSchedule
+            - key: ENC[AES256_GCM,data:lj9OurynU8pRt4/MhgyevjmV0b7Zq+WG/9+zRRvPeA+RYZsV+A==,iv:LjuhErDT0bDlLcpo/AlNykz8Xu89B+nx1ja+HkPNXwo=,tag:5xM6XHrCdEejE7ituQ8W4Q==,type:str]
+              operator: Exists
+              effect: NoSchedule
+# ---------------------------------------------------------------------------
+# kube-state-metrics + node-exporter (metrics sources)
+# ---------------------------------------------------------------------------
+kube-state-metrics:
+    enabled: true
+prometheus-node-exporter:
+    enabled: true
+# ---------------------------------------------------------------------------
+# Grafana (UI) — served behind ingress at grafana.lan.
+# ---------------------------------------------------------------------------
+grafana:
+    enabled: true
+    persistence:
+        enabled: true
+        storageClassName: longhorn
+        accessModes:
+            - ReadWriteOnce
+        size: 5Gi
+    adminUser: admin
+    # SOPS encrypts this when the file is processed.
+    adminPassword: ENC[AES256_GCM,data:vpPvJA/PktMnsg/IovYfsD4lDBndUw==,iv:b4eLMEcUMJj4BwyeYw1kw2sZuieyr2xL9et+R7ss0BU=,tag:u0RcTp6mkQNAGkblG4/xww==,type:str]
+    # Provision a Loki datasource alongside the chart's default VictoriaMetrics
+    # datasource so metric/log correlation works in one UI.
+    datasources:
+        datasources.yaml:
+            apiVersion: ENC[AES256_GCM,data:Uw==,iv:KYTMBV53yMbojMaqNinYpm/Uj5ylV86jcdJTdxuy+G0=,tag:IbIpiSBQFw7YSTQg0WRGqg==,type:int]
+            datasources:
+                - name: ENC[AES256_GCM,data:BPfR8Q==,iv:MkMhH3IJfBZzbIcqQw7y6bfJB/4Ew4feZNugjovk5O0=,tag:jaa43tzIXkE8kWZp2R/kcg==,type:str]
+                  type: ENC[AES256_GCM,data:WWncww==,iv:hPqJE0KCidSBnf11wxFH+Je+YKztWmeToBqi39Qf4ZQ=,tag:c9enJ2VgQqDJVbasvgG+Dw==,type:str]
+                  url: ENC[AES256_GCM,data:YkS66tpWXpW62Dtful0KNCwoZ3rG3nQtDBUsNQUwdQ+eqtG38JTrAudQqnsRD51q,iv:Mu3Sc6T2xypdot9dA1DHc8nD7UI8372nqlveLITR82E=,tag:1VmmIg+xR9qjWcvPKVYsJQ==,type:str]
+                  access: ENC[AES256_GCM,data:af3GpjE=,iv:MAYKfklF+wCC/XN+4snebjPAnW4KpYzaMMGYds7rG18=,tag:X+oFiDAiCb1f7cfREqFT7A==,type:str]
+                  isDefault: ENC[AES256_GCM,data:Zgu4Hn8=,iv:nnVB/in75nPAQos0Dfl6abBrHwT8tCt5X7lcCYBZbLc=,tag:u1UvkSj6SjcMm06NzKMGDA==,type:bool]
+                  jsonData:
+                    maxLines: ENC[AES256_GCM,data:88JJqA==,iv:lKsZa024I6223UEu0SHuZ3N9LZrzw0WBTfC+L2/3ljI=,tag:YvqieXK36xMmC/sSJ11NIg==,type:int]
+    ingress:
+        enabled: true
+        ingressClassName: traefik
+        annotations:
+            external-dns.alpha.kubernetes.io/enabled: "true"
+        hosts:
+            - grafana.lan
+        path: /
+        pathType: Prefix
+        tls: []
+# ---------------------------------------------------------------------------
+# vmalert — evaluates VMRule CRs against VictoriaMetrics, forwards firing
+# alerts to Alertmanager. selectAllByDefault picks up all VMRules in the
+# namespace (including our vmalert-rules.yaml VMRule CR).
+# ---------------------------------------------------------------------------
+vmalert:
+    enabled: true
+    spec:
+        selectAllByDefault: true
+        evaluationInterval: 20s
+        notifiers:
+            - url: http://vm-victoria-metrics-k8s-stack-alertmanager.observability.svc.cluster.local:9093
+        resources:
+            requests:
+                cpu: 50m
+                memory: 64Mi
+            limits:
+                cpu: 200m
+                memory: 128Mi
+# ---------------------------------------------------------------------------
+# Alertmanager — 1 replica; inline config with a single n8n webhook receiver.
+# n8n runs on the docker host and fans out to email/Telegram/whatever.
+# ---------------------------------------------------------------------------
+alertmanager:
+    enabled: true
+    spec:
+        replicaCount: 1
+        port: "9093"
+        selectAllByDefault: true
+        storage:
+            volumeClaimTemplate:
+                spec:
+                    storageClassName: longhorn
+                    accessModes:
+                        - ReadWriteOnce
+                    resources:
+                        requests:
+                            storage: 2Gi
+        resources:
+            requests:
+                cpu: 50m
+                memory: 64Mi
+            limits:
+                cpu: 200m
+                memory: 128Mi
+    config:
+        route:
+            receiver: n8n-webhook
+            group_wait: 30s
+            group_interval: 5m
+            repeat_interval: 4h
+        receivers:
+            - name: n8n-webhook
+              webhook_configs:
+                - url: http://n8n.lan/webhook/observability-alert
+                  send_resolved: true
+extraRules:
+    node-alerts:
+        groups:
+            - name: node
+              rules:
+                - alert: HighNodeCPU
+                  expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: High CPU on {{ $labels.instance }}
+                    description: CPU usage above 80% for 10 minutes.
+                - alert: HighNodeRAM
+                  expr: 100 - ((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100) > 85
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: High RAM on {{ $labels.instance }}
+                    description: RAM usage above 85% for 10 minutes.
+                - alert: LowDiskSpace
+                  expr: |
+                    100 - ((node_filesystem_avail_bytes{mountpoint!~"/run.*|/var/lib/docker.*"} /
+                            node_filesystem_size_bytes{mountpoint!~"/run.*|/var/lib/docker.*"}) * 100) > 85
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: Low disk space on {{ $labels.instance }} {{ $labels.mountpoint }}
+                    description: Disk usage above 85% for 10 minutes.
+sops:
+    lastmodified: "2026-07-28T16:39:09Z"
+    mac: ENC[AES256_GCM,data:Ymxmmkui0Nuv35Yq6v4BXM4xGhvjAufexW7fIm9fUIdTTXZ3e10PitNCZXKiaDwwCJfdEkNZri7jPAYzhTzKMIdcTKnOMImsMjdiJZwRwwbKcYVay0A6KSLJ8eFjDRRYhlCXbsAIRlQHpFh4UkssKdu40mIiudU0qIVm04ppbno=,iv:R8kpITPYSbQ1yZw5IK0fArssxUtIjNmvzknOrLJQcuk=,tag:/CRcr1VZAIKIPbcag05uTw==,type:str]
+    pgp:
+        - created_at: "2026-07-28T16:39:08Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA7uy4qQr71wiARAAmOxyx5eH5Jykp/GDL79REgf61iGujtPiUVy8CT3o7O5k
+            tTGma6rRXK5wZV/iGN3wOQehF19Oy2iYwMfxilRwpB69RFwyKPKdHDDWMWNXv5Th
+            wJnNjVjWZ0xiRautWPn/TjDOJZ6j6p7lQ3IcQ3vzuap1RVRIXv081DxJGR9JbeYl
+            khmrl1+zecwxS87JK+fLko9NIBb5JgFEtEf1RvLyFak8nZu5cNCu+PCykALvUaO7
+            V5NAaF1aV9cdz8G9TZWmHZsIlDDE9/oro2aLR7cr8uxnKQhWerttqvyDYNwYzNKW
+            qgsUplTxTmyuc9DdnLDfJ55Ncuv9qS5/4/p5GpsHrteD6oiPNjXaxR8cBVFLvGwp
+            kQb/VZgOGAnkYlMCUG8APiwRhbpAlyy51hKBqVh+a9iRlzZEWHKlxcvIrcFAQaHt
+            sDXyg36F0GCoHiMAqHKG0kRcbTV7Sq0JZVoLU/qxvaDfqvyCZO7X/U/YsSWNbWL0
+            emyBfeSvBX9h62NJx2aw8xKyiL+aeVfzSKAaXHMd/r4gKxBw/kVHAljKwRzbL85j
+            UHeLUvqVkXs1mtIqGgUnpCcAVuV83omgcOwF44FitG/f9GCx+UWTvScwaw5ywuG3
+            J/4fOaA3K0Z4e/mOejARd4jLSPktSRpA01k9kjYY8UBn7TzQ0LrMLj5dxh8BpWrS
+            XgECZapi+q6IjIses7+JqkWiP25hcglhqK3LAiUapA0c5UmAa5Vf7jfnkFm8eUdZ
+            bA7ZnhX8nxK9n7JdSxrZhrbPWMO8hF/XKi7KTZMvgXFlb4/qP96y7oeqzKYcnvE=
+            =pmSq
+            -----END PGP MESSAGE-----
+          fp: DC6910268E657FF70BA7EC289974494E76938DDC
+    encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
+    version: 3.10.2

kubernetes/kustomization.yaml

@@ -24,6 +24,7 @@ resources:
   # Infrastructure
   - infrastructure/authentik
   - infrastructure/mail/stalwart
+  - infrastructure/observability
   # Personal
   - personal/linkwarden
   - personal/notesnook

kubernetes/system/longhorn/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
   - namespace.yaml
   - repository.yaml
   - release.yaml
+  - recurring-jobs.yaml
 secretGenerator:
   - name: longhorn-values
     files:

kubernetes/system/longhorn/recurring-jobs.yaml

@@ -0,0 +1,34 @@
+# Recurring backup and snapshot schedules for Longhorn.
+apiVersion: longhorn.io/v1beta2
+kind: RecurringJob
+metadata:
+  name: daily-snapshot
+  namespace: longhorn-system
+spec:
+  # Fast local recovery from accidental deletes.
+  name: daily-snapshot
+  task: snapshot
+  cron: "0 2 * * *"
+  retain: 7
+  concurrency: 2
+  labels:
+    type: snapshot
+  groups:
+    - default
+---
+apiVersion: longhorn.io/v1beta2
+kind: RecurringJob
+metadata:
+  name: daily-backup
+  namespace: longhorn-system
+spec:
+  # Daily NFS backup for DR if a node is lost.
+  name: daily-backup
+  task: backup
+  cron: "0 3 * * *"
+  retain: 7
+  concurrency: 2
+  labels:
+    type: backup
+  groups:
+    - default

kubernetes/system/longhorn/values.yaml

@@ -2,8 +2,6 @@
 defaultSettings:
     # 2 replicas so volumes survive single-node loss
     defaultReplicaCount: 2
-    # NFS backup target on TrueNAS
-    backupTarget: nfs://truenas.lan:/mnt/fast_app_data/longhorn-backup
     # Allow full disk usage on single-node
     storageOverProvisioningPercentage: 100
     storageMinimalAvailablePercentage: 10
@@ -36,23 +34,23 @@ persistence:
 # R740, but keeps things correct if that ever changes).
 longhornManager:
     tolerations:
-        - key: ENC[AES256_GCM,data:72tSl2wrzA+rNFPSi6y1sXNdzFd2QI9/DIoLPYv600yIWre9Jg==,iv:6FJCiqSNlTTKb0qp7Zupq6RTcwA5de8uxrFM9o8XdAw=,tag:I1gnHYR0XNgq60tOnXYFGA==,type:str]
+        - key: ENC[AES256_GCM,data:DDNTXJiwP2LQiOmqgf/L6tLIZiFgqh42dLDXj8xomGv7rlUP9A==,iv:eu6vnl3EiLu1DWWDWlxkQNcN6/uEhSkpufw7L9Ikr14=,tag:DJOMo8LDGmzkqvIEkfCTHg==,type:str]
           operator: Exists
           effect: NoSchedule
         # Tolerate the failover taint so longhorn-manager runs on the P330
-        - key: ENC[AES256_GCM,data:Tru89mna++TT,iv:8gH3VLU6CCCC42SB6LfEaPY2MFHaD4cLwCWFkSHQV4A=,tag:H8gUvT97+GTgSJu7buUOjw==,type:str]
-          value: ENC[AES256_GCM,data:QYHvXr+rR44=,iv:jJrfJST2MG4pC71Qr2p4WAheDHcd49DbwrBVlLFqyk0=,tag:GJHrXSKfK6/N0IWmKemZxg==,type:str]
+        - key: ENC[AES256_GCM,data:rbWq0sruC6zC,iv:Eq5izQ4LXZ90Bhf1BJ7V9/Xo7HVMULeDHZ6NgItZfGQ=,tag:ixJt7YD4cBy+SrG2WVn0ng==,type:str]
+          value: ENC[AES256_GCM,data:wxGNkS97rg8=,iv:bBGGqcwibhXE0zkH+ZJEVgr20ogu7JwQxxDXqJcMdsQ=,tag:VLBQh8vRl1SiYB9S/vwLeg==,type:str]
           operator: Equal
           effect: NoSchedule
 # longhorn-driver (CSI provisioner/attacher deployer) — tolerate the same
 # taints so CSI operations work from the failover node.
 longhornDriver:
     tolerations:
-        - key: ENC[AES256_GCM,data:dQhIOMVadhzSbeJqzVBbikkKzk11GLxOHjSA1hdZkKRuKUONJA==,iv:nuurPvU/YbVdFxO+iPbWchqVNOWO+tssuF5ZhIHo8+Y=,tag:mbSWnxT6kP6ehZu1+kaAIA==,type:str]
+        - key: ENC[AES256_GCM,data:1EBSG4hL4heCnLFJYzTNaQWtFU3v6xwTlG+Y9/H4p5d0TZdFog==,iv:9ILcNXXzxzG5CYA4Ny7bKHBI581B1qJC3OBUObScxbQ=,tag:4B4JaFO0tfyVA6w7ypCY6g==,type:str]
           operator: Exists
           effect: NoSchedule
-        - key: ENC[AES256_GCM,data:9dhWE/kEwWr/,iv:pXHuR05UNdJlBaCMo0T4FHPN+8qGRXFGYQglP/P/VLc=,tag:KTbwQ4KW9zsSsftXQF4DRg==,type:str]
-          value: ENC[AES256_GCM,data:H6vDw998h28=,iv:IHNNLysSnMuqrmdPsCaXYFPUtGrxBFZdDFG+iOMBnvw=,tag:r1BZTI2PQ0wjhWaIi1yJZA==,type:str]
+        - key: ENC[AES256_GCM,data:ZSC5qg+ChOkI,iv:rIq8j7L7F641kWDzcSV9lrYxhHggT1FTZ8WF53vxvV0=,tag:Hj4RlF3usdIQi0G6Ecmdbg==,type:str]
+          value: ENC[AES256_GCM,data:YfiuCv2EKec=,iv:+6i5CT1ToCKy1gFtVVIFv5FLQaJaKHx4vQk8b0IBy3U=,tag:zHgzNuijbwI1QbvuLnZNvw==,type:str]
           operator: Equal
           effect: NoSchedule
 # longhorn-ui — tolerate the same taints so the UI is reachable even if it
@@ -60,58 +58,46 @@ longhornDriver:
 longhornUI:
     replicas: 1
     tolerations:
-        - key: ENC[AES256_GCM,data:deSbcZL4q+7tZaTg2pE1Ssq/WG0aS29EYAW+y1/Vl4Uq3E4tDQ==,iv:mE3AYFNrs7aAdKG1eRMJ21AwVdspJq42zWqytehn9z0=,tag:DsC7ehN8NhANSzWBQ3rMjQ==,type:str]
+        - key: ENC[AES256_GCM,data:n4UdwssGYhHQgDTpCpu+WgIPsdNc8uAaZtnURZt0/C3VfeLR4w==,iv:mqy0GqHN0KxtkETS7psu61Mr1oGmtPDhg2vQhVDas6c=,tag:uUd+NY90XK3i0aigc5ndCg==,type:str]
           operator: Exists
           effect: NoSchedule
-        - key: ENC[AES256_GCM,data:4UJN0oQsnPgz,iv:bnCs9OQzRNYxZS4LMVcI5Zgel4MES745e47/Wc+72Ps=,tag:+WD085TKD5CfFYJ3B0RotA==,type:str]
-          value: ENC[AES256_GCM,data:preRO+Om90c=,iv:Q6VF0hDh5Jz3ghB+VqUB/KO5i2wwxq68WWLq2ptnwOQ=,tag:ug333G7gCkSn60g/Lo5S7A==,type:str]
+        - key: ENC[AES256_GCM,data:isSkUNqax+AU,iv:jWgZyZL9VMmzYqoLCUkIiUuy1iBeOm2RE5CjZGCcXxQ=,tag:714V0YGb7jqebC4IRX/phw==,type:str]
+          value: ENC[AES256_GCM,data:Qbtu1wgV1oI=,iv:RihD43IA1+C3m9OkXACGgd5lY9lDWCyzIcw/MCoSMvg=,tag:WzQtC4rT+uovS6s36DJWUw==,type:str]
           operator: Equal
           effect: NoSchedule
 # Disable ingress for now
 ingress:
     enabled: false
-# Recurring backup and snapshot schedules
-# - Daily snapshots: fast local recovery from accidental deletes
-# - Daily NFS backups: DR if node is lost
-recurringJobs:
-    enable: true
-    jobs:
-        - name: daily-snapshot
-          task: snapshot
-          cron: 0 2 * * *
-          retain: 7
-          concurrency: 2
-          labels:
-            type: snapshot
-        - name: daily-backup
-          task: backup
-          cron: 0 3 * * *
-          retain: 7
-          concurrency: 2
-          labels:
-            type: backup
+# Default backup store (NFS target on TrueNAS).
+defaultBackupStore:
+    # NFS endpoint exposing the TrueNAS dataset as a backup target.
+    backupTarget: nfs://truenas.lan:/mnt/fast_app_data/longhorn-backup
+    # No credentials required for an anonymously-readable NFS export.
+    backupTargetCredentialSecret: ""
+    # Poll the backup target every 5 minutes for new/removed backups.
+    pollInterval: 300
 sops:
-    lastmodified: "2026-07-25T12:52:04Z"
-    mac: ENC[AES256_GCM,data:iYSrt+E3ZBR/x85S4A/Y4GalkJ5Y0NDwjSvh6Pfa8d5Qz+uFdGofnNISS7QgqmsGpLbLKMFHRxLV6L3R85V/o293SfNA/+iM8DXOoZTb8zQTLsPxjqxr8Hfl0cxBgV8LgZJzzG8pkeONDfqqtF6pGMgAz0SHfdAR2qdbCKTztis=,iv:YdWgVo9/YlusV1mqBWpVQVVdEQGSD7ytPVNJYjppmTw=,tag:XOxeIx5GWJcZD19zZTe5YQ==,type:str]
+    lastmodified: "2026-07-26T08:44:08Z"
+    mac: ENC[AES256_GCM,data:4shHlVWcVol/Z8b+5+tz6shT8x6gCZz2xhYv1VT7dHlgcYWWgq2V5mbGUtsVmWFzkGPC5rYvU3dUEiEkl0zb6Q1rP/fD6xQmlyIgH/ioHrfCvjorY5Rv8t4OFH/TyBK9hMwC+iFdAUJOdE9vYQoDmUMTGz6XwWiDY3PxRv1X9D4=,iv:DDDiStImg5qH2QP9lcSyW5+b+1P1mk6F0TWkt9p2syo=,tag:zJnZ455gsXPkE4Lfup2Rcg==,type:str]
     pgp:
-        - created_at: "2026-07-25T12:52:04Z"
+        - created_at: "2026-07-26T08:44:08Z"
           enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMA7uy4qQr71wiARAAgDIE7tPjU9uheakXKYvn6IyNKkZjXTnV2sBPdoo4l0SW
-            ZYAu+L9SKnVKpmuQxz3HKqk/9Cib7Q/g9HdrXsrLkQ0xS6BC+4z+4JynxWHN3CII
-            0w7SNKVFV4RIXWwMQe0Hg+2OtzXyF1g8ZmNZu/L+/kusEnl7WX9BoYOB5i5wssM/
-            Nfb7LTlMGqmyQoWxbTzkdNuMbw2oVjD6pgL/VEpfCmvVzMpXScmTT9tFdJRYDuj6
-            r9a8wkFzCHNUWy3z7+00cQVnyDBA/UY/9pskeR58QH4keXp5xeBIYKxUe28L9JRg
-            YkVJjy1XFmmdY6VWmMo/Sx89YUQ9kMpf9KI/D0WkgGLqTihzsX4v7xxvWJfYXiPd
-            RmkRR4f19HfUapLzosO65rtYAG3BptdRPBpvx77evLkPwps34+dxUrVESYxY/I01
-            0BNNLDthEhBzfmh/yz3BcYOigZp1ziiXq201pn+qiDbLSDeoidm0IrN+g3Kib25g
-            dBTpRD3iEoBXeW2kbwltoUwsNkv59Gf/fj3DUQFAclP56b7UzRLDCQKiKIL17l2O
-            nU6yvz9iARPYvT3gzqOw8yaB780RQejR+8qiZca0WetNcVn7b+DMGpKgQXnchc1g
-            WzI3MdpNtajQGwBtHic3BG4xYAoN6sNmVrz+aGBIkBxFK3nMBzeWQJBSm6iiEwzS
-            XgFcb1jlb7dC/g8ltmAIOyjwnFHkfVI2/v+u+E1mW2VWBQqtABAHobh6uEf1jFzj
-            DZubFTVfrI4ssrvsjhm84mVJND9taaLPU4qbVYJ8G4kmP8UyKdHd1fJifNdp9+o=
-            =qY6k
+            hQIMA7uy4qQr71wiAQ/9FsBtkggrKwHfh5FFRUFkyqCU596d/R7ZEvsMP7zzCH9H
+            VAAYJxl8D6/KY5EavpeZH4s2ZnlzZjlb/qS2CKpSzXuN7ADPnK10cBxxAOQLp6KF
+            pqdkv6x5MyWbfv87IzM8cI0D/jlG2/iZPfBZy/YU4Rg8cefIl82vNgNADW2O68bK
+            w1VS75AYKmUfXZ2vqz+wX46YcjdXkrvRwzM9lzfIm2Q2dKMqBKSYzxMICZ/hWGRj
+            t33KbZLKC1Kg0NmYOY4GCt8hi6Ck4TjKvzxktsEpXYFDhWzXe9nr+axtrfypLrAn
+            fcLCQV1FSovMK+P4JxB/dd0u+F3RR500F3tBJPQ1xs1Rc1f8Cd2Ynr/3LWtTstkq
+            JyOfLFAj4/LG8o7ub/FSvLZZI052iRLZqlIIe0cFTqCz31yVEJOnLJWmME36hDEm
+            rfd3X9IvixfwWVTa3K0TxZta7u0Q0DsqHIt3eLdeTJC3wY4J+A/vUvPFv137YEqp
+            O+/ALTbwK7ViSQIupjr4kEcklBtiu9myNi3u8JNg67/XHwahCh0tfGk4jCevve5S
+            dDk4Ud7469LuDwYSZw0RdwuU89jQivvsp8/mZpQIoSVzl//iEQ2EOAff1gMfiPoj
+            1MrwGtrrMAo+2Uoxacd7LfFn/l1zRNiGw1rX21NzswmzIU84US4Grv9LGealQ8TS
+            XgH+datMRHIAPKcYY0u/B/PfyoKArlPUzQsIQOm02+s1zuvK/dArKxgObKTAr/u4
+            uNuVH3ObUVMb6VX4iwlxfS+VKpjotTbgY2TJfV46kGC3JJ0w1LdjEyJbpYlIz/k=
+            =HWEp
             -----END PGP MESSAGE-----
           fp: DC6910268E657FF70BA7EC289974494E76938DDC
     encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$