observability: deploy observability stack

2026-07-28 16:40:49 +00:00 · 2026-07-28 15:46:52 +02:00
parent 3cfef81f89
commit 38ceb887a7
23 changed files with 795 additions and 0 deletions
@@ -0,0 +1,93 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alloy-config
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: alloy
+    app.kubernetes.io/component: log-collector
+data:
+  config.river: |
+    // -----------------------------------------------------------------------------
+    // Grafana Alloy — log collection only (metrics cluster+host collection is
+    // handled by vmagent/node-exporter/cAdvisor elsewhere).
+    // Forwards pod logs to the in-cluster Loki single-binary.
+    // -----------------------------------------------------------------------------
+
+    logging {
+      level = "info"
+      format = "logfmt"
+    }
+
+    // Tail every pod's log files written under /var/log/pods/* on the host.
+    local.file_match "pods" {
+      path_targets = [{
+        __path__ = "/var/log/pods/*/*/*.log",
+      }]
+    }
+
+    // Discover Kubernetes metadata for the tailed files (pod_name, namespace,
+    // container_name, uid).
+    discovery.kubernetes "pods" {
+      role = "pod"
+    }
+
+    // Stream-stage relabel: read the filename to extract namespace/pod/container
+    // metadata in line with Loki/Promtail's expectations.
+    lrelabel "pods" {
+      targets = local.file_match.pods.targets
+      rule {
+        source_labels = ["__path__"]
+        regex        = "/var/log/pods/(?P<namespace>[^/]+)/(?P<pod>[^/]+)/(?P<container>[^/]+)/(?P<uid>[^/]+)/.*"
+        target_label  = "namespace"
+        replacement   = "$namespace"
+      }
+      rule {
+        source_labels = ["__path__"]
+        regex        = "/var/log/pods/([^/]+)/([^/]+)/([^/]+)/[^/]+/.*"
+        target_label  = "pod"
+        replacement   = "$pod"
+      }
+      rule {
+        source_labels = ["__path__"]
+        regex        = "/var/log/pods/([^/]+)/([^/]+)/([^/]+)/[^/]+/.*"
+        target_label  = "container"
+        replacement   = "$container"
+      }
+    }
+
+    // Tail the actual log files.
+    local.file "pods_logs" {
+      targets    = lrelabel.pods.output
+      read_from  = "beginning"
+      tail_from_end = false
+    }
+
+    // Parse CRI-style lines ({"log":"...","stream":"stdout","time":"..."}).
+    loki.process "pods" {
+      forward_to = [loki.write.default.receiver]
+
+      stage.json {
+        expressions = {
+          log    = "log",
+          stream = "stream",
+          time   = "time",
+        }
+      }
+      stage.labels {
+        values = {
+          stream = "stream",
+        }
+      }
+      stage.timestamp {
+        source = "time"
+        format = "RFC3339Nano"
+      }
+    }
+
+    // Push to Loki in-cluster (no auth on .lan).
+    loki.write "default" {
+      endpoint {
+        url = "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push"
+      }
+    }