observability: deploy observability stack

docker/infrastructure/observability/alloy.river

@@ -0,0 +1,83 @@
+// Grafana Alloy config for the docker host
+// Tails Docker container logs via the docker_sd discovery +
+// file streaming, applies container_name/image labels, and pushes them to
+// the in-cluster Loki at loki-internal.lan (LAN-only Traefik Ingress).
+
+logging {
+  level  = "info"
+  format = "logfmt"
+}
+
+// Discover running Docker containers via the host socket.
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+}
+
+// Relabel rules: strip the leading slash from __meta_docker_container_name and
+// expose useful labels (container name, compose service, image).
+discovery.relabel "containers" {
+  targets = discovery.docker.containers.targets
+
+  // Strip leading / from container name.
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container_name"
+    replacement   = "$1"
+  }
+
+  // Compose service name (if label is set).
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "compose_service"
+  }
+
+  // Log stream (stdout/stderr).
+  rule {
+    source_labels = ["__meta_docker_container_log_stream"]
+    target_label  = "stream"
+  }
+
+  // Mark the source so dashboards/alerts can distinguish the docker host.
+  rule {
+    target_label = "source"
+    replacement  = "docker-host"
+  }
+}
+
+// Tail the container log files based on the docker_sd discovery. Each target
+// exposes a `__path__`idders pointing at /var/lib/docker/containers/<id>/<id>-json.log
+// (the default docker json-file driver path; we mount that host path read-only).
+local.file "container_logs" {
+  targets       = discovery.relabel.containers.output
+  tail_from_end = true
+}
+
+// Parse docker json-file logs ({"log":"...","stream":"stdout","time":"..."}).
+loki.process "containers" {
+  forward_to = [loki.write.default.receiver]
+
+  stage.json {
+    expressions = {
+      log    = "log",
+      stream = "stream",
+      time   = "time",
+    }
+  }
+  stage.labels {
+    values = {
+      stream = "stream",
+    }
+  }
+  stage.timestamp {
+    source = "time"
+    format = "RFC3339Nano"
+  }
+}
+
+// Push to Loki in-cluster via the LAN-only Traefik Ingress (port 80).
+loki.write "default" {
+  endpoint {
+    url = "http://loki-internal.lan/loki/api/v1/push"
+  }
+}

docker/infrastructure/observability/docker-compose.yml

@@ -0,0 +1,82 @@
+# Docker-host collectors for the central observability stack on Kubernetes.
+# Managed by swarm-cd (see .swarmcd/stacks.yaml).
+#
+# node-exporter + cAdvisor run locally (scraped by vmagent below).
+# vmagent remote_writes metrics to vm-internal.lan (LAN-only Traefik Ingress) ->
+#   vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428
+# alloy tails /var/lib/docker/containers + /var/log and pushes logs to
+#   loki-internal.lan -> loki.observability.svc.cluster.local:3100
+
+services:
+  node-exporter:
+    image: prom/node-exporter:v1.9.1
+    container_name: obs-node-exporter
+    network_mode: host
+    pid: host
+    command:
+      - "--path.rootfs=/host"
+    volumes:
+      - /:/host:ro,rslave
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.49.1
+    container_name: obs-cadvisor
+    ports:
+      - "8080"
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /dev/disk/:/dev/disk:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  vmagent:
+    image: victoriametrics/vmagent:v1.131.0
+    container_name: obs-vmagent
+    # vmagent listens on 8429 (its own metrics), scrapes node-exporter:9100
+    # and cadvisor:8080, remote_writes to vm-internal.lan
+    command:
+      - "-promscrape.config=/etc/vmagent/vmagent.yml"
+      - "-remoteWrite.url=http://vm-internal.lan/api/v1/write"
+    volumes:
+      - type: bind
+        source: /root/homeprod/docker/infrastructure/observability/vmagent.yml
+        target: /etc/vmagent/vmagent.yml
+        read_only: true
+    depends_on:
+      - node-exporter
+      - cadvisor
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+  alloy:
+    image: grafana/alloy:v1.18.0
+    container_name: obs-alloy
+    command:
+      - "run"
+      - "/etc/alloy/config.river"
+      - "--server.http.listen.address=0.0.0.0"
+      - "--server.http.listen.port=12345"
+    volumes:
+      - type: bind
+        source: /root/homeprod/docker/infrastructure/observability/alloy.river
+        target: /etc/alloy/config.river
+        read_only: true
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/log:/var/log:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+    labels:
+      - "traefik.enable=false"
+
+networks:
+  default:
+    name: proxy
+    external: true

docker/infrastructure/observability/vmagent.yml

@@ -0,0 +1,22 @@
+# vmagent scrape config for the docker host.
+# Scrapes the two local collectors, forwards everything to the central
+# VictoriaMetrics via the LAN-only Caddy route vm-internal.lan (Caddy proxies
+# it to vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428).
+
+scrape_configs:
+  - job_name: docker-host-node
+    static_configs:
+      - targets:
+          - localhost:9100
+        labels:
+          # Distinguishes the docker host from K8s nodes in dashboards/alerts.
+          instance_source: docker-host
+          host: docker-r740
+
+  - job_name: docker-host-cadvisor
+    static_configs:
+      - targets:
+          - localhost:8080
+        labels:
+          instance_source: docker-host
+          host: docker-r740