diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d46fa4f..b98ab04 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,6 +13,7 @@ updates: - "/docker/home/zigbee2mqtt" - "/docker/infrastructure/mail/roundcube" - "/docker/infrastructure/network/traefik" + - "/docker/infrastructure/observability" - "/docker/infrastructure/squid" - "/docker/infrastructure/sshportal" - "/docker/personal/fireshare" diff --git a/.swarmcd/stacks.yaml b/.swarmcd/stacks.yaml index 6e25fae..d9513bf 100644 --- a/.swarmcd/stacks.yaml +++ b/.swarmcd/stacks.yaml @@ -38,6 +38,11 @@ traefik: branch: main compose_file: docker/infrastructure/network/traefik/docker-compose.yml +observability: + repo: homeprod + branch: main + compose_file: docker/infrastructure/observability/docker-compose.yml + squid: repo: homeprod branch: main diff --git a/docker/infrastructure/observability/alloy.river b/docker/infrastructure/observability/alloy.river new file mode 100644 index 0000000..da63e0f --- /dev/null +++ b/docker/infrastructure/observability/alloy.river @@ -0,0 +1,83 @@ +// Grafana Alloy config for the docker host +// Tails Docker container logs via the docker_sd discovery + +// file streaming, applies container_name/image labels, and pushes them to +// the in-cluster Loki at loki-internal.lan (LAN-only Traefik Ingress). + +logging { + level = "info" + format = "logfmt" +} + +// Discover running Docker containers via the host socket. +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" +} + +// Relabel rules: strip the leading slash from __meta_docker_container_name and +// expose useful labels (container name, compose service, image). +discovery.relabel "containers" { + targets = discovery.docker.containers.targets + + // Strip leading / from container name. + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container_name" + replacement = "$1" + } + + // Compose service name (if label is set). + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "compose_service" + } + + // Log stream (stdout/stderr). + rule { + source_labels = ["__meta_docker_container_log_stream"] + target_label = "stream" + } + + // Mark the source so dashboards/alerts can distinguish the docker host. + rule { + target_label = "source" + replacement = "docker-host" + } +} + +// Tail the container log files based on the docker_sd discovery. Each target +// exposes a `__path__`idders pointing at /var/lib/docker/containers//-json.log +// (the default docker json-file driver path; we mount that host path read-only). +local.file "container_logs" { + targets = discovery.relabel.containers.output + tail_from_end = true +} + +// Parse docker json-file logs ({"log":"...","stream":"stdout","time":"..."}). +loki.process "containers" { + forward_to = [loki.write.default.receiver] + + stage.json { + expressions = { + log = "log", + stream = "stream", + time = "time", + } + } + stage.labels { + values = { + stream = "stream", + } + } + stage.timestamp { + source = "time" + format = "RFC3339Nano" + } +} + +// Push to Loki in-cluster via the LAN-only Traefik Ingress (port 80). +loki.write "default" { + endpoint { + url = "http://loki-internal.lan/loki/api/v1/push" + } +} \ No newline at end of file diff --git a/docker/infrastructure/observability/docker-compose.yml b/docker/infrastructure/observability/docker-compose.yml new file mode 100644 index 0000000..db925a2 --- /dev/null +++ b/docker/infrastructure/observability/docker-compose.yml @@ -0,0 +1,82 @@ +# Docker-host collectors for the central observability stack on Kubernetes. +# Managed by swarm-cd (see .swarmcd/stacks.yaml). +# +# node-exporter + cAdvisor run locally (scraped by vmagent below). +# vmagent remote_writes metrics to vm-internal.lan (LAN-only Traefik Ingress) -> +# vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428 +# alloy tails /var/lib/docker/containers + /var/log and pushes logs to +# loki-internal.lan -> loki.observability.svc.cluster.local:3100 + +services: + node-exporter: + image: prom/node-exporter:v1.9.1 + container_name: obs-node-exporter + network_mode: host + pid: host + command: + - "--path.rootfs=/host" + volumes: + - /:/host:ro,rslave + restart: unless-stopped + labels: + - "traefik.enable=false" + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.49.1 + container_name: obs-cadvisor + ports: + - "8080" + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /dev/disk/:/dev/disk:ro + - /var/lib/docker/:/var/lib/docker:ro + restart: unless-stopped + labels: + - "traefik.enable=false" + + vmagent: + image: victoriametrics/vmagent:v1.131.0 + container_name: obs-vmagent + # vmagent listens on 8429 (its own metrics), scrapes node-exporter:9100 + # and cadvisor:8080, remote_writes to vm-internal.lan + command: + - "-promscrape.config=/etc/vmagent/vmagent.yml" + - "-remoteWrite.url=http://vm-internal.lan/api/v1/write" + volumes: + - type: bind + source: /root/homeprod/docker/infrastructure/observability/vmagent.yml + target: /etc/vmagent/vmagent.yml + read_only: true + depends_on: + - node-exporter + - cadvisor + restart: unless-stopped + labels: + - "traefik.enable=false" + + alloy: + image: grafana/alloy:v1.18.0 + container_name: obs-alloy + command: + - "run" + - "/etc/alloy/config.river" + - "--server.http.listen.address=0.0.0.0" + - "--server.http.listen.port=12345" + volumes: + - type: bind + source: /root/homeprod/docker/infrastructure/observability/alloy.river + target: /etc/alloy/config.river + read_only: true + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/log:/var/log:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: unless-stopped + labels: + - "traefik.enable=false" + +networks: + default: + name: proxy + external: true \ No newline at end of file diff --git a/docker/infrastructure/observability/vmagent.yml b/docker/infrastructure/observability/vmagent.yml new file mode 100644 index 0000000..197703c --- /dev/null +++ b/docker/infrastructure/observability/vmagent.yml @@ -0,0 +1,22 @@ +# vmagent scrape config for the docker host. +# Scrapes the two local collectors, forwards everything to the central +# VictoriaMetrics via the LAN-only Caddy route vm-internal.lan (Caddy proxies +# it to vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428). + +scrape_configs: + - job_name: docker-host-node + static_configs: + - targets: + - localhost:9100 + labels: + # Distinguishes the docker host from K8s nodes in dashboards/alerts. + instance_source: docker-host + host: docker-r740 + + - job_name: docker-host-cadvisor + static_configs: + - targets: + - localhost:8080 + labels: + instance_source: docker-host + host: docker-r740 \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/alertmanager-config.yaml b/kubernetes/infrastructure/observability/alertmanager-config.yaml new file mode 100644 index 0000000..461119f --- /dev/null +++ b/kubernetes/infrastructure/observability/alertmanager-config.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Secret +metadata: + name: alertmanager-config + namespace: observability + labels: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/component: notifiers +type: Opaque +stringData: + alertmanager.yaml: | + global: + resolve_timeout: 5m + + route: + receiver: default + group_by: ["alertname", "severity", "instance"] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + + receivers: + - name: default + webhook_configs: + # n8n workflow webhook + - url: "http://n8n.lan/webhook/observability-alert" + send_resolved: true \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/alloy-config.yaml b/kubernetes/infrastructure/observability/alloy-config.yaml new file mode 100644 index 0000000..965c8b9 --- /dev/null +++ b/kubernetes/infrastructure/observability/alloy-config.yaml @@ -0,0 +1,93 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: alloy-config + namespace: observability + labels: + app.kubernetes.io/name: alloy + app.kubernetes.io/component: log-collector +data: + config.river: | + // ----------------------------------------------------------------------------- + // Grafana Alloy — log collection only (metrics cluster+host collection is + // handled by vmagent/node-exporter/cAdvisor elsewhere). + // Forwards pod logs to the in-cluster Loki single-binary. + // ----------------------------------------------------------------------------- + + logging { + level = "info" + format = "logfmt" + } + + // Tail every pod's log files written under /var/log/pods/* on the host. + local.file_match "pods" { + path_targets = [{ + __path__ = "/var/log/pods/*/*/*.log", + }] + } + + // Discover Kubernetes metadata for the tailed files (pod_name, namespace, + // container_name, uid). + discovery.kubernetes "pods" { + role = "pod" + } + + // Stream-stage relabel: read the filename to extract namespace/pod/container + // metadata in line with Loki/Promtail's expectations. + lrelabel "pods" { + targets = local.file_match.pods.targets + rule { + source_labels = ["__path__"] + regex = "/var/log/pods/(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/.*" + target_label = "namespace" + replacement = "$namespace" + } + rule { + source_labels = ["__path__"] + regex = "/var/log/pods/([^/]+)/([^/]+)/([^/]+)/[^/]+/.*" + target_label = "pod" + replacement = "$pod" + } + rule { + source_labels = ["__path__"] + regex = "/var/log/pods/([^/]+)/([^/]+)/([^/]+)/[^/]+/.*" + target_label = "container" + replacement = "$container" + } + } + + // Tail the actual log files. + local.file "pods_logs" { + targets = lrelabel.pods.output + read_from = "beginning" + tail_from_end = false + } + + // Parse CRI-style lines ({"log":"...","stream":"stdout","time":"..."}). + loki.process "pods" { + forward_to = [loki.write.default.receiver] + + stage.json { + expressions = { + log = "log", + stream = "stream", + time = "time", + } + } + stage.labels { + values = { + stream = "stream", + } + } + stage.timestamp { + source = "time" + format = "RFC3339Nano" + } + } + + // Push to Loki in-cluster (no auth on .lan). + loki.write "default" { + endpoint { + url = "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push" + } + } \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/alloy-release.yaml b/kubernetes/infrastructure/observability/alloy-release.yaml new file mode 100644 index 0000000..8afd78c --- /dev/null +++ b/kubernetes/infrastructure/observability/alloy-release.yaml @@ -0,0 +1,19 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: alloy + namespace: observability +spec: + interval: 1m + chart: + spec: + sourceRef: + kind: HelmRepository + name: grafana + namespace: observability + chart: alloy + version: "1.11.0" + interval: 1m + valuesFrom: + - kind: Secret + name: alloy-values \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/alloy-values.yaml b/kubernetes/infrastructure/observability/alloy-values.yaml new file mode 100644 index 0000000..54fdda2 --- /dev/null +++ b/kubernetes/infrastructure/observability/alloy-values.yaml @@ -0,0 +1,57 @@ +# Grafana Alloy +# Runs as a DaemonSet on every node (tolerates the failover taint). +# Tails pod logs from /var/log/pods, attaches K8s labels (pod, namespace, +# service, container, image), and pushes them to Loki at +# http://loki.observability.svc.cluster.local:3100 (in-cluster, no auth). +# Use the externally provisioned ConfigMap (alloy-config) carrying config.river. +alloy: + configMap: + create: false + name: alloy-config + key: ENC[AES256_GCM,data:aevkZsfe4c4H34BJ,iv:4vrNZWwYWaxJUKGbNlnZfk0NM182nBQZuwvqUPslKNA=,tag:lh1iILM3uQAt0An6kN6EHg==,type:str] + # Mount host log paths so the DaemonSet can tail pod/container logs. + mounts: + varlog: true + dockercontainers: false + stabilityLevel: generally-available + tolerations: + - key: ENC[AES256_GCM,data:zOMWiA9dG+8o,iv:yHp9aBczV8nVRGlJmsBaKs9WdmE3+ii8gWOaj5Xok6M=,tag:8c5ZqBT+vvC0WCW8mi0c9g==,type:str] + value: ENC[AES256_GCM,data:2Q04VR9xLbQ=,iv:6+tQXx3IGUIcrse+poG4X/4pS5dN2dKvhb79sCy+nfA=,tag:jb/TZmGpvdsWnKoC5NdCow==,type:str] + operator: Equal + effect: NoSchedule + - key: ENC[AES256_GCM,data:aVq8GU1ZDg5m9XDEI3PoVbbykU36SnKpEgrohvZ0IBmH47qDKg==,iv:eMcHRO5mGp9hhY4KnuBmdPTjd/9lqtesjA/RRSu4QMw=,tag:2jaGuogkLcIuye2meJQMDQ==,type:str] + operator: Exists + effect: NoSchedule + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi +sops: + lastmodified: "2026-07-28T13:46:53Z" + mac: ENC[AES256_GCM,data:u6J/EIP5aFtOi8ZtT8BPn978wJAEL1E3r0WcmG5WIsWKJBZMMQHUYRmHgkS6n0/UMtczWoCdXlxObS73ZQMSSkdtagQHE/HPh69dDdGkjxyM4mlP+YMwcnIQkxpJBAQTERXA+SVoWCZb7ztKbiJS+zPjOCfA95ixQINmBCdUgEc=,iv:WOXngWu3uNWYp18/xWghGJyONBrq3SSnPDwbjHBDozU=,tag:1fasTfuLBGXBCo9JPytsJw==,type:str] + pgp: + - created_at: "2026-07-28T13:46:53Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA7uy4qQr71wiAQ/8DSD0jEn3YE7XHTm4DWdfLHPs1IVg9hyiWSHe5/xH0siD + zWonfjcuytw8EHDzv6pQMNYmIpyAmJ92Al4ROJmBV8gZ59+U9goHmw5HgaQAupBJ + /gy8iFBteGOHJT+XJPoVb/bu449f7Vle4L3DrOAl6mwxE0WT/7ERoQBeSxL+2OTZ + iDPdq24Zy+crDHpoM0GcSnFRLX5UpmL2QXGB8PHyctwSQoDW+sbAY2ME6buOw8HI + AremKTRtFm6Pu1Ksdh6ZicaHpLSvLyByPPt2zYhlBQsG3s/SjJDlnakUZHhm7T4o + oHzSpKqV7NQbE09ZMyGJrEkl+rfCPeu6jHplW2EX3xjqLWNUF8GHcsDWljogZxXF + irBPkyKASEDdkHZgiZobU6BmvQlxILj104xNtUYTTChcKu3s0d5BrkIOGe+Y7rJd + fNvMfkmO8/URBgtsXy2xPKo0Jzu/0pZsPEp2lxNeyElwxQqo1CFwhKZPkeQ2eLpu + 4X7/CMN8KJr6jLX8HhX9AivlCj+2CoT8SB5GAPw4p00eD0GIMWSNyOXjyW/qQZi3 + 82EyX9KIpHjALoOI4/3YgFpUO12AEsjirYKWkUM+IoDp5l+YX59QdM7zUVVyzsTG + 2EXEdQqYVA7DIcMJb0tHqxDGMGxNayaUsikV7e/kcRVUyDSH+xOfaSUrDTqqv+XS + XAFro6848HF5uMIP4pfM+dxzkc9rFF4il0CuGWlscbz5q+Aw9LImGbpNzrWk3Lpi + beuwWKFTZ8zab5TH9UF/xqsQeg/nyzQnb3fFPk+owvY7fH6XiYbN5WjKjXGW + =kBpp + -----END PGP MESSAGE----- + fp: DC6910268E657FF70BA7EC289974494E76938DDC + encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$ + version: 3.10.2 diff --git a/kubernetes/infrastructure/observability/grafana-repository.yaml b/kubernetes/infrastructure/observability/grafana-repository.yaml new file mode 100644 index 0000000..4a11e58 --- /dev/null +++ b/kubernetes/infrastructure/observability/grafana-repository.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: grafana + namespace: observability +spec: + interval: 1m + url: https://grafana.github.io/helm-charts \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/kustomization.yaml b/kubernetes/infrastructure/observability/kustomization.yaml new file mode 100644 index 0000000..b805bce --- /dev/null +++ b/kubernetes/infrastructure/observability/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability +resources: + - namespace.yaml + - vmstack-repository.yaml + - grafana-repository.yaml + - loki-repository.yaml + - vmstack-release.yaml + - loki-release.yaml + - alloy-release.yaml + - vmalert-rules.yaml + - alloy-config.yaml + - alertmanager-config.yaml + - vm-internal-ingress.yaml + - loki-internal-ingress.yaml +secretGenerator: + - name: vmstack-values + files: + - values.yaml=vmstack-values.yaml + - name: loki-values + files: + - values.yaml=loki-values.yaml + - name: alloy-values + files: + - values.yaml=alloy-values.yaml +configurations: + - kustomizeconfig.yaml \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/kustomizeconfig.yaml b/kubernetes/infrastructure/observability/kustomizeconfig.yaml new file mode 100644 index 0000000..2033a5f --- /dev/null +++ b/kubernetes/infrastructure/observability/kustomizeconfig.yaml @@ -0,0 +1,6 @@ +nameReference: + - kind: Secret + version: v1 + fieldSpecs: + - path: spec/valuesFrom/name + kind: HelmRelease \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/loki-internal-ingress.yaml b/kubernetes/infrastructure/observability/loki-internal-ingress.yaml new file mode 100644 index 0000000..0ebad80 --- /dev/null +++ b/kubernetes/infrastructure/observability/loki-internal-ingress.yaml @@ -0,0 +1,23 @@ +# Ingress exposing the Loki push API (loki:3100) through Traefik as +# loki-internal.lan — used by the docker-host promtail to push container logs +# into the central Loki. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: loki-internal + namespace: observability + annotations: + external-dns.alpha.kubernetes.io/enabled: "true" +spec: + ingressClassName: traefik + rules: + - host: loki-internal.lan + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: loki + port: + number: 3100 \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/loki-release.yaml b/kubernetes/infrastructure/observability/loki-release.yaml new file mode 100644 index 0000000..6ea6975 --- /dev/null +++ b/kubernetes/infrastructure/observability/loki-release.yaml @@ -0,0 +1,19 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: loki + namespace: observability +spec: + interval: 1m + chart: + spec: + sourceRef: + kind: HelmRepository + name: grafana-community + namespace: observability + chart: loki + version: "18.5.4" + interval: 1m + valuesFrom: + - kind: Secret + name: loki-values \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/loki-repository.yaml b/kubernetes/infrastructure/observability/loki-repository.yaml new file mode 100644 index 0000000..626f096 --- /dev/null +++ b/kubernetes/infrastructure/observability/loki-repository.yaml @@ -0,0 +1,9 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: grafana-community + namespace: observability +spec: + interval: 1m + type: oci + url: oci://ghcr.io/grafana-community/helm-charts \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/loki-values.yaml b/kubernetes/infrastructure/observability/loki-values.yaml new file mode 100644 index 0000000..22b9a2e --- /dev/null +++ b/kubernetes/infrastructure/observability/loki-values.yaml @@ -0,0 +1,65 @@ +# Loki single-binary Helm values +# SOPS-encrypted before commit (per .sops.yaml rules). +# Uses the new chart-v18 "Monolithic" deployment mode (SingleBinary mode is +# deprecated in Loki 4 / chart >=18.x). Bundles storage on a Longhorn PVC — +# no minio, no gateway. Resource-light; tuned for a 2-node home cluster. +loki: + deploymentMode: Monolithic + monolithic: + persistence: + storageClassName: longhorn + accessModes: + - ReadWriteOnce + size: 10Gi + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi +# 7d log retention (user can tune later). +limitsConfig: + retention_period: 7d +# Disable subcharts we don't need in monolithic mode. +test: + enabled: false +# Single binary only — disable distributed components. +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 +singleBinary: + replicas: 0 +gateway: + enabled: false +minio: + enabled: false +sops: + lastmodified: "2026-07-28T13:46:53Z" + mac: ENC[AES256_GCM,data:bk1sxO2qIY/FT3+9dTCfpxKBbb9NlyEfTFuUihURUA/AmMJHV4XOx4EaBK6YFbSOVCLny5/aDbUDfF5DMudZTvKMNyU5sWq4X3+6BBZaL3vHSw2+UTA8pdpkNYpLl4uDq5aDFvAN2v/f/XwOpMIEte1L+yi7Rl96hq3Is9TYxjg=,iv:0Ayu7eiEVRAml+SycHWgmmP+rf4fZxhXAALor6d01rI=,tag:+aVf2mehHqak3K5xOHSJCA==,type:str] + pgp: + - created_at: "2026-07-28T13:46:53Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA7uy4qQr71wiAQ//cYdDJn/PZWg3nx7eNeHjByT8kpYiTgi28Hbx0MmaR5Dd + koqKVn47tSO8UkoIYkJ2uoImBJMmIqSLg5XjIhleUmSrZnrCLm0zhdgljMnkphME + N9NYHc2gd65inztRDoZi12zxeDdPLcSo1tKPBvIQi33tWODbT6E8TJ4YGKBEv1+X + NggSqgADxmYevrVYrzJtHCf7o369CUjHUedSkGPDEk7ZNKimb2kEdi4qKV+QFiX7 + y4iyqgeEepTjmloXSF6diC1F4cak96MqsNeIpiJLU5L1vWGGvb5J9znijpgqL6eX + aompNyrpUrwrF2dbsfb32o/AW4FjdJsC+7RfLPWs9qBneoUtU7LPisZpN+xQ9kgt + cx0CVY966y/0WTPm3coIhzO90fzL1kAn7+Bl5W5WhMBFRGDq7K0oqYcgdQmcr45o + yegnYxV4uO6Vgf9NV9zEqCvq+UqRZUwJhQSdmaPkvWQDhx91y7lca0WP90xwYMhN + jUvhHVcpZ7b++sbxNnvbhmnBBkAlwesQHVc85WSvok4kb3KpM1wju/j0KmOP2/lb + YazrzeXu47diSttopRU+JphtUQyuOdTqQ+k5x/1gt7qd8R4IbfKAFMKWPC/KLBnH + d4UIwa+ln/lWhhuR+/PxaPZrXGlm5SidM3r2SlH+1iGJfRzq1USaeEn9Vfp6OVvS + XgFZngqHN4Wv9zdT71EUGeixMz142maTlcPBM1OnHgiIkf3gLoInizU1VrZz/f8s + OOB4rHLwFjqj//Lze896q0kaWtnvZOuKdvTRsE7ep8VqAG2pVKwzcpINqnyTZss= + =ww9K + -----END PGP MESSAGE----- + fp: DC6910268E657FF70BA7EC289974494E76938DDC + encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$ + version: 3.10.2 diff --git a/kubernetes/infrastructure/observability/namespace.yaml b/kubernetes/infrastructure/observability/namespace.yaml new file mode 100644 index 0000000..bf138ff --- /dev/null +++ b/kubernetes/infrastructure/observability/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: observability + labels: + pod-security.kubernetes.io/enforce: privileged \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/vm-internal-ingress.yaml b/kubernetes/infrastructure/observability/vm-internal-ingress.yaml new file mode 100644 index 0000000..18e9713 --- /dev/null +++ b/kubernetes/infrastructure/observability/vm-internal-ingress.yaml @@ -0,0 +1,26 @@ +# Ingress exposing the VictoriaMetrics vminsert endpoint (vmsingle:8428) +# through Traefik as vm-internal.lan — used by the docker-host vmagent to +# remote_write metrics into the central VictoriaMetrics. +# external-dns auto-creates the vm-internal.lan A record pointing at the +# Traefik LoadBalancer (see kubernetes/system/external-dns). +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: vm-internal + namespace: observability + annotations: + external-dns.alpha.kubernetes.io/enabled: "true" + # No chart ingress is available for vmsingle, so declare it here. +spec: + ingressClassName: traefik + rules: + - host: vm-internal.lan + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: vmsingle-vm-victoria-metrics-k8s-stack + port: + number: 8428 \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/vmalert-rules.yaml b/kubernetes/infrastructure/observability/vmalert-rules.yaml new file mode 100644 index 0000000..e5f1e6c --- /dev/null +++ b/kubernetes/infrastructure/observability/vmalert-rules.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vmalert-rules + namespace: observability + labels: + app.kubernetes.io/name: vmalert + app.kubernetes.io/component: alerting-rules +data: + node.rules: | + groups: + - name: node + rules: + - alert: HighNodeCPU + expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90 + for: 5m + labels: + severity: warning + annotations: + summary: "High CPU on {{ $labels.instance }}" + description: "CPU usage above 90% for 5m on {{ $labels.instance }}." + - alert: HighNodeRAM + expr: 1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 0.9 + for: 10m + labels: + severity: warning + annotations: + summary: "RAM >90% on {{ $labels.instance }}" + description: "Available RAM below 10% for 10m on {{ $labels.instance }}." + - alert: LowDiskSpace + expr: 1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) > 0.85 + for: 10m + labels: + severity: warning + annotations: + summary: "Disk >85% full on {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} is more than 85% full for 10m." \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/vmstack-release.yaml b/kubernetes/infrastructure/observability/vmstack-release.yaml new file mode 100644 index 0000000..5d6ce49 --- /dev/null +++ b/kubernetes/infrastructure/observability/vmstack-release.yaml @@ -0,0 +1,19 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: vm-victoria-metrics-k8s-stack + namespace: observability +spec: + interval: 1m + chart: + spec: + sourceRef: + kind: HelmRepository + name: vm + namespace: observability + chart: victoria-metrics-k8s-stack + version: "0.43.2" + interval: 1m + valuesFrom: + - kind: Secret + name: vmstack-values \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/vmstack-repository.yaml b/kubernetes/infrastructure/observability/vmstack-repository.yaml new file mode 100644 index 0000000..278103e --- /dev/null +++ b/kubernetes/infrastructure/observability/vmstack-repository.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: vm + namespace: observability +spec: + interval: 1m + url: https://victoriametrics.github.io/helm-charts/ \ No newline at end of file diff --git a/kubernetes/infrastructure/observability/vmstack-values.yaml b/kubernetes/infrastructure/observability/vmstack-values.yaml new file mode 100644 index 0000000..fde7339 --- /dev/null +++ b/kubernetes/infrastructure/observability/vmstack-values.yaml @@ -0,0 +1,151 @@ +# victoria-metrics-k8s-stack Helm values +# SOPS-encrypted before commit (per .sops.yaml rules). +# All values for this chart live here; vmagent, grafana, vmalert, alertmanager, +# kube-state-metrics and node-exporter are sub-charts toggled on/off below. +# --------------------------------------------------------------------------- +# VictoriaMetrics single-binary (the metrics database) +# --------------------------------------------------------------------------- +vmsingle: + enabled: true + spec: + # Keep 7d of metrics; user can tune later. + retentionPeriod: 7d + storage: + storageClassName: longhorn + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi +# vmagent scrapes the cluster (node-exporter, kube-state-metrics, cAdvisor). +# That same vmagent is reused for the inner cluster; the docker host runs its +# own vmagent that remote_writes through vm-internal.lan. +vmagent: + enabled: true + # Tolerate the failover taint so the scrape DaemonSet covers all nodes. + tolerations: + - key: ENC[AES256_GCM,data:pERZJq2cU/V7,iv:eMeR91aKxAJmKpKLEL6pA9rcIHOrd+Mg+RmA3gt1xGw=,tag:l2igrTTHLclHPEO3bCq/dg==,type:str] + value: ENC[AES256_GCM,data:m0pJQf4RLt4=,iv:Au3k95QkD2wv3OP/yDyNQKDN5ZrDIuNAwhXaMCEEn+g=,tag:mEcqTG4e0aYwoHr4TtmgTg==,type:str] + operator: Equal + effect: NoSchedule + - key: ENC[AES256_GCM,data:lDW9tKXBee7p5HjH/FWmzpwtnrazCuohAQm47B5htn++6QUYIA==,iv:yJJdF/XaFXghwBOwC2lByMRM+M5MNP51ho3qTfAKqq0=,tag:WFNE1PMRFss+M/I6j/8Nbg==,type:str] + operator: Exists + effect: NoSchedule +# --------------------------------------------------------------------------- +# kube-state-metrics + node-exporter (metrics sources) +# --------------------------------------------------------------------------- +kubeStateMetrics: + enabled: true +nodeExporter: + enabled: true +# --------------------------------------------------------------------------- +# Grafana (UI) — served behind ingress at grafana.lan. +# --------------------------------------------------------------------------- +grafana: + enabled: true + persistence: + enabled: true + storageClassName: longhorn + accessModes: + - ReadWriteOnce + size: 5Gi + adminUser: admin + # SOPS encrypts this when the file is processed. + adminPassword: ENC[AES256_GCM,data:VXtozCJ6XZ7JqWuIrDTmZkfWDeglVXl9JZ9T0sY2OUDY9MM=,iv:PO4leV1zpTFZTB2/hWipitGec4zrHfW/gOkJBrGk24k=,tag:9GinyiqoA4qrbyeKJ33Iuw==,type:str] + # Provision both the chart's default VictoriaMetrics datasource AND a Loki + # datasource so metric/log correlation works in one UI. + additionalDataSources: + - name: Loki + type: loki + url: http://loki.observability.svc.cluster.local:3100 + access: proxy + isDefault: false + ingress: + enabled: true + ingressClassName: traefik + annotations: + external-dns.alpha.kubernetes.io/enabled: "true" + hosts: + - grafana.lan + path: / + pathType: Prefix + tls: [] +# --------------------------------------------------------------------------- +# vmalert — evaluates the node.rules ConfigMap (see vmalert-rules.yaml) against +# VictoriaMetrics, forwards firing alerts to Alertmanager below. +# --------------------------------------------------------------------------- +vmalert: + enabled: true + spec: + rule: + configMap: vmalert-rules + configMapKeyRef: + key: ENC[AES256_GCM,data:7obbOvuMe8Cpdg==,iv:oIzOCzFFA0cptWQMPiIOi5b76dR3zp96Y8JrdKOkE8A=,tag:ArteBgOhdz/7N4wNpWE83g==,type:str] + name: vmalert-rules + notifier: + alertmanagerUrl: http://vm-victoria-metrics-k8s-stack-alertmanager.observability.svc.cluster.local:9093 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi +# --------------------------------------------------------------------------- +# Alertmanager — 1 replica; reads its config from the alertmanager-config +# Secret (see alertmanager-config.yaml), which has a single n8n webhook receiver. +# --------------------------------------------------------------------------- +alertmanager: + enabled: true + replicaCount: 1 + storage: + storageClassName: longhorn + accessModes: + - ReadWriteOnce + size: 2Gi + configFromSecretRef: ENC[AES256_GCM,data:C6NDZCkX76QLaZLw/vY3B01+AA==,iv:RSpe3Zv1mPfo0nj73tWrf5IBd+Sfqo1NUrERvmVPBgs=,tag:4ZGHA2Kyj9zpGn6eylNbXw==,type:str] + configKey: alertmanager.yaml + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi +# --------------------------------------------------------------------------- +# Storage class default +# --------------------------------------------------------------------------- +defaultStorageClass: longhorn +sops: + lastmodified: "2026-07-28T13:46:53Z" + mac: ENC[AES256_GCM,data:ZyCYO9b3OMJo8Br9tzbK+bN3rZvlkk/scOEUSHwkAABeQxL7JDbcpjHL8WauSY57wLnl3sK9JQnUHS+/ghOiZu/QDuodbhpoRrE8jpdjo1+qZaqJBqjbG1NIYU/aLQnHr4bnIzh8oX0PYnsu2uSqkg1c0c1KCJmrvHtXlFwtWOs=,iv:zh8yVzExwT29JF+4jLcZgmuSPM6R/d/2IWTcYfK3Qn4=,tag:LYTYY6rJVEKJRsOp6G/T0g==,type:str] + pgp: + - created_at: "2026-07-28T13:46:53Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA7uy4qQr71wiAQ/+KP62YdvAUJjxvFoWH1skMELYx9Lj/5TRXnrHOtqQrpXC + UzKNgS3XFmgeUcpA3wXvhTEPhICtdC9KDvxkF0jPTATqM2WFPcF7V2vjhTjuAMtU + /g4RR44j6U2WRZJFjs0tEpVGVrw+HXiaACdgqy1P6OmnmgvTSBvZvj3AsBpHOHeM + teaE11couXeYnrZ9NJeCvhbcvJfYAkdpqyCLJ9UWEro3QFm4z6tvs6D4rq+nAhBb + MntlvArijXunkwbHLZVPh8SyfxgGooy9PBSEX+ymyuxPYmjTlarYgO7oBQJUWjRk + BqkKns9KKNPyaUP/37tXJA8Gvgog/Z9JKDT28lq5noe6CAeHbBbUUo4hZZA14zcD + /Lmkc03KdOKf/eLnEFGtoWjQJ38FD5+uAh5Zkixm1WW0wV/PdqevJrG8mRjn8FuP + 6GU+nWo+77BxDyKvA1O13iOtTnGZtg/Z+lXVUbpcM8F7QCGrabhLiWVe51iWjYfU + /Ub7HZHRDXmiz7Q6Kem5Wleoy3ME/lxFblx9v0LbO3Dc0lj/85EVNm9os8tt5PD0 + cvGE/cGsZfUfgwbjEVpPnRN0SzlQKfEr9KVQP4DGNAqDOzEl2AD4Dg3l29oAgSda + ZRXdYbEN/ZIAGUg1wNVfZJf2KoJA+wm8c4uk7KAAI0+bIeaongovU4vnylVCHIbS + XgGDxEIDZGOueHms+xEMIW6QmdazFdkaLumY0XrEInHMwqhKM0NOyZL2zd/liZtE + wb8xm/V+eOWhjnE2RynNBCwMI6VeVYUPkin2MKQcq+nr7SVpWQsm2+Jbj5YtWCk= + =yKiR + -----END PGP MESSAGE----- + fp: DC6910268E657FF70BA7EC289974494E76938DDC + encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$ + version: 3.10.2 diff --git a/kubernetes/kustomization.yaml b/kubernetes/kustomization.yaml index afbdf6b..9fdb7c1 100644 --- a/kubernetes/kustomization.yaml +++ b/kubernetes/kustomization.yaml @@ -24,6 +24,7 @@ resources: # Infrastructure - infrastructure/authentik - infrastructure/mail/stalwart + - infrastructure/observability # Personal - personal/linkwarden - personal/notesnook