Compare commits

...

3 Commits

Author SHA1 Message Date
dependabot[bot] 372cf772aa build(deps): bump victoriametrics/vmagent
Bumps victoriametrics/vmagent from v1.131.0 to v1.148.0.

---
updated-dependencies:
- dependency-name: victoriametrics/vmagent
  dependency-version: v1.148.0
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-07-28 13:48:29 +00:00
vhaudiquet 38ceb887a7 observability: deploy observability stack 2026-07-28 15:46:52 +02:00
vhaudiquet 3cfef81f89 longhorn: tryfix backups 2026-07-26 10:44:08 +02:00
26 changed files with 864 additions and 48 deletions
+1
View File
@@ -13,6 +13,7 @@ updates:
- "/docker/home/zigbee2mqtt"
- "/docker/infrastructure/mail/roundcube"
- "/docker/infrastructure/network/traefik"
- "/docker/infrastructure/observability"
- "/docker/infrastructure/squid"
- "/docker/infrastructure/sshportal"
- "/docker/personal/fireshare"
+5
View File
@@ -38,6 +38,11 @@ traefik:
branch: main
compose_file: docker/infrastructure/network/traefik/docker-compose.yml
observability:
repo: homeprod
branch: main
compose_file: docker/infrastructure/observability/docker-compose.yml
squid:
repo: homeprod
branch: main
@@ -0,0 +1,83 @@
// Grafana Alloy config for the docker host
// Tails Docker container logs via the docker_sd discovery +
// file streaming, applies container_name/image labels, and pushes them to
// the in-cluster Loki at loki-internal.lan (LAN-only Traefik Ingress).
logging {
level = "info"
format = "logfmt"
}
// Discover running Docker containers via the host socket.
discovery.docker "containers" {
host = "unix:///var/run/docker.sock"
}
// Relabel rules: strip the leading slash from __meta_docker_container_name and
// expose useful labels (container name, compose service, image).
discovery.relabel "containers" {
targets = discovery.docker.containers.targets
// Strip leading / from container name.
rule {
source_labels = ["__meta_docker_container_name"]
regex = "/(.*)"
target_label = "container_name"
replacement = "$1"
}
// Compose service name (if label is set).
rule {
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
target_label = "compose_service"
}
// Log stream (stdout/stderr).
rule {
source_labels = ["__meta_docker_container_log_stream"]
target_label = "stream"
}
// Mark the source so dashboards/alerts can distinguish the docker host.
rule {
target_label = "source"
replacement = "docker-host"
}
}
// Tail the container log files based on the docker_sd discovery. Each target
// exposes a `__path__`idders pointing at /var/lib/docker/containers/<id>/<id>-json.log
// (the default docker json-file driver path; we mount that host path read-only).
local.file "container_logs" {
targets = discovery.relabel.containers.output
tail_from_end = true
}
// Parse docker json-file logs ({"log":"...","stream":"stdout","time":"..."}).
loki.process "containers" {
forward_to = [loki.write.default.receiver]
stage.json {
expressions = {
log = "log",
stream = "stream",
time = "time",
}
}
stage.labels {
values = {
stream = "stream",
}
}
stage.timestamp {
source = "time"
format = "RFC3339Nano"
}
}
// Push to Loki in-cluster via the LAN-only Traefik Ingress (port 80).
loki.write "default" {
endpoint {
url = "http://loki-internal.lan/loki/api/v1/push"
}
}
@@ -0,0 +1,82 @@
# Docker-host collectors for the central observability stack on Kubernetes.
# Managed by swarm-cd (see .swarmcd/stacks.yaml).
#
# node-exporter + cAdvisor run locally (scraped by vmagent below).
# vmagent remote_writes metrics to vm-internal.lan (LAN-only Traefik Ingress) ->
# vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428
# alloy tails /var/lib/docker/containers + /var/log and pushes logs to
# loki-internal.lan -> loki.observability.svc.cluster.local:3100
services:
node-exporter:
image: prom/node-exporter:v1.9.1
container_name: obs-node-exporter
network_mode: host
pid: host
command:
- "--path.rootfs=/host"
volumes:
- /:/host:ro,rslave
restart: unless-stopped
labels:
- "traefik.enable=false"
cadvisor:
image: gcr.io/cadvisor/cadvisor:v0.49.1
container_name: obs-cadvisor
ports:
- "8080"
volumes:
- /:/rootfs:ro
- /var/run:/var/run:ro
- /sys:/sys:ro
- /dev/disk/:/dev/disk:ro
- /var/lib/docker/:/var/lib/docker:ro
restart: unless-stopped
labels:
- "traefik.enable=false"
vmagent:
image: victoriametrics/vmagent:v1.148.0
container_name: obs-vmagent
# vmagent listens on 8429 (its own metrics), scrapes node-exporter:9100
# and cadvisor:8080, remote_writes to vm-internal.lan
command:
- "-promscrape.config=/etc/vmagent/vmagent.yml"
- "-remoteWrite.url=http://vm-internal.lan/api/v1/write"
volumes:
- type: bind
source: /root/homeprod/docker/infrastructure/observability/vmagent.yml
target: /etc/vmagent/vmagent.yml
read_only: true
depends_on:
- node-exporter
- cadvisor
restart: unless-stopped
labels:
- "traefik.enable=false"
alloy:
image: grafana/alloy:v1.18.0
container_name: obs-alloy
command:
- "run"
- "/etc/alloy/config.river"
- "--server.http.listen.address=0.0.0.0"
- "--server.http.listen.port=12345"
volumes:
- type: bind
source: /root/homeprod/docker/infrastructure/observability/alloy.river
target: /etc/alloy/config.river
read_only: true
- /var/lib/docker/containers:/var/lib/docker/containers:ro
- /var/log:/var/log:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
restart: unless-stopped
labels:
- "traefik.enable=false"
networks:
default:
name: proxy
external: true
@@ -0,0 +1,22 @@
# vmagent scrape config for the docker host.
# Scrapes the two local collectors, forwards everything to the central
# VictoriaMetrics via the LAN-only Caddy route vm-internal.lan (Caddy proxies
# it to vmsingle-vm-victoria-metrics-k8s-stack.observability.svc.cluster.local:8428).
scrape_configs:
- job_name: docker-host-node
static_configs:
- targets:
- localhost:9100
labels:
# Distinguishes the docker host from K8s nodes in dashboards/alerts.
instance_source: docker-host
host: docker-r740
- job_name: docker-host-cadvisor
static_configs:
- targets:
- localhost:8080
labels:
instance_source: docker-host
host: docker-r740
@@ -0,0 +1,27 @@
apiVersion: v1
kind: Secret
metadata:
name: alertmanager-config
namespace: observability
labels:
app.kubernetes.io/name: alertmanager
app.kubernetes.io/component: notifiers
type: Opaque
stringData:
alertmanager.yaml: |
global:
resolve_timeout: 5m
route:
receiver: default
group_by: ["alertname", "severity", "instance"]
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
receivers:
- name: default
webhook_configs:
# n8n workflow webhook
- url: "http://n8n.lan/webhook/observability-alert"
send_resolved: true
@@ -0,0 +1,93 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: alloy-config
namespace: observability
labels:
app.kubernetes.io/name: alloy
app.kubernetes.io/component: log-collector
data:
config.river: |
// -----------------------------------------------------------------------------
// Grafana Alloy — log collection only (metrics cluster+host collection is
// handled by vmagent/node-exporter/cAdvisor elsewhere).
// Forwards pod logs to the in-cluster Loki single-binary.
// -----------------------------------------------------------------------------
logging {
level = "info"
format = "logfmt"
}
// Tail every pod's log files written under /var/log/pods/* on the host.
local.file_match "pods" {
path_targets = [{
__path__ = "/var/log/pods/*/*/*.log",
}]
}
// Discover Kubernetes metadata for the tailed files (pod_name, namespace,
// container_name, uid).
discovery.kubernetes "pods" {
role = "pod"
}
// Stream-stage relabel: read the filename to extract namespace/pod/container
// metadata in line with Loki/Promtail's expectations.
lrelabel "pods" {
targets = local.file_match.pods.targets
rule {
source_labels = ["__path__"]
regex = "/var/log/pods/(?P<namespace>[^/]+)/(?P<pod>[^/]+)/(?P<container>[^/]+)/(?P<uid>[^/]+)/.*"
target_label = "namespace"
replacement = "$namespace"
}
rule {
source_labels = ["__path__"]
regex = "/var/log/pods/([^/]+)/([^/]+)/([^/]+)/[^/]+/.*"
target_label = "pod"
replacement = "$pod"
}
rule {
source_labels = ["__path__"]
regex = "/var/log/pods/([^/]+)/([^/]+)/([^/]+)/[^/]+/.*"
target_label = "container"
replacement = "$container"
}
}
// Tail the actual log files.
local.file "pods_logs" {
targets = lrelabel.pods.output
read_from = "beginning"
tail_from_end = false
}
// Parse CRI-style lines ({"log":"...","stream":"stdout","time":"..."}).
loki.process "pods" {
forward_to = [loki.write.default.receiver]
stage.json {
expressions = {
log = "log",
stream = "stream",
time = "time",
}
}
stage.labels {
values = {
stream = "stream",
}
}
stage.timestamp {
source = "time"
format = "RFC3339Nano"
}
}
// Push to Loki in-cluster (no auth on .lan).
loki.write "default" {
endpoint {
url = "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push"
}
}
@@ -0,0 +1,19 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: alloy
namespace: observability
spec:
interval: 1m
chart:
spec:
sourceRef:
kind: HelmRepository
name: grafana
namespace: observability
chart: alloy
version: "1.11.0"
interval: 1m
valuesFrom:
- kind: Secret
name: alloy-values
@@ -0,0 +1,57 @@
# Grafana Alloy
# Runs as a DaemonSet on every node (tolerates the failover taint).
# Tails pod logs from /var/log/pods, attaches K8s labels (pod, namespace,
# service, container, image), and pushes them to Loki at
# http://loki.observability.svc.cluster.local:3100 (in-cluster, no auth).
# Use the externally provisioned ConfigMap (alloy-config) carrying config.river.
alloy:
configMap:
create: false
name: alloy-config
key: ENC[AES256_GCM,data:aevkZsfe4c4H34BJ,iv:4vrNZWwYWaxJUKGbNlnZfk0NM182nBQZuwvqUPslKNA=,tag:lh1iILM3uQAt0An6kN6EHg==,type:str]
# Mount host log paths so the DaemonSet can tail pod/container logs.
mounts:
varlog: true
dockercontainers: false
stabilityLevel: generally-available
tolerations:
- key: ENC[AES256_GCM,data:zOMWiA9dG+8o,iv:yHp9aBczV8nVRGlJmsBaKs9WdmE3+ii8gWOaj5Xok6M=,tag:8c5ZqBT+vvC0WCW8mi0c9g==,type:str]
value: ENC[AES256_GCM,data:2Q04VR9xLbQ=,iv:6+tQXx3IGUIcrse+poG4X/4pS5dN2dKvhb79sCy+nfA=,tag:jb/TZmGpvdsWnKoC5NdCow==,type:str]
operator: Equal
effect: NoSchedule
- key: ENC[AES256_GCM,data:aVq8GU1ZDg5m9XDEI3PoVbbykU36SnKpEgrohvZ0IBmH47qDKg==,iv:eMcHRO5mGp9hhY4KnuBmdPTjd/9lqtesjA/RRSu4QMw=,tag:2jaGuogkLcIuye2meJQMDQ==,type:str]
operator: Exists
effect: NoSchedule
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
sops:
lastmodified: "2026-07-28T13:46:53Z"
mac: ENC[AES256_GCM,data:u6J/EIP5aFtOi8ZtT8BPn978wJAEL1E3r0WcmG5WIsWKJBZMMQHUYRmHgkS6n0/UMtczWoCdXlxObS73ZQMSSkdtagQHE/HPh69dDdGkjxyM4mlP+YMwcnIQkxpJBAQTERXA+SVoWCZb7ztKbiJS+zPjOCfA95ixQINmBCdUgEc=,iv:WOXngWu3uNWYp18/xWghGJyONBrq3SSnPDwbjHBDozU=,tag:1fasTfuLBGXBCo9JPytsJw==,type:str]
pgp:
- created_at: "2026-07-28T13:46:53Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hQIMA7uy4qQr71wiAQ/8DSD0jEn3YE7XHTm4DWdfLHPs1IVg9hyiWSHe5/xH0siD
zWonfjcuytw8EHDzv6pQMNYmIpyAmJ92Al4ROJmBV8gZ59+U9goHmw5HgaQAupBJ
/gy8iFBteGOHJT+XJPoVb/bu449f7Vle4L3DrOAl6mwxE0WT/7ERoQBeSxL+2OTZ
iDPdq24Zy+crDHpoM0GcSnFRLX5UpmL2QXGB8PHyctwSQoDW+sbAY2ME6buOw8HI
AremKTRtFm6Pu1Ksdh6ZicaHpLSvLyByPPt2zYhlBQsG3s/SjJDlnakUZHhm7T4o
oHzSpKqV7NQbE09ZMyGJrEkl+rfCPeu6jHplW2EX3xjqLWNUF8GHcsDWljogZxXF
irBPkyKASEDdkHZgiZobU6BmvQlxILj104xNtUYTTChcKu3s0d5BrkIOGe+Y7rJd
fNvMfkmO8/URBgtsXy2xPKo0Jzu/0pZsPEp2lxNeyElwxQqo1CFwhKZPkeQ2eLpu
4X7/CMN8KJr6jLX8HhX9AivlCj+2CoT8SB5GAPw4p00eD0GIMWSNyOXjyW/qQZi3
82EyX9KIpHjALoOI4/3YgFpUO12AEsjirYKWkUM+IoDp5l+YX59QdM7zUVVyzsTG
2EXEdQqYVA7DIcMJb0tHqxDGMGxNayaUsikV7e/kcRVUyDSH+xOfaSUrDTqqv+XS
XAFro6848HF5uMIP4pfM+dxzkc9rFF4il0CuGWlscbz5q+Aw9LImGbpNzrWk3Lpi
beuwWKFTZ8zab5TH9UF/xqsQeg/nyzQnb3fFPk+owvY7fH6XiYbN5WjKjXGW
=kBpp
-----END PGP MESSAGE-----
fp: DC6910268E657FF70BA7EC289974494E76938DDC
encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
version: 3.10.2
@@ -0,0 +1,8 @@
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: grafana
namespace: observability
spec:
interval: 1m
url: https://grafana.github.io/helm-charts
@@ -0,0 +1,28 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: observability
resources:
- namespace.yaml
- vmstack-repository.yaml
- grafana-repository.yaml
- loki-repository.yaml
- vmstack-release.yaml
- loki-release.yaml
- alloy-release.yaml
- vmalert-rules.yaml
- alloy-config.yaml
- alertmanager-config.yaml
- vm-internal-ingress.yaml
- loki-internal-ingress.yaml
secretGenerator:
- name: vmstack-values
files:
- values.yaml=vmstack-values.yaml
- name: loki-values
files:
- values.yaml=loki-values.yaml
- name: alloy-values
files:
- values.yaml=alloy-values.yaml
configurations:
- kustomizeconfig.yaml
@@ -0,0 +1,6 @@
nameReference:
- kind: Secret
version: v1
fieldSpecs:
- path: spec/valuesFrom/name
kind: HelmRelease
@@ -0,0 +1,23 @@
# Ingress exposing the Loki push API (loki:3100) through Traefik as
# loki-internal.lan — used by the docker-host promtail to push container logs
# into the central Loki.
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: loki-internal
namespace: observability
annotations:
external-dns.alpha.kubernetes.io/enabled: "true"
spec:
ingressClassName: traefik
rules:
- host: loki-internal.lan
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: loki
port:
number: 3100
@@ -0,0 +1,19 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: loki
namespace: observability
spec:
interval: 1m
chart:
spec:
sourceRef:
kind: HelmRepository
name: grafana-community
namespace: observability
chart: loki
version: "18.5.4"
interval: 1m
valuesFrom:
- kind: Secret
name: loki-values
@@ -0,0 +1,9 @@
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: grafana-community
namespace: observability
spec:
interval: 1m
type: oci
url: oci://ghcr.io/grafana-community/helm-charts
@@ -0,0 +1,65 @@
# Loki single-binary Helm values
# SOPS-encrypted before commit (per .sops.yaml rules).
# Uses the new chart-v18 "Monolithic" deployment mode (SingleBinary mode is
# deprecated in Loki 4 / chart >=18.x). Bundles storage on a Longhorn PVC —
# no minio, no gateway. Resource-light; tuned for a 2-node home cluster.
loki:
deploymentMode: Monolithic
monolithic:
persistence:
storageClassName: longhorn
accessModes:
- ReadWriteOnce
size: 10Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
# 7d log retention (user can tune later).
limitsConfig:
retention_period: 7d
# Disable subcharts we don't need in monolithic mode.
test:
enabled: false
# Single binary only — disable distributed components.
backend:
replicas: 0
read:
replicas: 0
write:
replicas: 0
singleBinary:
replicas: 0
gateway:
enabled: false
minio:
enabled: false
sops:
lastmodified: "2026-07-28T13:46:53Z"
mac: ENC[AES256_GCM,data:bk1sxO2qIY/FT3+9dTCfpxKBbb9NlyEfTFuUihURUA/AmMJHV4XOx4EaBK6YFbSOVCLny5/aDbUDfF5DMudZTvKMNyU5sWq4X3+6BBZaL3vHSw2+UTA8pdpkNYpLl4uDq5aDFvAN2v/f/XwOpMIEte1L+yi7Rl96hq3Is9TYxjg=,iv:0Ayu7eiEVRAml+SycHWgmmP+rf4fZxhXAALor6d01rI=,tag:+aVf2mehHqak3K5xOHSJCA==,type:str]
pgp:
- created_at: "2026-07-28T13:46:53Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hQIMA7uy4qQr71wiAQ//cYdDJn/PZWg3nx7eNeHjByT8kpYiTgi28Hbx0MmaR5Dd
koqKVn47tSO8UkoIYkJ2uoImBJMmIqSLg5XjIhleUmSrZnrCLm0zhdgljMnkphME
N9NYHc2gd65inztRDoZi12zxeDdPLcSo1tKPBvIQi33tWODbT6E8TJ4YGKBEv1+X
NggSqgADxmYevrVYrzJtHCf7o369CUjHUedSkGPDEk7ZNKimb2kEdi4qKV+QFiX7
y4iyqgeEepTjmloXSF6diC1F4cak96MqsNeIpiJLU5L1vWGGvb5J9znijpgqL6eX
aompNyrpUrwrF2dbsfb32o/AW4FjdJsC+7RfLPWs9qBneoUtU7LPisZpN+xQ9kgt
cx0CVY966y/0WTPm3coIhzO90fzL1kAn7+Bl5W5WhMBFRGDq7K0oqYcgdQmcr45o
yegnYxV4uO6Vgf9NV9zEqCvq+UqRZUwJhQSdmaPkvWQDhx91y7lca0WP90xwYMhN
jUvhHVcpZ7b++sbxNnvbhmnBBkAlwesQHVc85WSvok4kb3KpM1wju/j0KmOP2/lb
YazrzeXu47diSttopRU+JphtUQyuOdTqQ+k5x/1gt7qd8R4IbfKAFMKWPC/KLBnH
d4UIwa+ln/lWhhuR+/PxaPZrXGlm5SidM3r2SlH+1iGJfRzq1USaeEn9Vfp6OVvS
XgFZngqHN4Wv9zdT71EUGeixMz142maTlcPBM1OnHgiIkf3gLoInizU1VrZz/f8s
OOB4rHLwFjqj//Lze896q0kaWtnvZOuKdvTRsE7ep8VqAG2pVKwzcpINqnyTZss=
=ww9K
-----END PGP MESSAGE-----
fp: DC6910268E657FF70BA7EC289974494E76938DDC
encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
version: 3.10.2
@@ -0,0 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
name: observability
labels:
pod-security.kubernetes.io/enforce: privileged
@@ -0,0 +1,26 @@
# Ingress exposing the VictoriaMetrics vminsert endpoint (vmsingle:8428)
# through Traefik as vm-internal.lan — used by the docker-host vmagent to
# remote_write metrics into the central VictoriaMetrics.
# external-dns auto-creates the vm-internal.lan A record pointing at the
# Traefik LoadBalancer (see kubernetes/system/external-dns).
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: vm-internal
namespace: observability
annotations:
external-dns.alpha.kubernetes.io/enabled: "true"
# No chart ingress is available for vmsingle, so declare it here.
spec:
ingressClassName: traefik
rules:
- host: vm-internal.lan
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: vmsingle-vm-victoria-metrics-k8s-stack
port:
number: 8428
@@ -0,0 +1,37 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: vmalert-rules
namespace: observability
labels:
app.kubernetes.io/name: vmalert
app.kubernetes.io/component: alerting-rules
data:
node.rules: |
groups:
- name: node
rules:
- alert: HighNodeCPU
expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
for: 5m
labels:
severity: warning
annotations:
summary: "High CPU on {{ $labels.instance }}"
description: "CPU usage above 90% for 5m on {{ $labels.instance }}."
- alert: HighNodeRAM
expr: 1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 0.9
for: 10m
labels:
severity: warning
annotations:
summary: "RAM >90% on {{ $labels.instance }}"
description: "Available RAM below 10% for 10m on {{ $labels.instance }}."
- alert: LowDiskSpace
expr: 1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) > 0.85
for: 10m
labels:
severity: warning
annotations:
summary: "Disk >85% full on {{ $labels.instance }} {{ $labels.mountpoint }}"
description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} is more than 85% full for 10m."
@@ -0,0 +1,19 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: vm-victoria-metrics-k8s-stack
namespace: observability
spec:
interval: 1m
chart:
spec:
sourceRef:
kind: HelmRepository
name: vm
namespace: observability
chart: victoria-metrics-k8s-stack
version: "0.43.2"
interval: 1m
valuesFrom:
- kind: Secret
name: vmstack-values
@@ -0,0 +1,8 @@
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: vm
namespace: observability
spec:
interval: 1m
url: https://victoriametrics.github.io/helm-charts/
@@ -0,0 +1,151 @@
# victoria-metrics-k8s-stack Helm values
# SOPS-encrypted before commit (per .sops.yaml rules).
# All values for this chart live here; vmagent, grafana, vmalert, alertmanager,
# kube-state-metrics and node-exporter are sub-charts toggled on/off below.
# ---------------------------------------------------------------------------
# VictoriaMetrics single-binary (the metrics database)
# ---------------------------------------------------------------------------
vmsingle:
enabled: true
spec:
# Keep 7d of metrics; user can tune later.
retentionPeriod: 7d
storage:
storageClassName: longhorn
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
# vmagent scrapes the cluster (node-exporter, kube-state-metrics, cAdvisor).
# That same vmagent is reused for the inner cluster; the docker host runs its
# own vmagent that remote_writes through vm-internal.lan.
vmagent:
enabled: true
# Tolerate the failover taint so the scrape DaemonSet covers all nodes.
tolerations:
- key: ENC[AES256_GCM,data:pERZJq2cU/V7,iv:eMeR91aKxAJmKpKLEL6pA9rcIHOrd+Mg+RmA3gt1xGw=,tag:l2igrTTHLclHPEO3bCq/dg==,type:str]
value: ENC[AES256_GCM,data:m0pJQf4RLt4=,iv:Au3k95QkD2wv3OP/yDyNQKDN5ZrDIuNAwhXaMCEEn+g=,tag:mEcqTG4e0aYwoHr4TtmgTg==,type:str]
operator: Equal
effect: NoSchedule
- key: ENC[AES256_GCM,data:lDW9tKXBee7p5HjH/FWmzpwtnrazCuohAQm47B5htn++6QUYIA==,iv:yJJdF/XaFXghwBOwC2lByMRM+M5MNP51ho3qTfAKqq0=,tag:WFNE1PMRFss+M/I6j/8Nbg==,type:str]
operator: Exists
effect: NoSchedule
# ---------------------------------------------------------------------------
# kube-state-metrics + node-exporter (metrics sources)
# ---------------------------------------------------------------------------
kubeStateMetrics:
enabled: true
nodeExporter:
enabled: true
# ---------------------------------------------------------------------------
# Grafana (UI) — served behind ingress at grafana.lan.
# ---------------------------------------------------------------------------
grafana:
enabled: true
persistence:
enabled: true
storageClassName: longhorn
accessModes:
- ReadWriteOnce
size: 5Gi
adminUser: admin
# SOPS encrypts this when the file is processed.
adminPassword: ENC[AES256_GCM,data:VXtozCJ6XZ7JqWuIrDTmZkfWDeglVXl9JZ9T0sY2OUDY9MM=,iv:PO4leV1zpTFZTB2/hWipitGec4zrHfW/gOkJBrGk24k=,tag:9GinyiqoA4qrbyeKJ33Iuw==,type:str]
# Provision both the chart's default VictoriaMetrics datasource AND a Loki
# datasource so metric/log correlation works in one UI.
additionalDataSources:
- name: Loki
type: loki
url: http://loki.observability.svc.cluster.local:3100
access: proxy
isDefault: false
ingress:
enabled: true
ingressClassName: traefik
annotations:
external-dns.alpha.kubernetes.io/enabled: "true"
hosts:
- grafana.lan
path: /
pathType: Prefix
tls: []
# ---------------------------------------------------------------------------
# vmalert — evaluates the node.rules ConfigMap (see vmalert-rules.yaml) against
# VictoriaMetrics, forwards firing alerts to Alertmanager below.
# ---------------------------------------------------------------------------
vmalert:
enabled: true
spec:
rule:
configMap: vmalert-rules
configMapKeyRef:
key: ENC[AES256_GCM,data:7obbOvuMe8Cpdg==,iv:oIzOCzFFA0cptWQMPiIOi5b76dR3zp96Y8JrdKOkE8A=,tag:ArteBgOhdz/7N4wNpWE83g==,type:str]
name: vmalert-rules
notifier:
alertmanagerUrl: http://vm-victoria-metrics-k8s-stack-alertmanager.observability.svc.cluster.local:9093
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 128Mi
# ---------------------------------------------------------------------------
# Alertmanager — 1 replica; reads its config from the alertmanager-config
# Secret (see alertmanager-config.yaml), which has a single n8n webhook receiver.
# ---------------------------------------------------------------------------
alertmanager:
enabled: true
replicaCount: 1
storage:
storageClassName: longhorn
accessModes:
- ReadWriteOnce
size: 2Gi
configFromSecretRef: ENC[AES256_GCM,data:C6NDZCkX76QLaZLw/vY3B01+AA==,iv:RSpe3Zv1mPfo0nj73tWrf5IBd+Sfqo1NUrERvmVPBgs=,tag:4ZGHA2Kyj9zpGn6eylNbXw==,type:str]
configKey: alertmanager.yaml
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 128Mi
# ---------------------------------------------------------------------------
# Storage class default
# ---------------------------------------------------------------------------
defaultStorageClass: longhorn
sops:
lastmodified: "2026-07-28T13:46:53Z"
mac: ENC[AES256_GCM,data:ZyCYO9b3OMJo8Br9tzbK+bN3rZvlkk/scOEUSHwkAABeQxL7JDbcpjHL8WauSY57wLnl3sK9JQnUHS+/ghOiZu/QDuodbhpoRrE8jpdjo1+qZaqJBqjbG1NIYU/aLQnHr4bnIzh8oX0PYnsu2uSqkg1c0c1KCJmrvHtXlFwtWOs=,iv:zh8yVzExwT29JF+4jLcZgmuSPM6R/d/2IWTcYfK3Qn4=,tag:LYTYY6rJVEKJRsOp6G/T0g==,type:str]
pgp:
- created_at: "2026-07-28T13:46:53Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hQIMA7uy4qQr71wiAQ/+KP62YdvAUJjxvFoWH1skMELYx9Lj/5TRXnrHOtqQrpXC
UzKNgS3XFmgeUcpA3wXvhTEPhICtdC9KDvxkF0jPTATqM2WFPcF7V2vjhTjuAMtU
/g4RR44j6U2WRZJFjs0tEpVGVrw+HXiaACdgqy1P6OmnmgvTSBvZvj3AsBpHOHeM
teaE11couXeYnrZ9NJeCvhbcvJfYAkdpqyCLJ9UWEro3QFm4z6tvs6D4rq+nAhBb
MntlvArijXunkwbHLZVPh8SyfxgGooy9PBSEX+ymyuxPYmjTlarYgO7oBQJUWjRk
BqkKns9KKNPyaUP/37tXJA8Gvgog/Z9JKDT28lq5noe6CAeHbBbUUo4hZZA14zcD
/Lmkc03KdOKf/eLnEFGtoWjQJ38FD5+uAh5Zkixm1WW0wV/PdqevJrG8mRjn8FuP
6GU+nWo+77BxDyKvA1O13iOtTnGZtg/Z+lXVUbpcM8F7QCGrabhLiWVe51iWjYfU
/Ub7HZHRDXmiz7Q6Kem5Wleoy3ME/lxFblx9v0LbO3Dc0lj/85EVNm9os8tt5PD0
cvGE/cGsZfUfgwbjEVpPnRN0SzlQKfEr9KVQP4DGNAqDOzEl2AD4Dg3l29oAgSda
ZRXdYbEN/ZIAGUg1wNVfZJf2KoJA+wm8c4uk7KAAI0+bIeaongovU4vnylVCHIbS
XgGDxEIDZGOueHms+xEMIW6QmdazFdkaLumY0XrEInHMwqhKM0NOyZL2zd/liZtE
wb8xm/V+eOWhjnE2RynNBCwMI6VeVYUPkin2MKQcq+nr7SVpWQsm2+Jbj5YtWCk=
=yKiR
-----END PGP MESSAGE-----
fp: DC6910268E657FF70BA7EC289974494E76938DDC
encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
version: 3.10.2
+1
View File
@@ -24,6 +24,7 @@ resources:
# Infrastructure
- infrastructure/authentik
- infrastructure/mail/stalwart
- infrastructure/observability
# Personal
- personal/linkwarden
- personal/notesnook
@@ -5,6 +5,7 @@ resources:
- namespace.yaml
- repository.yaml
- release.yaml
- recurring-jobs.yaml
secretGenerator:
- name: longhorn-values
files:
@@ -0,0 +1,34 @@
# Recurring backup and snapshot schedules for Longhorn.
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: daily-snapshot
namespace: longhorn-system
spec:
# Fast local recovery from accidental deletes.
name: daily-snapshot
task: snapshot
cron: "0 2 * * *"
retain: 7
concurrency: 2
labels:
type: snapshot
groups:
- default
---
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: daily-backup
namespace: longhorn-system
spec:
# Daily NFS backup for DR if a node is lost.
name: daily-backup
task: backup
cron: "0 3 * * *"
retain: 7
concurrency: 2
labels:
type: backup
groups:
- default
+34 -48
View File
@@ -2,8 +2,6 @@
defaultSettings:
# 2 replicas so volumes survive single-node loss
defaultReplicaCount: 2
# NFS backup target on TrueNAS
backupTarget: nfs://truenas.lan:/mnt/fast_app_data/longhorn-backup
# Allow full disk usage on single-node
storageOverProvisioningPercentage: 100
storageMinimalAvailablePercentage: 10
@@ -36,23 +34,23 @@ persistence:
# R740, but keeps things correct if that ever changes).
longhornManager:
tolerations:
- key: ENC[AES256_GCM,data:72tSl2wrzA+rNFPSi6y1sXNdzFd2QI9/DIoLPYv600yIWre9Jg==,iv:6FJCiqSNlTTKb0qp7Zupq6RTcwA5de8uxrFM9o8XdAw=,tag:I1gnHYR0XNgq60tOnXYFGA==,type:str]
- key: ENC[AES256_GCM,data:DDNTXJiwP2LQiOmqgf/L6tLIZiFgqh42dLDXj8xomGv7rlUP9A==,iv:eu6vnl3EiLu1DWWDWlxkQNcN6/uEhSkpufw7L9Ikr14=,tag:DJOMo8LDGmzkqvIEkfCTHg==,type:str]
operator: Exists
effect: NoSchedule
# Tolerate the failover taint so longhorn-manager runs on the P330
- key: ENC[AES256_GCM,data:Tru89mna++TT,iv:8gH3VLU6CCCC42SB6LfEaPY2MFHaD4cLwCWFkSHQV4A=,tag:H8gUvT97+GTgSJu7buUOjw==,type:str]
value: ENC[AES256_GCM,data:QYHvXr+rR44=,iv:jJrfJST2MG4pC71Qr2p4WAheDHcd49DbwrBVlLFqyk0=,tag:GJHrXSKfK6/N0IWmKemZxg==,type:str]
- key: ENC[AES256_GCM,data:rbWq0sruC6zC,iv:Eq5izQ4LXZ90Bhf1BJ7V9/Xo7HVMULeDHZ6NgItZfGQ=,tag:ixJt7YD4cBy+SrG2WVn0ng==,type:str]
value: ENC[AES256_GCM,data:wxGNkS97rg8=,iv:bBGGqcwibhXE0zkH+ZJEVgr20ogu7JwQxxDXqJcMdsQ=,tag:VLBQh8vRl1SiYB9S/vwLeg==,type:str]
operator: Equal
effect: NoSchedule
# longhorn-driver (CSI provisioner/attacher deployer) — tolerate the same
# taints so CSI operations work from the failover node.
longhornDriver:
tolerations:
- key: ENC[AES256_GCM,data:dQhIOMVadhzSbeJqzVBbikkKzk11GLxOHjSA1hdZkKRuKUONJA==,iv:nuurPvU/YbVdFxO+iPbWchqVNOWO+tssuF5ZhIHo8+Y=,tag:mbSWnxT6kP6ehZu1+kaAIA==,type:str]
- key: ENC[AES256_GCM,data:1EBSG4hL4heCnLFJYzTNaQWtFU3v6xwTlG+Y9/H4p5d0TZdFog==,iv:9ILcNXXzxzG5CYA4Ny7bKHBI581B1qJC3OBUObScxbQ=,tag:4B4JaFO0tfyVA6w7ypCY6g==,type:str]
operator: Exists
effect: NoSchedule
- key: ENC[AES256_GCM,data:9dhWE/kEwWr/,iv:pXHuR05UNdJlBaCMo0T4FHPN+8qGRXFGYQglP/P/VLc=,tag:KTbwQ4KW9zsSsftXQF4DRg==,type:str]
value: ENC[AES256_GCM,data:H6vDw998h28=,iv:IHNNLysSnMuqrmdPsCaXYFPUtGrxBFZdDFG+iOMBnvw=,tag:r1BZTI2PQ0wjhWaIi1yJZA==,type:str]
- key: ENC[AES256_GCM,data:ZSC5qg+ChOkI,iv:rIq8j7L7F641kWDzcSV9lrYxhHggT1FTZ8WF53vxvV0=,tag:Hj4RlF3usdIQi0G6Ecmdbg==,type:str]
value: ENC[AES256_GCM,data:YfiuCv2EKec=,iv:+6i5CT1ToCKy1gFtVVIFv5FLQaJaKHx4vQk8b0IBy3U=,tag:zHgzNuijbwI1QbvuLnZNvw==,type:str]
operator: Equal
effect: NoSchedule
# longhorn-ui — tolerate the same taints so the UI is reachable even if it
@@ -60,58 +58,46 @@ longhornDriver:
longhornUI:
replicas: 1
tolerations:
- key: ENC[AES256_GCM,data:deSbcZL4q+7tZaTg2pE1Ssq/WG0aS29EYAW+y1/Vl4Uq3E4tDQ==,iv:mE3AYFNrs7aAdKG1eRMJ21AwVdspJq42zWqytehn9z0=,tag:DsC7ehN8NhANSzWBQ3rMjQ==,type:str]
- key: ENC[AES256_GCM,data:n4UdwssGYhHQgDTpCpu+WgIPsdNc8uAaZtnURZt0/C3VfeLR4w==,iv:mqy0GqHN0KxtkETS7psu61Mr1oGmtPDhg2vQhVDas6c=,tag:uUd+NY90XK3i0aigc5ndCg==,type:str]
operator: Exists
effect: NoSchedule
- key: ENC[AES256_GCM,data:4UJN0oQsnPgz,iv:bnCs9OQzRNYxZS4LMVcI5Zgel4MES745e47/Wc+72Ps=,tag:+WD085TKD5CfFYJ3B0RotA==,type:str]
value: ENC[AES256_GCM,data:preRO+Om90c=,iv:Q6VF0hDh5Jz3ghB+VqUB/KO5i2wwxq68WWLq2ptnwOQ=,tag:ug333G7gCkSn60g/Lo5S7A==,type:str]
- key: ENC[AES256_GCM,data:isSkUNqax+AU,iv:jWgZyZL9VMmzYqoLCUkIiUuy1iBeOm2RE5CjZGCcXxQ=,tag:714V0YGb7jqebC4IRX/phw==,type:str]
value: ENC[AES256_GCM,data:Qbtu1wgV1oI=,iv:RihD43IA1+C3m9OkXACGgd5lY9lDWCyzIcw/MCoSMvg=,tag:WzQtC4rT+uovS6s36DJWUw==,type:str]
operator: Equal
effect: NoSchedule
# Disable ingress for now
ingress:
enabled: false
# Recurring backup and snapshot schedules
# - Daily snapshots: fast local recovery from accidental deletes
# - Daily NFS backups: DR if node is lost
recurringJobs:
enable: true
jobs:
- name: daily-snapshot
task: snapshot
cron: 0 2 * * *
retain: 7
concurrency: 2
labels:
type: snapshot
- name: daily-backup
task: backup
cron: 0 3 * * *
retain: 7
concurrency: 2
labels:
type: backup
# Default backup store (NFS target on TrueNAS).
defaultBackupStore:
# NFS endpoint exposing the TrueNAS dataset as a backup target.
backupTarget: nfs://truenas.lan:/mnt/fast_app_data/longhorn-backup
# No credentials required for an anonymously-readable NFS export.
backupTargetCredentialSecret: ""
# Poll the backup target every 5 minutes for new/removed backups.
pollInterval: 300
sops:
lastmodified: "2026-07-25T12:52:04Z"
mac: ENC[AES256_GCM,data:iYSrt+E3ZBR/x85S4A/Y4GalkJ5Y0NDwjSvh6Pfa8d5Qz+uFdGofnNISS7QgqmsGpLbLKMFHRxLV6L3R85V/o293SfNA/+iM8DXOoZTb8zQTLsPxjqxr8Hfl0cxBgV8LgZJzzG8pkeONDfqqtF6pGMgAz0SHfdAR2qdbCKTztis=,iv:YdWgVo9/YlusV1mqBWpVQVVdEQGSD7ytPVNJYjppmTw=,tag:XOxeIx5GWJcZD19zZTe5YQ==,type:str]
lastmodified: "2026-07-26T08:44:08Z"
mac: ENC[AES256_GCM,data:4shHlVWcVol/Z8b+5+tz6shT8x6gCZz2xhYv1VT7dHlgcYWWgq2V5mbGUtsVmWFzkGPC5rYvU3dUEiEkl0zb6Q1rP/fD6xQmlyIgH/ioHrfCvjorY5Rv8t4OFH/TyBK9hMwC+iFdAUJOdE9vYQoDmUMTGz6XwWiDY3PxRv1X9D4=,iv:DDDiStImg5qH2QP9lcSyW5+b+1P1mk6F0TWkt9p2syo=,tag:zJnZ455gsXPkE4Lfup2Rcg==,type:str]
pgp:
- created_at: "2026-07-25T12:52:04Z"
- created_at: "2026-07-26T08:44:08Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hQIMA7uy4qQr71wiARAAgDIE7tPjU9uheakXKYvn6IyNKkZjXTnV2sBPdoo4l0SW
ZYAu+L9SKnVKpmuQxz3HKqk/9Cib7Q/g9HdrXsrLkQ0xS6BC+4z+4JynxWHN3CII
0w7SNKVFV4RIXWwMQe0Hg+2OtzXyF1g8ZmNZu/L+/kusEnl7WX9BoYOB5i5wssM/
Nfb7LTlMGqmyQoWxbTzkdNuMbw2oVjD6pgL/VEpfCmvVzMpXScmTT9tFdJRYDuj6
r9a8wkFzCHNUWy3z7+00cQVnyDBA/UY/9pskeR58QH4keXp5xeBIYKxUe28L9JRg
YkVJjy1XFmmdY6VWmMo/Sx89YUQ9kMpf9KI/D0WkgGLqTihzsX4v7xxvWJfYXiPd
RmkRR4f19HfUapLzosO65rtYAG3BptdRPBpvx77evLkPwps34+dxUrVESYxY/I01
0BNNLDthEhBzfmh/yz3BcYOigZp1ziiXq201pn+qiDbLSDeoidm0IrN+g3Kib25g
dBTpRD3iEoBXeW2kbwltoUwsNkv59Gf/fj3DUQFAclP56b7UzRLDCQKiKIL17l2O
nU6yvz9iARPYvT3gzqOw8yaB780RQejR+8qiZca0WetNcVn7b+DMGpKgQXnchc1g
WzI3MdpNtajQGwBtHic3BG4xYAoN6sNmVrz+aGBIkBxFK3nMBzeWQJBSm6iiEwzS
XgFcb1jlb7dC/g8ltmAIOyjwnFHkfVI2/v+u+E1mW2VWBQqtABAHobh6uEf1jFzj
DZubFTVfrI4ssrvsjhm84mVJND9taaLPU4qbVYJ8G4kmP8UyKdHd1fJifNdp9+o=
=qY6k
hQIMA7uy4qQr71wiAQ/9FsBtkggrKwHfh5FFRUFkyqCU596d/R7ZEvsMP7zzCH9H
VAAYJxl8D6/KY5EavpeZH4s2ZnlzZjlb/qS2CKpSzXuN7ADPnK10cBxxAOQLp6KF
pqdkv6x5MyWbfv87IzM8cI0D/jlG2/iZPfBZy/YU4Rg8cefIl82vNgNADW2O68bK
w1VS75AYKmUfXZ2vqz+wX46YcjdXkrvRwzM9lzfIm2Q2dKMqBKSYzxMICZ/hWGRj
t33KbZLKC1Kg0NmYOY4GCt8hi6Ck4TjKvzxktsEpXYFDhWzXe9nr+axtrfypLrAn
fcLCQV1FSovMK+P4JxB/dd0u+F3RR500F3tBJPQ1xs1Rc1f8Cd2Ynr/3LWtTstkq
JyOfLFAj4/LG8o7ub/FSvLZZI052iRLZqlIIe0cFTqCz31yVEJOnLJWmME36hDEm
rfd3X9IvixfwWVTa3K0TxZta7u0Q0DsqHIt3eLdeTJC3wY4J+A/vUvPFv137YEqp
O+/ALTbwK7ViSQIupjr4kEcklBtiu9myNi3u8JNg67/XHwahCh0tfGk4jCevve5S
dDk4Ud7469LuDwYSZw0RdwuU89jQivvsp8/mZpQIoSVzl//iEQ2EOAff1gMfiPoj
1MrwGtrrMAo+2Uoxacd7LfFn/l1zRNiGw1rX21NzswmzIU84US4Grv9LGealQ8TS
XgH+datMRHIAPKcYY0u/B/PfyoKArlPUzQsIQOm02+s1zuvK/dArKxgObKTAr/u4
uNuVH3ObUVMb6VX4iwlxfS+VKpjotTbgY2TJfV46kGC3JJ0w1LdjEyJbpYlIz/k=
=HWEp
-----END PGP MESSAGE-----
fp: DC6910268E657FF70BA7EC289974494E76938DDC
encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$