Files
homeprod/kubernetes/infrastructure/observability/vmstack-values.yaml
T
vhaudiquet c183f17606 fix(observability,photoprism): add resource limits to prevent OOM kills and sidecar crash-loops
- photoprism: add requests (200m/1Gi) and limits (2/4Gi) for RAW indexing
- grafana: add requests (100m/256Mi) and limits (500m/512Mi) to prevent OOMKill
- kube-state-metrics: add requests (50m/128Mi) and limits (200m/256Mi)
- node-exporter: add requests (50m/50Mi) and limits (200m/128Mi)
- vm-operator: add requests (50m/64Mi) and limits (200m/256Mi)
- loki sidecar (rules): add REQ_TIMEOUT=120, retry env vars, and small resources
- grafana sidecars (dashboards+datasources): add REQ_TIMEOUT=120 and retry env vars
2026-08-10 22:52:47 +02:00

250 lines
11 KiB
YAML

# victoria-metrics-k8s-stack Helm values
# The chart installs the VictoriaMetrics operator + CRDs and creates VMSingle,
# VMAgent, VMAlert, VMAlertmanager CRs.
# ---------------------------------------------------------------------------
# VictoriaMetrics single-binary (the metrics database)
# ---------------------------------------------------------------------------
vmsingle:
enabled: true
spec:
retentionPeriod: 7d
storage:
storageClassName: longhorn
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
# ---------------------------------------------------------------------------
# vmagent — scrapes node-exporter, kube-state-metrics, kubelet, etc.
# The docker host runs its own vmagent that remote_writes through
# vm-internal.lan (the Traefik Ingress → this vmsingle).
# ---------------------------------------------------------------------------
vmagent:
enabled: true
spec:
selectAllByDefault: true
scrapeInterval: 20s
tolerations:
- key: ENC[AES256_GCM,data:u/hZlRGlQsjm,iv:EIP1fb4hFx22j2pBmlmtLpUWcCqP1mZgLdgtG0UHnD0=,tag:ZDc67BaFIOzgobSOxzwoSg==,type:str]
value: ENC[AES256_GCM,data:37vtp0nJ+e0=,iv:v0Sch84ysia+ka9foooh+lqAQTc4mYhDxMRiiimIWy4=,tag:sNEHQI89QvMGj60dgoJy4g==,type:str]
operator: Equal
effect: NoSchedule
- key: ENC[AES256_GCM,data:xBpooUuWDs5ji8gvswL9dBc4hOsqlGpfYtsjGj0t8k8ott/Hdg==,iv:vXQyiFjim3d6qUsgl+lyVaolpvkY0OC6S0JraGuyWaM=,tag:dM6H8evZAw9BTa5UmETmxA==,type:str]
operator: Exists
effect: NoSchedule
# ---------------------------------------------------------------------------
# kube-state-metrics + node-exporter (metrics sources)
# ---------------------------------------------------------------------------
kube-state-metrics:
enabled: true
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 200m
memory: 256Mi
prometheus-node-exporter:
enabled: true
resources:
requests:
cpu: 50m
memory: 50Mi
limits:
cpu: 200m
memory: 128Mi
# ---------------------------------------------------------------------------
# VM operator (the Deployment created by this chart, not a CR). Without
# requests, when the API server stalls and elects a new leader, the operator
# pod gets killed by the kubelet on node pressure before it can re-acquire
# the lease. Small request keeps it alive.
# ---------------------------------------------------------------------------
operator:
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
# ---------------------------------------------------------------------------
# Grafana (UI) — served behind ingress at grafana.lan.
# ---------------------------------------------------------------------------
grafana:
enabled: true
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
sidecar:
dashboards:
env:
REQ_TIMEOUT: "120"
REQ_RETRY_CONNECT: "10"
REQ_RETRY_READ: "10"
REQ_RETRY_TOTAL: "20"
IGNORE_ALREADY_PROCESSED: "true"
datasources:
env:
REQ_TIMEOUT: "120"
REQ_RETRY_CONNECT: "10"
REQ_RETRY_READ: "10"
REQ_RETRY_TOTAL: "20"
IGNORE_ALREADY_PROCESSED: "true"
persistence:
enabled: true
storageClassName: longhorn
accessModes:
- ReadWriteOnce
size: 5Gi
adminUser: admin
# SOPS encrypts this when the file is processed.
adminPassword: ENC[AES256_GCM,data:ufl2eX7Q7cyIhXUeZxWgVeHa+5Fj9g==,iv:FxaR2v2PhfF+zR7+ScAsEzPQhpVsGly55ByXw6/q2jc=,tag:5w2M3TjMkN7LUHcHezpGhw==,type:str]
# Provision a Loki datasource alongside the chart's default VictoriaMetrics
# datasource so metric/log correlation works in one UI.
datasources:
datasources.yaml:
apiVersion: ENC[AES256_GCM,data:1w==,iv:XkLdUqFlwufQBD7ThJ+kPlhsWcY+0cqWkK1Mk9XOK6o=,tag:xk+jVn82cmPqTBqYarzfZg==,type:int]
datasources:
- name: ENC[AES256_GCM,data:v8Ub8A==,iv:PETLfpJPxLv9UcYAqWb+gaJTIcNixXRd7rztjhNnH+o=,tag:sdNcDdf7/g0x1P0X7u7egw==,type:str]
type: ENC[AES256_GCM,data:mYW4yQ==,iv:003u1YybknNt3pGVLtZNLfCWqi/gcq1cXjzgHa0wVnM=,tag:5B7Zo3KDgTmGrytpMll+Gg==,type:str]
url: ENC[AES256_GCM,data:5+SeDxRQk6qTV3YZ6H5Lw7qgvjHNU2riXfnxXYkx/MVi3DBmm5ugL9PpEprYYt9S,iv:cD/G0SUhuwxX5pjWuyQ9eUdTLekRbzEVo/0AgtN99xs=,tag:0K2BS4IGBNHcTWLUFYaw2Q==,type:str]
access: ENC[AES256_GCM,data:MrPS+4o=,iv:AZ0Jo0jhUhqETZnD1PhkZuQN/GqlHmI7nwv+ket5KOY=,tag:edCTNtS+fsjpN67fiAuoMA==,type:str]
isDefault: ENC[AES256_GCM,data:tL41ZSg=,iv:FSfXJumXRrUJeOdupCRuii/aNY3bbb9tjkMA8JUcV8o=,tag:s/FOwc6s2C7X9KKJo+CKng==,type:bool]
jsonData:
maxLines: ENC[AES256_GCM,data:NKwyqg==,iv:t2Rsefq9jsXpH92b/SiF4mh1Sy/bQ9SQBhg6xqc8nbU=,tag:w4o2v9s2BkK+T0PlEyBwBw==,type:int]
ingress:
enabled: true
ingressClassName: traefik
annotations:
external-dns.alpha.kubernetes.io/enabled: "true"
hosts:
- grafana.lan
path: /
pathType: Prefix
tls: []
# ---------------------------------------------------------------------------
# vmalert — evaluates VMRule CRs against VictoriaMetrics, forwards firing
# alerts to Alertmanager. selectAllByDefault picks up all VMRules in the
# namespace (including our vmalert-rules.yaml VMRule CR).
# ---------------------------------------------------------------------------
vmalert:
enabled: true
spec:
selectAllByDefault: true
evaluationInterval: 20s
notifiers:
- url: http://vm-victoria-metrics-k8s-stack-alertmanager.observability.svc.cluster.local:9093
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 128Mi
# ---------------------------------------------------------------------------
# Alertmanager — 1 replica; inline config with a single n8n webhook receiver.
# n8n runs on the docker host and fans out to email/Telegram/whatever.
# ---------------------------------------------------------------------------
alertmanager:
enabled: true
spec:
replicaCount: 1
port: "9093"
selectAllByDefault: true
storage:
volumeClaimTemplate:
spec:
storageClassName: longhorn
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 128Mi
config:
route:
receiver: n8n-webhook
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
receivers:
- name: n8n-webhook
webhook_configs:
- url: http://n8n.lan/webhook/observability-alert
send_resolved: true
extraRules:
node-alerts:
groups:
- name: node
rules:
- alert: HighNodeCPU
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
for: 10m
labels:
severity: warning
annotations:
summary: High CPU on {{ $labels.instance }}
description: CPU usage above 80% for 10 minutes.
- alert: HighNodeRAM
expr: 100 - ((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100) > 85
for: 10m
labels:
severity: warning
annotations:
summary: High RAM on {{ $labels.instance }}
description: RAM usage above 85% for 10 minutes.
- alert: LowDiskSpace
expr: |
100 - ((node_filesystem_avail_bytes{mountpoint!~"/run.*|/var/lib/docker.*"} /
node_filesystem_size_bytes{mountpoint!~"/run.*|/var/lib/docker.*"}) * 100) > 85
for: 10m
labels:
severity: warning
annotations:
summary: Low disk space on {{ $labels.instance }} {{ $labels.mountpoint }}
description: Disk usage above 85% for 10 minutes.
sops:
lastmodified: "2026-08-10T20:52:47Z"
mac: ENC[AES256_GCM,data:9R6bXpoKxowWpkoSOcjxkVI1EkKaoCiy7/c3/LYE81j50ThgPUXlcAnf024D0tXHv0LYCJm54qXNDwkg1Jzb3SYOt15Xbw9wPHyxtUFHKDb12BWGtyrE7TRw6VYdXs1P+TzZDJZPJJjSyTOW+hdB2WxNu7QG1M27Bg3DE28iOdg=,iv:OQisyRio+fqzdSU16whSJ26rvGibqP4FardJ0p3Ubeo=,tag:fHPd1TnzsmBWKLb6QTP7Fg==,type:str]
pgp:
- created_at: "2026-08-10T20:52:47Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hQIMA7uy4qQr71wiAQ/+Kc8P/t7IZ4shbHhrh25wdm8mI27QZ/h3uujubjgoWooD
oh+6zF7sqP36hOxyFjGBnmyHCp1Qqk4lxt3mBT0f5TR/q8DkBTbUDm8PTouZTG8/
dbEt29r7dzVrH+O7vMf2QWGt7wpvv5iOJsOXBJXvdaiKJCM0noaLryHPnzTbt1BQ
tAN5vk1wlDMZ3lPekKBRQe8W5dWAcfo5WSf5dPH0cVcXtDvO86uCPN+hN9dSp0KZ
LeC1WPRrNCLoHYxF4uFl/DYlnDbS0mg9DJqRqSvLdMqTkAIZUHEv/5Qwg1nsNGjk
5K2OWdbBz2vU/8hoN/9Qmw3gfZHfA4NW+DFzW6bv+5siATSNTObkLCBOHhgWd1oY
hRRhJeD6o6KXZhwai5OnTH3VMH8+OYp9wxYVhVDQ4y3+JOtbgCuDI18q7dkU7mAy
9nJMn0IgmqkRw7v8hfzjV8T7Ktv/b+HlJq+PB6/UXGaUutgrPFa70RLLAkFfFKUj
2oV3yhRHvPTTDXx6bw5knBDRh/XvtX2JJVgp9x7hnKbHr905qithP+tABNAloX3V
zOeEBV4EnIZk/sMB5CQcR0A8gznGpskYtowEVVr9pouNaC8o8D2qfCRIYvcaDCFU
t5N4te0vg6uLbW9PNokO6rzhJ+7lweC4Ou2zRAxRhSgzmi81ylIdNMKGle+5sn7S
XAHRSz9TKJf061m2kk+Ku/yxnzN2CoH4z+x65VvuK4qpVy2dSC00xyXdR4HxGzVO
jpRlAuWFlMu32gTa9cCXQjVPkYE0NbniGtlZkdsCUhydnVtNqqdm0JXTNvPS
=U+mY
-----END PGP MESSAGE-----
fp: DC6910268E657FF70BA7EC289974494E76938DDC
encrypted_regex: ^(password|value|ssh-key|api-key|user|username|privateKey|clientSecret|clientId|apiKey|extraArgs.*|.*Secret.*|extraEnvVars|.*SECRET.*|.*secret.*|key|.*Password|.*\.ya?ml)$
version: 3.10.2