homeprod/kubernetes/infrastructure/observability/vmalert-rules.yaml

# VMRule CR for custom alerting rules (node CPU/RAM/disk).
# The VictoriaMetrics operator's vmalert picks this up automatically via
# selectAllByDefault
apiVersion: victoriametrics.com/v1beta1
kind: VMRule
metadata:
  name: node-alerts
  namespace: observability
spec:
  groups:
    - name: node
      rules:
        - alert: HighNodeCPU
          expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "High CPU on {{ $labels.instance }}"
            description: "CPU usage above 80% for 10 minutes."
        - alert: HighNodeRAM
          expr: 100 - ((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100) > 85
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "High RAM on {{ $labels.instance }}"
            description: "RAM usage above 85% for 10 minutes."
        - alert: LowDiskSpace
          expr: |
            100 - ((node_filesystem_avail_bytes{mountpoint!~"/run.*|/var/lib/docker.*"} /
                    node_filesystem_size_bytes{mountpoint!~"/run.*|/var/lib/docker.*"}) * 100) > 85
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Low disk space on {{ $labels.instance }} {{ $labels.mountpoint }}"
            description: "Disk usage above 85% for 10 minutes."