diff --git a/kubernetes/system/flux/kustomization.yaml b/kubernetes/system/flux/kustomization.yaml index c7842b6..69ac2cc 100644 --- a/kubernetes/system/flux/kustomization.yaml +++ b/kubernetes/system/flux/kustomization.yaml @@ -13,3 +13,4 @@ resources: - longhorn-jobs.yaml - openclaw-operator.yaml - openclaw.yaml + - kyverno.yaml diff --git a/kubernetes/system/flux/kyverno.yaml b/kubernetes/system/flux/kyverno.yaml new file mode 100644 index 0000000..bb8233c --- /dev/null +++ b/kubernetes/system/flux/kyverno.yaml @@ -0,0 +1,24 @@ +--- +# Flux Kustomization for Kyverno +# Separate from main homeprod because Kyverno CRDs (installed by the HelmRelease) +# must be present before the ClusterPolicy can be applied. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: kyverno + namespace: flux-system +spec: + interval: 5m + prune: true + sourceRef: + kind: GitRepository + name: homeprod + path: ./kubernetes/system/kyverno + # Wait for the admission controller Deployment to be Ready before Flux + # considers this Kustomization healthy — this ensures the ClusterPolicy + # is applied after the CRDs and controller are live. + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: kyverno-admission-controller + namespace: kyverno diff --git a/kubernetes/system/kyverno/kustomization.yaml b/kubernetes/system/kyverno/kustomization.yaml new file mode 100644 index 0000000..25aa8c5 --- /dev/null +++ b/kubernetes/system/kyverno/kustomization.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kyverno +resources: + - namespace.yaml + - repository.yaml + - release.yaml + # ClusterPolicy that strips CPU requests for emergency relief (toggleable). + # See policy-strip-cpu-requests.yaml header for the toggle instructions. + - policy-strip-cpu-requests.yaml +configurations: + - kustomizeconfig.yaml diff --git a/kubernetes/system/kyverno/kustomizeconfig.yaml b/kubernetes/system/kyverno/kustomizeconfig.yaml new file mode 100644 index 0000000..165ed1d --- /dev/null +++ b/kubernetes/system/kyverno/kustomizeconfig.yaml @@ -0,0 +1,6 @@ +nameReference: + - kind: HelmRepository + version: v1 + fieldSpecs: + - path: spec/chart/spec/sourceRef/name + kind: HelmRelease diff --git a/kubernetes/system/kyverno/namespace.yaml b/kubernetes/system/kyverno/namespace.yaml new file mode 100644 index 0000000..3bcedeb --- /dev/null +++ b/kubernetes/system/kyverno/namespace.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kyverno + labels: + # Pod Security Admission: Kyverno's own pods run privileged-sidecars/agent. + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged diff --git a/kubernetes/system/kyverno/policy-strip-cpu-requests.yaml b/kubernetes/system/kyverno/policy-strip-cpu-requests.yaml new file mode 100644 index 0000000..4dbcced --- /dev/null +++ b/kubernetes/system/kyverno/policy-strip-cpu-requests.yaml @@ -0,0 +1,114 @@ +--- +# ============================================================================ +# CPU-REQUEST RELIEF — TOGGLEABLE CLUSTER POLICY +# ============================================================================ +# Emergency relief for CPU-saturated scenarios +# +# When ENABLED, this Kyverno ClusterPolicy removes the `resources.requests.cpu` +# field from every Pod created/updated in the cluster (except Kyverno's own pods, +# critical kube-system control-plane pods, and the Longhorn instance-manager / +# engine-image / longhorn-manager, which must keep guarantees so admission and +# the storage layer keep working). With no CPU request, pods schedule onto +# whatever capacity is free and burst freely. +# +# === TOGGLE === +# The policy is ACTIVE by default (failurePolicy: Ignore = mutate, never block). +# +# DISABLE relief (restore the per-app CPU requests on next Flux reconcile): +# kubectl patch clusterpolicy.kyverno.io strip-cpu-requests --type=merge \ +# -p '{"spec":{"failurePolicy":"Never"}}' +# # failurePolicy: Never tells Kyverno to skip the rule without mutating, so the +# # original CPU requests from each HelmRelease are restored on next reconcile. +# +# RE-ENABLE relief (strip requests again): +# kubectl patch clusterpolicy.kyverno.io strip-cpu-requests --type=merge \ +# -p '{"spec":{"failurePolicy":"Ignore"}}' +# ============================================================================ + +apiVersion: kyverno.io/v2 +kind: ClusterPolicy +metadata: + name: strip-cpu-requests + annotations: + policies.kyverno.io/title: Strip CPU Requests (Emergency Relief) + policies.kyverno.io/category: Other + policies.kyverno.io/severity: low + policies.kyverno.io/subject: Pod + policies.kyverno.io/description: >- + Removes resources.requests.cpu from pods so they can be scheduled on an + overloaded single-node cluster. Toggle via spec.failurePolicy: + Ignore = active (strip), Never = inactive (restore). +spec: + # Never block a pod if Kyverno admission is itself under pressure. + failurePolicy: Ignore + # background: true so the backgroundController also mutates ALREADY-EXISTING + # pods (not just new/updated ones at admission). This makes the relief take + # effect immediately on flip-on, without waiting for a pod restart. + background: true + rules: + - name: remove-cpu-request-from-containers + match: + any: + - resources: + kinds: + - Pod + # Keep guarantees for things that must keep running so admission and the + # storage layer keep working even while everything else is bursting. + exclude: + any: + - resources: + namespaces: + - kyverno + - resources: + namespaces: + - kube-system + names: + # Control-plane static pods (apiserver, scheduler, controller-manager) + - kube-apiserver-* + - kube-scheduler-* + - kube-controller-manager-* + - resources: + namespaces: + - longhorn-system + selector: + matchLabels: + app: longhorn-manager + - resources: + namespaces: + - longhorn-system + selector: + matchLabels: + app: instance-manager + - resources: + namespaces: + - longhorn-system + selector: + matchLabels: + app: engine-image + mutate: + # Use `foreach` over the pod's own containers (relative paths, NOT + # request.object.*) so the rule works in BOTH admission AND background + # modes. The backgroundController has no admission request, so + # request.object.* is unavailable in background scanning. + foreach: + - list: "spec.containers[]" + patchStrategicMerge: | + spec: + containers: + - name: "{{ element.name }}" + resources: + requests: + cpu: null + - list: "spec.initContainers[]" + preconditions: + all: + - key: "{{ spec.initContainers[] || `[]` | length(@) }}" + operator: GreaterThan + value: 0 + patchStrategicMerge: | + spec: + initContainers: + - name: "{{ element.name }}" + resources: + requests: + cpu: null diff --git a/kubernetes/system/kyverno/release.yaml b/kubernetes/system/kyverno/release.yaml new file mode 100644 index 0000000..f3a9e79 --- /dev/null +++ b/kubernetes/system/kyverno/release.yaml @@ -0,0 +1,70 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: kyverno + namespace: kyverno +spec: + interval: 5m + chart: + spec: + sourceRef: + kind: HelmRepository + name: kyverno + namespace: kyverno + chart: kyverno + version: '3.3.7' + interval: 5m + # Minimal footprint for a single-node cluster: + # - admissionReports disabled (no extra Redis/Postgres) + # - admissionController + backgroundController only (no reports, no + # cleanupController) — the backgroundController lets the ClusterPolicy also + # mutate ALREADY-RUNNING pods (not just new/updated ones), so flipping the + # relief on immediately strips CPU requests from existing workloads. + # - tolerates the control-plane node (allowSchedulingOnMasters=true on p330) + # - low CPU request so Kyverno's own admission doesn't itself deadlock the node + values: + admissionController: + rbac: + create: true + # Scale to 1 replica on this single-node cluster + replicas: 1 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + # The background controller re-applies policies to already-existing resources, + # so the CPU-request strip also affects pods that were created BEFORE the policy + # (and pods Flux re-applies with their original requests between toggles). + backgroundController: + enabled: true + replicas: 1 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + # Reports/reportsController are for policy-report auditing — not needed for a + # one-off relief toggle. + reportsController: + enabled: false + cleanupController: + enabled: false + # No autoscaling on a single node. + autoscaling: + admissionController: + enabled: false + backgroundController: + enabled: false diff --git a/kubernetes/system/kyverno/repository.yaml b/kubernetes/system/kyverno/repository.yaml new file mode 100644 index 0000000..2c2e04c --- /dev/null +++ b/kubernetes/system/kyverno/repository.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: kyverno + namespace: kyverno +spec: + interval: 5m + url: https://kyverno.github.io/kyverno/