diff --git a/.gitignore b/.gitignore index e25fa4d..03eee87 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ terraform.tfstate.backup kubeconfig talosconfig +# Ignore rendered Talos machine configs (may contain cluster secrets) +p330.yaml + diff --git a/infra/p330/kube/main.tf b/infra/p330/kube/main.tf new file mode 100644 index 0000000..a338f70 --- /dev/null +++ b/infra/p330/kube/main.tf @@ -0,0 +1,195 @@ +# Talos node for the P330 — joins the r740 "kube" cluster. +terraform { + required_providers { + talos = { + source = "siderolabs/talos" + version = "0.9.0" + } + null = { + source = "hashicorp/null" + version = "3.2.3" + } + } +} + +# Read the r740 kube module state to reuse the cluster secrets & endpoint. +# The r740 module exposes: client_configuration, machine_secrets, cluster_name, +# cluster_endpoint, kube_host. +data "terraform_remote_state" "r740_kube" { + backend = var.r740_backend + + config = var.r740_backend == "local" ? { + path = "${var.r740_state_path}/terraform.tfstate" + } : var.r740_backend_config +} + +locals { + cluster_name = data.terraform_remote_state.r740_kube.outputs.cluster_name + cluster_endpoint = data.terraform_remote_state.r740_kube.outputs.cluster_endpoint + machine_secrets = data.terraform_remote_state.r740_kube.outputs.machine_secrets + client_config = data.terraform_remote_state.r740_kube.outputs.client_configuration + + # kubeconfig produced by the r740 kube module — used to wait for the node and + # apply labels/taints. There is no in-tree kubernetes provider here on + # purpose: managing a `kubernetes_node` resource conflicts with the node + # object that kubelet itself creates, so we use a null_resource with kubectl + # to wait + label + taint idempotently. + kubeconfig_path = "${var.r740_state_path}/kubeconfig" + + # Network config: static if node_subnet is provided, otherwise Talos DHCPs. + static_network = var.node_subnet == null ? {} : { + interfaces = [{ + interface = var.network_interface + addresses = [var.node_subnet] + routes = var.node_gateway == null ? [] : [{ gateway = var.node_gateway }] + }] + } + + network_patch = { + nameservers = var.nameservers + } + network_patch_merged = merge(local.network_patch, local.static_network) + + machine_patch = { + install = { + image = var.installer_image + disk = var.install_disk + } + network = merge(local.network_patch_merged, { + # Pin the Kubernetes node name. Talos otherwise auto-generates a hostname + # (e.g. "talos-8ec-vd1"), so the node registers with that random name + # instead of var.p330_node_name — and our label/taint null_resource waits + # for the wrong node. Setting machine.network.hostname fixes the node name. + hostname = var.p330_node_name + }) + # Kernel modules required by Longhorn (iSCSI + ext4) — must match the + # control-plane nodes so Longhorn can schedule replicas on the failover node. + kernel = { + modules = [ + { name = "iscsi_tcp" }, + { name = "libiscsi" }, + { name = "scsi_transport_iscsi" }, + { name = "ext4" }, + ] + } + sysctls = { + "fs.inotify.max_user_instances" = "1024" + "fs.inotify.max_user_watches" = "1048576" + } + kubelet = { + # Keep the failover node from accumulating non-essential DaemonSet pods + # via the regular scheduler; the taint does the heavy lifting, this is + # belt-and-braces. + extraArgs = { + "register-with-taints" = "${var.failover_taint_key}=${var.failover_taint_value}:${var.failover_taint_effect}" + } + } + } +} + +# Control-plane machine configuration. machine_type = "controlplane" makes +# Talos generate a join config that runs the apiserver/controller-manager/ +# scheduler AND joins the existing etcd cluster as a new member (the cluster +# was already bootstrapped by the r740 module's talos_machine_bootstrap). +data "talos_machine_configuration" "p330" { + cluster_name = local.cluster_name + machine_type = "controlplane" + cluster_endpoint = local.cluster_endpoint + machine_secrets = local.machine_secrets + config_patches = [ + yamlencode({ + machine = local.machine_patch + }) + ] +} + +# Rendered config is written to disk so it can also be applied manually with +# `talosctl apply-config --nodes --file p330.yaml` if needed. +resource "local_file" "p330_machine_config" { + filename = "${path.module}/p330.yaml" + content = data.talos_machine_configuration.p330.machine_configuration +} + +# Apply the machine config to the running (maintenance-mode) node over the +# Talos API. Because the config patch contains a `machine.install` block, when +# Talos receives this config on a node booted from the USB (maintenance) image +# it installs itself to install.disk and reboots into the installed system. +# For a controlplane node it then joins the existing etcd cluster as a new +# member and runs the control-plane components; for a worker it just registers +# via kubelet. +resource "talos_machine_configuration_apply" "p330" { + client_configuration = local.client_config + machine_configuration_input = data.talos_machine_configuration.p330.machine_configuration + node = var.p330_host + depends_on = [local_file.p330_machine_config] +} + +# Emit a talosconfig scoped to this node for ad-hoc `talosctl` use. +data "talos_client_configuration" "p330" { + cluster_name = local.cluster_name + client_configuration = local.client_config + nodes = [var.p330_host] +} + +resource "local_file" "talosconfig" { + content = data.talos_client_configuration.p330.talos_config + filename = "${path.module}/talosconfig" + depends_on = [data.talos_client_configuration.p330] +} + +# Wait for the node to register with Kubernetes (kubelet creates the Node +# object after Talos installs and reboots), then label it and (re)apply the +# failover taint. This is idempotent: kubectl exits 0 if the label/taint already +# exists. The taint is also set via kubelet `register-with-taints`, so this +# null_resource is a safety net for manual edits / drift. +resource "null_resource" "p330_node_label_and_taint" { + triggers = { + node = var.p330_node_name + key = var.failover_taint_key + value = var.failover_taint_value + effect = var.failover_taint_effect + kubeconfig = local.kubeconfig_path + } + + provisioner "local-exec" { + # Wait for the node to show up, then label + taint. The wait loop is bounded + # by kubectl --timeout; tune it via TF_LOG / re-run if the node is slow to + # join (a controlplane node must first complete the etcd join handshake). + command = <<-EOT + set -euo pipefail + KUBECONFIG="${local.kubeconfig_path}" + export KUBECONFIG + NODE="${var.p330_node_name}" + + echo "Waiting for node $NODE to be registered (kubelet creates the Node object once Talos has installed, rebooted and joined etcd)..." + # kubectl wait --for=condition=Ready fails instantly with NotFound if the + # node object doesn't exist yet, so poll for existence first. + # /bin/sh (dash) has no $SECONDS, so count iterations with a bounded loop. + tries=240 # 240 * 5s = 20 minutes max + until kubectl get node "$NODE" >/dev/null 2>&1; do + tries=$((tries - 1)) + if [ "$tries" -le 0 ]; then + echo "Timed out waiting for node $NODE to register." >&2 + exit 1 + fi + sleep 5 + done + echo "Node $NODE registered. Waiting for it to become Ready..." + + # Now wait for Ready (a controlplane node needs etcd joined + apiserver up). + kubectl wait --for=condition=Ready "node/$NODE" --timeout=20m || \ + kubectl wait --for=jsonpath='{.status.conditions[?(@.reason=="KubeletReady")].status}'=True "node/$NODE" --timeout=20m + + # Failover marker + taint (applied to both controlplane and worker nodes). + kubectl label --overwrite node "$NODE" homeprod.io/failover=true + + # Apply the taint idempotently (kubectl taint --overwrite is a no-op if it exists). + kubectl taint --overwrite node "$NODE" \ + "${var.failover_taint_key}=${var.failover_taint_value}:${var.failover_taint_effect}" + + echo "Node $NODE ready, labeled and tainted for failover-only scheduling." + EOT + } + + depends_on = [talos_machine_configuration_apply.p330] +} diff --git a/infra/p330/kube/variables.tf b/infra/p330/kube/variables.tf new file mode 100644 index 0000000..a63cf8a --- /dev/null +++ b/infra/p330/kube/variables.tf @@ -0,0 +1,118 @@ +# Variables for the P330 Talos worker node that joins the r740 cluster. + +variable "p330_host" { + description = "Reachable IP/hostname of the P330 Talos node (for Talos API access)." + type = string +} + +variable "p330_node_name" { + description = "Kubernetes/Talos node name for the P330 (e.g. p330)." + type = string + default = "p330" +} + +variable "r740_state_path" { + description = <