mirror of
https://github.com/vhaudiquet/homeprod.git
synced 2026-07-21 21:17:08 +00:00
infra/p330: add p330 talos node
This commit is contained in:
@@ -9,3 +9,6 @@ terraform.tfstate.backup
|
||||
kubeconfig
|
||||
talosconfig
|
||||
|
||||
# Ignore rendered Talos machine configs (may contain cluster secrets)
|
||||
p330.yaml
|
||||
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
# Talos node for the P330 — joins the r740 "kube" cluster.
|
||||
terraform {
|
||||
required_providers {
|
||||
talos = {
|
||||
source = "siderolabs/talos"
|
||||
version = "0.9.0"
|
||||
}
|
||||
null = {
|
||||
source = "hashicorp/null"
|
||||
version = "3.2.3"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Read the r740 kube module state to reuse the cluster secrets & endpoint.
|
||||
# The r740 module exposes: client_configuration, machine_secrets, cluster_name,
|
||||
# cluster_endpoint, kube_host.
|
||||
data "terraform_remote_state" "r740_kube" {
|
||||
backend = var.r740_backend
|
||||
|
||||
config = var.r740_backend == "local" ? {
|
||||
path = "${var.r740_state_path}/terraform.tfstate"
|
||||
} : var.r740_backend_config
|
||||
}
|
||||
|
||||
locals {
|
||||
cluster_name = data.terraform_remote_state.r740_kube.outputs.cluster_name
|
||||
cluster_endpoint = data.terraform_remote_state.r740_kube.outputs.cluster_endpoint
|
||||
machine_secrets = data.terraform_remote_state.r740_kube.outputs.machine_secrets
|
||||
client_config = data.terraform_remote_state.r740_kube.outputs.client_configuration
|
||||
|
||||
# kubeconfig produced by the r740 kube module — used to wait for the node and
|
||||
# apply labels/taints. There is no in-tree kubernetes provider here on
|
||||
# purpose: managing a `kubernetes_node` resource conflicts with the node
|
||||
# object that kubelet itself creates, so we use a null_resource with kubectl
|
||||
# to wait + label + taint idempotently.
|
||||
kubeconfig_path = "${var.r740_state_path}/kubeconfig"
|
||||
|
||||
# Network config: static if node_subnet is provided, otherwise Talos DHCPs.
|
||||
static_network = var.node_subnet == null ? {} : {
|
||||
interfaces = [{
|
||||
interface = var.network_interface
|
||||
addresses = [var.node_subnet]
|
||||
routes = var.node_gateway == null ? [] : [{ gateway = var.node_gateway }]
|
||||
}]
|
||||
}
|
||||
|
||||
network_patch = {
|
||||
nameservers = var.nameservers
|
||||
}
|
||||
network_patch_merged = merge(local.network_patch, local.static_network)
|
||||
|
||||
machine_patch = {
|
||||
install = {
|
||||
image = var.installer_image
|
||||
disk = var.install_disk
|
||||
}
|
||||
network = merge(local.network_patch_merged, {
|
||||
# Pin the Kubernetes node name. Talos otherwise auto-generates a hostname
|
||||
# (e.g. "talos-8ec-vd1"), so the node registers with that random name
|
||||
# instead of var.p330_node_name — and our label/taint null_resource waits
|
||||
# for the wrong node. Setting machine.network.hostname fixes the node name.
|
||||
hostname = var.p330_node_name
|
||||
})
|
||||
# Kernel modules required by Longhorn (iSCSI + ext4) — must match the
|
||||
# control-plane nodes so Longhorn can schedule replicas on the failover node.
|
||||
kernel = {
|
||||
modules = [
|
||||
{ name = "iscsi_tcp" },
|
||||
{ name = "libiscsi" },
|
||||
{ name = "scsi_transport_iscsi" },
|
||||
{ name = "ext4" },
|
||||
]
|
||||
}
|
||||
sysctls = {
|
||||
"fs.inotify.max_user_instances" = "1024"
|
||||
"fs.inotify.max_user_watches" = "1048576"
|
||||
}
|
||||
kubelet = {
|
||||
# Keep the failover node from accumulating non-essential DaemonSet pods
|
||||
# via the regular scheduler; the taint does the heavy lifting, this is
|
||||
# belt-and-braces.
|
||||
extraArgs = {
|
||||
"register-with-taints" = "${var.failover_taint_key}=${var.failover_taint_value}:${var.failover_taint_effect}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Control-plane machine configuration. machine_type = "controlplane" makes
|
||||
# Talos generate a join config that runs the apiserver/controller-manager/
|
||||
# scheduler AND joins the existing etcd cluster as a new member (the cluster
|
||||
# was already bootstrapped by the r740 module's talos_machine_bootstrap).
|
||||
data "talos_machine_configuration" "p330" {
|
||||
cluster_name = local.cluster_name
|
||||
machine_type = "controlplane"
|
||||
cluster_endpoint = local.cluster_endpoint
|
||||
machine_secrets = local.machine_secrets
|
||||
config_patches = [
|
||||
yamlencode({
|
||||
machine = local.machine_patch
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
# Rendered config is written to disk so it can also be applied manually with
|
||||
# `talosctl apply-config --nodes <p330_host> --file p330.yaml` if needed.
|
||||
resource "local_file" "p330_machine_config" {
|
||||
filename = "${path.module}/p330.yaml"
|
||||
content = data.talos_machine_configuration.p330.machine_configuration
|
||||
}
|
||||
|
||||
# Apply the machine config to the running (maintenance-mode) node over the
|
||||
# Talos API. Because the config patch contains a `machine.install` block, when
|
||||
# Talos receives this config on a node booted from the USB (maintenance) image
|
||||
# it installs itself to install.disk and reboots into the installed system.
|
||||
# For a controlplane node it then joins the existing etcd cluster as a new
|
||||
# member and runs the control-plane components; for a worker it just registers
|
||||
# via kubelet.
|
||||
resource "talos_machine_configuration_apply" "p330" {
|
||||
client_configuration = local.client_config
|
||||
machine_configuration_input = data.talos_machine_configuration.p330.machine_configuration
|
||||
node = var.p330_host
|
||||
depends_on = [local_file.p330_machine_config]
|
||||
}
|
||||
|
||||
# Emit a talosconfig scoped to this node for ad-hoc `talosctl` use.
|
||||
data "talos_client_configuration" "p330" {
|
||||
cluster_name = local.cluster_name
|
||||
client_configuration = local.client_config
|
||||
nodes = [var.p330_host]
|
||||
}
|
||||
|
||||
resource "local_file" "talosconfig" {
|
||||
content = data.talos_client_configuration.p330.talos_config
|
||||
filename = "${path.module}/talosconfig"
|
||||
depends_on = [data.talos_client_configuration.p330]
|
||||
}
|
||||
|
||||
# Wait for the node to register with Kubernetes (kubelet creates the Node
|
||||
# object after Talos installs and reboots), then label it and (re)apply the
|
||||
# failover taint. This is idempotent: kubectl exits 0 if the label/taint already
|
||||
# exists. The taint is also set via kubelet `register-with-taints`, so this
|
||||
# null_resource is a safety net for manual edits / drift.
|
||||
resource "null_resource" "p330_node_label_and_taint" {
|
||||
triggers = {
|
||||
node = var.p330_node_name
|
||||
key = var.failover_taint_key
|
||||
value = var.failover_taint_value
|
||||
effect = var.failover_taint_effect
|
||||
kubeconfig = local.kubeconfig_path
|
||||
}
|
||||
|
||||
provisioner "local-exec" {
|
||||
# Wait for the node to show up, then label + taint. The wait loop is bounded
|
||||
# by kubectl --timeout; tune it via TF_LOG / re-run if the node is slow to
|
||||
# join (a controlplane node must first complete the etcd join handshake).
|
||||
command = <<-EOT
|
||||
set -euo pipefail
|
||||
KUBECONFIG="${local.kubeconfig_path}"
|
||||
export KUBECONFIG
|
||||
NODE="${var.p330_node_name}"
|
||||
|
||||
echo "Waiting for node $NODE to be registered (kubelet creates the Node object once Talos has installed, rebooted and joined etcd)..."
|
||||
# kubectl wait --for=condition=Ready fails instantly with NotFound if the
|
||||
# node object doesn't exist yet, so poll for existence first.
|
||||
# /bin/sh (dash) has no $SECONDS, so count iterations with a bounded loop.
|
||||
tries=240 # 240 * 5s = 20 minutes max
|
||||
until kubectl get node "$NODE" >/dev/null 2>&1; do
|
||||
tries=$((tries - 1))
|
||||
if [ "$tries" -le 0 ]; then
|
||||
echo "Timed out waiting for node $NODE to register." >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "Node $NODE registered. Waiting for it to become Ready..."
|
||||
|
||||
# Now wait for Ready (a controlplane node needs etcd joined + apiserver up).
|
||||
kubectl wait --for=condition=Ready "node/$NODE" --timeout=20m || \
|
||||
kubectl wait --for=jsonpath='{.status.conditions[?(@.reason=="KubeletReady")].status}'=True "node/$NODE" --timeout=20m
|
||||
|
||||
# Failover marker + taint (applied to both controlplane and worker nodes).
|
||||
kubectl label --overwrite node "$NODE" homeprod.io/failover=true
|
||||
|
||||
# Apply the taint idempotently (kubectl taint --overwrite is a no-op if it exists).
|
||||
kubectl taint --overwrite node "$NODE" \
|
||||
"${var.failover_taint_key}=${var.failover_taint_value}:${var.failover_taint_effect}"
|
||||
|
||||
echo "Node $NODE ready, labeled and tainted for failover-only scheduling."
|
||||
EOT
|
||||
}
|
||||
|
||||
depends_on = [talos_machine_configuration_apply.p330]
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
# Variables for the P330 Talos worker node that joins the r740 cluster.
|
||||
|
||||
variable "p330_host" {
|
||||
description = "Reachable IP/hostname of the P330 Talos node (for Talos API access)."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "p330_node_name" {
|
||||
description = "Kubernetes/Talos node name for the P330 (e.g. p330)."
|
||||
type = string
|
||||
default = "p330"
|
||||
}
|
||||
|
||||
variable "r740_state_path" {
|
||||
description = <<EOT
|
||||
Path to the Terraform state of the r740 kube module, used by terraform_remote_state
|
||||
to read the cluster secrets and endpoint so this node can join the existing cluster.
|
||||
Path is resolved by terraform_remote_state relative to the working directory where
|
||||
terraform runs (this module dir). The default points two levels up to the repo
|
||||
root and back down to the r740 kube module.
|
||||
EOT
|
||||
type = string
|
||||
default = "../../r740/kube"
|
||||
}
|
||||
|
||||
variable "r740_backend" {
|
||||
description = <<EOT
|
||||
Terraform backend type used by the r740 kube module.
|
||||
Set to "local" (default) when r740 uses a local tfstate file in its own directory,
|
||||
or the matching remote backend name ("s3", "remote", ...) if the r740 module uses
|
||||
a configured backend.
|
||||
EOT
|
||||
type = string
|
||||
default = "local"
|
||||
}
|
||||
|
||||
variable "r740_backend_config" {
|
||||
description = <<EOT
|
||||
Backend configuration map passed to terraform_remote_state when r740_backend is
|
||||
not "local". For a local backend this is ignored.
|
||||
EOT
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "installer_image" {
|
||||
description = <<EOT
|
||||
Talos installer image to use on the P330 (bare metal).
|
||||
Must be a **metal** Image Factory build that includes ixgbe.allow_unsupported_sfp=1
|
||||
in the kernel command line (sd-boot/UKI ignores machine.install.extraKernelArgs, so
|
||||
the param must be baked into the image). The default is a custom factory build
|
||||
(a18165114...).
|
||||
EOT
|
||||
type = string
|
||||
default = "factory.talos.dev/installer/a18165114f80c28601d05bc4ff1f6ea6d6b214882c5b9af7928aaf4d09741beb:v1.13.6"
|
||||
}
|
||||
|
||||
variable "install_disk" {
|
||||
description = "Block device path to install Talos on (e.g. /dev/sda, /dev/nvme0n1)."
|
||||
type = string
|
||||
default = "/dev/nvme0n1"
|
||||
}
|
||||
|
||||
variable "node_subnet" {
|
||||
description = <<EOT
|
||||
Static IPv4 address in CIDR notation for the P330 node (e.g. 10.1.2.132/24).
|
||||
Set to null to use DHCP. A static address is recommended for a failover node so
|
||||
DNS/affinity rules stay stable.
|
||||
EOT
|
||||
type = string
|
||||
default = "10.1.2.132/24"
|
||||
}
|
||||
|
||||
variable "node_gateway" {
|
||||
description = "IPv4 gateway for the P330 node. Ignored when node_subnet is null."
|
||||
type = string
|
||||
default = "10.1.2.1"
|
||||
}
|
||||
|
||||
variable "network_interface" {
|
||||
description = <<EOT
|
||||
Primary network interface name on the P330. Defaults to enp3s0f1 (the 10G Intel
|
||||
X520 NIC), which must be on the same L2/subnet as the r740 control plane so etcd
|
||||
peer traffic (TLS-verified against the r740's etcd cert SANs) doesn't cross a
|
||||
router. eno1 (1G) is left unconfigured.
|
||||
EOT
|
||||
type = string
|
||||
default = "enp3s0f1"
|
||||
}
|
||||
|
||||
variable "nameservers" {
|
||||
description = "DNS nameservers configured on the node (must work independently of kube)."
|
||||
type = list(string)
|
||||
default = ["10.1.2.148", "1.1.1.1"]
|
||||
}
|
||||
|
||||
variable "failover_taint_key" {
|
||||
description = "Taint key applied to the node to reserve it for failover workloads."
|
||||
type = string
|
||||
default = "dedicated"
|
||||
}
|
||||
|
||||
variable "failover_taint_value" {
|
||||
description = "Taint value applied to the node."
|
||||
type = string
|
||||
default = "failover"
|
||||
}
|
||||
|
||||
variable "failover_taint_effect" {
|
||||
description = "Taint effect applied to the node (NoSchedule / NoExecute)."
|
||||
type = string
|
||||
default = "NoSchedule"
|
||||
|
||||
validation {
|
||||
condition = contains(["NoSchedule", "PreferNoSchedule", "NoExecute"], var.failover_taint_effect)
|
||||
error_message = "failover_taint_effect must be NoSchedule, PreferNoSchedule or NoExecute."
|
||||
}
|
||||
}
|
||||
+52
-1
@@ -40,7 +40,9 @@ data "talos_machine_configuration" "kube" {
|
||||
yamlencode({
|
||||
machine = {
|
||||
install = {
|
||||
image = "factory.talos.dev/installer/ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515:v1.11.5"
|
||||
# Image Factory image with iSCSI extension for Longhorn.
|
||||
# Generated at https://factory.talos.dev — siderolabs/iscsi-tools + qemu-guest-agent
|
||||
image = "factory.talos.dev/installer/dc7b152cb3ea99b821fcb7340ce7168313ce393d663740b791c36f6e95fc8586:v1.13.6"
|
||||
}
|
||||
network = {
|
||||
nameservers = [
|
||||
@@ -53,6 +55,28 @@ data "talos_machine_configuration" "kube" {
|
||||
certSANs = [
|
||||
"${var.kube_host}", "${var.kube_hostname}"
|
||||
]
|
||||
# Kernel modules required by Longhorn (iSCSI + ext4)
|
||||
kernel = {
|
||||
modules = [
|
||||
{
|
||||
name = "iscsi_tcp"
|
||||
},
|
||||
{
|
||||
name = "libiscsi"
|
||||
},
|
||||
{
|
||||
name = "scsi_transport_iscsi"
|
||||
},
|
||||
{
|
||||
name = "ext4"
|
||||
},
|
||||
]
|
||||
}
|
||||
# Sysctls for Longhorn
|
||||
sysctls = {
|
||||
"fs.inotify.max_user_instances" = "1024"
|
||||
"fs.inotify.max_user_watches" = "1048576"
|
||||
}
|
||||
}
|
||||
cluster = {
|
||||
clusterName = "kube-${var.physical_hostname}"
|
||||
@@ -106,6 +130,33 @@ output "kubeconfig" {
|
||||
value = talos_cluster_kubeconfig.kube.kubeconfig_raw
|
||||
}
|
||||
|
||||
output "client_configuration" {
|
||||
description = "Talos client configuration (sensitive) used to manage nodes."
|
||||
sensitive = true
|
||||
value = talos_machine_secrets.kube.client_configuration
|
||||
}
|
||||
|
||||
output "machine_secrets" {
|
||||
description = "Talos machine secrets (sensitive) used to generate node configs."
|
||||
sensitive = true
|
||||
value = talos_machine_secrets.kube.machine_secrets
|
||||
}
|
||||
|
||||
output "cluster_name" {
|
||||
description = "Name of the Talos cluster the worker joins."
|
||||
value = "kube-${var.physical_hostname}"
|
||||
}
|
||||
|
||||
output "cluster_endpoint" {
|
||||
description = "Endpoint (host:port) of the Talos/Kubernetes API on the cluster."
|
||||
value = "https://${var.kube_host}:6443"
|
||||
}
|
||||
|
||||
output "kube_host" {
|
||||
description = "Reachable IP/hostname of the control-plane node."
|
||||
value = var.kube_host
|
||||
}
|
||||
|
||||
resource "local_file" "kubeconfig" {
|
||||
content = "${talos_cluster_kubeconfig.kube.kubeconfig_raw}"
|
||||
filename = "${path.module}/kubeconfig"
|
||||
|
||||
@@ -30,6 +30,9 @@ data:
|
||||
docker-r740 IN A 10.1.2.212
|
||||
truenas IN A 10.1.2.139
|
||||
|
||||
; P330
|
||||
p330 IN A 10.1.2.132
|
||||
|
||||
; PVE
|
||||
pve IN A 10.1.2.10
|
||||
docker-homeprod IN A 10.1.2.12
|
||||
|
||||
Reference in New Issue
Block a user