Add encryption to in-cluster pod traffic

assets/charts/control-plane/calico-host-protection/templates/host-protection.yaml

@@ -1,15 +1,3 @@
-# General
-# =======
----
-# Disable default failsafe inbound rules
-apiVersion: crd.projectcalico.org/v1
-kind: FelixConfiguration
-metadata:
-  name: default
-spec:
-  failsafeInboundHostPorts: []
-  bpfLogLevel: Info
-
 # Firewall policy
 # ===============
 ---

assets/charts/control-plane/calico/templates/config.yaml

@@ -13,7 +13,11 @@ data:
   # - Otherwise, if VXLAN or BPF mode is enabled, set to your network MTU - 50
   # - Otherwise, if IPIP is enabled, set to your network MTU - 20
   # - Otherwise, if not using any encapsulation, set to your network MTU.
-  veth_mtu: "{{ .Values.calico.networkMTU }}"
+  {{- if .Values.calico.encryptPodTraffic }}
+  veth_mtu: "{{ sub .Values.calico.networkMTU 60 }}" # Wireguard overhead.
+  {{- else }}
+  veth_mtu: "{{ sub .Values.calico.networkMTU 20 }}" # IP in IP overhead.
+  {{- end}}
   # The CNI network configuration to install on each node. The special
   # values in this config will be automatically populated.
   cni_network_config: |-

assets/charts/control-plane/calico/templates/felixconfig.yaml

@@ -0,0 +1,10 @@
+apiVersion: crd.projectcalico.org/v1
+kind: FelixConfiguration
+metadata:
+  name: default
+spec:
+  {{- if hasKey .Values.calico "failsafeInboundHostPorts"  }}
+  failsafeInboundHostPorts: {{ .Values.calico.failsafeInboundHostPorts }}
+  {{- end }}
+  bpfLogLevel: Info
+  wireguardEnabled: {{ .Values.calico.encryptPodTraffic }}

assets/charts/control-plane/calico/values.yaml

@@ -13,3 +13,6 @@ calico:
   podCIDR: 10.2.0.0/16
   networkEncapsulation: "ipipMode: Always"
   blockedMetadataCIDRs: []
+  # Lokomotive specific change.
+  # failsafeInboundHostPorts:
+  encryptPodTraffic: false

assets/terraform-modules/aws/flatcar-linux/kubernetes/bootkube.tf

@@ -30,4 +30,5 @@ module "bootkube" {
 
   bootstrap_tokens     = var.enable_tls_bootstrap ? concat([local.controller_bootstrap_token], var.worker_bootstrap_tokens) : []
   enable_tls_bootstrap = var.enable_tls_bootstrap
+  encrypt_pod_traffic  = var.encrypt_pod_traffic
 }

assets/terraform-modules/aws/flatcar-linux/kubernetes/variables.tf

@@ -111,9 +111,8 @@ variable "asset_dir" {
 }
 
 variable "network_mtu" {
-  description = "CNI interface MTU. Use 8981 if using instances types with Jumbo frames."
+  description = "Physical Network MTU. Use 9001 if using instances types with Jumbo frames."
   type        = number
-  default     = 1480
 }
 
 variable "host_cidr" {
@@ -185,3 +184,9 @@ variable "kube_apiserver_extra_flags" {
   type        = list(string)
   default     = []
 }
+
+variable "encrypt_pod_traffic" {
+  description = "Enable in-cluster pod traffic encryption."
+  type        = bool
+  default     = false
+}

assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/bootkube.tf

@@ -23,4 +23,5 @@ module "bootkube" {
 
   bootstrap_tokens     = var.enable_tls_bootstrap ? [local.controller_bootstrap_token, local.worker_bootstrap_token] : []
   enable_tls_bootstrap = var.enable_tls_bootstrap
+  encrypt_pod_traffic  = var.encrypt_pod_traffic
 }

assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/variables.tf

@@ -83,9 +83,8 @@ variable "asset_dir" {
 }
 
 variable "network_mtu" {
-  description = "CNI interface MTU"
+  description = "Physical Network MTU."
   type        = number
-  default     = 1480
 }
 
 variable "network_ip_autodetection_method" {
@@ -184,3 +183,9 @@ variable "kube_apiserver_extra_flags" {
   type        = list(string)
   default     = []
 }
+
+variable "encrypt_pod_traffic" {
+  description = "Enable in-cluster pod traffic encryption."
+  type        = bool
+  default     = false
+}

assets/terraform-modules/bootkube/conditional.tf

@@ -15,6 +15,14 @@ resource "local_file" "calico" {
     pod_cidr                        = var.pod_cidr
     enable_reporting                = var.enable_reporting
     blocked_metadata_cidrs          = var.blocked_metadata_cidrs
+    failsafe_inbound_host_ports = var.failsafe_inbound_host_ports != null ? [
+      for protoport in var.failsafe_inbound_host_ports :
+      {
+        protocol = protoport.protocol
+        port     = protoport.port
+      }
+    ] : null
+    encrypt_pod_traffic = var.encrypt_pod_traffic
   })
   filename = "${var.asset_dir}/charts/kube-system/calico.yaml"
 }

assets/terraform-modules/bootkube/resources/charts/calico.yaml

@@ -16,3 +16,14 @@ calico:
   - ${cidr}
   %{~ endfor ~}
   %{~ endif ~}
+  %{~ if failsafe_inbound_host_ports != null && failsafe_inbound_host_ports != [] ~}
+  failsafeInboundHostPorts:
+  %{~ for protoport in failsafe_inbound_host_ports ~}
+  - protocol: ${protoport.protocol}
+    port: ${protoport.port}
+  %{~ endfor ~}
+  %{~ endif ~}
+  %{~ if failsafe_inbound_host_ports == [] ~}
+  failsafeInboundHostPorts: []
+  %{~ endif ~}
+  encryptPodTraffic: ${encrypt_pod_traffic}

assets/terraform-modules/bootkube/variables.tf

@@ -173,3 +173,15 @@ variable "enable_tls_bootstrap" {
   description = "Enable TLS Bootstrap for Kubelet."
   type        = bool
 }
+
+variable "failsafe_inbound_host_ports" {
+  description = "UDP/TCP/SCTP protocol/port pairs to allow incoming traffic on regardless of the security policy."
+  type        = list(any)
+  default     = null
+}
+
+variable "encrypt_pod_traffic" {
+  description = "Enable in-cluster pod traffic encryption."
+  type        = bool
+  default     = false
+}

assets/terraform-modules/google-cloud/flatcar-linux/kubernetes/bootkube.tf

@@ -12,6 +12,7 @@ module "bootkube" {
   cluster_domain_suffix = var.cluster_domain_suffix
   enable_reporting      = var.enable_reporting
   enable_aggregation    = var.enable_aggregation
+  encrypt_pod_traffic   = var.encrypt_pod_traffic
 
   // temporary
   external_apiserver_port = 443

assets/terraform-modules/google-cloud/flatcar-linux/kubernetes/variables.tf

@@ -130,3 +130,9 @@ variable "certs_validity_period_hours" {
   type        = number
   default     = 8760
 }
+
+variable "encrypt_pod_traffic" {
+  description = "Enable in-cluster pod traffic encryption."
+  type        = bool
+  default     = false
+}

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/bootkube.tf

@@ -17,6 +17,7 @@ module "bootkube" {
   cluster_domain_suffix = var.cluster_domain_suffix
   enable_reporting      = var.enable_reporting
   enable_aggregation    = var.enable_aggregation
+  encrypt_pod_traffic   = var.encrypt_pod_traffic
 
   certs_validity_period_hours = var.certs_validity_period_hours
 }

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/variables.tf

@@ -112,3 +112,9 @@ variable "certs_validity_period_hours" {
   type        = number
   default     = 8760
 }
+
+variable "encrypt_pod_traffic" {
+  description = "Enable in-cluster pod traffic encryption."
+  type        = bool
+  default     = false
+}

assets/terraform-modules/packet/flatcar-linux/kubernetes/bootkube.tf

@@ -44,4 +44,8 @@ module "bootkube" {
 
   bootstrap_tokens     = var.enable_tls_bootstrap ? concat([local.controller_bootstrap_token], var.worker_bootstrap_tokens) : []
   enable_tls_bootstrap = var.enable_tls_bootstrap
+
+  # We install calico-host-protection chart on Packet which ships GNPs, so we can disable failsafe ports in Calico.
+  failsafe_inbound_host_ports = []
+  encrypt_pod_traffic         = var.encrypt_pod_traffic
 }

assets/terraform-modules/packet/flatcar-linux/kubernetes/variables.tf

@@ -86,9 +86,8 @@ variable "asset_dir" {
 }
 
 variable "network_mtu" {
-  description = "CNI interface MTU"
+  description = "Physical Network MTU."
   type        = number
-  default     = 1480
 }
 
 variable "network_ip_autodetection_method" {
@@ -202,3 +201,9 @@ variable "kube_apiserver_extra_flags" {
   type        = list(string)
   default     = []
 }
+
+variable "encrypt_pod_traffic" {
+  description = "Enable in-cluster pod traffic encryption."
+  type        = bool
+  default     = false
+}

docs/configuration-reference/platforms/aws.md

@@ -107,13 +107,15 @@ cluster "aws" {
 
   enable_tls_bootstrap = true
 
+  encrypt_pod_traffic = true
+
   disk_size = var.disk_size
 
   disk_type = var.disk_type
 
   disk_iops = var.disk_iops
 
-  network_mtu = 1480
+  network_mtu = 1500
 
   host_cidr = ""
 
@@ -219,10 +221,11 @@ worker_pool "my-worker-pool" {
 | `region`                      | AWS region to use for deploying the cluster.                                                                                                                                               | "eu-central-1"  |    string    |  false   |
 | `enable_aggregation`          | Enable the Kubernetes Aggregation Layer.                                                                                                                                                   |      true       |     bool     |  false   |
 | `enable_tls_bootstrap`        | Enable TLS bootstraping for Kubelet.                                                                                                                                                       |      true       |     bool     |  false   |
+| `encrypt_pod_traffic`         | Enable in-cluster pod traffic encryption. If true `network_mtu` is reduced by 60 to make room for the encryption header.                                                                   |      false      |     bool     |  false   |
 | `disk_size`                   | Size of the EBS volume in GB.                                                                                                                                                              |       40        |    number    |  false   |
 | `disk_type`                   | Type of the EBS volume (e.g. standard, gp2, io1).                                                                                                                                          |      "gp2"      |    string    |  false   |
 | `disk_iops`                   | IOPS of the EBS volume (e.g 100).                                                                                                                                                          |        0        |    number    |  false   |
-| `network_mtu`                 | CNI interface MTU. Use 8981 if using instances types with Jumbo frames.                                                                                                                    |      1480       |    number    |  false   |
+| `network_mtu`                 | Physical Network MTU. When using instance types with Jumbo frames, use 9001.                                                                                                               |      1500       |    number    |  false   |
 | `host_cidr`                   | CIDR IPv4 range to assign to EC2 nodes.                                                                                                                                                    |  "10.0.0.0/16"  |    string    |  false   |
 | `pod_cidr`                    | CIDR IPv4 range to assign Kubernetes pods.                                                                                                                                                 |  "10.2.0.0/16"  |    string    |  false   |
 | `service_cidr`                | CIDR IPv4 range to assign Kubernetes services.                                                                                                                                             |  "10.3.0.0/16"  |    string    |  false   |

docs/configuration-reference/platforms/baremetal.md

@@ -76,7 +76,7 @@ cluster "bare-metal" {
     "testlabel" = ""
   }
 
-  network_mtu = 1480
+  network_mtu = 1500
 
   controller_domains = var.controller_domains
 
@@ -96,6 +96,8 @@ cluster "bare-metal" {
 
   enable_tls_bootstrap = true
 
+  encrypt_pod_traffic = true
+
   oidc {
     issuer_url     = var.oidc_issuer_url
     client_id      = var.oidc_client_id
@@ -145,14 +147,15 @@ os_version = var.custom_default_os_version
 | `matchbox_client_key_path`  | Path to the server TLS key file.                                                                                                                                             |        -         |    string    |   true   |
 | `matchbox_endpoint`         | Matchbox API endpoint.                                                                                                                                                       |        -         |    string    |   true   |
 | `matchbox_http_endpoint`    | Matchbox HTTP read-only endpoint. Example: "http://matchbox.example.com:8080"                                                                                                |        -         |    string    |   true   |
-| `network_mtu`               | CNI interface MTU.                                                                                                                                                           |       1480       |    number    |  false   |
+| `network_mtu`               | Physical Network MTU.                                                                                                                                                        |       1500       |    number    |  false   |
 | `worker_names`              | Ordered list of worker names. Example: ["node2", "node3"]                                                                                                                    |        -         | list(string) |   true   |
 | `worker_macs`               | Ordered list of worker identifying MAC addresses. Example ["52:54:00:b2:2f:86", "52:54:00:c3:61:77"]                                                                         |        -         | list(string) |   true   |
 | `worker_domains`            | Ordered list of worker FQDNs. Example ["node2.example.com", "node3.example.com"]                                                                                             |        -         | list(string) |   true   |
 | `ssh_pubkeys`               | List of SSH public keys for user `core`. Each element must be specified in a valid OpenSSH public key format, as defined in RFC 4253 Section 6.6, e.g. "ssh-rsa AAAAB3N...". |        -         | list(string) |   true   |
 | `os_version`                | Flatcar Container Linux version to install. Version such as "2303.3.1" or "current".                                                                                         |    "current"     |    string    |  false   |
 | `os_channel`                | Flatcar Container Linux channel to install from ("flatcar-stable", "flatcar-beta", "flatcar-alpha", "flatcar-edge").                                                         | "flatcar-stable" |    string    |  false   |
 | `enable_tls_bootstrap`      | Enable TLS bootstraping for Kubelet.                                                                                                                                         |       true       |     bool     |  false   |
+| `encrypt_pod_traffic`       | Enable in-cluster pod traffic encryption. If true `network_mtu` is reduced by 60 to make room for the encryption header.                                                     |      false       |     bool     |  false   |
 | `oidc`                      | OIDC configuration block.                                                                                                                                                    |        -         |    object    |  false   |
 | `oidc.issuer_url`           | URL of the provider which allows the API server to discover public signing keys. Only URLs which use the https:// scheme are accepted.                                       |        -         |    string    |  false   |
 | `oidc.client_id`            | A client id that all tokens must be issued for.                                                                                                                              |    "gangway"     |    string    |  false   |

docs/configuration-reference/platforms/packet.md

@@ -100,7 +100,7 @@ cluster "packet" {
 
   cluster_domain_suffix = "cluster.local"
 
-  network_mtu = 1480
+  network_mtu = 1500
 
   tags {
     key1 = "value1"
@@ -111,6 +111,8 @@ cluster "packet" {
 
   enable_tls_bootstrap = true
 
+  encrypt_pod_traffic = true
+
   enable_reporting = false
 
   network_ip_autodetection_method = "first-found"
@@ -221,7 +223,8 @@ node_type = var.custom_default_worker_type
 | `node_private_cidr`                   | Private IPv4 CIDR of the nodes used to allow inter-node traffic. Example "10.0.0.0/8"                                                                                                                                                                                             |        -        |    string    |   true   |
 | `enable_aggregation`                  | Enable the Kubernetes Aggregation Layer.                                                                                                                                                                                                                                          |      true       |     bool     |  false   |
 | `enable_tls_bootstrap`                | Enable TLS bootstraping for Kubelet.                                                                                                                                                                                                                                              |      true       |     bool     |  false   |
-| `network_mtu`                         | CNI interface MTU                                                                                                                                                                                                                                                                 |      1480       |    number    |  false   |
+| `encrypt_pod_traffic`                 | Enable in-cluster pod traffic encryption. If true `network_mtu` is reduced by 60 to make room for the encryption header.                                                                                                                                                          |      false      |     bool     |  false   |
+| `network_mtu`                         | Physical Network MTU.                                                                                                                                                                                                                                                             |      1500       |    number    |  false   |
 | `pod_cidr`                            | CIDR IPv4 range to assign Kubernetes pods.                                                                                                                                                                                                                                        |  "10.2.0.0/16"  |    string    |  false   |
 | `service_cidr`                        | CIDR IPv4 range to assign Kubernetes services.                                                                                                                                                                                                                                    |  "10.3.0.0/16"  |    string    |  false   |
 | `cluster_domain_suffix`               | Cluster's DNS domain.                                                                                                                                                                                                                                                             | "cluster.local" |    string    |  false   |