packet: enable CCM

This commit enables CCM for Packet platform to allow dropping hacky way
of labeling Node objects on Packet via Ignition script for MetalLB
autodiscovery to work, which we are doing right now.

Closes #548

Signed-off-by: Mateusz Gozdek <mateusz@kinvolk.io>

Author: invidian

assets/terraform-modules/packet/flatcar-linux/kubernetes/bootkube.tf

@@ -48,4 +48,5 @@ module "bootkube" {
   encrypt_pod_traffic         = var.encrypt_pod_traffic
 
   ignore_x509_cn_check = var.ignore_x509_cn_check
+  cloud_provider       = "external"
 }

assets/terraform-modules/packet/flatcar-linux/kubernetes/cl/controller.yaml.tmpl

@@ -131,6 +131,7 @@ systemd:
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${k8s_dns_service_ip} \
           --cluster_domain=${cluster_domain_suffix} \
+          --cloud-provider=external \
           --cni-conf-dir=/etc/cni/net.d \
           --config=/etc/kubernetes/kubelet.config \
           --exit-on-lock-contention \

assets/terraform-modules/packet/flatcar-linux/kubernetes/packet-ccm.tf

@@ -0,0 +1,7 @@
+resource "local_file" "packet-ccm" {
+  filename = "${var.asset_dir}/charts/kube-system/packet-ccm.yaml"
+  content = templatefile("${path.module}/packet-ccm.yaml.tmpl", {
+    api_key    = var.auth_token
+    project_id = var.project_id
+  })
+}

assets/terraform-modules/packet/flatcar-linux/kubernetes/packet-ccm.yaml.tmpl

@@ -0,0 +1,2 @@
+apiKey: ${api_key}
+projectID: ${project_id}

assets/terraform-modules/packet/flatcar-linux/kubernetes/ssh.tf

@@ -71,6 +71,7 @@ resource "null_resource" "copy-assets-dir" {
     null_resource.copy-controller-secrets,
     local_file.calico_host_protection,
     local_file.calico_crds,
+    local_file.packet-ccm,
   ]
 
   connection {

assets/terraform-modules/packet/flatcar-linux/kubernetes/variables.tf

@@ -14,6 +14,10 @@ variable "project_id" {
   description = "Packet project ID (e.g. 405efe9c-cce9-4c71-87c1-949c290b27dc)"
 }
 
+variable "auth_token" {
+  description = "Packet Auth token (e.g. 4KUDZICtnKfNF4uJSGxxLi4GhWxhzKPj)"
+}
+
 # Nodes
 
 variable "os_arch" {

assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -34,20 +34,6 @@ systemd:
         ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done; /opt/wait-for-dns ${dns_zone} ${cluster_name}-private 3600'
         [Install]
         RequiredBy=kubelet.service
-    %{~ if bgp_node_labels != "" ~}
-    - name: bgp-metadata.service
-      enable: true
-      contents: |
-        [Unit]
-        Description=Write BGP metadata to disk
-        Before=kubelet.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=true
-        ExecStart=/opt/bgp-metadata
-        [Install]
-        RequiredBy=kubelet.service
-    %{~ endif ~}
     - name: coreos-metadata.service
       enable: true
       contents: |
@@ -69,7 +55,6 @@ systemd:
         Wants=rpc-statd.service
         [Service]
         EnvironmentFile=/run/metadata/flatcar
-        EnvironmentFile=-/run/metadata/bgp
         EnvironmentFile=/etc/kubernetes/kubelet.env
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -107,6 +92,7 @@ systemd:
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${k8s_dns_service_ip} \
           --cluster_domain=${cluster_domain_suffix} \
+          --cloud-provider=external \
           --cni-conf-dir=/etc/cni/net.d \
           --config=/etc/kubernetes/kubelet.config \
           --exit-on-lock-contention \
@@ -121,11 +107,7 @@ systemd:
           --network-plugin=cni \
           --node-labels=$${NODE_LABELS} \
           --node-labels=lokomotive.alpha.kinvolk.io/public-ipv4=$${COREOS_PACKET_IPV4_PUBLIC_0} \
-          %{~ if bgp_node_labels != "" ~}
-          --node-labels=$${BGP_NODE_LABELS} \
-          --node-labels=metallb.lokomotive.io/peer-address=$${BGP_PEER_ADDRESS_0} \
           --node-labels=metallb.lokomotive.io/src-address=$${COREOS_PACKET_IPV4_PRIVATE_0} \
-          %{~ endif ~}
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
           --register-with-taints=$${NODE_TAINTS} \
@@ -302,7 +284,6 @@ storage:
           KUBELET_IMAGE_URL=quay.io/poseidon/kubelet
           KUBELET_IMAGE_TAG=v1.19.3-${os_arch}
           NODE_LABELS="${join(",", [for k, v in node_labels : "${k}=${v}"])}"
-          BGP_NODE_LABELS="${bgp_node_labels}"
           NODE_TAINTS="${join(",", [for k, v in taints : "${k}=${v}"])}"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
@@ -429,41 +410,6 @@ storage:
           done
           echo "$record.$zone is available on all nameservers"
           exit 0
-    - path: /opt/bgp-metadata
-      filesystem: root
-      mode: 0544
-      contents:
-        inline: |
-          #!/bin/bash
-          set -o pipefail
-          max_attempts=3600
-          target=/run/metadata/bgp
-          echo "Polling metadata service for BGP information"
-          counter=0
-          while [[ $counter -lt $max_attempts ]]; do
-              out=$(curl -s -f --connect-timeout 5 \
-                  https://metadata.packet.net/metadata | jq -r .bgp_neighbors[0].peer_ips[0])
-              ret=$?
-              if [[ $ret -ne 0 ]]; then
-                  echo "Non-zero exit code: $ret"
-              elif [[ "$out" = "" ]]; then
-                  echo "Empty response"
-              elif [[ "$out" = "null" ]]; then
-                  echo "Null response"
-              else
-                  echo "BGP metadata is available!"
-                  if [[ ! "$out" =~ ^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4}$ ]]; then
-                      echo "Invalid IP $out"
-                      exit 1
-                  fi
-                  echo "BGP_PEER_ADDRESS_0=$out" > "$target"
-                  exit $?
-              fi
-              sleep 1
-              counter=$((counter+1))
-          done
-          echo "BGP metadata did not become available in time"
-          exit 1
 passwd:
   users:
   - name: core

assets/terraform-modules/packet/flatcar-linux/kubernetes/workers/workers.tf

@@ -43,19 +43,6 @@ data "ct_config" "install-ignitions" {
   })
 }
 
-resource "packet_bgp_session" "bgp" {
-  count          = var.disable_bgp == true ? 0 : var.worker_count
-  device_id      = packet_device.nodes[count.index].id
-  address_family = "ipv4"
-}
-
-# BGP node labels.
-locals {
-  my_asn = format("metallb.lokomotive.io/my-asn=%d", data.packet_project.project.bgp_config.0.asn)
-  # Packet always uses ASN 65530 as the remote ASN for local BGP.
-  peer_asn = format("metallb.lokomotive.io/peer-asn=%d", 65530)
-}
-
 data "ct_config" "ignitions" {
   content = templatefile(
     "${path.module}/cl/worker.yaml.tmpl",
@@ -74,7 +61,6 @@ data "ct_config" "ignitions" {
         "node.kubernetes.io/node"                 = "",
         "lokomotive.alpha.kinvolk.io/bgp-enabled" = format("%t", ! var.disable_bgp),
       }, var.labels)
-      bgp_node_labels      = var.disable_bgp ? "" : format("%s,%s", local.my_asn, local.peer_asn)
       taints               = var.taints
       setup_raid           = var.setup_raid
       setup_raid_hdd       = var.setup_raid_hdd

pkg/assets/generated_assets.go

@@ -196,6 +196,17 @@ func (c *config) Initialize(ex *terraform.Executor) error {
 		}
 	}
 
+	// TODO: A transient change which shall be reverted in a follow up PR to handle
+	// https://github.com/kinvolk/lokomotive/issues/716.
+	// Extract host protection chart.
+	src = filepath.Join(assets.ControlPlaneSource, "packet-ccm")
+	dst = filepath.Join(assetDir,
+		"cluster-assets", "charts", "kube-system", "packet-ccm")
+
+	if err := assets.Extract(src, dst); err != nil {
+		return fmt.Errorf("extracting packet-ccm chart: %w", err)
+	}
+
 	terraformRootDir := terraform.GetTerraformRootDir(assetDir)
 
 	return createTerraformConfigFile(c, terraformRootDir)