feat: CRD installer init container

Makefile

@@ -7,10 +7,12 @@ endif
 HUB_NET_CONTROLLER_MANAGER_IMAGE_VERSION ?= $(TAG)
 MEMBER_NET_CONTROLLER_MANAGER_IMAGE_VERSION ?= $(TAG)
 MCS_CONTROLLER_MANAGER_IMAGE_VERSION ?= $(TAG)
+NET_CRD_INSTALLER_IMAGE_VERSION ?= $(TAG)
 
 HUB_NET_CONTROLLER_MANAGER_IMAGE_NAME ?= hub-net-controller-manager
 MEMBER_NET_CONTROLLER_MANAGER_IMAGE_NAME ?= member-net-controller-manager
 MCS_CONTROLLER_MANAGER_IMAGE_NAME ?= mcs-controller-manager
+NET_CRD_INSTALLER_IMAGE_NAME ?= net-crd-installer
 
 # Directories
 ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
@@ -24,7 +26,7 @@ CONTROLLER_GEN_VER := v0.16.0
 CONTROLLER_GEN_BIN := controller-gen
 CONTROLLER_GEN := $(abspath $(TOOLS_BIN_DIR)/$(CONTROLLER_GEN_BIN)-$(CONTROLLER_GEN_VER))
 
-STATICCHECK_VER := 2023.1.7
+STATICCHECK_VER := latest
 STATICCHECK_BIN := staticcheck
 STATICCHECK := $(abspath $(TOOLS_BIN_DIR)/$(STATICCHECK_BIN)-$(STATICCHECK_VER))
 
@@ -106,12 +108,12 @@ test: manifests generate fmt vet local-unit-test integration-test
 
 .PHONY: local-unit-test
 local-unit-test: $(ENVTEST) ## Run tests.
-	CGO_ENABLED=1 KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./pkg/... -race -coverprofile=coverage.xml -covermode=atomic -v
+	CGO_ENABLED=1 KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./cmd/... ./pkg/... -race -coverprofile=coverage.xml -covermode=atomic -v
 
 .PHONY: integration-test
 integration-test: $(ENVTEST) ## Run integration tests.
 	CGO_ENABLED=1 KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" \
-	ginkgo -v -p --race --cover --coverpkg=./... ./test/apis/...
+	ginkgo -v -p --race --cover --coverpkg=./... ./test/apis/... ./test/netcrdinstaller
 
 .PHONY: e2e-setup
 e2e-setup:
@@ -186,11 +188,11 @@ vendor:
 
 .PHONY: image
 image:
-	$(MAKE) OUTPUT_TYPE="type=docker" docker-build-hub-net-controller-manager docker-build-member-net-controller-manager docker-build-mcs-controller-manager
+	$(MAKE) OUTPUT_TYPE="type=docker" docker-build-hub-net-controller-manager docker-build-member-net-controller-manager docker-build-mcs-controller-manager docker-build-net-crd-installer
 
 .PHONY: push
 push:
-	$(MAKE) OUTPUT_TYPE="type=registry" docker-build-hub-net-controller-manager docker-build-member-net-controller-manager docker-build-mcs-controller-manager
+	$(MAKE) OUTPUT_TYPE="type=registry" docker-build-hub-net-controller-manager docker-build-member-net-controller-manager docker-build-mcs-controller-manager docker-build-net-crd-installer
 
 # By default, docker buildx create will pull image moby/buildkit:buildx-stable-1 and hit the too many requests error.
 .PHONY: docker-buildx-builder
@@ -228,6 +230,15 @@ docker-build-mcs-controller-manager: docker-buildx-builder vendor
 		--pull \
 		--tag $(REGISTRY)/$(MCS_CONTROLLER_MANAGER_IMAGE_NAME):$(MCS_CONTROLLER_MANAGER_IMAGE_VERSION) .
 
+.PHONY: docker-build-net-crd-installer
+docker-build-net-crd-installer: docker-buildx-builder vendor
+	docker buildx build \
+		--file docker/$(NET_CRD_INSTALLER_IMAGE_NAME).Dockerfile \
+		--output=$(OUTPUT_TYPE) \
+		--platform="linux/amd64" \
+		--pull \
+		--tag $(REGISTRY)/$(NET_CRD_INSTALLER_IMAGE_NAME):$(NET_CRD_INSTALLER_IMAGE_VERSION) .
+
 ## -----------------------------------
 ## Cleanup
 ## -----------------------------------

charts/hub-net-controller-manager/templates/deployment.yaml

@@ -20,6 +20,16 @@ spec:
         {{- include "hub-net-controller-manager.selectorLabels" . | nindent 8 }}
     spec:
       serviceAccountName:  {{ include "hub-net-controller-manager.fullname" . }}-sa
+      {{- if .Values.crdInstaller.enabled }}
+      initContainers:
+        - name: net-crd-installer
+          image: "{{ .Values.crdInstaller.image.repository }}:{{ .Values.crdInstaller.image.tag }}"
+          imagePullPolicy: {{ .Values.crdInstaller.image.pullPolicy }}
+          args:
+            - --mode=hub
+            - --v={{ .Values.crdInstaller.logVerbosity }}
+            - --e2e-test={{ .Values.crdInstaller.isE2ETest }}
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"

charts/hub-net-controller-manager/templates/rbac.yaml

@@ -137,6 +137,17 @@ rules:
     - get
     - list
     - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 {{- if .Values.enableTrafficManagerFeature }}
 - apiGroups:
     - networking.fleet.azure.com

charts/hub-net-controller-manager/values.yaml

@@ -10,6 +10,17 @@ image:
   # Overrides the image tag whose default is the chart appVersion.
   tag: "v0.1.0"
 
+# CRD installer configuration.
+crdInstaller:
+  enabled: false
+  image:
+    repository: ghcr.io/azure/fleet-networking/net-crd-installer
+    pullPolicy: Always
+    tag: main
+  logVerbosity: 2
+  # Set to true when running E2E tests to avoid adding Azure managed labels
+  isE2ETest: false
+
 logVerbosity: 2
 
 leaderElectionNamespace: fleet-system

charts/member-net-controller-manager/templates/deployment.yaml

@@ -20,6 +20,16 @@ spec:
         {{- include "member-net-controller-manager.selectorLabels" . | nindent 8 }}
     spec:
       serviceAccountName: {{ include "member-net-controller-manager.fullname" . }}-sa
+      {{- if .Values.crdInstaller.enabled }}
+      initContainers:
+        - name: net-crd-installer
+          image: "{{ .Values.crdInstaller.image.repository }}:{{ .Values.crdInstaller.image.tag }}"
+          imagePullPolicy: {{ .Values.crdInstaller.image.pullPolicy }}
+          args:
+            - --mode=member
+            - --v={{ .Values.crdInstaller.logVerbosity }}
+            - --e2e-test={{ .Values.crdInstaller.isE2ETest }}
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:

charts/member-net-controller-manager/templates/rbac.yaml

@@ -114,6 +114,17 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

charts/member-net-controller-manager/values.yaml

@@ -10,6 +10,17 @@ image:
   # Overrides the image tag whose default is the chart appVersion.
   tag: "v0.1.0"
 
+# CRD installer configuration.
+crdInstaller:
+  enabled: false
+  image:
+    repository: ghcr.io/azure/fleet-networking/net-crd-installer
+    pullPolicy: Always
+    tag: main
+  logVerbosity: 2
+  # Set to true when running E2E tests to avoid adding Azure managed labels
+  isE2ETest: false
+
 fleetSystemNamespace:  fleet-system
 leaderElectionNamespace: fleet-system
 

cmd/net-crd-installer/main.go

@@ -0,0 +1,99 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+*/
+
+// Package main contains the CRD installer utility for KubeFleet.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	"go.goms.io/fleet-networking/cmd/net-crd-installer/utils"
+)
+
+var (
+	mode      = flag.String("mode", "", "Mode to run in: 'hub' or 'member' (required)")
+	isE2ETest = flag.Bool("e2e-test", false, "Whether this is running as part of E2E tests (default: false)")
+)
+
+func main() {
+	klog.InitFlags(nil)
+	flag.Parse()
+
+	// Validate required flags.
+	if *mode != "hub" && *mode != "member" {
+		klog.Fatal("--mode flag must be either 'hub' or 'member'")
+	}
+
+	klog.Infof("Starting network CRD installer in %s mode", *mode)
+	klog.Infof("E2E test mode: %t", *isE2ETest)
+
+	// Print all flags for debugging.
+	flag.VisitAll(func(f *flag.Flag) {
+		klog.V(2).InfoS("flag:", "name", f.Name, "value", f.Value)
+	})
+
+	// Set up controller-runtime logger.
+	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
+
+	// Create context for API operations.
+	ctx := ctrl.SetupSignalHandler()
+
+	// Get Kubernetes config using controller-runtime.
+	config := ctrl.GetConfigOrDie()
+
+	// Create a scheme that knows about CRD types.
+	scheme := runtime.NewScheme()
+	if err := apiextensionsv1.AddToScheme(scheme); err != nil {
+		klog.Fatalf("Failed to add apiextensions scheme: %v", err)
+	}
+	client, err := client.New(config, client.Options{
+		Scheme: scheme,
+	})
+
+	if err != nil {
+		klog.Fatalf("Failed to create Kubernetes client: %v", err)
+	}
+
+	// Install CRDs from the fixed location.
+	const crdPath = "/workspace/config/crd/bases"
+	if err := installCRDs(ctx, client, crdPath, *mode, *isE2ETest); err != nil {
+		klog.Fatalf("Failed to install CRDs: %v", err)
+	}
+
+	klog.Infof("Successfully installed %s CRDs", *mode)
+}
+
+// installCRDs installs the CRDs from the specified directory based on the mode.
+func installCRDs(ctx context.Context, client client.Client, crdPath, mode string, isE2ETest bool) error {
+	// List of CRDs to install based on mode.
+	crdsToInstall, err := utils.CollectCRDs(crdPath, mode, client.Scheme())
+	if err != nil {
+		return err
+	}
+
+	if len(crdsToInstall) == 0 {
+		return fmt.Errorf("no CRDs found for mode %s in directory %s", mode, crdPath)
+	}
+
+	klog.Infof("Found %d CRDs to install for mode %s", len(crdsToInstall), mode)
+
+	// Install each CRD.
+	for i := range crdsToInstall {
+		if err := utils.InstallCRD(ctx, client, &crdsToInstall[i], isE2ETest); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}