Extra port added for test cluster to make ingress port reachable (#1308)

* Extra port added to make ingress port reachable

Signed-off-by: Alexey Makhov <amakhov@mirantis.com>
Signed-off-by: makhov <amakhov@mirantis.com>

* Extra port added to make ingress port reachable. Change hosting cluster api port to non-default

Signed-off-by: Alexey Makhov <amakhov@mirantis.com>
Signed-off-by: makhov <amakhov@mirantis.com>

---------

Signed-off-by: Alexey Makhov <amakhov@mirantis.com>
Signed-off-by: makhov <amakhov@mirantis.com>

Author: makhov

e2e/data/infrastructure-docker/main/cluster-template-ingress/cluster-with-ingress.yaml

@@ -28,6 +28,8 @@ metadata:
   name: ${CLUSTER_NAME}-cp
 spec:
   version: v1.34.1-k0s.0
+  service:
+    type: NodePort
   ingress:
     apiHost: kube-api.${KIND_IP}.nip.io
     konnectivityHost: konnectivity.${KIND_IP}.nip.io

e2e/data/infrastructure-docker/main/cluster-template-remote-hcp/hcp-with-external-cluster-patch.yaml

@@ -12,6 +12,7 @@ spec:
   version: v1.30.1-k0s.0
   service:
     type: NodePort
+    apiPort: 31443
 
   k0sConfig:
     apiVersion: k0s.k0sproject.io/v1beta1

e2e/ingress_test.go

@@ -23,10 +23,10 @@ import (
 	"fmt"
 	"github.com/k0sproject/k0s/inttest/common"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"os/exec"
 	"path/filepath"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
-	"sigs.k8s.io/cluster-api/controllers/remote"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"testing"
 	"time"
@@ -156,9 +156,10 @@ func ingressSupportSpec(t *testing.T) {
 		clusterv1.ClusterNameLabel: workloadClusterName,
 	}), "Should list machines")
 
-	wrc, err := remote.RESTConfig(ctx, "ingress-test", bootstrapClusterProxy.GetClient(), client.ObjectKey{Namespace: workloadClusterNamespace, Name: workloadClusterName})
-	require.NoError(t, err, "Should get workload rest config")
-	wcs, err := kubernetes.NewForConfig(wrc)
+	workloadCluster := bootstrapClusterProxy.GetWorkloadCluster(ctx, workloadClusterNamespace, workloadClusterName, capiframework.WithRESTConfigModifier(func(config *rest.Config) {
+		config.Host = "https://localhost:30443"
+	}))
+	wcs, err := kubernetes.NewForConfig(workloadCluster.GetRESTConfig())
 	require.NoError(t, err, "Should get workload clientset")
 	require.NoError(t, common.WaitForDaemonSet(ctx, wcs, "konnectivity-agent"))
 
@@ -167,7 +168,8 @@ func ingressSupportSpec(t *testing.T) {
 	require.NoError(t, err, "Should list k0smotron pods")
 	out, err := podexec.PodExecCmdOutput(ctx, bootstrapClusterProxy.GetClientSet(), bootstrapClusterProxy.GetRESTConfig(), podList.Items[0].Name, testNamespace.Name, "k0s kc logs -n kube-system ds/konnectivity-agent")
 	require.NoError(t, err)
-	t.Logf("Konnectivity agent logs:\n%s", string(out))
+	t.Logf("Konnectivity agent logs:\n%s", out)
+	require.Contains(t, out, "change detected in proxy")
 
 	for _, m := range machineList.Items {
 		var (

e2e/remote_hcp_test.go

@@ -241,8 +241,8 @@ func deployHostingCluster() {
 		LogFolder:          filepath.Join(artifactFolder, "kind"),
 		ExtraPortMappings: []v1alpha4.PortMapping{
 			{
-				ContainerPort: 30443,
-				HostPort:      30443,
+				ContainerPort: 31443,
+				HostPort:      31443,
 			},
 		},
 	})

e2e/setup.go

@@ -31,16 +31,17 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/klog/v2"
-	"sigs.k8s.io/yaml"
-
-	cpv1beta1 "github.com/k0sproject/k0smotron/api/controlplane/v1beta1"
-	"github.com/k0sproject/k0smotron/e2e/mothership"
-	"github.com/k0sproject/k0smotron/e2e/util"
 	"sigs.k8s.io/cluster-api/test/framework"
 	capiframework "sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/bootstrap"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
+	"sigs.k8s.io/yaml"
+
+	cpv1beta1 "github.com/k0sproject/k0smotron/api/controlplane/v1beta1"
+	"github.com/k0sproject/k0smotron/e2e/mothership"
+	"github.com/k0sproject/k0smotron/e2e/util"
 )
 
 // Test suite constants for e2e config variables.
@@ -153,6 +154,16 @@ func setupMothership() error {
 			RequiresDockerSock: e2eConfig.HasDockerProvider(),
 			IPFamily:           e2eConfig.MustGetVariable(IPFamily),
 			LogFolder:          filepath.Join(artifactFolder, "kind"),
+			ExtraPortMappings: []v1alpha4.PortMapping{
+				{
+					ContainerPort: 32143, // haproxy ingress port
+					HostPort:      32143,
+				},
+				{
+					ContainerPort: 30443, // HCP api nodeport
+					HostPort:      30443,
+				},
+			},
 		})
 		if bootstrapClusterProvider == nil {
 			return errors.New("failed to create a management cluster")