Skip to content
This repository was archived by the owner on Jun 29, 2022. It is now read-only.

Commit b6dffe4

Browse files
invidianinvidian
authored
Merge pull request #1193 from kinvolk/invidian/fix-updating-kubernetes-controlplane
Fix controlplane running as DaemonSet on single node clusters
2 parents 9761c7c + 87fde27 commit b6dffe4

File tree

8 files changed

+177
-26
lines changed

8 files changed

+177
-26
lines changed

‎assets/terraform-modules/bootkube/assets.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ resource "local_file" "kubernetes" {
6969
kube_scheduler_image = var.container_images["kube_scheduler"]
7070
kube_proxy_image = var.container_images["kube_proxy"]
7171
coredns_image = var.container_images["coredns"]
72-
control_plane_replicas = max(2, length(var.etcd_servers))
72+
control_plane_replicas = length(var.etcd_servers)
7373
cloud_provider = var.cloud_provider
7474
pod_cidr = var.pod_cidr
7575
service_cidr = var.service_cidr

‎pkg/assets/generated_assets.go‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/components/kubernetes/controlplane-multi-node_disruptive_test.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright 2020 The Lokomotive Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// +build aws aws_edge
16+
// +build disruptivee2e
17+
18+
package kubernetes_test
19+
20+
import (
21+
"context"
22+
"testing"
23+
"time"
24+
25+
testutil "github.com/kinvolk/lokomotive/test/components/util"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
)
28+
29+
func TestControlplaneComponentsDaemonSetsCanBeGracefullyUpdated(t *testing.T) {
30+
client := testutil.CreateKubeClient(t)
31+
dsClient := client.AppsV1().DaemonSets(namespace)
32+
33+
components := components()
34+
components["kube-proxy"] = testutil.RetryInterval
35+
36+
for c, waitTime := range components {
37+
c := c
38+
waitTime := waitTime
39+
40+
t.Run(c, func(t *testing.T) {
41+
ds, err := dsClient.Get(context.TODO(), c, metav1.GetOptions{})
42+
if err != nil {
43+
t.Fatalf("Getting DaemonSet %q: %v", c, err)
44+
}
45+
46+
// Use current time to have different value on each test run.
47+
ds.Spec.Template.Annotations["update-test"] = time.Now().String()
48+
49+
if _, err := dsClient.Update(context.TODO(), ds, metav1.UpdateOptions{}); err != nil {
50+
t.Fatalf("Updating DaemonSet %q: %v", c, err)
51+
}
52+
53+
// Wait a bit to let Kubernetes trigger pod updates.
54+
time.Sleep(waitTime)
55+
56+
testutil.WaitForDaemonSet(t, client, namespace, c, testutil.RetryInterval, testutil.Timeout)
57+
})
58+
}
59+
}

test/components/coredns/coredns_test.go renamed to test/components/kubernetes/controlplane-multi-node_test.go

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,24 +12,27 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
// +build aws aws_edge packet
15+
// +build aws aws_edge
1616
// +build e2e
1717

18-
package coredns
18+
package kubernetes_test
1919

2020
import (
2121
"testing"
2222

2323
testutil "github.com/kinvolk/lokomotive/test/components/util"
2424
)
2525

26-
func TestCoreDNSDaemonSet(t *testing.T) {
27-
t.Parallel()
26+
func TestControlplaneComponentsAreDaemonSetsOnMultiControllerNodeCluster(t *testing.T) {
27+
client := testutil.CreateKubeClient(t)
2828

29-
namespace := "kube-system"
30-
daemonset := "coredns"
29+
for c := range components() {
30+
c := c
3131

32-
client := testutil.CreateKubeClient(t)
32+
t.Run(c, func(t *testing.T) {
33+
t.Parallel()
3334

34-
testutil.WaitForDaemonSet(t, client, namespace, daemonset, testutil.RetryInterval, testutil.Timeout)
35+
testutil.WaitForDaemonSet(t, client, namespace, c, testutil.RetryInterval, testutil.Timeout)
36+
})
37+
}
3538
}

‎test/components/kubernetes/controlplane-single-node_disruptive_test.go‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// Copyright 2020 The Lokomotive Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// +build packet baremetal
16+
// +build disruptivee2e
17+
18+
package kubernetes_test
19+
20+
import (
21+
"context"
22+
"testing"
23+
"time"
24+
25+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
26+
27+
testutil "github.com/kinvolk/lokomotive/test/components/util"
28+
)
29+
30+
func TestControlplaneComponentsDeploymentsCanBeGracefullyUpdated(t *testing.T) {
31+
client := testutil.CreateKubeClient(t)
32+
deployClient := client.AppsV1().Deployments(namespace)
33+
34+
for c, waitTime := range components() {
35+
c := c
36+
waitTime := waitTime
37+
38+
t.Run(c, func(t *testing.T) {
39+
deploy, err := deployClient.Get(context.TODO(), c, metav1.GetOptions{})
40+
if err != nil {
41+
t.Fatalf("Getting Deployment %q: %v", c, err)
42+
}
43+
44+
// Use current time to have different value on each test run.
45+
deploy.Spec.Template.Annotations["update-test"] = time.Now().String()
46+
47+
if _, err := deployClient.Update(context.TODO(), deploy, metav1.UpdateOptions{}); err != nil {
48+
t.Fatalf("Updating Deployment %q: %v", c, err)
49+
}
50+
51+
// Wait a bit to let Kubernetes trigger pod updates.
52+
time.Sleep(waitTime)
53+
54+
testutil.WaitForDeployment(t, client, namespace, c, testutil.RetryInterval, testutil.Timeout)
55+
})
56+
}
57+
}

test/components/kubernetes/controller-manager_test.go renamed to test/components/kubernetes/controlplane-single-node_test.go

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,24 +12,27 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
// +build aws aws_edge packet
15+
// +build packet baremetal
1616
// +build e2e
1717

18-
package kubernetes
18+
package kubernetes_test
1919

2020
import (
2121
"testing"
2222

2323
testutil "github.com/kinvolk/lokomotive/test/components/util"
2424
)
2525

26-
func TestControllerManagerDaemonSet(t *testing.T) {
27-
t.Parallel()
26+
func TestControlplaneComponentsAreDeploymentsOnSingleControllerNodeCluster(t *testing.T) {
27+
client := testutil.CreateKubeClient(t)
2828

29-
namespace := "kube-system"
30-
daemonset := "kube-controller-manager"
29+
for c := range components() {
30+
c := c
3131

32-
client := testutil.CreateKubeClient(t)
32+
t.Run(c, func(t *testing.T) {
33+
t.Parallel()
3334

34-
testutil.WaitForDaemonSet(t, client, namespace, daemonset, testutil.RetryInterval, testutil.Timeout)
35+
testutil.WaitForDeployment(t, client, namespace, c, testutil.RetryInterval, testutil.Timeout)
36+
})
37+
}
3538
}

‎test/components/kubernetes/controlplane_test.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package kubernetes_test
2+
3+
import (
4+
"time"
5+
6+
testutil "github.com/kinvolk/lokomotive/test/components/util"
7+
)
8+
9+
const (
10+
namespace = "kube-system"
11+
)
12+
13+
func components() map[string]time.Duration {
14+
return map[string]time.Duration{
15+
// If we kill active kube-controller-manager, it takes time for new one to kick in
16+
// so we need to wait longer.
17+
"kube-controller-manager": 1 * time.Minute,
18+
"kube-scheduler": testutil.RetryInterval,
19+
"kube-apiserver": testutil.RetryInterval,
20+
"coredns": testutil.RetryInterval,
21+
}
22+
}

0 commit comments

Comments
 (0)