Skip to content
This repository was archived by the owner on Sep 2, 2024. It is now read-only.

Commit 7a5c418

Browse files
committed
support kind clusters
Before setting the node role assert that it is running a supported container runtime, a.k.a "containerd", and auto-detect reasonable defaults for the container socket and storage host path (parent). Added `k3s builder install --containerd-volume` flag to support kind and non-standard containerd installations. Removed `k3s builder install --buildkit-namespace` flag (because it is not something you can set on buildkitd). Fixes #49 Signed-off-by: Jacob Blain Christen <[email protected]>
1 parent ac0a8eb commit 7a5c418

File tree

3 files changed

+52
-21
lines changed

3 files changed

+52
-21
lines changed

pkg/client/builder/install.go

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"context"
55
"fmt"
66
"net"
7+
"net/url"
8+
"strings"
79
"time"
810

911
"github.com/pkg/errors"
@@ -298,7 +300,7 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
298300
{Name: "_PATH", Value: "/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux"},
299301
},
300302
Command: []string{"sh", "-c"},
301-
Args: []string{"(if mountpoint $_DIR; then nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
303+
Args: []string{"(if mountpoint $_DIR; set -x; nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
302304
SecurityContext: &corev1.SecurityContext{
303305
Privileged: &privileged,
304306
},
@@ -313,27 +315,27 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
313315
{Name: "_PATH", Value: "/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux"},
314316
},
315317
Command: []string{"sh", "-c"},
316-
Args: []string{"(if mountpoint $_DIR; then nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
318+
Args: []string{"(if mountpoint $_DIR; then set -x; nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
317319
SecurityContext: &corev1.SecurityContext{
318320
Privileged: &privileged,
319321
},
320322
VolumeMounts: []corev1.VolumeMount{
321323
{Name: "host-var-lib-buildkit", MountPath: "/var/lib/buildkit"},
322324
},
323325
}, {
324-
Name: "rshared-rancher",
326+
Name: "rshared-containerd",
325327
Image: buildkitImage,
326328
Env: []corev1.EnvVar{
327-
{Name: "_DIR", Value: "/var/lib/rancher"},
329+
{Name: "_DIR", Value: a.ContainerdVolume},
328330
{Name: "_PATH", Value: "/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux"},
329331
},
330332
Command: []string{"sh", "-c"},
331-
Args: []string{"(if mountpoint $_DIR; then nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
333+
Args: []string{"(if mountpoint $_DIR; set -x; nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
332334
SecurityContext: &corev1.SecurityContext{
333335
Privileged: &privileged,
334336
},
335337
VolumeMounts: []corev1.VolumeMount{
336-
{Name: "host-var-lib-rancher", MountPath: "/var/lib/rancher"},
338+
{Name: "host-containerd", MountPath: a.ContainerdVolume},
337339
},
338340
}},
339341
Containers: []corev1.Container{{
@@ -361,7 +363,7 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
361363
{Name: "host-run", MountPath: "/run"},
362364
{Name: "host-tmp", MountPath: "/tmp", MountPropagation: &mountPropagationBidirectional},
363365
{Name: "host-var-lib-buildkit", MountPath: "/var/lib/buildkit", MountPropagation: &mountPropagationBidirectional},
364-
{Name: "host-var-lib-rancher", MountPath: "/var/lib/rancher", MountPropagation: &mountPropagationBidirectional},
366+
{Name: "host-containerd", MountPath: a.ContainerdVolume, MountPropagation: &mountPropagationBidirectional},
365367
{Name: "certs-ca", MountPath: "/certs/ca", ReadOnly: true},
366368
{Name: "certs-server", MountPath: "/certs/server", ReadOnly: true},
367369
},
@@ -373,7 +375,6 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
373375
Command: []string{"kim", "--debug", "agent"},
374376
Args: []string{
375377
fmt.Sprintf("--agent-port=%d", a.AgentPort),
376-
fmt.Sprintf("--buildkit-namespace=%s", a.BuildkitNamespace),
377378
fmt.Sprintf("--buildkit-socket=%s", a.BuildkitSocket),
378379
fmt.Sprintf("--buildkit-port=%d", a.BuildkitPort),
379380
fmt.Sprintf("--containerd-socket=%s", a.ContainerdSocket),
@@ -388,12 +389,12 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
388389
Privileged: &privileged,
389390
},
390391
VolumeMounts: []corev1.VolumeMount{
392+
{Name: "host-containerd", MountPath: a.ContainerdVolume, MountPropagation: &mountPropagationBidirectional},
391393
{Name: "host-ctl", MountPath: "/sys/fs/cgroup"},
392394
{Name: "host-etc-pki", MountPath: "/etc/pki", ReadOnly: true},
393395
{Name: "host-etc-ssl", MountPath: "/etc/ssl", ReadOnly: true},
394396
{Name: "host-run", MountPath: "/run"},
395397
{Name: "host-var-lib-buildkit", MountPath: "/var/lib/buildkit", MountPropagation: &mountPropagationBidirectional},
396-
{Name: "host-var-lib-rancher", MountPath: "/var/lib/rancher", MountPropagation: &mountPropagationBidirectional},
397398
{Name: "certs-ca", MountPath: "/certs/ca", ReadOnly: true},
398399
{Name: "certs-server", MountPath: "/certs/server", ReadOnly: true},
399400
},
@@ -442,9 +443,9 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
442443
},
443444
},
444445
{
445-
Name: "host-var-lib-rancher", VolumeSource: corev1.VolumeSource{
446+
Name: "host-containerd", VolumeSource: corev1.VolumeSource{
446447
HostPath: &corev1.HostPathVolumeSource{
447-
Path: "/var/lib/rancher", Type: &hostPathDirectoryOrCreate,
448+
Path: a.ContainerdVolume, Type: &hostPathDirectoryOrCreate,
448449
},
449450
},
450451
},
@@ -474,6 +475,7 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
474475
return err
475476
}
476477

478+
// NodeRole asserts that the node can run KIM and labels it with the builder role
477479
func (a *Install) NodeRole(_ context.Context, k *client.Interface) error {
478480
nodeList, err := k.Core.Node().List(metav1.ListOptions{
479481
LabelSelector: a.Selector,
@@ -488,6 +490,29 @@ func (a *Install) NodeRole(_ context.Context, k *client.Interface) error {
488490
if err != nil {
489491
return err
490492
}
493+
// detect container runtime and adjust defaults
494+
crv, err := url.Parse(node.Status.NodeInfo.ContainerRuntimeVersion)
495+
if err != nil {
496+
return errors.Wrap(err, "failed to parse container runtime version")
497+
}
498+
switch {
499+
case crv.Scheme == "containerd" && strings.Contains(crv.Host, "-k3s"): // embedded containerd
500+
if a.ContainerdSocket == "" {
501+
a.ContainerdSocket = server.K3sContainerdSocket
502+
}
503+
if a.ContainerdVolume == "" {
504+
a.ContainerdVolume = server.K3sContainerdVolume
505+
}
506+
case crv.Scheme == "containerd": // external containerd
507+
if a.ContainerdSocket == "" {
508+
a.ContainerdSocket = server.StockContainerdSocket
509+
}
510+
if a.ContainerdVolume == "" {
511+
a.ContainerdVolume = server.StockContainerdVolume
512+
}
513+
default:
514+
return errors.Errorf("container runtime `%s` not supported", crv.Scheme)
515+
}
491516
node.Labels = labels.Merge(node.Labels, labels.Set{
492517
"node-role.kubernetes.io/builder": "true",
493518
})

pkg/server/agent_linux.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func (a *Agent) Run(ctx context.Context) error {
3333
}
3434
defer backend.Close()
3535

36-
go a.syncImageContent(namespaces.WithNamespace(ctx, a.BuildkitNamespace), backend.Containerd)
36+
go a.syncImageContent(namespaces.WithNamespace(ctx, buildkitNamespace), backend.Containerd)
3737
go a.listenAndServe(ctx, backend)
3838

3939
select {
@@ -94,7 +94,7 @@ func (a *Agent) syncImageContent(ctx context.Context, ctr *containerd.Client) {
9494
if !ok {
9595
return
9696
}
97-
if evt.Namespace != a.BuildkitNamespace {
97+
if evt.Namespace != buildkitNamespace {
9898
continue
9999
}
100100
if err := handleImageEvent(ctx, ctr, evt.Event); err != nil {

pkg/server/config.go

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,12 @@ const (
2020
defaultAgentPort = 1233
2121
defaultAgentImage = "docker.io/rancher/kim"
2222
defaultBuildkitImage = "docker.io/moby/buildkit:v0.8.3"
23+
buildkitNamespace = "buildkit"
24+
25+
K3sContainerdSocket = "/run/k3s/containerd/containerd.sock"
26+
K3sContainerdVolume = "/var/lib/rancher"
27+
StockContainerdSocket = "/run/containerd/containerd.sock"
28+
StockContainerdVolume = "/var/lib/containerd"
2329
)
2430

2531
var (
@@ -29,13 +35,13 @@ var (
2935
)
3036

3137
type Config struct {
32-
AgentImage string `usage:"Image to run the agent w/ missing tag inferred from version"`
33-
AgentPort int `usage:"Port that the agent will listen on" default:"1233"`
34-
BuildkitImage string `usage:"BuildKit image for running buildkitd" default:"docker.io/moby/buildkit:v0.8.3"`
35-
BuildkitNamespace string `usage:"BuildKit namespace in containerd (not 'k8s.io')" default:"buildkit"`
36-
BuildkitPort int `usage:"BuildKit service port" default:"1234"`
37-
BuildkitSocket string `usage:"BuildKit socket address" default:"unix:///run/buildkit/buildkitd.sock"`
38-
ContainerdSocket string `usage:"Containerd socket address" default:"/run/k3s/containerd/containerd.sock"`
38+
AgentImage string `usage:"Image to run the agent w/ missing tag inferred from version"`
39+
AgentPort int `usage:"Port that the agent will listen on" default:"1233"`
40+
BuildkitImage string `usage:"BuildKit image for running buildkitd" default:"docker.io/moby/buildkit:v0.8.3"`
41+
BuildkitPort int `usage:"BuildKit service port" default:"1234"`
42+
BuildkitSocket string `usage:"BuildKit socket address" default:"unix:///run/buildkit/buildkitd.sock"`
43+
ContainerdSocket string `usage:"Containerd socket address (default on k3s \"/run/k3s/containerd/containerd.sock\")"`
44+
ContainerdVolume string `usage:"Containerd storage volume (default on k3s \"/var/lib/rancher\")"`
3945
}
4046

4147
func (c *Config) GetAgentImage() (string, error) {
@@ -86,7 +92,7 @@ func (c *Config) Interface(ctx context.Context, config *client.Config) (*images.
8692
return nil, err
8793
}
8894
server.Containerd, err = containerd.NewWithConn(conn,
89-
containerd.WithDefaultNamespace(c.BuildkitNamespace),
95+
containerd.WithDefaultNamespace(buildkitNamespace),
9096
containerd.WithTimeout(5*time.Second),
9197
)
9298
if err != nil {

0 commit comments

Comments
 (0)