Skip to content
This repository was archived by the owner on Sep 2, 2024. It is now read-only.

support kind clusters #55

Merged
merged 1 commit into from
May 25, 2021
Merged

support kind clusters #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 38 additions & 11 deletions pkg/client/builder/install.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"fmt"
"net"
"net/url"
"strings"
"time"

"github.com/pkg/errors"
Expand Down Expand Up @@ -298,7 +300,7 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
{Name: "_PATH", Value: "/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux"},
},
Command: []string{"sh", "-c"},
Args: []string{"(if mountpoint $_DIR; then nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
Args: []string{"(if mountpoint $_DIR; then set -x; nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
SecurityContext: &corev1.SecurityContext{
Privileged: &privileged,
},
Expand All @@ -313,27 +315,27 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
{Name: "_PATH", Value: "/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux"},
},
Command: []string{"sh", "-c"},
Args: []string{"(if mountpoint $_DIR; then nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
Args: []string{"(if mountpoint $_DIR; then set -x; nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
SecurityContext: &corev1.SecurityContext{
Privileged: &privileged,
},
VolumeMounts: []corev1.VolumeMount{
{Name: "host-var-lib-buildkit", MountPath: "/var/lib/buildkit"},
},
}, {
Name: "rshared-rancher",
Name: "rshared-containerd",
Image: buildkitImage,
Env: []corev1.EnvVar{
{Name: "_DIR", Value: "/var/lib/rancher"},
{Name: "_DIR", Value: a.ContainerdVolume},
{Name: "_PATH", Value: "/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux"},
},
Command: []string{"sh", "-c"},
Args: []string{"(if mountpoint $_DIR; then nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
Args: []string{"(if mountpoint $_DIR; then set -x; nsenter -m -p -t 1 -- env PATH=$_PATH sh -c 'mount --make-rshared $_DIR'; fi) || true"},
SecurityContext: &corev1.SecurityContext{
Privileged: &privileged,
},
VolumeMounts: []corev1.VolumeMount{
{Name: "host-var-lib-rancher", MountPath: "/var/lib/rancher"},
{Name: "host-containerd", MountPath: a.ContainerdVolume},
},
}},
Containers: []corev1.Container{{
Expand Down Expand Up @@ -361,7 +363,7 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
{Name: "host-run", MountPath: "/run"},
{Name: "host-tmp", MountPath: "/tmp", MountPropagation: &mountPropagationBidirectional},
{Name: "host-var-lib-buildkit", MountPath: "/var/lib/buildkit", MountPropagation: &mountPropagationBidirectional},
{Name: "host-var-lib-rancher", MountPath: "/var/lib/rancher", MountPropagation: &mountPropagationBidirectional},
{Name: "host-containerd", MountPath: a.ContainerdVolume, MountPropagation: &mountPropagationBidirectional},
{Name: "certs-ca", MountPath: "/certs/ca", ReadOnly: true},
{Name: "certs-server", MountPath: "/certs/server", ReadOnly: true},
},
Expand All @@ -373,7 +375,6 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
Command: []string{"kim", "--debug", "agent"},
Args: []string{
fmt.Sprintf("--agent-port=%d", a.AgentPort),
fmt.Sprintf("--buildkit-namespace=%s", a.BuildkitNamespace),
fmt.Sprintf("--buildkit-socket=%s", a.BuildkitSocket),
fmt.Sprintf("--buildkit-port=%d", a.BuildkitPort),
fmt.Sprintf("--containerd-socket=%s", a.ContainerdSocket),
Expand All @@ -388,12 +389,12 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
Privileged: &privileged,
},
VolumeMounts: []corev1.VolumeMount{
{Name: "host-containerd", MountPath: a.ContainerdVolume, MountPropagation: &mountPropagationBidirectional},
{Name: "host-ctl", MountPath: "/sys/fs/cgroup"},
{Name: "host-etc-pki", MountPath: "/etc/pki", ReadOnly: true},
{Name: "host-etc-ssl", MountPath: "/etc/ssl", ReadOnly: true},
{Name: "host-run", MountPath: "/run"},
{Name: "host-var-lib-buildkit", MountPath: "/var/lib/buildkit", MountPropagation: &mountPropagationBidirectional},
{Name: "host-var-lib-rancher", MountPath: "/var/lib/rancher", MountPropagation: &mountPropagationBidirectional},
{Name: "certs-ca", MountPath: "/certs/ca", ReadOnly: true},
{Name: "certs-server", MountPath: "/certs/server", ReadOnly: true},
},
Expand Down Expand Up @@ -442,9 +443,9 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
},
},
{
Name: "host-var-lib-rancher", VolumeSource: corev1.VolumeSource{
Name: "host-containerd", VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/var/lib/rancher", Type: &hostPathDirectoryOrCreate,
Path: a.ContainerdVolume, Type: &hostPathDirectoryOrCreate,
},
},
},
Expand Down Expand Up @@ -474,6 +475,7 @@ func (a *Install) DaemonSet(_ context.Context, k *client.Interface) error {
return err
}

// NodeRole asserts that the node can run KIM and labels it with the builder role
func (a *Install) NodeRole(_ context.Context, k *client.Interface) error {
nodeList, err := k.Core.Node().List(metav1.ListOptions{
LabelSelector: a.Selector,
Expand All @@ -488,6 +490,31 @@ func (a *Install) NodeRole(_ context.Context, k *client.Interface) error {
if err != nil {
return err
}
// detect container runtime and adjust defaults
crv, err := url.Parse(node.Status.NodeInfo.ContainerRuntimeVersion)
if err != nil {
return errors.Wrap(err, "failed to parse container runtime version")
}
switch {
// embedded containerd
case crv.Scheme == "containerd" && strings.Contains(crv.Host, "-k3s"):
if a.ContainerdSocket == "" {
a.ContainerdSocket = server.K3sContainerdSocket
}
if a.ContainerdVolume == "" {
a.ContainerdVolume = server.K3sContainerdVolume
}
// external containerd
case crv.Scheme == "containerd" /* && !strings.Contains(crv.Host, "-k3s") */ :
if a.ContainerdSocket == "" {
a.ContainerdSocket = server.StockContainerdSocket
}
if a.ContainerdVolume == "" {
a.ContainerdVolume = server.StockContainerdVolume
}
default:
return errors.Errorf("container runtime `%s` not supported", crv.Scheme)
}
node.Labels = labels.Merge(node.Labels, labels.Set{
"node-role.kubernetes.io/builder": "true",
})
Expand Down
4 changes: 2 additions & 2 deletions pkg/server/agent_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func (a *Agent) Run(ctx context.Context) error {
}
defer backend.Close()

go a.syncImageContent(namespaces.WithNamespace(ctx, a.BuildkitNamespace), backend.Containerd)
go a.syncImageContent(namespaces.WithNamespace(ctx, buildkitNamespace), backend.Containerd)
go a.listenAndServe(ctx, backend)

select {
Expand Down Expand Up @@ -94,7 +94,7 @@ func (a *Agent) syncImageContent(ctx context.Context, ctr *containerd.Client) {
if !ok {
return
}
if evt.Namespace != a.BuildkitNamespace {
if evt.Namespace != buildkitNamespace {
continue
}
if err := handleImageEvent(ctx, ctr, evt.Event); err != nil {
Expand Down
22 changes: 14 additions & 8 deletions pkg/server/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ const (
defaultAgentPort = 1233
defaultAgentImage = "docker.io/rancher/kim"
defaultBuildkitImage = "docker.io/moby/buildkit:v0.8.3"
buildkitNamespace = "buildkit"

K3sContainerdSocket = "/run/k3s/containerd/containerd.sock"
K3sContainerdVolume = "/var/lib/rancher"
StockContainerdSocket = "/run/containerd/containerd.sock"
StockContainerdVolume = "/var/lib/containerd"
)

var (
Expand All @@ -29,13 +35,13 @@ var (
)

type Config struct {
AgentImage string `usage:"Image to run the agent w/ missing tag inferred from version"`
AgentPort int `usage:"Port that the agent will listen on" default:"1233"`
BuildkitImage string `usage:"BuildKit image for running buildkitd" default:"docker.io/moby/buildkit:v0.8.3"`
BuildkitNamespace string `usage:"BuildKit namespace in containerd (not 'k8s.io')" default:"buildkit"`
BuildkitPort int `usage:"BuildKit service port" default:"1234"`
BuildkitSocket string `usage:"BuildKit socket address" default:"unix:///run/buildkit/buildkitd.sock"`
ContainerdSocket string `usage:"Containerd socket address" default:"/run/k3s/containerd/containerd.sock"`
AgentImage string `usage:"Image to run the agent w/ missing tag inferred from version"`
AgentPort int `usage:"Port that the agent will listen on" default:"1233"`
BuildkitImage string `usage:"BuildKit image for running buildkitd" default:"docker.io/moby/buildkit:v0.8.3"`
BuildkitPort int `usage:"BuildKit service port" default:"1234"`
BuildkitSocket string `usage:"BuildKit socket address" default:"unix:///run/buildkit/buildkitd.sock"`
ContainerdSocket string `usage:"Containerd socket address (default on k3s \"/run/k3s/containerd/containerd.sock\")"`
ContainerdVolume string `usage:"Containerd storage volume (default on k3s \"/var/lib/rancher\")"`
}

func (c *Config) GetAgentImage() (string, error) {
Expand Down Expand Up @@ -86,7 +92,7 @@ func (c *Config) Interface(ctx context.Context, config *client.Config) (*images.
return nil, err
}
server.Containerd, err = containerd.NewWithConn(conn,
containerd.WithDefaultNamespace(c.BuildkitNamespace),
containerd.WithDefaultNamespace(buildkitNamespace),
containerd.WithTimeout(5*time.Second),
)
if err != nil {
Expand Down