Add sandbox /dev/shm.

Signed-off-by: Lantao Liu <[email protected]>

Author: Random-Liu

pkg/server/container_start.go

@@ -309,8 +309,16 @@ func (c *criContainerdService) generateContainerMounts(sandboxRootDir string, co
 		HostPath:      getSandboxHosts(sandboxRootDir),
 		Readonly:      securityContext.ReadonlyRootfs,
 	})
+	sandboxDevShm := getSandboxDevShm(sandboxRootDir)
+	if securityContext.GetNamespaceOptions().GetHostIpc() {
+		sandboxDevShm = devShm
+	}
+	mounts = append(mounts, &runtime.Mount{
+		ContainerPath: devShm,
+		HostPath:      sandboxDevShm,
+		Readonly:      false,
+	})
 	// TODO(random-liu): [P0] Mount sandbox resolv.config.
-	// TODO(random-liu): [P0] Mount sandbox /dev/shm.
 	return mounts
 }
 
@@ -349,6 +357,8 @@ func addImageEnvs(g *generate.Generator, imageEnvs []string) error {
 }
 
 // addOCIBindMounts adds bind mounts.
+// TODO(random-liu): Figure out whether we need to change all CRI mounts to readonly when
+// rootfs is readonly. (https://github.com/moby/moby/blob/master/daemon/oci_linux.go)
 func addOCIBindMounts(g *generate.Generator, mounts []*runtime.Mount) {
 	for _, mount := range mounts {
 		dst := mount.GetContainerPath()

pkg/server/container_start_test.go

@@ -313,19 +313,50 @@ func TestGenerateContainerMounts(t *testing.T) {
 			securityContext: &runtime.LinuxContainerSecurityContext{
 				ReadonlyRootfs: true,
 			},
-			expectedMounts: []*runtime.Mount{{
-				ContainerPath: "/etc/hosts",
-				HostPath:      testSandboxRootDir + "/hosts",
-				Readonly:      true,
-			}},
+			expectedMounts: []*runtime.Mount{
+				{
+					ContainerPath: "/etc/hosts",
+					HostPath:      testSandboxRootDir + "/hosts",
+					Readonly:      true,
+				},
+				{
+					ContainerPath: "/dev/shm",
+					HostPath:      testSandboxRootDir + "/shm",
+					Readonly:      false,
+				},
+			},
 		},
 		"should setup rw /etc/hosts mount when rootfs is read-write": {
 			securityContext: &runtime.LinuxContainerSecurityContext{},
-			expectedMounts: []*runtime.Mount{{
-				ContainerPath: "/etc/hosts",
-				HostPath:      testSandboxRootDir + "/hosts",
-				Readonly:      false,
-			}},
+			expectedMounts: []*runtime.Mount{
+				{
+					ContainerPath: "/etc/hosts",
+					HostPath:      testSandboxRootDir + "/hosts",
+					Readonly:      false,
+				},
+				{
+					ContainerPath: "/dev/shm",
+					HostPath:      testSandboxRootDir + "/shm",
+					Readonly:      false,
+				},
+			},
+		},
+		"should use host /dev/shm when host ipc is set": {
+			securityContext: &runtime.LinuxContainerSecurityContext{
+				NamespaceOptions: &runtime.NamespaceOption{HostIpc: true},
+			},
+			expectedMounts: []*runtime.Mount{
+				{
+					ContainerPath: "/etc/hosts",
+					HostPath:      testSandboxRootDir + "/hosts",
+					Readonly:      false,
+				},
+				{
+					ContainerPath: "/dev/shm",
+					HostPath:      "/dev/shm",
+					Readonly:      false,
+				},
+			},
 		},
 	} {
 		config := &runtime.ContainerConfig{

pkg/server/helpers.go

@@ -58,6 +58,8 @@ const (
 	// defaultSandboxImage is the image used by sandbox container.
 	// TODO(random-liu): [P1] Build schema 2 pause image and use it here.
 	defaultSandboxImage = "gcr.io/google.com/noogler-kubernetes/pause-amd64:3.0"
+	// defaultShmSize is the default size of the sandbox shm.
+	defaultShmSize = int64(1024 * 1024 * 64)
 	// relativeRootfsPath is the rootfs path relative to bundle path.
 	relativeRootfsPath = "rootfs"
 	// defaultRuntime is the runtime to use in containerd. We may support
@@ -85,6 +87,8 @@ const (
 	utsNSFormat = "/proc/%v/ns/uts"
 	// pidNSFormat is the format of pid namespace of a process.
 	pidNSFormat = "/proc/%v/ns/pid"
+	// devShm is the default path of /dev/shm.
+	devShm = "/dev/shm"
 	// etcHosts is the default path of /etc/hosts file.
 	etcHosts = "/etc/hosts"
 )
@@ -149,6 +153,11 @@ func getSandboxHosts(sandboxRootDir string) string {
 	return filepath.Join(sandboxRootDir, "hosts")
 }
 
+// getSandboxDevShm returns the shm file path inside the sandbox root directory.
+func getSandboxDevShm(sandboxRootDir string) string {
+	return filepath.Join(sandboxRootDir, "shm")
+}
+
 // prepareStreamingPipes prepares stream named pipe for container. returns nil
 // streaming handler if corresponding stream path is empty.
 func (c *criContainerdService) prepareStreamingPipes(ctx context.Context, stdin, stdout, stderr string) (

pkg/server/sandbox_run.go

@@ -19,6 +19,7 @@ package server
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/containerd/containerd/api/services/execution"
@@ -30,6 +31,7 @@ import (
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
 	"golang.org/x/net/context"
+	"golang.org/x/sys/unix"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
 
 	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
@@ -133,11 +135,14 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 	}
 
 	// Setup sandbox /dev/shm, /etc/hosts and /etc/resolv.conf.
-	if err = c.setupSandboxFiles(sandboxRootDir, config); err != nil {
+	if err = c.setupSandboxFiles(sandboxRootDir, config.GetLinux().GetSecurityContext()); err != nil {
 		return nil, fmt.Errorf("failed to setup sandbox files: %v", err)
 	}
-	// No need to cleanup on error, because the whole sandbox root directory will be removed
-	// on error.
+	defer func() {
+		if retErr != nil {
+			c.cleanupSandboxFiles(sandboxRootDir, config.GetLinux().GetSecurityContext())
+		}
+	}()
 
 	// Start sandbox container.
 	spec, err := c.generateSandboxContainerSpec(id, config, imageMeta.Config)
@@ -297,14 +302,38 @@ func (c *criContainerdService) generateSandboxContainerSpec(id string, config *r
 
 // setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts
 // and /etc/resolv.conf.
-func (c *criContainerdService) setupSandboxFiles(rootDir string, config *runtime.PodSandboxConfig) error {
+func (c *criContainerdService) setupSandboxFiles(rootDir string, sc *runtime.LinuxSandboxSecurityContext) error {
 	// TODO(random-liu): Consider whether we should maintain /etc/hosts and /etc/resolv.conf in kubelet.
 	sandboxEtcHosts := getSandboxHosts(rootDir)
 	if err := c.os.CopyFile(etcHosts, sandboxEtcHosts, 0666); err != nil {
 		return fmt.Errorf("failed to generate sandbox hosts file %q: %v", sandboxEtcHosts, err)
 	}
+	// Setup sandbox /dev/shm.
+	if sc.GetNamespaceOptions().GetHostIpc() {
+		if _, err := c.os.Stat(devShm); err != nil {
+			return fmt.Errorf("host %q is not available for host ipc: %v", devShm, err)
+		}
+	} else {
+		sandboxDevShm := getSandboxDevShm(rootDir)
+		if err := c.os.MkdirAll(sandboxDevShm, 0700); err != nil {
+			return fmt.Errorf("failed to create sandbox shm: %v", err)
+		}
+		shmproperty := fmt.Sprintf("mode=1777,size=%d", defaultShmSize)
+		if err := c.os.Mount("shm", sandboxDevShm, "tmpfs", uintptr(unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV), shmproperty); err != nil {
+			return fmt.Errorf("failed to mount sandbox shm: %v", err)
+		}
+	}
+
 	// TODO(random-liu): [P0] Set DNS options. Maintain a resolv.conf for the sandbox.
-	// TODO(random-liu): [P0] Deal with /dev/shm. Use host for HostIpc, and create and mount for
-	// non-HostIpc. What about mqueue?
 	return nil
 }
+
+// cleanupSandboxFiles only unmount files, we rely on the removal of sandbox root directory to remove files.
+// Each cleanup task should log error instead of returning, so as to keep on cleanup on error.
+func (c *criContainerdService) cleanupSandboxFiles(rootDir string, sc *runtime.LinuxSandboxSecurityContext) {
+	if !sc.GetNamespaceOptions().GetHostIpc() {
+		if err := c.os.Unmount(getSandboxDevShm(rootDir), unix.MNT_DETACH); err != nil && os.IsNotExist(err) {
+			glog.Errorf("failed to unmount sandbox shm: %v", err)
+		}
+	}
+}

pkg/server/sandbox_run_test.go

@@ -159,17 +159,65 @@ func TestGenerateSandboxContainerSpec(t *testing.T) {
 
 func TestSetupSandboxFiles(t *testing.T) {
 	testRootDir := "test-sandbox-root"
-	expectedCopys := [][]interface{}{
-		{"/etc/hosts", testRootDir + "/hosts", os.FileMode(0666)},
-	}
-	c := newTestCRIContainerdService()
-	var copys [][]interface{}
-	c.os.(*ostesting.FakeOS).CopyFileFn = func(src string, dest string, perm os.FileMode) error {
-		copys = append(copys, []interface{}{src, dest, perm})
-		return nil
+	for desc, test := range map[string]struct {
+		hostIpc       bool
+		expectedCalls []ostesting.CalledDetail
+	}{
+		"should check host /dev/shm existence when hostIpc is true": {
+			hostIpc: true,
+			expectedCalls: []ostesting.CalledDetail{
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/hosts", testRootDir + "/hosts", os.FileMode(0666),
+					},
+				},
+				{
+					Name:      "Stat",
+					Arguments: []interface{}{"/dev/shm"},
+				},
+			},
+		},
+		"should create sandbox shm when hostIpc is false": {
+			hostIpc: false,
+			expectedCalls: []ostesting.CalledDetail{
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/hosts", testRootDir + "/hosts", os.FileMode(0666),
+					},
+				},
+				{
+					Name: "MkdirAll",
+					Arguments: []interface{}{
+						testRootDir + "/shm", os.FileMode(0700),
+					},
+				},
+				{
+					Name: "Mount",
+					// Ignore arguments which are too complex to check.
+				},
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		c := newTestCRIContainerdService()
+		cfg := &runtime.LinuxSandboxSecurityContext{
+			NamespaceOptions: &runtime.NamespaceOption{
+				HostIpc: test.hostIpc,
+			},
+		}
+		c.setupSandboxFiles(testRootDir, cfg)
+		calls := c.os.(*ostesting.FakeOS).GetCalls()
+		assert.Len(t, calls, len(test.expectedCalls))
+		for i, expected := range test.expectedCalls {
+			if expected.Arguments == nil {
+				// Ignore arguments.
+				expected.Arguments = calls[i].Arguments
+			}
+			assert.Equal(t, expected, calls[i])
+		}
 	}
-	c.setupSandboxFiles(testRootDir, nil)
-	assert.Equal(t, expectedCopys, copys, "should copy /etc/hosts for sandbox")
 }
 
 func TestRunPodSandbox(t *testing.T) {
@@ -183,7 +231,6 @@ func TestRunPodSandbox(t *testing.T) {
 	var pipes []string
 	fakeOS.MkdirAllFn = func(path string, perm os.FileMode) error {
 		dirs = append(dirs, path)
-		assert.Equal(t, os.FileMode(0755), perm)
 		return nil
 	}
 	fakeOS.OpenFifoFn = func(ctx context.Context, fn string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) {
@@ -210,11 +257,11 @@ func TestRunPodSandbox(t *testing.T) {
 	require.NotNil(t, res)
 	id := res.GetPodSandboxId()
 
-	assert.Len(t, dirs, 1)
-	assert.Equal(t, getSandboxRootDir(c.rootDir, id), dirs[0], "sandbox root directory should be created")
+	sandboxRootDir := getSandboxRootDir(c.rootDir, id)
+	assert.Contains(t, dirs[0], sandboxRootDir, "sandbox root directory should be created")
 
 	assert.Len(t, pipes, 2)
-	_, stdout, stderr := getStreamingPipes(getSandboxRootDir(c.rootDir, id))
+	_, stdout, stderr := getStreamingPipes(sandboxRootDir)
 	assert.Contains(t, pipes, stdout, "sandbox stdout pipe should be created")
 	assert.Contains(t, pipes, stderr, "sandbox stderr pipe should be created")