Pull container image and apply image config.

Signed-off-by: Lantao Liu <[email protected]>

Author: Random-Liu

cmd/cri-containerd/cri_containerd.go

@@ -48,6 +48,9 @@ func main() {
 	if err != nil {
 		glog.Exitf("Failed to create CRI containerd service %+v: %v", o, err)
 	}
+	if err := service.Start(); err != nil {
+		glog.Exitf("Failed to start CRI containerd service: %v", err)
+	}
 
 	s := server.NewCRIContainerdServer(o.SocketPath, service, service)
 	if err := s.Run(); err != nil {

pkg/server/container_create.go

@@ -20,9 +20,10 @@ import (
 	"fmt"
 	"time"
 
+	rootfsapi "github.com/containerd/containerd/api/services/rootfs"
 	"github.com/golang/glog"
+	imagedigest "github.com/opencontainers/go-digest"
 	"golang.org/x/net/context"
-
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
 
 	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
@@ -68,9 +69,23 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 		Config:    config,
 	}
 
-	// TODO(random-liu): [P0] Prepare container rootfs.
-
-	// TODO(random-liu): [P0] Set ImageRef in ContainerMetadata with image id.
+	// Prepare container image snapshot. For container, the image should have
+	// been pulled before creating the container, so do not ensure the image.
+	image := config.GetImage().GetImage()
+	imageMeta, err := c.getImageMetadata(ctx, image, false /*ensure*/)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get container image %q: %v", image, err)
+	}
+	if _, err := c.rootfsService.Prepare(ctx, &rootfsapi.PrepareRequest{
+		Name: id,
+		// We are sure that ChainID must be a digest.
+		ChainID:  imagedigest.Digest(imageMeta.ChainID),
+		Readonly: config.GetLinux().GetSecurityContext().GetReadonlyRootfs(),
+	}); err != nil {
+		return nil, fmt.Errorf("failed to prepare container rootfs %q: %v", imageMeta.ChainID, err)
+	}
+	// TODO(random-liu): [P0] Cleanup snapshot on failure after switching to new rootfs api.
+	meta.ImageRef = imageMeta.ID
 
 	// Create container root directory.
 	containerRootDir := getContainerRootDir(c.rootDir, id)

pkg/server/container_create_test.go

@@ -21,13 +21,17 @@ import (
 	"os"
 	"testing"
 
+	rootfsapi "github.com/containerd/containerd/api/services/rootfs"
+	imagedigest "github.com/opencontainers/go-digest"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
 
 	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	ostesting "github.com/kubernetes-incubator/cri-containerd/pkg/os/testing"
+	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
 )
 
 func TestCreateContainer(t *testing.T) {
@@ -42,10 +46,20 @@ func TestCreateContainer(t *testing.T) {
 		Namespace: "test-sandbox-namespace",
 		Attempt:   2,
 	}
+	// Use an image id to avoid image name resolution.
+	// TODO(random-liu): Change this to image name after we have complete image
+	// management unit test framework.
+	testImage := "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113799"
+	testChainID := imagedigest.Digest("test-chain-id")
+	testImageMetadata := metadata.ImageMetadata{
+		ID:      testImage,
+		ChainID: testChainID.String(),
+		Config:  &imagespec.ImageConfig{},
+	}
 	testConfig := &runtime.ContainerConfig{
 		Metadata: testNameMeta,
 		Image: &runtime.ImageSpec{
-			Image: "test-image",
+			Image: testImage,
 		},
 		Labels:      map[string]string{"a": "b"},
 		Annotations: map[string]string{"c": "d"},
@@ -55,12 +69,14 @@ func TestCreateContainer(t *testing.T) {
 	}
 
 	for desc, test := range map[string]struct {
-		sandboxMetadata   *metadata.SandboxMetadata
-		reserveNameErr    bool
-		createRootDirErr  error
-		createMetadataErr bool
-		expectErr         bool
-		expectMeta        *metadata.ContainerMetadata
+		sandboxMetadata    *metadata.SandboxMetadata
+		reserveNameErr     bool
+		imageMetadataErr   bool
+		prepareSnapshotErr error
+		createRootDirErr   error
+		createMetadataErr  bool
+		expectErr          bool
+		expectMeta         *metadata.ContainerMetadata
 	}{
 		"should return error if sandbox does not exist": {
 			sandboxMetadata: nil,
@@ -84,6 +100,24 @@ func TestCreateContainer(t *testing.T) {
 			createRootDirErr: errors.New("random error"),
 			expectErr:        true,
 		},
+		"should return error if image is not pulled": {
+			sandboxMetadata: &metadata.SandboxMetadata{
+				ID:     testSandboxID,
+				Name:   makeSandboxName(testSandboxNameMeta),
+				Config: testSandboxConfig,
+			},
+			imageMetadataErr: true,
+			expectErr:        true,
+		},
+		"should return error if prepare snapshot fails": {
+			sandboxMetadata: &metadata.SandboxMetadata{
+				ID:     testSandboxID,
+				Name:   makeSandboxName(testSandboxNameMeta),
+				Config: testSandboxConfig,
+			},
+			prepareSnapshotErr: errors.New("random error"),
+			expectErr:          true,
+		},
 		"should be able to create container successfully": {
 			sandboxMetadata: &metadata.SandboxMetadata{
 				ID:     testSandboxID,
@@ -94,12 +128,14 @@ func TestCreateContainer(t *testing.T) {
 			expectMeta: &metadata.ContainerMetadata{
 				Name:      makeContainerName(testNameMeta, testSandboxNameMeta),
 				SandboxID: testSandboxID,
+				ImageRef:  testImage,
 				Config:    testConfig,
 			},
 		},
 	} {
 		t.Logf("TestCase %q", desc)
 		c := newTestCRIContainerdService()
+		fakeRootfsClient := c.rootfsService.(*servertesting.FakeRootfsClient)
 		fakeOS := c.os.(*ostesting.FakeOS)
 		if test.sandboxMetadata != nil {
 			assert.NoError(t, c.sandboxStore.Create(*test.sandboxMetadata))
@@ -108,6 +144,13 @@ func TestCreateContainer(t *testing.T) {
 		if test.reserveNameErr {
 			assert.NoError(t, c.containerNameIndex.Reserve(containerName, "random id"))
 		}
+		if !test.imageMetadataErr {
+			assert.NoError(t, c.imageMetadataStore.Create(testImageMetadata))
+		}
+		if test.prepareSnapshotErr != nil {
+			fakeRootfsClient.InjectError("prepare", test.prepareSnapshotErr)
+		}
+		fakeRootfsClient.SetFakeChainIDs([]imagedigest.Digest{testChainID})
 		rootExists := false
 		rootPath := ""
 		fakeOS.MkdirAllFn = func(path string, perm os.FileMode) error {
@@ -153,5 +196,15 @@ func TestCreateContainer(t *testing.T) {
 		// TODO(random-liu): Use fake clock to test CreatedAt.
 		test.expectMeta.CreatedAt = meta.CreatedAt
 		assert.Equal(t, test.expectMeta, meta, "container metadata should be created")
+
+		assert.Equal(t, []string{"prepare"}, fakeRootfsClient.GetCalledNames(), "prepare should be called")
+		calls := fakeRootfsClient.GetCalledDetails()
+		prepareOpts := calls[0].Argument.(*rootfsapi.PrepareRequest)
+		assert.Equal(t, &rootfsapi.PrepareRequest{
+			Name:    id,
+			ChainID: testChainID,
+			// TODO(random-liu): Test readonly rootfs.
+			Readonly: false,
+		}, prepareOpts, "prepare request should be correct")
 	}
 }

pkg/server/container_remove.go

@@ -69,7 +69,7 @@ func (c *criContainerdService) RemoveContainer(ctx context.Context, r *runtime.R
 	// kubelet implementation, we'll never start a container once we decide to remove it,
 	// so we don't need the "Dead" state for now.
 
-	// TODO(random-liu): [P0] Cleanup container rootfs.
+	// TODO(random-liu): [P0] Cleanup snapshot after switching to new snapshot api.
 
 	// Cleanup container root directory.
 	containerRootDir := getContainerRootDir(c.rootDir, id)

pkg/server/container_start.go

@@ -23,16 +23,15 @@ import (
 	"os"
 	"time"
 
+	"github.com/containerd/containerd/api/services/execution"
+	rootfsapi "github.com/containerd/containerd/api/services/rootfs"
+	"github.com/containerd/containerd/api/types/container"
 	prototypes "github.com/gogo/protobuf/types"
 	"github.com/golang/glog"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
 	"golang.org/x/net/context"
-
-	"github.com/containerd/containerd/api/services/execution"
-	"github.com/containerd/containerd/api/types/container"
-	"github.com/containerd/containerd/api/types/mount"
-
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
 
 	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
@@ -114,12 +113,11 @@ func (c *criContainerdService) startContainer(ctx context.Context, id string, me
 	glog.V(2).Infof("Sandbox container %q is running with pid %d", sandboxID, sandboxPid)
 
 	// Generate containerd container create options.
-	// TODO(random-liu): [P0] Create container rootfs with image ref.
-	// TODO(random-liu): [P0] Apply default image config.
-	// Use fixed rootfs path for now.
-	const rootPath = "/"
-
-	spec, err := c.generateContainerSpec(id, sandboxPid, config, sandboxConfig)
+	imageMeta, err := c.imageMetadataStore.Get(meta.ImageRef)
+	if err != nil {
+		return fmt.Errorf("failed to get container image %q: %v", meta.ImageRef, err)
+	}
+	spec, err := c.generateContainerSpec(id, sandboxPid, config, sandboxConfig, imageMeta.Config)
 	if err != nil {
 		return fmt.Errorf("failed to generate container %q spec: %v", id, err)
 	}
@@ -169,31 +167,27 @@ func (c *criContainerdService) startContainer(ctx context.Context, id string, me
 		}(stderrPipe)
 	}
 
+	// Get rootfs mounts.
+	mountsResp, err := c.rootfsService.Mounts(ctx, &rootfsapi.MountsRequest{Name: id})
+	if err != nil {
+		return fmt.Errorf("failed to get rootfs mounts %q: %v", id, err)
+	}
+
 	// Create containerd container.
 	createOpts := &execution.CreateRequest{
 		ID: id,
 		Spec: &prototypes.Any{
 			TypeUrl: runtimespec.Version,
 			Value:   rawSpec,
 		},
-		// TODO(random-liu): [P0] Get rootfs mount from containerd.
-		Rootfs: []*mount.Mount{
-			{
-				Type:   "bind",
-				Source: rootPath,
-				Options: []string{
-					"rw",
-					"rbind",
-				},
-			},
-		},
+		Rootfs:   mountsResp.Mounts,
 		Runtime:  defaultRuntime,
 		Stdin:    stdin,
 		Stdout:   stdout,
 		Stderr:   stderr,
 		Terminal: config.GetTty(),
 	}
-	glog.V(2).Infof("Create containerd container (id=%q, name=%q) with options %+v.",
+	glog.V(5).Infof("Create containerd container (id=%q, name=%q) with options %+v.",
 		id, meta.Name, createOpts)
 	createResp, err := c.containerService.Create(ctx, createOpts)
 	if err != nil {
@@ -219,7 +213,8 @@ func (c *criContainerdService) startContainer(ctx context.Context, id string, me
 	return nil
 }
 
-func (c *criContainerdService) generateContainerSpec(id string, sandboxPid uint32, config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig) (*runtimespec.Spec, error) {
+func (c *criContainerdService) generateContainerSpec(id string, sandboxPid uint32, config *runtime.ContainerConfig,
+	sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) (*runtimespec.Spec, error) {
 	// Creates a spec Generator with the default spec.
 	// TODO(random-liu): [P2] Move container runtime spec generation into a helper function.
 	g := generate.New()
@@ -228,14 +223,21 @@ func (c *criContainerdService) generateContainerSpec(id string, sandboxPid uint3
 	// pre-defined directory.
 	g.SetRootPath(relativeRootfsPath)
 
-	if len(config.GetCommand()) != 0 || len(config.GetArgs()) != 0 {
-		g.SetProcessArgs(append(config.GetCommand(), config.GetArgs()...))
+	if err := setOCIProcessArgs(&g, config, imageConfig); err != nil {
+		return nil, err
 	}
 
 	if config.GetWorkingDir() != "" {
 		g.SetProcessCwd(config.GetWorkingDir())
+	} else if imageConfig.WorkingDir != "" {
+		g.SetProcessCwd(imageConfig.WorkingDir)
 	}
 
+	// Apply envs from image config first, so that envs from container config
+	// can override them.
+	if err := addImageEnvs(&g, imageConfig.Env); err != nil {
+		return nil, err
+	}
 	for _, e := range config.GetEnvs() {
 		g.AddProcessEnv(e.GetKey(), e.GetValue())
 	}
@@ -288,6 +290,27 @@ func (c *criContainerdService) generateContainerSpec(id string, sandboxPid uint3
 	return g.Spec(), nil
 }
 
+// setOCIProcessArgs sets process args. It returns error if the final arg list
+// is empty.
+func setOCIProcessArgs(g *generate.Generator, config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) error {
+	command, args := config.GetCommand(), config.GetArgs()
+	// The following logic is migrated from https://github.com/moby/moby/blob/master/daemon/commit.go
+	// TODO(random-liu): Figure out whether the following logic is
+	if len(command) == 0 {
+		if len(args) == 0 {
+			args = imageConfig.Cmd
+		}
+		if command == nil {
+			command = imageConfig.Entrypoint
+		}
+	}
+	if len(command) == 0 && len(args) == 0 {
+		return fmt.Errorf("no command specified")
+	}
+	g.SetProcessArgs(append(command, args...))
+	return nil
+}
+
 // addOCIBindMounts adds bind mounts.
 func addOCIBindMounts(g *generate.Generator, mounts []*runtime.Mount) {
 	for _, mount := range mounts {