Skip to content

Avoid adding ipfs dependencies to github.com/containerd/stargz-snapshotter #511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AkihiroSuda merged 1 commit into from
Nov 4, 2021
Merged

Avoid adding ipfs dependencies to github.com/containerd/stargz-snapshotter #511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# Base path used to install.
CMD_DESTDIR ?= /usr/local
GO111MODULE_VALUE=auto
PREFIX ?= out/
PREFIX ?= $(CURDIR)/out/

PKG=github.com/containerd/stargz-snapshotter
VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
Expand All @@ -27,7 +27,7 @@ CMD=containerd-stargz-grpc ctr-remote stargz-store

CMD_BINARIES=$(addprefix $(PREFIX),$(CMD))

.PHONY: all build check install-check-tools install uninstall clean test test-root test-all integration test-optimize benchmark test-kind test-cri-containerd test-cri-o test-criauth generate validate-generated test-k3s test-k3s-argo-workflow
.PHONY: all build check install-check-tools install uninstall clean test test-root test-all integration test-optimize benchmark test-kind test-cri-containerd test-cri-o test-criauth generate validate-generated test-k3s test-k3s-argo-workflow vendor

all: build

Expand All @@ -36,18 +36,20 @@ build: $(CMD)
FORCE:

containerd-stargz-grpc: FORCE
GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/containerd-stargz-grpc
cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./containerd-stargz-grpc

ctr-remote: FORCE
GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/ctr-remote
cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./ctr-remote

stargz-store: FORCE
GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/stargz-store
cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./stargz-store

check:
@echo "$@"
@GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run

install-check-tools:
@curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.42.1
Expand All @@ -71,10 +73,18 @@ generate:
validate-generated:
@./script/generated-files/generate.sh validate

vendor:
@GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy

test:
@echo "$@"
@GO111MODULE=$(GO111MODULE_VALUE) go test -race ./...
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 20m -race ./...
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 30m -race ./...
@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 20m -race ./...
@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 20m -race ./...

test-root:
@echo "$@"
Expand Down
38 changes: 18 additions & 20 deletions fs/remote/ipfs/reader.go → ...ainerd-stargz-grpc/ipfs/resolvehandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,24 +22,17 @@ import (
"fmt"
"io"

"github.com/containerd/stargz-snapshotter/fs/remote"
"github.com/containerd/stargz-snapshotter/ipfs"
httpapi "github.com/ipfs/go-ipfs-http-client"
iface "github.com/ipfs/interface-go-ipfs-core"
ipath "github.com/ipfs/interface-go-ipfs-core/path"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

type Reader struct {
api iface.CoreAPI
path ipath.Path
}

func Supported(desc ocispec.Descriptor) bool {
_, err := ipfs.GetPath(desc)
return err == nil
}
type ResolveHandler struct{}

func NewReader(ctx context.Context, desc ocispec.Descriptor) (*Reader, int64, error) {
func (r *ResolveHandler) Handle(ctx context.Context, desc ocispec.Descriptor) (remote.Fetcher, int64, error) {
p, err := ipfs.GetPath(desc)
if err != nil {
return nil, 0, err
Expand All @@ -62,11 +55,16 @@ func NewReader(ctx context.Context, desc ocispec.Descriptor) (*Reader, int64, er
if err != nil {
return nil, 0, err
}
return &Reader{client, p}, s, nil
return &fetcher{client, p}, s, nil
}

type fetcher struct {
api iface.CoreAPI
path ipath.Path
}

func (r *Reader) Reader(ctx context.Context, off int64, size int64) (io.ReadCloser, error) {
n, err := r.api.Unixfs().Get(ctx, r.path)
func (f *fetcher) Fetch(ctx context.Context, off int64, size int64) (io.ReadCloser, error) {
n, err := f.api.Unixfs().Get(ctx, f.path)
if err != nil {
return nil, err
}
Expand All @@ -76,14 +74,14 @@ func (r *Reader) Reader(ctx context.Context, off int64, size int64) (io.ReadClos
if !ok {
return nil, fmt.Errorf("ReaderAt is not implemented")
}
return &reader{
return &readCloser{
Reader: io.NewSectionReader(ra, off, size),
closeFunc: n.Close,
}, nil
}

func (r *Reader) Check() error {
n, err := r.api.Unixfs().Get(context.Background(), r.path)
func (f *fetcher) Check() error {
n, err := f.api.Unixfs().Get(context.Background(), f.path)
if err != nil {
return err
}
Expand All @@ -95,14 +93,14 @@ func (r *Reader) Check() error {
return n.Close()
}

func (r *Reader) GenID(off int64, size int64) string {
sum := sha256.Sum256([]byte(fmt.Sprintf("%s-%d-%d", r.path.String(), off, size)))
func (f *fetcher) GenID(off int64, size int64) string {
sum := sha256.Sum256([]byte(fmt.Sprintf("%s-%d-%d", f.path.String(), off, size)))
return fmt.Sprintf("%x", sum)
}

type reader struct {
type readCloser struct {
io.Reader
closeFunc func() error
}

func (r *reader) Close() error { return r.closeFunc() }
func (r *readCloser) Close() error { return r.closeFunc() }
12 changes: 11 additions & 1 deletion cmd/containerd-stargz-grpc/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ import (
"github.com/containerd/containerd/pkg/dialer"
"github.com/containerd/containerd/snapshots"
"github.com/containerd/containerd/sys"
ipfs "github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/ipfs"
"github.com/containerd/stargz-snapshotter/fs"
"github.com/containerd/stargz-snapshotter/service"
"github.com/containerd/stargz-snapshotter/service/keychain/cri"
"github.com/containerd/stargz-snapshotter/service/keychain/dockerconfig"
Expand Down Expand Up @@ -80,6 +82,9 @@ type snapshotterConfig struct {

// DebugAddress is a Unix domain socket address where the snapshotter exposes /debug/ endpoints.
DebugAddress string `toml:"debug_address"`

// IPFS is a flag to enbale lazy pulling from IPFS.
IPFS bool `toml:"ipfs"`
}

func main() {
Expand Down Expand Up @@ -162,7 +167,12 @@ func main() {
runtime.RegisterImageServiceServer(rpc, criServer)
credsFuncs = append(credsFuncs, f)
}
rs, err := service.NewStargzSnapshotterService(ctx, *rootDir, &config.Config, service.WithCredsFuncs(credsFuncs...))
var fsOpts []fs.Option
if config.IPFS {
fsOpts = append(fsOpts, fs.WithResolveHandler("ipfs", new(ipfs.ResolveHandler)))
}
rs, err := service.NewStargzSnapshotterService(ctx, *rootDir, &config.Config,
service.WithCredsFuncs(credsFuncs...), service.WithFilesystemOptions(fsOpts...))
if err != nil {
log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
}
Expand Down
37 changes: 37 additions & 0 deletions cmd/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
module github.com/containerd/stargz-snapshotter/cmd

go 1.16

require (
github.com/containerd/containerd v1.5.7
github.com/containerd/go-cni v1.1.0
github.com/containerd/stargz-snapshotter v0.9.0
github.com/containerd/stargz-snapshotter/estargz v0.9.0
github.com/containerd/stargz-snapshotter/ipfs v0.9.0
github.com/coreos/go-systemd/v22 v22.3.2
github.com/docker/go-metrics v0.0.1
github.com/hashicorp/go-multierror v1.1.1
github.com/ipfs/go-ipfs-http-client v0.1.0
github.com/ipfs/interface-go-ipfs-core v0.5.2
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
github.com/pelletier/go-toml v1.9.4
github.com/pkg/errors v0.9.1
github.com/rs/xid v1.3.0
github.com/sirupsen/logrus v1.8.1
github.com/urfave/cli v1.22.4
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164
google.golang.org/grpc v1.42.0
k8s.io/cri-api v0.22.3
)

replace (
// Import local packages.
github.com/containerd/stargz-snapshotter => ../
github.com/containerd/stargz-snapshotter/estargz => ../estargz
github.com/containerd/stargz-snapshotter/ipfs => ../ipfs

// Temporary fork for avoiding importing patent-protected code: https://github.com/hashicorp/golang-lru/issues/73
github.com/hashicorp/golang-lru => github.com/ktock/golang-lru v0.5.5-0.20211029085301-ec551be6f75c
)
Loading