Skip to content

Commit 6b6ca38

Browse files
AkihiroSudaAkihiroSuda
authored
Merge pull request #511 from ktock/ipfsdeps
Avoid adding ipfs dependencies to `github.com/containerd/stargz-snapshotter`
2 parents 20c9e08 + abde477 commit 6b6ca38

File tree

22 files changed

+3959
-1027
lines changed

22 files changed

+3959
-1027
lines changed

‎Makefile‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
# Base path used to install.
1717
CMD_DESTDIR ?= /usr/local
1818
GO111MODULE_VALUE=auto
19-
PREFIX ?= out/
19+
PREFIX ?= $(CURDIR)/out/
2020

2121
PKG=github.com/containerd/stargz-snapshotter
2222
VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
@@ -27,7 +27,7 @@ CMD=containerd-stargz-grpc ctr-remote stargz-store
2727

2828
CMD_BINARIES=$(addprefix $(PREFIX),$(CMD))
2929

30-
.PHONY: all build check install-check-tools install uninstall clean test test-root test-all integration test-optimize benchmark test-kind test-cri-containerd test-cri-o test-criauth generate validate-generated test-k3s test-k3s-argo-workflow
30+
.PHONY: all build check install-check-tools install uninstall clean test test-root test-all integration test-optimize benchmark test-kind test-cri-containerd test-cri-o test-criauth generate validate-generated test-k3s test-k3s-argo-workflow vendor
3131

3232
all: build
3333

@@ -36,18 +36,20 @@ build: $(CMD)
3636
FORCE:
3737

3838
containerd-stargz-grpc: FORCE
39-
GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/containerd-stargz-grpc
39+
cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./containerd-stargz-grpc
4040

4141
ctr-remote: FORCE
42-
GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/ctr-remote
42+
cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./ctr-remote
4343

4444
stargz-store: FORCE
45-
GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/stargz-store
45+
cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./stargz-store
4646

4747
check:
4848
@echo "$@"
4949
@GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
5050
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
51+
@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
52+
@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
5153

5254
install-check-tools:
5355
@curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.42.1
@@ -71,10 +73,18 @@ generate:
7173
validate-generated:
7274
@./script/generated-files/generate.sh validate
7375

76+
vendor:
77+
@GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
78+
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
79+
@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
80+
@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
81+
7482
test:
7583
@echo "$@"
7684
@GO111MODULE=$(GO111MODULE_VALUE) go test -race ./...
77-
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 20m -race ./...
85+
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 30m -race ./...
86+
@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 20m -race ./...
87+
@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 20m -race ./...
7888

7989
test-root:
8090
@echo "$@"

fs/remote/ipfs/reader.go renamed to cmd/containerd-stargz-grpc/ipfs/resolvehandler.go

Lines changed: 18 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -22,24 +22,17 @@ import (
2222
"fmt"
2323
"io"
2424

25+
"github.com/containerd/stargz-snapshotter/fs/remote"
2526
"github.com/containerd/stargz-snapshotter/ipfs"
2627
httpapi "github.com/ipfs/go-ipfs-http-client"
2728
iface "github.com/ipfs/interface-go-ipfs-core"
2829
ipath "github.com/ipfs/interface-go-ipfs-core/path"
2930
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
3031
)
3132

32-
type Reader struct {
33-
api iface.CoreAPI
34-
path ipath.Path
35-
}
36-
37-
func Supported(desc ocispec.Descriptor) bool {
38-
_, err := ipfs.GetPath(desc)
39-
return err == nil
40-
}
33+
type ResolveHandler struct{}
4134

42-
func NewReader(ctx context.Context, desc ocispec.Descriptor) (*Reader, int64, error) {
35+
func (r *ResolveHandler) Handle(ctx context.Context, desc ocispec.Descriptor) (remote.Fetcher, int64, error) {
4336
p, err := ipfs.GetPath(desc)
4437
if err != nil {
4538
return nil, 0, err
@@ -62,11 +55,16 @@ func NewReader(ctx context.Context, desc ocispec.Descriptor) (*Reader, int64, er
6255
if err != nil {
6356
return nil, 0, err
6457
}
65-
return &Reader{client, p}, s, nil
58+
return &fetcher{client, p}, s, nil
59+
}
60+
61+
type fetcher struct {
62+
api iface.CoreAPI
63+
path ipath.Path
6664
}
6765

68-
func (r *Reader) Reader(ctx context.Context, off int64, size int64) (io.ReadCloser, error) {
69-
n, err := r.api.Unixfs().Get(ctx, r.path)
66+
func (f *fetcher) Fetch(ctx context.Context, off int64, size int64) (io.ReadCloser, error) {
67+
n, err := f.api.Unixfs().Get(ctx, f.path)
7068
if err != nil {
7169
return nil, err
7270
}
@@ -76,14 +74,14 @@ func (r *Reader) Reader(ctx context.Context, off int64, size int64) (io.ReadClos
7674
if !ok {
7775
return nil, fmt.Errorf("ReaderAt is not implemented")
7876
}
79-
return &reader{
77+
return &readCloser{
8078
Reader: io.NewSectionReader(ra, off, size),
8179
closeFunc: n.Close,
8280
}, nil
8381
}
8482

85-
func (r *Reader) Check() error {
86-
n, err := r.api.Unixfs().Get(context.Background(), r.path)
83+
func (f *fetcher) Check() error {
84+
n, err := f.api.Unixfs().Get(context.Background(), f.path)
8785
if err != nil {
8886
return err
8987
}
@@ -95,14 +93,14 @@ func (r *Reader) Check() error {
9593
return n.Close()
9694
}
9795

98-
func (r *Reader) GenID(off int64, size int64) string {
99-
sum := sha256.Sum256([]byte(fmt.Sprintf("%s-%d-%d", r.path.String(), off, size)))
96+
func (f *fetcher) GenID(off int64, size int64) string {
97+
sum := sha256.Sum256([]byte(fmt.Sprintf("%s-%d-%d", f.path.String(), off, size)))
10098
return fmt.Sprintf("%x", sum)
10199
}
102100

103-
type reader struct {
101+
type readCloser struct {
104102
io.Reader
105103
closeFunc func() error
106104
}
107105

108-
func (r *reader) Close() error { return r.closeFunc() }
106+
func (r *readCloser) Close() error { return r.closeFunc() }

‎cmd/containerd-stargz-grpc/main.go‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ import (
3636
"github.com/containerd/containerd/pkg/dialer"
3737
"github.com/containerd/containerd/snapshots"
3838
"github.com/containerd/containerd/sys"
39+
ipfs "github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/ipfs"
40+
"github.com/containerd/stargz-snapshotter/fs"
3941
"github.com/containerd/stargz-snapshotter/service"
4042
"github.com/containerd/stargz-snapshotter/service/keychain/cri"
4143
"github.com/containerd/stargz-snapshotter/service/keychain/dockerconfig"
@@ -80,6 +82,9 @@ type snapshotterConfig struct {
8082

8183
// DebugAddress is a Unix domain socket address where the snapshotter exposes /debug/ endpoints.
8284
DebugAddress string `toml:"debug_address"`
85+
86+
// IPFS is a flag to enbale lazy pulling from IPFS.
87+
IPFS bool `toml:"ipfs"`
8388
}
8489

8590
func main() {
@@ -162,7 +167,12 @@ func main() {
162167
runtime.RegisterImageServiceServer(rpc, criServer)
163168
credsFuncs = append(credsFuncs, f)
164169
}
165-
rs, err := service.NewStargzSnapshotterService(ctx, *rootDir, &config.Config, service.WithCredsFuncs(credsFuncs...))
170+
var fsOpts []fs.Option
171+
if config.IPFS {
172+
fsOpts = append(fsOpts, fs.WithResolveHandler("ipfs", new(ipfs.ResolveHandler)))
173+
}
174+
rs, err := service.NewStargzSnapshotterService(ctx, *rootDir, &config.Config,
175+
service.WithCredsFuncs(credsFuncs...), service.WithFilesystemOptions(fsOpts...))
166176
if err != nil {
167177
log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
168178
}

‎cmd/go.mod‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
module github.com/containerd/stargz-snapshotter/cmd
2+
3+
go 1.16
4+
5+
require (
6+
github.com/containerd/containerd v1.5.7
7+
github.com/containerd/go-cni v1.1.0
8+
github.com/containerd/stargz-snapshotter v0.9.0
9+
github.com/containerd/stargz-snapshotter/estargz v0.9.0
10+
github.com/containerd/stargz-snapshotter/ipfs v0.9.0
11+
github.com/coreos/go-systemd/v22 v22.3.2
12+
github.com/docker/go-metrics v0.0.1
13+
github.com/hashicorp/go-multierror v1.1.1
14+
github.com/ipfs/go-ipfs-http-client v0.1.0
15+
github.com/ipfs/interface-go-ipfs-core v0.5.2
16+
github.com/opencontainers/go-digest v1.0.0
17+
github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
18+
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
19+
github.com/pelletier/go-toml v1.9.4
20+
github.com/pkg/errors v0.9.1
21+
github.com/rs/xid v1.3.0
22+
github.com/sirupsen/logrus v1.8.1
23+
github.com/urfave/cli v1.22.4
24+
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164
25+
google.golang.org/grpc v1.42.0
26+
k8s.io/cri-api v0.22.3
27+
)
28+
29+
replace (
30+
// Import local packages.
31+
github.com/containerd/stargz-snapshotter => ../
32+
github.com/containerd/stargz-snapshotter/estargz => ../estargz
33+
github.com/containerd/stargz-snapshotter/ipfs => ../ipfs
34+
35+
// Temporary fork for avoiding importing patent-protected code: https://github.com/hashicorp/golang-lru/issues/73
36+
github.com/hashicorp/golang-lru => github.com/ktock/golang-lru v0.5.5-0.20211029085301-ec551be6f75c
37+
)

0 commit comments

Comments
 (0)