storage: shallow storage

common.go

@@ -15,6 +15,7 @@ import (
 type Storer interface {
 	storer.ObjectStorer
 	storer.ReferenceStorer
+	storer.ShallowStorer
 	config.ConfigStorer
 }
 

examples/storage/aerospike/storage.go

@@ -248,6 +248,44 @@ func (s *Storage) buildConfigKey() (*driver.Key, error) {
 	return driver.NewKey(s.ns, configSet, fmt.Sprintf("%s|config", s.url))
 }
 
+func (s *Storage) Shallow() ([]plumbing.Hash, error) {
+	key, err := s.buildShallowKey()
+	if err != nil {
+		return nil, err
+	}
+
+	rec, err := s.client.Get(nil, key)
+	if err != nil {
+		return nil, err
+	}
+
+	var h []plumbing.Hash
+	return h, json.Unmarshal(rec.Bins["blob"].([]byte), h)
+}
+
+func (s *Storage) SetShallow(hash []plumbing.Hash) error {
+	key, err := s.buildShallowKey()
+	if err != nil {
+		return err
+	}
+
+	json, err := json.Marshal(hash)
+	if err != nil {
+		return err
+	}
+
+	bins := driver.BinMap{
+		urlField: s.url,
+		"blob":   json,
+	}
+
+	return s.client.Put(nil, key, bins)
+}
+
+func (s *Storage) buildShallowKey() (*driver.Key, error) {
+	return driver.NewKey(s.ns, configSet, fmt.Sprintf("%s|config", s.url))
+}
+
 func createIndexes(c *driver.Client, ns string) error {
 	for _, set := range [...]string{
 		referencesSet,

plumbing/storer/shallow.go

@@ -0,0 +1,10 @@
+package storer
+
+import "gopkg.in/src-d/go-git.v4/plumbing"
+
+// ShallowStorer storage of shallow commits, meaning that it does not have the
+// parents of a commit (explanation from git documentation)
+type ShallowStorer interface {
+	SetShallow([]plumbing.Hash) error
+	Shallow() ([]plumbing.Hash, error)
+}

remote.go

@@ -128,6 +128,10 @@ func (r *Remote) Fetch(o *FetchOptions) (err error) {
 
 	defer checkClose(reader, &err)
 
+	if err := r.updateShallow(o, reader); err != nil {
+		return err
+	}
+
 	if err = r.updateObjectStorage(
 		r.buildSidebandIfSupported(req.Capabilities, reader),
 	); err != nil {
@@ -294,6 +298,14 @@ func (r *Remote) buildFetchedTags() error {
 	})
 }
 
+func (r *Remote) updateShallow(o *FetchOptions, resp *packp.UploadPackResponse) error {
+	if o.Depth == 0 {
+		return nil
+	}
+
+	return r.s.SetShallow(resp.Shallows)
+}
+
 // Head returns the Reference of the HEAD
 func (r *Remote) Head() *plumbing.Reference {
 	ref, err := storer.ResolveReference(r.refs, plumbing.HEAD)

remote_test.go

@@ -124,6 +124,14 @@ func (s *RemoteSuite) TestFetchDepth(c *C) {
 		r, _ := sto.Reference(exp.Name())
 		c.Assert(exp.String(), Equals, r.String())
 	}
+
+	h, err := sto.Shallow()
+	c.Assert(err, IsNil)
+	c.Assert(h, HasLen, 2)
+	c.Assert(h, DeepEquals, []plumbing.Hash{
+		plumbing.NewHash("e8d3ffab552895c19b9fcf7aa264d277cde33881"),
+		plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"),
+	})
 }
 
 func (s *RemoteSuite) TestFetchWithProgress(c *C) {

storage/filesystem/internal/dotgit/dotgit.go

@@ -17,6 +17,7 @@ const (
 	suffix         = ".git"
 	packedRefsPath = "packed-refs"
 	configPath     = "config"
+	shallowPath    = "shallow"
 
 	objectsPath = "objects"
 	packPath    = "pack"
@@ -60,15 +61,35 @@ func New(fs fs.Filesystem) *DotGit {
 	return &DotGit{fs: fs}
 }
 
+// ConfigWriter returns a file pointer for write to the config file
 func (d *DotGit) ConfigWriter() (fs.File, error) {
 	return d.fs.Create(configPath)
 }
 
-// Config returns the path of the config file
+// Config returns a file pointer for read to the config file
 func (d *DotGit) Config() (fs.File, error) {
 	return d.fs.Open(configPath)
 }
 
+// ShallowWriter returns a file pointer for write to the shallow file
+func (d *DotGit) ShallowWriter() (fs.File, error) {
+	return d.fs.Create(shallowPath)
+}
+
+// Shallow returns a file pointer for read to the shallow file
+func (d *DotGit) Shallow() (fs.File, error) {
+	f, err := d.fs.Open(shallowPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+
+		return nil, err
+	}
+
+	return f, nil
+}
+
 // NewObjectPack return a writer for a new packfile, it saves the packfile to
 // disk and also generates and save the index for the given packfile.
 func (d *DotGit) NewObjectPack() (*PackWriter, error) {

storage/filesystem/internal/dotgit/dotgit_test.go

@@ -130,6 +130,61 @@ func (s *SuiteDotGit) TestConfig(c *C) {
 	c.Assert(filepath.Base(file.Filename()), Equals, "config")
 }
 
+func (s *SuiteDotGit) TestConfigWriteAndConfig(c *C) {
+	tmp, err := ioutil.TempDir("", "dot-git")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(tmp)
+
+	fs := osfs.New(tmp)
+	dir := New(fs)
+
+	f, err := dir.ConfigWriter()
+	c.Assert(err, IsNil)
+
+	_, err = f.Write([]byte("foo"))
+	c.Assert(err, IsNil)
+
+	f, err = dir.Config()
+	c.Assert(err, IsNil)
+
+	cnt, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+
+	c.Assert(string(cnt), Equals, "foo")
+}
+
+func (s *SuiteDotGit) TestShallow(c *C) {
+	fs := fixtures.Basic().ByTag(".git").One().DotGit()
+	dir := New(fs)
+
+	file, err := dir.Shallow()
+	c.Assert(err, IsNil)
+	c.Assert(file, IsNil)
+}
+
+func (s *SuiteDotGit) TestShallowWriteAndShallow(c *C) {
+	tmp, err := ioutil.TempDir("", "dot-git")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(tmp)
+
+	fs := osfs.New(tmp)
+	dir := New(fs)
+
+	f, err := dir.ShallowWriter()
+	c.Assert(err, IsNil)
+
+	_, err = f.Write([]byte("foo"))
+	c.Assert(err, IsNil)
+
+	f, err = dir.Shallow()
+	c.Assert(err, IsNil)
+
+	cnt, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+
+	c.Assert(string(cnt), Equals, "foo")
+}
+
 func findReference(refs []*plumbing.Reference, name string) *plumbing.Reference {
 	n := plumbing.ReferenceName(name)
 	for _, ref := range refs {
@@ -222,6 +277,18 @@ func (s *SuiteDotGit) TestObjects(c *C) {
 	c.Assert(hashes[2].String(), Equals, "03db8e1fbe133a480f2867aac478fd866686d69e")
 }
 
+func (s *SuiteDotGit) TestObjectsNoFolder(c *C) {
+	tmp, err := ioutil.TempDir("", "dot-git")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(tmp)
+
+	fs := osfs.New(tmp)
+	dir := New(fs)
+	hash, err := dir.Objects()
+	c.Assert(err, IsNil)
+	c.Assert(hash, HasLen, 0)
+}
+
 func (s *SuiteDotGit) TestObject(c *C) {
 	fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
 	dir := New(fs)

storage/filesystem/shallow.go

@@ -0,0 +1,52 @@
+package filesystem
+
+import (
+	"fmt"
+
+	"bufio"
+
+	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
+)
+
+// ShallowStorage where the shallow commits are stored, an internal to
+// manipulate the shallow file
+type ShallowStorage struct {
+	dir *dotgit.DotGit
+}
+
+// SetShallow save the shallows in the shallow file in the .git folder as one
+// commit per line represented by 40-byte hexadecimal object terminated by a
+// newline.
+func (s *ShallowStorage) SetShallow(commits []plumbing.Hash) error {
+	f, err := s.dir.ShallowWriter()
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+	for _, h := range commits {
+		if _, err := fmt.Fprintf(f, "%s\n", h); err != err {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Shallow return the shallow commits reading from shallo file from .git
+func (s *ShallowStorage) Shallow() ([]plumbing.Hash, error) {
+	f, err := s.dir.Shallow()
+	if err != nil {
+		return nil, err
+	}
+
+	var hash []plumbing.Hash
+
+	scn := bufio.NewScanner(f)
+	for scn.Scan() {
+		hash = append(hash, plumbing.NewHash(scn.Text()))
+	}
+
+	return hash, scn.Err()
+}

storage/filesystem/storage.go

@@ -12,6 +12,7 @@ import (
 type Storage struct {
 	ObjectStorage
 	ReferenceStorage
+	ShallowStorage
 	ConfigStorage
 }
 
@@ -26,6 +27,7 @@ func NewStorage(fs fs.Filesystem) (*Storage, error) {
 	return &Storage{
 		ObjectStorage:    o,
 		ReferenceStorage: ReferenceStorage{dir: dir},
+		ShallowStorage:   ShallowStorage{dir: dir},
 		ConfigStorage:    ConfigStorage{dir: dir},
 	}, nil
 }

storage/memory/storage.go

@@ -18,6 +18,7 @@ var ErrUnsupportedObjectType = fmt.Errorf("unsupported object type")
 type Storage struct {
 	ConfigStorage
 	ObjectStorage
+	ShallowStorage
 	ReferenceStorage
 }
 
@@ -26,6 +27,7 @@ func NewStorage() *Storage {
 	return &Storage{
 		ReferenceStorage: make(ReferenceStorage, 0),
 		ConfigStorage:    ConfigStorage{},
+		ShallowStorage:   ShallowStorage{},
 		ObjectStorage: ObjectStorage{
 			Objects: make(map[plumbing.Hash]plumbing.Object, 0),
 			Commits: make(map[plumbing.Hash]plumbing.Object, 0),
@@ -195,3 +197,14 @@ func (r ReferenceStorage) IterReferences() (storer.ReferenceIter, error) {
 
 	return storer.NewReferenceSliceIter(refs), nil
 }
+
+type ShallowStorage []plumbing.Hash
+
+func (s *ShallowStorage) SetShallow(commits []plumbing.Hash) error {
+	*s = commits
+	return nil
+}
+
+func (s ShallowStorage) Shallow() ([]plumbing.Hash, error) {
+	return s, nil
+}

storage/test/storage_suite.go

@@ -17,6 +17,7 @@ import (
 type Storer interface {
 	storer.ObjectStorer
 	storer.ReferenceStorer
+	storer.ShallowStorer
 	config.ConfigStorer
 }
 
@@ -263,6 +264,21 @@ func (s *BaseStorageSuite) TestIterReferences(c *C) {
 	c.Assert(err, Equals, io.EOF)
 }
 
+func (s *BaseStorageSuite) TestSetShallowAndShallow(c *C) {
+	expected := []plumbing.Hash{
+		plumbing.NewHash("b66c08ba28aa1f81eb06a1127aa3936ff77e5e2c"),
+		plumbing.NewHash("c3f4688a08fd86f1bf8e055724c84b7a40a09733"),
+		plumbing.NewHash("c78874f116be67ecf54df225a613162b84cc6ebf"),
+	}
+
+	err := s.Storer.SetShallow(expected)
+	c.Assert(err, IsNil)
+
+	result, err := s.Storer.Shallow()
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, expected)
+}
+
 func (s *BaseStorageSuite) TestSetConfigAndConfig(c *C) {
 	expected := config.NewConfig()
 	expected.Core.IsBare = true