From ae2d6e4612b19cda7d6629f0749c45be1aa151b8 Mon Sep 17 00:00:00 2001 From: Antonio Jesus Navarro Perez Date: Tue, 22 Nov 2016 09:38:49 +0100 Subject: [PATCH 1/4] plumbing/packfile: PACK encoder - Added simple PACK encoder, deltas not supported by now --- plumbing/format/packfile/common.go | 20 +++ plumbing/format/packfile/encoder.go | 103 +++++++++++++++ plumbing/format/packfile/encoder_test.go | 152 +++++++++++++++++++++++ plumbing/format/packfile/scanner.go | 16 +-- 4 files changed, 276 insertions(+), 15 deletions(-) create mode 100644 plumbing/format/packfile/common.go create mode 100644 plumbing/format/packfile/encoder.go create mode 100644 plumbing/format/packfile/encoder_test.go diff --git a/plumbing/format/packfile/common.go b/plumbing/format/packfile/common.go new file mode 100644 index 000000000..483a38c23 --- /dev/null +++ b/plumbing/format/packfile/common.go @@ -0,0 +1,20 @@ +package packfile + +const ( + // VersionSupported is the packfile version supported by this parser. + VersionSupported uint32 = 2 +) + +const ( + maskType = uint8(112) // 0111 0000 +) + +var signature = []byte{'P', 'A', 'C', 'K'} + +const ( + lengthBits = uint8(7) // subsequent bytes has 7 bits to store the length + firstLengthBits = uint8(4) // the first byte has 4 bits to store the length + maskFirstLength = 15 // 0000 1111 + maskContinue = 0x80 // 1000 0000 + maskLength = uint8(127) // 0111 1111 +) diff --git a/plumbing/format/packfile/encoder.go b/plumbing/format/packfile/encoder.go new file mode 100644 index 000000000..d20ec48e4 --- /dev/null +++ b/plumbing/format/packfile/encoder.go @@ -0,0 +1,103 @@ +package packfile + +import ( + "crypto/sha1" + "fmt" + "hash" + "io" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/storer" + "gopkg.in/src-d/go-git.v4/utils/binary" + + "github.com/klauspost/compress/zlib" +) + +// Encoder gets the data from the storage and write it into the writer in PACK format +type Encoder struct { + storage storer.ObjectStorer + w io.Writer + hash hash.Hash +} + +// Creates a new packfile Encoder using a specific Writer +func NewEncoder(w io.Writer, storage storer.ObjectStorer) *Encoder { + h := sha1.New() + mw := io.MultiWriter(w, h) + return &Encoder{storage, mw, h} +} + +// List of hashes into the storage that we want to encode into a pack file +func (e *Encoder) Encode(hashes []plumbing.Hash) (plumbing.Hash, error) { + if err := e.head(len(hashes)); err != nil { + return plumbing.ZeroHash, err + } + + for _, h := range hashes { + o, err := e.storage.Object(plumbing.AnyObject, h) + if err != nil { + return plumbing.ZeroHash, err + } + + if err := e.entry(o); err != nil { + return plumbing.ZeroHash, err + } + } + + return e.footer() +} + +func (e *Encoder) head(numEntries int) error { + return binary.Write(e.w, signature, int32(VersionSupported), int32(numEntries)) +} + +func (e *Encoder) entry(o plumbing.Object) error { + t := o.Type() + if t == plumbing.OFSDeltaObject || t == plumbing.REFDeltaObject { + // TODO implements delta objects + return fmt.Errorf("delta object not supported: %v", t) + } else { + if err := e.writeStandardHeader(t, o.Size()); err != nil { + return err + } + } + + zw := zlib.NewWriter(e.w) + or, err := o.Reader() + if err != nil { + return err + } + _, err = io.Copy(zw, or) + if err != nil { + return err + } + + return zw.Close() +} + +func (e *Encoder) writeStandardHeader(typeNum plumbing.ObjectType, size int64) error { + t := int64(typeNum) + header := []byte{} + c := (t << firstLengthBits) | (size & maskFirstLength) + size >>= firstLengthBits + for { + if size == 0 { + break + } + header = append(header, byte(c|maskContinue)) + c = size & int64(maskLength) + size >>= lengthBits + } + + header = append(header, byte(c)) + _, err := e.w.Write(header) + + return err +} + +func (e *Encoder) footer() (plumbing.Hash, error) { + b := e.hash.Sum(nil) + var h plumbing.Hash + copy(h[:], b) + return h, binary.Write(e.w, h) +} diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go new file mode 100644 index 000000000..0c4664e0a --- /dev/null +++ b/plumbing/format/packfile/encoder_test.go @@ -0,0 +1,152 @@ +package packfile + +import ( + "bytes" + + "gopkg.in/src-d/go-git.v4/fixtures" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/storage/memory" + + . "gopkg.in/check.v1" +) + +type EncoderSuite struct { + fixtures.Suite + buf *bytes.Buffer + store *memory.Storage + enc *Encoder +} + +var _ = Suite(&EncoderSuite{}) + +func (s *EncoderSuite) SetUpTest(c *C) { + s.buf = bytes.NewBuffer(nil) + s.store = memory.NewStorage() + s.enc = NewEncoder(s.buf, s.store) +} + +func (s *EncoderSuite) TestCorrectPackHeader(c *C) { + hash, err := s.enc.Encode([]plumbing.Hash{}) + c.Assert(err, IsNil) + + hb := [20]byte(hash) + + // PACK + VERSION + OBJECTS + HASH + expectedResult := []byte{'P', 'A', 'C', 'K', 0, 0, 0, 2, 0, 0, 0, 0} + expectedResult = append(expectedResult, hb[:]...) + + result := s.buf.Bytes() + + c.Assert(result, DeepEquals, expectedResult) +} + +func (s *EncoderSuite) TestCorrectPackWithOneEmptyObject(c *C) { + o := &plumbing.MemoryObject{} + o.SetType(plumbing.CommitObject) + o.SetSize(0) + s.store.SetObject(o) + + hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}) + c.Assert(err, IsNil) + + // PACK + VERSION(2) + OBJECT NUMBER(1) + expectedResult := []byte{'P', 'A', 'C', 'K', 0, 0, 0, 2, 0, 0, 0, 1} + // OBJECT HEADER(TYPE + SIZE)= 0001 0000 + expectedResult = append(expectedResult, []byte{16}...) + + // Zlib header + expectedResult = append(expectedResult, []byte{120, 156, 1, 0, 0, 255, 255, 0, 0, 0, 1}...) + + // + HASH + hb := [20]byte(hash) + expectedResult = append(expectedResult, hb[:]...) + + result := s.buf.Bytes() + + c.Assert(result, DeepEquals, expectedResult) +} + +func (s *EncoderSuite) TestMaxObjectSize(c *C) { + o := s.store.NewObject() + o.SetSize(9223372036854775807) + o.SetType(plumbing.CommitObject) + s.store.SetObject(o) + hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}) + c.Assert(err, IsNil) + c.Assert(hash.IsZero(), Not(Equals), true) +} + +func (s *EncoderSuite) TestDeltaNotSupported(c *C) { + oOne := s.store.NewObject() + oOne.SetType(plumbing.OFSDeltaObject) + oTwo := s.store.NewObject() + oTwo.SetType(plumbing.REFDeltaObject) + s.store.SetObject(oOne) + s.store.SetObject(oTwo) + hash, err := s.enc.Encode([]plumbing.Hash{oOne.Hash()}) + c.Assert(err, NotNil) + c.Assert(hash.IsZero(), Equals, true) + + hash, err = s.enc.Encode([]plumbing.Hash{oTwo.Hash()}) + c.Assert(err, NotNil) + c.Assert(hash.IsZero(), Equals, true) +} + +func (s *EncoderSuite) TestDecodeEncodeDecode(c *C) { + fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { + scanner := NewScanner(f.Packfile()) + storage := memory.NewStorage() + + d, err := NewDecoder(scanner, storage) + c.Assert(err, IsNil) + + ch, err := d.Decode() + c.Assert(err, IsNil) + c.Assert(ch, Equals, f.PackfileHash) + + commitIter, err := d.o.IterObjects(plumbing.AnyObject) + c.Assert(err, IsNil) + + objects := []plumbing.Object{} + hashes := []plumbing.Hash{} + err = commitIter.ForEach(func(o plumbing.Object) error { + objects = append(objects, o) + hash, err := s.store.SetObject(o) + hashes = append(hashes, hash) + + return err + + }) + c.Assert(err, IsNil) + _, err = s.enc.Encode(hashes) + c.Assert(err, IsNil) + + scanner = NewScanner(s.buf) + storage = memory.NewStorage() + d, err = NewDecoder(scanner, storage) + c.Assert(err, IsNil) + _, err = d.Decode() + c.Assert(err, IsNil) + + commitIter, err = d.o.IterObjects(plumbing.AnyObject) + c.Assert(err, IsNil) + obtainedObjects := []plumbing.Object{} + err = commitIter.ForEach(func(o plumbing.Object) error { + obtainedObjects = append(obtainedObjects, o) + + return nil + }) + c.Assert(len(obtainedObjects), Equals, len(objects)) + + equals := 0 + for _, oo := range obtainedObjects { + for _, o := range objects { + if o.Hash() == oo.Hash() { + equals++ + } + } + } + + c.Assert(len(obtainedObjects), Equals, equals) + }) +} diff --git a/plumbing/format/packfile/scanner.go b/plumbing/format/packfile/scanner.go index 130bb943b..3adc26ae5 100644 --- a/plumbing/format/packfile/scanner.go +++ b/plumbing/format/packfile/scanner.go @@ -26,11 +26,6 @@ var ( ErrSeekNotSupported = NewError("not seek support") ) -const ( - // VersionSupported is the packfile version supported by this parser. - VersionSupported uint32 = 2 -) - // ObjectHeader contains the information related to the object, this information // is collected from the previous bytes to the content of the object. type ObjectHeader struct { @@ -124,7 +119,7 @@ func (s *Scanner) readSignature() ([]byte, error) { // isValidSignature returns if sig is a valid packfile signature. func (s *Scanner) isValidSignature(sig []byte) bool { - return bytes.Equal(sig, []byte{'P', 'A', 'C', 'K'}) + return bytes.Equal(sig, signature) } // readVersion reads and returns the version field of a packfile. @@ -230,15 +225,6 @@ func (s *Scanner) readObjectTypeAndLength() (plumbing.ObjectType, int64, error) return t, l, err } -const ( - maskType = uint8(112) // 0111 0000 - maskFirstLength = uint8(15) // 0000 1111 - maskContinue = uint8(128) // 1000 000 - firstLengthBits = uint8(4) // the first byte has 4 bits to store the length - maskLength = uint8(127) // 0111 1111 - lengthBits = uint8(7) // subsequent bytes has 7 bits to store the length -) - func (s *Scanner) readType() (plumbing.ObjectType, byte, error) { var c byte var err error From a616d71095fe2664c229c53a60e1d3dc7aa21aa6 Mon Sep 17 00:00:00 2001 From: Antonio Jesus Navarro Perez Date: Tue, 22 Nov 2016 11:58:29 +0100 Subject: [PATCH 2/4] Requested changes --- plumbing/format/packfile/encoder.go | 34 ++++++++++++++++-------- plumbing/format/packfile/encoder_test.go | 16 +++++++---- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/plumbing/format/packfile/encoder.go b/plumbing/format/packfile/encoder.go index d20ec48e4..ca25a4ee6 100644 --- a/plumbing/format/packfile/encoder.go +++ b/plumbing/format/packfile/encoder.go @@ -13,21 +13,28 @@ import ( "github.com/klauspost/compress/zlib" ) -// Encoder gets the data from the storage and write it into the writer in PACK format +// Encoder gets the data from the storage and write it into the writer in PACK +// format type Encoder struct { storage storer.ObjectStorer w io.Writer hash hash.Hash } -// Creates a new packfile Encoder using a specific Writer -func NewEncoder(w io.Writer, storage storer.ObjectStorer) *Encoder { +// NewEncoder creates a new packfile encoder using a specific Writer and +// ObjectStorer +func NewEncoder(w io.Writer, s storer.ObjectStorer) *Encoder { h := sha1.New() mw := io.MultiWriter(w, h) - return &Encoder{storage, mw, h} + return &Encoder{ + storage: s, + w: mw, + hash: h, + } } -// List of hashes into the storage that we want to encode into a pack file +// Encode encodes objects specified using a list of hashes. This objects must +// exists into the storer func (e *Encoder) Encode(hashes []plumbing.Hash) (plumbing.Hash, error) { if err := e.head(len(hashes)); err != nil { return plumbing.ZeroHash, err @@ -48,7 +55,12 @@ func (e *Encoder) Encode(hashes []plumbing.Hash) (plumbing.Hash, error) { } func (e *Encoder) head(numEntries int) error { - return binary.Write(e.w, signature, int32(VersionSupported), int32(numEntries)) + return binary.Write( + e.w, + signature, + int32(VersionSupported), + int32(numEntries), + ) } func (e *Encoder) entry(o plumbing.Object) error { @@ -56,10 +68,10 @@ func (e *Encoder) entry(o plumbing.Object) error { if t == plumbing.OFSDeltaObject || t == plumbing.REFDeltaObject { // TODO implements delta objects return fmt.Errorf("delta object not supported: %v", t) - } else { - if err := e.writeStandardHeader(t, o.Size()); err != nil { - return err - } + } + + if err := e.entryHead(t, o.Size()); err != nil { + return err } zw := zlib.NewWriter(e.w) @@ -75,7 +87,7 @@ func (e *Encoder) entry(o plumbing.Object) error { return zw.Close() } -func (e *Encoder) writeStandardHeader(typeNum plumbing.ObjectType, size int64) error { +func (e *Encoder) entryHead(typeNum plumbing.ObjectType, size int64) error { t := int64(typeNum) header := []byte{} c := (t << firstLengthBits) | (size & maskFirstLength) diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go index 0c4664e0a..f68062416 100644 --- a/plumbing/format/packfile/encoder_test.go +++ b/plumbing/format/packfile/encoder_test.go @@ -44,7 +44,8 @@ func (s *EncoderSuite) TestCorrectPackWithOneEmptyObject(c *C) { o := &plumbing.MemoryObject{} o.SetType(plumbing.CommitObject) o.SetSize(0) - s.store.SetObject(o) + _, err := s.store.SetObject(o) + c.Assert(err, IsNil) hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}) c.Assert(err, IsNil) @@ -55,7 +56,8 @@ func (s *EncoderSuite) TestCorrectPackWithOneEmptyObject(c *C) { expectedResult = append(expectedResult, []byte{16}...) // Zlib header - expectedResult = append(expectedResult, []byte{120, 156, 1, 0, 0, 255, 255, 0, 0, 0, 1}...) + expectedResult = append(expectedResult, + []byte{120, 156, 1, 0, 0, 255, 255, 0, 0, 0, 1}...) // + HASH hb := [20]byte(hash) @@ -70,7 +72,8 @@ func (s *EncoderSuite) TestMaxObjectSize(c *C) { o := s.store.NewObject() o.SetSize(9223372036854775807) o.SetType(plumbing.CommitObject) - s.store.SetObject(o) + _, err := s.store.SetObject(o) + c.Assert(err, IsNil) hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}) c.Assert(err, IsNil) c.Assert(hash.IsZero(), Not(Equals), true) @@ -81,8 +84,10 @@ func (s *EncoderSuite) TestDeltaNotSupported(c *C) { oOne.SetType(plumbing.OFSDeltaObject) oTwo := s.store.NewObject() oTwo.SetType(plumbing.REFDeltaObject) - s.store.SetObject(oOne) - s.store.SetObject(oTwo) + _, err := s.store.SetObject(oOne) + c.Assert(err, IsNil) + _, err = s.store.SetObject(oTwo) + c.Assert(err, IsNil) hash, err := s.enc.Encode([]plumbing.Hash{oOne.Hash()}) c.Assert(err, NotNil) c.Assert(hash.IsZero(), Equals, true) @@ -136,6 +141,7 @@ func (s *EncoderSuite) TestDecodeEncodeDecode(c *C) { return nil }) + c.Assert(err, IsNil) c.Assert(len(obtainedObjects), Equals, len(objects)) equals := 0 From 7cd093a65e99922fb17b2a938adf7373e2c08921 Mon Sep 17 00:00:00 2001 From: Antonio Jesus Navarro Perez Date: Wed, 23 Nov 2016 16:03:18 +0100 Subject: [PATCH 3/4] Requested changes --- plumbing/format/packfile/common.go | 15 +++++---------- plumbing/format/packfile/encoder.go | 13 ++++++++----- plumbing/format/packfile/encoder_test.go | 18 ------------------ 3 files changed, 13 insertions(+), 33 deletions(-) diff --git a/plumbing/format/packfile/common.go b/plumbing/format/packfile/common.go index 483a38c23..1656551f3 100644 --- a/plumbing/format/packfile/common.go +++ b/plumbing/format/packfile/common.go @@ -1,20 +1,15 @@ package packfile -const ( - // VersionSupported is the packfile version supported by this parser. - VersionSupported uint32 = 2 -) - -const ( - maskType = uint8(112) // 0111 0000 -) - var signature = []byte{'P', 'A', 'C', 'K'} const ( + // VersionSupported is the packfile version supported by this package + VersionSupported uint32 = 2 + + firstLengthBits = uint8(4) // the first byte into object header has 4 bits to store the length lengthBits = uint8(7) // subsequent bytes has 7 bits to store the length - firstLengthBits = uint8(4) // the first byte has 4 bits to store the length maskFirstLength = 15 // 0000 1111 maskContinue = 0x80 // 1000 0000 maskLength = uint8(127) // 0111 1111 + maskType = uint8(112) // 0111 0000 ) diff --git a/plumbing/format/packfile/encoder.go b/plumbing/format/packfile/encoder.go index ca25a4ee6..28ed25447 100644 --- a/plumbing/format/packfile/encoder.go +++ b/plumbing/format/packfile/encoder.go @@ -18,6 +18,7 @@ import ( type Encoder struct { storage storer.ObjectStorer w io.Writer + zw *zlib.Writer hash hash.Hash } @@ -26,15 +27,17 @@ type Encoder struct { func NewEncoder(w io.Writer, s storer.ObjectStorer) *Encoder { h := sha1.New() mw := io.MultiWriter(w, h) + zw := zlib.NewWriter(mw) return &Encoder{ storage: s, w: mw, + zw: zw, hash: h, } } -// Encode encodes objects specified using a list of hashes. This objects must -// exists into the storer +// Encode creates a packfile containing all the objects referenced in hashes +// and writes it to the writer in the Encoder. func (e *Encoder) Encode(hashes []plumbing.Hash) (plumbing.Hash, error) { if err := e.head(len(hashes)); err != nil { return plumbing.ZeroHash, err @@ -74,17 +77,17 @@ func (e *Encoder) entry(o plumbing.Object) error { return err } - zw := zlib.NewWriter(e.w) + e.zw.Reset(e.w) or, err := o.Reader() if err != nil { return err } - _, err = io.Copy(zw, or) + _, err = io.Copy(e.zw, or) if err != nil { return err } - return zw.Close() + return e.zw.Close() } func (e *Encoder) entryHead(typeNum plumbing.ObjectType, size int64) error { diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go index f68062416..729843d06 100644 --- a/plumbing/format/packfile/encoder_test.go +++ b/plumbing/format/packfile/encoder_test.go @@ -79,24 +79,6 @@ func (s *EncoderSuite) TestMaxObjectSize(c *C) { c.Assert(hash.IsZero(), Not(Equals), true) } -func (s *EncoderSuite) TestDeltaNotSupported(c *C) { - oOne := s.store.NewObject() - oOne.SetType(plumbing.OFSDeltaObject) - oTwo := s.store.NewObject() - oTwo.SetType(plumbing.REFDeltaObject) - _, err := s.store.SetObject(oOne) - c.Assert(err, IsNil) - _, err = s.store.SetObject(oTwo) - c.Assert(err, IsNil) - hash, err := s.enc.Encode([]plumbing.Hash{oOne.Hash()}) - c.Assert(err, NotNil) - c.Assert(hash.IsZero(), Equals, true) - - hash, err = s.enc.Encode([]plumbing.Hash{oTwo.Hash()}) - c.Assert(err, NotNil) - c.Assert(hash.IsZero(), Equals, true) -} - func (s *EncoderSuite) TestDecodeEncodeDecode(c *C) { fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { scanner := NewScanner(f.Packfile()) From 5c287cf832a3aed094f7d0dfb42d95443b9a542c Mon Sep 17 00:00:00 2001 From: Antonio Jesus Navarro Perez Date: Thu, 24 Nov 2016 09:35:16 +0100 Subject: [PATCH 4/4] Requested changes --- plumbing/format/packfile/encoder.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/plumbing/format/packfile/encoder.go b/plumbing/format/packfile/encoder.go index 28ed25447..1404dbed7 100644 --- a/plumbing/format/packfile/encoder.go +++ b/plumbing/format/packfile/encoder.go @@ -1,16 +1,14 @@ package packfile import ( + "compress/zlib" "crypto/sha1" "fmt" - "hash" "io" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/utils/binary" - - "github.com/klauspost/compress/zlib" ) // Encoder gets the data from the storage and write it into the writer in PACK @@ -19,20 +17,22 @@ type Encoder struct { storage storer.ObjectStorer w io.Writer zw *zlib.Writer - hash hash.Hash + hasher plumbing.Hasher } // NewEncoder creates a new packfile encoder using a specific Writer and // ObjectStorer func NewEncoder(w io.Writer, s storer.ObjectStorer) *Encoder { - h := sha1.New() + h := plumbing.Hasher{ + Hash: sha1.New(), + } mw := io.MultiWriter(w, h) zw := zlib.NewWriter(mw) return &Encoder{ storage: s, w: mw, zw: zw, - hash: h, + hasher: h, } } @@ -111,8 +111,6 @@ func (e *Encoder) entryHead(typeNum plumbing.ObjectType, size int64) error { } func (e *Encoder) footer() (plumbing.Hash, error) { - b := e.hash.Sum(nil) - var h plumbing.Hash - copy(h[:], b) + h := e.hasher.Sum() return h, binary.Write(e.w, h) }