Skip to content
This repository was archived by the owner on Sep 11, 2020. It is now read-only.

Commit dc0337a

Browse files
ajnavarromcuadros
ajnavarro
authored and
mcuadros
committed
plumbing/packfile: PACK encoder (#131)
* plumbing/packfile: PACK encoder - Added simple PACK encoder, deltas not supported by now * Requested changes * Requested changes * Requested changes
1 parent b41ac9b commit dc0337a

File tree

4 files changed

+272
-15
lines changed

4 files changed

+272
-15
lines changed

‎plumbing/format/packfile/common.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package packfile
2+
3+
var signature = []byte{'P', 'A', 'C', 'K'}
4+
5+
const (
6+
// VersionSupported is the packfile version supported by this package
7+
VersionSupported uint32 = 2
8+
9+
firstLengthBits = uint8(4) // the first byte into object header has 4 bits to store the length
10+
lengthBits = uint8(7) // subsequent bytes has 7 bits to store the length
11+
maskFirstLength = 15 // 0000 1111
12+
maskContinue = 0x80 // 1000 0000
13+
maskLength = uint8(127) // 0111 1111
14+
maskType = uint8(112) // 0111 0000
15+
)

‎plumbing/format/packfile/encoder.go

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package packfile
2+
3+
import (
4+
"compress/zlib"
5+
"crypto/sha1"
6+
"fmt"
7+
"io"
8+
9+
"gopkg.in/src-d/go-git.v4/plumbing"
10+
"gopkg.in/src-d/go-git.v4/plumbing/storer"
11+
"gopkg.in/src-d/go-git.v4/utils/binary"
12+
)
13+
14+
// Encoder gets the data from the storage and write it into the writer in PACK
15+
// format
16+
type Encoder struct {
17+
storage storer.ObjectStorer
18+
w io.Writer
19+
zw *zlib.Writer
20+
hasher plumbing.Hasher
21+
}
22+
23+
// NewEncoder creates a new packfile encoder using a specific Writer and
24+
// ObjectStorer
25+
func NewEncoder(w io.Writer, s storer.ObjectStorer) *Encoder {
26+
h := plumbing.Hasher{
27+
Hash: sha1.New(),
28+
}
29+
mw := io.MultiWriter(w, h)
30+
zw := zlib.NewWriter(mw)
31+
return &Encoder{
32+
storage: s,
33+
w: mw,
34+
zw: zw,
35+
hasher: h,
36+
}
37+
}
38+
39+
// Encode creates a packfile containing all the objects referenced in hashes
40+
// and writes it to the writer in the Encoder.
41+
func (e *Encoder) Encode(hashes []plumbing.Hash) (plumbing.Hash, error) {
42+
if err := e.head(len(hashes)); err != nil {
43+
return plumbing.ZeroHash, err
44+
}
45+
46+
for _, h := range hashes {
47+
o, err := e.storage.Object(plumbing.AnyObject, h)
48+
if err != nil {
49+
return plumbing.ZeroHash, err
50+
}
51+
52+
if err := e.entry(o); err != nil {
53+
return plumbing.ZeroHash, err
54+
}
55+
}
56+
57+
return e.footer()
58+
}
59+
60+
func (e *Encoder) head(numEntries int) error {
61+
return binary.Write(
62+
e.w,
63+
signature,
64+
int32(VersionSupported),
65+
int32(numEntries),
66+
)
67+
}
68+
69+
func (e *Encoder) entry(o plumbing.Object) error {
70+
t := o.Type()
71+
if t == plumbing.OFSDeltaObject || t == plumbing.REFDeltaObject {
72+
// TODO implements delta objects
73+
return fmt.Errorf("delta object not supported: %v", t)
74+
}
75+
76+
if err := e.entryHead(t, o.Size()); err != nil {
77+
return err
78+
}
79+
80+
e.zw.Reset(e.w)
81+
or, err := o.Reader()
82+
if err != nil {
83+
return err
84+
}
85+
_, err = io.Copy(e.zw, or)
86+
if err != nil {
87+
return err
88+
}
89+
90+
return e.zw.Close()
91+
}
92+
93+
func (e *Encoder) entryHead(typeNum plumbing.ObjectType, size int64) error {
94+
t := int64(typeNum)
95+
header := []byte{}
96+
c := (t << firstLengthBits) | (size & maskFirstLength)
97+
size >>= firstLengthBits
98+
for {
99+
if size == 0 {
100+
break
101+
}
102+
header = append(header, byte(c|maskContinue))
103+
c = size & int64(maskLength)
104+
size >>= lengthBits
105+
}
106+
107+
header = append(header, byte(c))
108+
_, err := e.w.Write(header)
109+
110+
return err
111+
}
112+
113+
func (e *Encoder) footer() (plumbing.Hash, error) {
114+
h := e.hasher.Sum()
115+
return h, binary.Write(e.w, h)
116+
}

‎plumbing/format/packfile/encoder_test.go

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
package packfile
2+
3+
import (
4+
"bytes"
5+
6+
"gopkg.in/src-d/go-git.v4/fixtures"
7+
"gopkg.in/src-d/go-git.v4/plumbing"
8+
"gopkg.in/src-d/go-git.v4/storage/memory"
9+
10+
. "gopkg.in/check.v1"
11+
)
12+
13+
type EncoderSuite struct {
14+
fixtures.Suite
15+
buf *bytes.Buffer
16+
store *memory.Storage
17+
enc *Encoder
18+
}
19+
20+
var _ = Suite(&EncoderSuite{})
21+
22+
func (s *EncoderSuite) SetUpTest(c *C) {
23+
s.buf = bytes.NewBuffer(nil)
24+
s.store = memory.NewStorage()
25+
s.enc = NewEncoder(s.buf, s.store)
26+
}
27+
28+
func (s *EncoderSuite) TestCorrectPackHeader(c *C) {
29+
hash, err := s.enc.Encode([]plumbing.Hash{})
30+
c.Assert(err, IsNil)
31+
32+
hb := [20]byte(hash)
33+
34+
// PACK + VERSION + OBJECTS + HASH
35+
expectedResult := []byte{'P', 'A', 'C', 'K', 0, 0, 0, 2, 0, 0, 0, 0}
36+
expectedResult = append(expectedResult, hb[:]...)
37+
38+
result := s.buf.Bytes()
39+
40+
c.Assert(result, DeepEquals, expectedResult)
41+
}
42+
43+
func (s *EncoderSuite) TestCorrectPackWithOneEmptyObject(c *C) {
44+
o := &plumbing.MemoryObject{}
45+
o.SetType(plumbing.CommitObject)
46+
o.SetSize(0)
47+
_, err := s.store.SetObject(o)
48+
c.Assert(err, IsNil)
49+
50+
hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()})
51+
c.Assert(err, IsNil)
52+
53+
// PACK + VERSION(2) + OBJECT NUMBER(1)
54+
expectedResult := []byte{'P', 'A', 'C', 'K', 0, 0, 0, 2, 0, 0, 0, 1}
55+
// OBJECT HEADER(TYPE + SIZE)= 0001 0000
56+
expectedResult = append(expectedResult, []byte{16}...)
57+
58+
// Zlib header
59+
expectedResult = append(expectedResult,
60+
[]byte{120, 156, 1, 0, 0, 255, 255, 0, 0, 0, 1}...)
61+
62+
// + HASH
63+
hb := [20]byte(hash)
64+
expectedResult = append(expectedResult, hb[:]...)
65+
66+
result := s.buf.Bytes()
67+
68+
c.Assert(result, DeepEquals, expectedResult)
69+
}
70+
71+
func (s *EncoderSuite) TestMaxObjectSize(c *C) {
72+
o := s.store.NewObject()
73+
o.SetSize(9223372036854775807)
74+
o.SetType(plumbing.CommitObject)
75+
_, err := s.store.SetObject(o)
76+
c.Assert(err, IsNil)
77+
hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()})
78+
c.Assert(err, IsNil)
79+
c.Assert(hash.IsZero(), Not(Equals), true)
80+
}
81+
82+
func (s *EncoderSuite) TestDecodeEncodeDecode(c *C) {
83+
fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) {
84+
scanner := NewScanner(f.Packfile())
85+
storage := memory.NewStorage()
86+
87+
d, err := NewDecoder(scanner, storage)
88+
c.Assert(err, IsNil)
89+
90+
ch, err := d.Decode()
91+
c.Assert(err, IsNil)
92+
c.Assert(ch, Equals, f.PackfileHash)
93+
94+
commitIter, err := d.o.IterObjects(plumbing.AnyObject)
95+
c.Assert(err, IsNil)
96+
97+
objects := []plumbing.Object{}
98+
hashes := []plumbing.Hash{}
99+
err = commitIter.ForEach(func(o plumbing.Object) error {
100+
objects = append(objects, o)
101+
hash, err := s.store.SetObject(o)
102+
hashes = append(hashes, hash)
103+
104+
return err
105+
106+
})
107+
c.Assert(err, IsNil)
108+
_, err = s.enc.Encode(hashes)
109+
c.Assert(err, IsNil)
110+
111+
scanner = NewScanner(s.buf)
112+
storage = memory.NewStorage()
113+
d, err = NewDecoder(scanner, storage)
114+
c.Assert(err, IsNil)
115+
_, err = d.Decode()
116+
c.Assert(err, IsNil)
117+
118+
commitIter, err = d.o.IterObjects(plumbing.AnyObject)
119+
c.Assert(err, IsNil)
120+
obtainedObjects := []plumbing.Object{}
121+
err = commitIter.ForEach(func(o plumbing.Object) error {
122+
obtainedObjects = append(obtainedObjects, o)
123+
124+
return nil
125+
})
126+
c.Assert(err, IsNil)
127+
c.Assert(len(obtainedObjects), Equals, len(objects))
128+
129+
equals := 0
130+
for _, oo := range obtainedObjects {
131+
for _, o := range objects {
132+
if o.Hash() == oo.Hash() {
133+
equals++
134+
}
135+
}
136+
}
137+
138+
c.Assert(len(obtainedObjects), Equals, equals)
139+
})
140+
}

‎plumbing/format/packfile/scanner.go

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,6 @@ var (
2626
ErrSeekNotSupported = NewError("not seek support")
2727
)
2828

29-
const (
30-
// VersionSupported is the packfile version supported by this parser.
31-
VersionSupported uint32 = 2
32-
)
33-
3429
// ObjectHeader contains the information related to the object, this information
3530
// is collected from the previous bytes to the content of the object.
3631
type ObjectHeader struct {
@@ -124,7 +119,7 @@ func (s *Scanner) readSignature() ([]byte, error) {
124119

125120
// isValidSignature returns if sig is a valid packfile signature.
126121
func (s *Scanner) isValidSignature(sig []byte) bool {
127-
return bytes.Equal(sig, []byte{'P', 'A', 'C', 'K'})
122+
return bytes.Equal(sig, signature)
128123
}
129124

130125
// readVersion reads and returns the version field of a packfile.
@@ -230,15 +225,6 @@ func (s *Scanner) readObjectTypeAndLength() (plumbing.ObjectType, int64, error)
230225
return t, l, err
231226
}
232227

233-
const (
234-
maskType = uint8(112) // 0111 0000
235-
maskFirstLength = uint8(15) // 0000 1111
236-
maskContinue = uint8(128) // 1000 000
237-
firstLengthBits = uint8(4) // the first byte has 4 bits to store the length
238-
maskLength = uint8(127) // 0111 1111
239-
lengthBits = uint8(7) // subsequent bytes has 7 bits to store the length
240-
)
241-
242228
func (s *Scanner) readType() (plumbing.ObjectType, byte, error) {
243229
var c byte
244230
var err error

0 commit comments

Comments
 (0)