Skip to content
This repository was archived by the owner on Sep 11, 2020. It is now read-only.

Commit 009f106

Browse files
erizocosmicoerizocosmico
committed
plumbing/format/idxfile: add new Index and MemoryIndex
Signed-off-by: Miguel Molina <[email protected]>
1 parent 9f00789 commit 009f106

File tree

10 files changed

+484
-387
lines changed

10 files changed

+484
-387
lines changed

‎plumbing/format/idxfile/decoder.go

Lines changed: 64 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@ var (
1717
ErrMalformedIdxFile = errors.New("Malformed IDX file")
1818
)
1919

20+
const (
21+
fanout = 256
22+
objectIDLength = 20
23+
)
24+
2025
// Decoder reads and decodes idx files from an input stream.
2126
type Decoder struct {
2227
*bufio.Reader
@@ -27,13 +32,13 @@ func NewDecoder(r io.Reader) *Decoder {
2732
return &Decoder{bufio.NewReader(r)}
2833
}
2934

30-
// Decode reads from the stream and decode the content into the Idxfile struct.
31-
func (d *Decoder) Decode(idx *Idxfile) error {
35+
// Decode reads from the stream and decode the content into the MemoryIndex struct.
36+
func (d *Decoder) Decode(idx *MemoryIndex) error {
3237
if err := validateHeader(d); err != nil {
3338
return err
3439
}
3540

36-
flow := []func(*Idxfile, io.Reader) error{
41+
flow := []func(*MemoryIndex, io.Reader) error{
3742
readVersion,
3843
readFanout,
3944
readObjectNames,
@@ -48,10 +53,6 @@ func (d *Decoder) Decode(idx *Idxfile) error {
4853
}
4954
}
5055

51-
if !idx.isValid() {
52-
return ErrMalformedIdxFile
53-
}
54-
5556
return nil
5657
}
5758

@@ -68,7 +69,7 @@ func validateHeader(r io.Reader) error {
6869
return nil
6970
}
7071

71-
func readVersion(idx *Idxfile, r io.Reader) error {
72+
func readVersion(idx *MemoryIndex, r io.Reader) error {
7273
v, err := binary.ReadUint32(r)
7374
if err != nil {
7475
return err
@@ -82,74 +83,92 @@ func readVersion(idx *Idxfile, r io.Reader) error {
8283
return nil
8384
}
8485

85-
func readFanout(idx *Idxfile, r io.Reader) error {
86-
var err error
87-
for i := 0; i < 255; i++ {
88-
idx.Fanout[i], err = binary.ReadUint32(r)
86+
func readFanout(idx *MemoryIndex, r io.Reader) error {
87+
for k := 0; k < fanout; k++ {
88+
n, err := binary.ReadUint32(r)
8989
if err != nil {
9090
return err
9191
}
92+
93+
idx.Fanout[k] = n
94+
idx.FanoutMapping[k] = noMapping
9295
}
9396

94-
idx.ObjectCount, err = binary.ReadUint32(r)
95-
return err
97+
return nil
9698
}
9799

98-
func readObjectNames(idx *Idxfile, r io.Reader) error {
99-
c := int(idx.ObjectCount)
100-
new := make([]Entry, c)
101-
for i := 0; i < c; i++ {
102-
e := &new[i]
103-
if _, err := io.ReadFull(r, e.Hash[:]); err != nil {
100+
func readObjectNames(idx *MemoryIndex, r io.Reader) error {
101+
for k := 0; k < fanout; k++ {
102+
var buckets uint32
103+
if k == 0 {
104+
buckets = idx.Fanout[k]
105+
} else {
106+
buckets = idx.Fanout[k] - idx.Fanout[k-1]
107+
}
108+
109+
if buckets == 0 {
110+
continue
111+
}
112+
113+
if buckets < 0 {
114+
return ErrMalformedIdxFile
115+
}
116+
117+
idx.FanoutMapping[k] = len(idx.Names)
118+
119+
nameLen := int(buckets * objectIDLength)
120+
bin := make([]byte, nameLen)
121+
if _, err := io.ReadFull(r, bin); err != nil {
104122
return err
105123
}
106124

107-
idx.Entries = append(idx.Entries, e)
125+
idx.Names = append(idx.Names, bin)
126+
idx.Offset32 = append(idx.Offset32, make([]byte, buckets*4))
127+
idx.Crc32 = append(idx.Crc32, make([]byte, buckets*4))
108128
}
109129

110130
return nil
111131
}
112132

113-
func readCRC32(idx *Idxfile, r io.Reader) error {
114-
c := int(idx.ObjectCount)
115-
for i := 0; i < c; i++ {
116-
if err := binary.Read(r, &idx.Entries[i].CRC32); err != nil {
117-
return err
133+
func readCRC32(idx *MemoryIndex, r io.Reader) error {
134+
for k := 0; k < fanout; k++ {
135+
if pos := idx.FanoutMapping[k]; pos != noMapping {
136+
if _, err := io.ReadFull(r, idx.Crc32[pos]); err != nil {
137+
return err
138+
}
118139
}
119140
}
120141

121142
return nil
122143
}
123144

124-
func readOffsets(idx *Idxfile, r io.Reader) error {
125-
c := int(idx.ObjectCount)
126-
127-
for i := 0; i < c; i++ {
128-
o, err := binary.ReadUint32(r)
129-
if err != nil {
130-
return err
145+
func readOffsets(idx *MemoryIndex, r io.Reader) error {
146+
var o64cnt int
147+
for k := 0; k < fanout; k++ {
148+
if pos := idx.FanoutMapping[k]; pos != noMapping {
149+
if _, err := io.ReadFull(r, idx.Offset32[pos]); err != nil {
150+
return err
151+
}
152+
153+
for p := 0; p < len(idx.Offset32[pos]); p += 4 {
154+
if idx.Offset32[pos][p]&(byte(1)<<7) > 0 {
155+
o64cnt++
156+
}
157+
}
131158
}
132-
133-
idx.Entries[i].Offset = uint64(o)
134159
}
135160

136-
for i := 0; i < c; i++ {
137-
if idx.Entries[i].Offset <= offsetLimit {
138-
continue
139-
}
140-
141-
o, err := binary.ReadUint64(r)
142-
if err != nil {
161+
if o64cnt > 0 {
162+
idx.Offset64 = make([]byte, o64cnt*8)
163+
if _, err := io.ReadFull(r, idx.Offset64); err != nil {
143164
return err
144165
}
145-
146-
idx.Entries[i].Offset = o
147166
}
148167

149168
return nil
150169
}
151170

152-
func readChecksums(idx *Idxfile, r io.Reader) error {
171+
func readChecksums(idx *MemoryIndex, r io.Reader) error {
153172
if _, err := io.ReadFull(r, idx.PackfileChecksum[:]); err != nil {
154173
return err
155174
}

‎plumbing/format/idxfile/decoder_test.go

Lines changed: 55 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ import (
44
"bytes"
55
"encoding/base64"
66
"fmt"
7+
"io"
8+
"io/ioutil"
79
"testing"
810

11+
"gopkg.in/src-d/go-git.v4/plumbing"
912
. "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
10-
"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
11-
"gopkg.in/src-d/go-git.v4/storage/memory"
1213

1314
. "gopkg.in/check.v1"
1415
"gopkg.in/src-d/go-git-fixtures.v3"
@@ -26,51 +27,34 @@ func (s *IdxfileSuite) TestDecode(c *C) {
2627
f := fixtures.Basic().One()
2728

2829
d := NewDecoder(f.Idx())
29-
idx := &Idxfile{}
30+
idx := new(MemoryIndex)
3031
err := d.Decode(idx)
3132
c.Assert(err, IsNil)
3233

33-
c.Assert(idx.Entries, HasLen, 31)
34-
c.Assert(idx.Entries[0].Hash.String(), Equals, "1669dce138d9b841a518c64b10914d88f5e488ea")
35-
c.Assert(idx.Entries[0].Offset, Equals, uint64(615))
36-
c.Assert(idx.Entries[0].CRC32, Equals, uint32(3645019190))
34+
count, _ := idx.Count()
35+
c.Assert(count, Equals, int64(31))
3736

38-
c.Assert(fmt.Sprintf("%x", idx.IdxChecksum), Equals, "fb794f1ec720b9bc8e43257451bd99c4be6fa1c9")
39-
c.Assert(fmt.Sprintf("%x", idx.PackfileChecksum), Equals, f.PackfileHash.String())
40-
}
41-
42-
func (s *IdxfileSuite) TestDecodeCRCs(c *C) {
43-
f := fixtures.Basic().ByTag("ofs-delta").One()
44-
45-
scanner := packfile.NewScanner(f.Packfile())
46-
storage := memory.NewStorage()
47-
48-
pd, err := packfile.NewDecoder(scanner, storage)
37+
hash := plumbing.NewHash("1669dce138d9b841a518c64b10914d88f5e488ea")
38+
ok, err := idx.Contains(hash)
4939
c.Assert(err, IsNil)
50-
_, err = pd.Decode()
51-
c.Assert(err, IsNil)
52-
53-
i := pd.Index().ToIdxFile()
54-
i.Version = VersionSupported
40+
c.Assert(ok, Equals, true)
5541

56-
buf := bytes.NewBuffer(nil)
57-
e := NewEncoder(buf)
58-
_, err = e.Encode(i)
42+
offset, err := idx.FindOffset(hash)
5943
c.Assert(err, IsNil)
44+
c.Assert(offset, Equals, int64(615))
6045

61-
idx := &Idxfile{}
62-
63-
d := NewDecoder(buf)
64-
err = d.Decode(idx)
46+
crc32, err := idx.FindCRC32(hash)
6547
c.Assert(err, IsNil)
48+
c.Assert(crc32, Equals, uint32(3645019190))
6649

67-
c.Assert(idx.Entries, DeepEquals, i.Entries)
50+
c.Assert(fmt.Sprintf("%x", idx.IdxChecksum), Equals, "fb794f1ec720b9bc8e43257451bd99c4be6fa1c9")
51+
c.Assert(fmt.Sprintf("%x", idx.PackfileChecksum), Equals, f.PackfileHash.String())
6852
}
6953

7054
func (s *IdxfileSuite) TestDecode64bitsOffsets(c *C) {
7155
f := bytes.NewBufferString(fixtureLarge4GB)
7256

73-
idx := &Idxfile{}
57+
idx := new(MemoryIndex)
7458

7559
d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f))
7660
err := d.Decode(idx)
@@ -88,29 +72,22 @@ func (s *IdxfileSuite) TestDecode64bitsOffsets(c *C) {
8872
"35858be9c6f5914cbe6768489c41eb6809a2bceb": 5924278919,
8973
}
9074

91-
for _, e := range idx.Entries {
92-
c.Assert(expected[e.Hash.String()], Equals, e.Offset)
93-
}
94-
}
95-
96-
func (s *IdxfileSuite) TestDecode64bitsOffsetsIdempotent(c *C) {
97-
f := bytes.NewBufferString(fixtureLarge4GB)
98-
99-
expected := &Idxfile{}
100-
101-
d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f))
102-
err := d.Decode(expected)
75+
iter, err := idx.Entries()
10376
c.Assert(err, IsNil)
10477

105-
buf := bytes.NewBuffer(nil)
106-
_, err = NewEncoder(buf).Encode(expected)
107-
c.Assert(err, IsNil)
78+
var entries int
79+
for {
80+
e, err := iter.Next()
81+
if err == io.EOF {
82+
break
83+
}
84+
c.Assert(err, IsNil)
85+
entries++
10886

109-
idx := &Idxfile{}
110-
err = NewDecoder(buf).Decode(idx)
111-
c.Assert(err, IsNil)
87+
c.Assert(expected[e.Hash.String()], Equals, e.Offset)
88+
}
11289

113-
c.Assert(idx.Entries, DeepEquals, expected.Entries)
90+
c.Assert(entries, Equals, len(expected))
11491
}
11592

11693
const fixtureLarge4GB = `/3RPYwAAAAIAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEA
@@ -139,3 +116,30 @@ AAAAAAAMgAAAAQAAAI6AAAACgAAAA4AAAASAAAAFAAAAAV9Qam8AAAABYR1ShwAAAACdxfYxAAAA
139116
ANz1Di4AAAABPUnxJAAAAADNxzlGr6vCJpIFz4XaG/fi/f9C9zgQ8ptKSQpfQ1NMJBGTDTxxYGGp
140117
ch2xUA==
141118
`
119+
120+
func BenchmarkDecode(b *testing.B) {
121+
if err := fixtures.Init(); err != nil {
122+
b.Errorf("unexpected error initializing fixtures: %s", err)
123+
}
124+
125+
f := fixtures.Basic().One()
126+
fixture, err := ioutil.ReadAll(f.Idx())
127+
if err != nil {
128+
b.Errorf("unexpected error reading idx file: %s", err)
129+
}
130+
131+
defer func() {
132+
if err := fixtures.Clean(); err != nil {
133+
b.Errorf("unexpected error cleaning fixtures: %s", err)
134+
}
135+
}()
136+
137+
for i := 0; i < b.N; i++ {
138+
f := bytes.NewBuffer(fixture)
139+
idx := new(MemoryIndex)
140+
d := NewDecoder(f)
141+
if err := d.Decode(idx); err != nil {
142+
b.Errorf("unexpected error decoding: %s", err)
143+
}
144+
}
145+
}

0 commit comments

Comments
 (0)