Skip to content
This repository was archived by the owner on Sep 11, 2020. It is now read-only.

plumbing: format/idxfile, avoid creating temporary buffers to decode integers #1136

Merged
merged 3 commits into from
Apr 26, 2019
Merged

plumbing: format/idxfile, avoid creating temporary buffers to decode integers #1136

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3abbe14
plumbing: format/idxfile, avoid creating temporary buffers to decode …
filipnavara Apr 25, 2019
1f19576
plumbing: format/idxfile, save another 18% of time in genOffsetHash b…
filipnavara Apr 25, 2019
5943b45
plumbing: format/idxfile, avoid looking up the fanout mapping in the …
filipnavara Apr 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 28 additions & 51 deletions plumbing/format/idxfile/idxfile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ import (
"io"
"sort"

encbin "encoding/binary"

"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/utils/binary"
)

const (
Expand Down Expand Up @@ -122,41 +123,32 @@ func (idx *MemoryIndex) FindOffset(h plumbing.Hash) (int64, error) {
return 0, plumbing.ErrObjectNotFound
}

offset, err := idx.getOffset(k, i)
offset := idx.getOffset(k, i)

if !idx.offsetHashIsFull {
// Save the offset for reverse lookup
if idx.offsetHash == nil {
idx.offsetHash = make(map[int64]plumbing.Hash)
}
idx.offsetHash[offset] = h
idx.offsetHash[int64(offset)] = h
}

return offset, err
return int64(offset), nil
}

const isO64Mask = uint64(1) << 31

func (idx *MemoryIndex) getOffset(firstLevel, secondLevel int) (int64, error) {
func (idx *MemoryIndex) getOffset(firstLevel, secondLevel int) uint64 {
offset := secondLevel << 2
buf := bytes.NewBuffer(idx.Offset32[firstLevel][offset : offset+4])
ofs, err := binary.ReadUint32(buf)
if err != nil {
return -1, err
}
ofs := encbin.BigEndian.Uint32(idx.Offset32[firstLevel][offset : offset+4])

if (uint64(ofs) & isO64Mask) != 0 {
offset := 8 * (uint64(ofs) & ^isO64Mask)
buf := bytes.NewBuffer(idx.Offset64[offset : offset+8])
n, err := binary.ReadUint64(buf)
if err != nil {
return -1, err
}

return int64(n), nil
n := encbin.BigEndian.Uint64(idx.Offset64[offset : offset+8])
return n
}

return int64(ofs), nil
return uint64(ofs)
}

// FindCRC32 implements the Index interface.
Expand All @@ -167,13 +159,12 @@ func (idx *MemoryIndex) FindCRC32(h plumbing.Hash) (uint32, error) {
return 0, plumbing.ErrObjectNotFound
}

return idx.getCRC32(k, i)
return idx.getCRC32(k, i), nil
}

func (idx *MemoryIndex) getCRC32(firstLevel, secondLevel int) (uint32, error) {
func (idx *MemoryIndex) getCRC32(firstLevel, secondLevel int) uint32 {
offset := secondLevel << 2
buf := bytes.NewBuffer(idx.CRC32[firstLevel][offset : offset+4])
return binary.ReadUint32(buf)
return encbin.BigEndian.Uint32(idx.CRC32[firstLevel][offset : offset+4])
}

// FindHash implements the Index interface.
Expand Down Expand Up @@ -213,22 +204,19 @@ func (idx *MemoryIndex) genOffsetHash() error {
idx.offsetHash = make(map[int64]plumbing.Hash, count)
idx.offsetHashIsFull = true

iter, err := idx.Entries()
if err != nil {
return err
}

for {
entry, err := iter.Next()
if err != nil {
if err == io.EOF {
return nil
}
return err
var hash plumbing.Hash
i := uint32(0)
for firstLevel, fanoutValue := range idx.Fanout {
mappedFirstLevel := idx.FanoutMapping[firstLevel]
for secondLevel := uint32(0); i < fanoutValue; i++ {
copy(hash[:], idx.Names[mappedFirstLevel][secondLevel*objectIDLength:])
offset := int64(idx.getOffset(mappedFirstLevel, int(secondLevel)))
idx.offsetHash[offset] = hash
secondLevel++
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this a leftover?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope. The loop is a bit weird since it's iterating over two variables. Maybe there's some better Go syntax for it?

secondLevel goes from 0 to maximum value on that fanout level (not checked directly). i goes from 0 over both the inner and outer loop and bounds the inner loop.

}

idx.offsetHash[int64(entry.Offset)] = entry.Hash
}

return nil
}

// Count implements the Index interface.
Expand Down Expand Up @@ -297,22 +285,11 @@ func (i *idxfileEntryIter) Next() (*Entry, error) {
continue
}

mappedFirstLevel := i.idx.FanoutMapping[i.firstLevel]
entry := new(Entry)
ofs := i.secondLevel * objectIDLength
copy(entry.Hash[:], i.idx.Names[i.idx.FanoutMapping[i.firstLevel]][ofs:])

pos := i.idx.FanoutMapping[entry.Hash[0]]

offset, err := i.idx.getOffset(pos, i.secondLevel)
if err != nil {
return nil, err
}
entry.Offset = uint64(offset)

entry.CRC32, err = i.idx.getCRC32(pos, i.secondLevel)
if err != nil {
return nil, err
}
copy(entry.Hash[:], i.idx.Names[mappedFirstLevel][i.secondLevel*objectIDLength:])
entry.Offset = i.idx.getOffset(mappedFirstLevel, i.secondLevel)
entry.CRC32 = i.idx.getCRC32(mappedFirstLevel, i.secondLevel)

i.secondLevel++
i.total++
Expand Down