Skip to content
This repository was archived by the owner on Sep 11, 2020. It is now read-only.

Commit f6f3148

Browse files
mcuadrosmcuadros
committed
format: packfile fix ReadObjectAt without decode
1 parent 00ba614 commit f6f3148

File tree

14 files changed

+270
-89
lines changed

14 files changed

+270
-89
lines changed

‎.vscode/launch.json

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"version": "0.2.0",
3+
"configurations": [
4+
{
5+
"name": "Launch",
6+
"type": "go",
7+
"request": "launch",
8+
"mode": "debug",
9+
"remotePath": "",
10+
"port": 2345,
11+
"host": "127.0.0.1",
12+
"program": "${workspaceRoot}",
13+
"env": {},
14+
"args": [],
15+
"showLog": true
16+
}
17+
]
18+
}

‎examples/basic/main.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
)
1111

1212
func main() {
13-
r := git.NewMemoryRepository()
13+
r, _ := git.NewFilesystemRepository(".git")
1414

1515
// Clone the given repository, creating the remote, the local branches
1616
// and fetching the objects, exactly as:
@@ -28,8 +28,8 @@ func main() {
2828
// ... retrieving the branch being pointed by HEAD
2929
ref, _ := r.Head()
3030
// ... retrieving the commit object
31-
commit, _ := r.Commit(ref.Hash())
32-
fmt.Println(commit)
31+
commit, err := r.Commit(ref.Hash())
32+
fmt.Println(commit, err)
3333

3434
// List the tree from HEAD
3535
// > git ls-tree -r HEAD

‎fixtures/data/git-7cbde0ca02f13aedd5ec8b358ca17b1c0bf5ee64.tgz

83.8 KB
Binary file not shown.

‎fixtures/data/pack-c544593473465e6315ad4182d04d366c4592b829.idx

0 Bytes
Binary file not shown.

‎fixtures/fixtures.go

Lines changed: 34 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,21 @@ var RootFolder = ""
1717

1818
const DataFolder = "data"
1919

20-
var fixtures = []*Fixture{{
20+
var folders []string
21+
22+
var fixtures = Fixtures{{
2123
Tags: []string{"packfile", "ofs-delta", ".git"},
2224
URL: "https://github.com/git-fixtures/basic",
2325
Head: core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"),
2426
PackfileHash: core.NewHash("a3fed42da1e8189a077c0e6846c040dcf73fc9dd"),
2527
DotGitHash: core.NewHash("0a00a25543e6d732dbf4e8e9fec55c8e65fc4e8d"),
2628
ObjectsCount: 31,
2729
}, {
28-
Tags: []string{"packfile", "ref-delta"},
30+
Tags: []string{"packfile", "ref-delta", ".git"},
2931
URL: "https://github.com/git-fixtures/basic",
3032
Head: core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"),
3133
PackfileHash: core.NewHash("c544593473465e6315ad4182d04d366c4592b829"),
34+
DotGitHash: core.NewHash("7cbde0ca02f13aedd5ec8b358ca17b1c0bf5ee64"),
3235
ObjectsCount: 31,
3336
}, {
3437
Tags: []string{".git", "unpacked", "multi-packfile"},
@@ -51,27 +54,11 @@ func Basic() Fixtures {
5154
}
5255

5356
func ByURL(url string) Fixtures {
54-
r := make(Fixtures, 0)
55-
for _, f := range fixtures {
56-
if f.URL == url {
57-
r = append(r, f)
58-
}
59-
}
60-
61-
return r
57+
return fixtures.ByURL(url)
6258
}
6359

6460
func ByTag(tag string) Fixtures {
65-
r := make(Fixtures, 0)
66-
for _, f := range fixtures {
67-
for _, t := range f.Tags {
68-
if t == tag {
69-
r = append(r, f)
70-
}
71-
}
72-
}
73-
74-
return r
61+
return fixtures.ByTag(tag)
7562
}
7663

7764
type Fixture struct {
@@ -83,6 +70,16 @@ type Fixture struct {
8370
ObjectsCount int32
8471
}
8572

73+
func (f *Fixture) Is(tag string) bool {
74+
for _, t := range f.Tags {
75+
if t == tag {
76+
return true
77+
}
78+
}
79+
80+
return false
81+
}
82+
8683
func (f *Fixture) Packfile() io.ReadSeeker {
8784
fn := filepath.Join(RootFolder, DataFolder, fmt.Sprintf("pack-%s.pack", f.PackfileHash))
8885
file, err := os.Open(fn)
@@ -109,6 +106,8 @@ func (f *Fixture) DotGit() fs.Filesystem {
109106
if err != nil {
110107
panic(err)
111108
}
109+
110+
folders = append(folders, path)
112111
return fs.NewOS(path)
113112
}
114113

@@ -125,14 +124,23 @@ func (g Fixtures) One() *Fixture {
125124
return g[0]
126125
}
127126

128-
func (g Fixtures) ByTag(tag string) *Fixture {
127+
func (g Fixtures) ByTag(tag string) Fixtures {
128+
r := make(Fixtures, 0)
129129
for _, f := range g {
130-
for _, t := range f.Tags {
131-
if t == tag {
132-
return f
133-
}
130+
if f.Is(tag) {
131+
r = append(r, f)
134132
}
135133
}
136134

137-
return nil
135+
return r
136+
}
137+
func (g Fixtures) ByURL(url string) Fixtures {
138+
r := make(Fixtures, 0)
139+
for _, f := range g {
140+
if f.URL == url {
141+
r = append(r, f)
142+
}
143+
}
144+
145+
return r
138146
}

‎formats/idxfile/count.out

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
mode: count
2+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:18.39,22.2 3 5
3+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:25.53,38.25 4 5
4+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:47.2,47.16 1 5
5+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:38.25,42.17 3 30
6+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:42.17,44.4 1 0
7+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:50.59,52.16 2 5
8+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:56.2,56.41 1 5
9+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:52.16,54.3 1 0
10+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:59.59,61.27 2 5
11+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:67.2,67.18 1 5
12+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:61.27,62.41 1 1280
13+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:62.41,64.4 1 0
14+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:70.59,72.34 2 5
15+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:81.2,81.16 1 5
16+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:72.34,76.17 3 4051
17+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:76.17,78.4 1 0
18+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:84.58,86.34 2 5
19+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:95.2,95.16 1 5
20+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:86.34,90.17 3 4051
21+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:90.17,92.4 1 0
22+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:98.60,100.34 2 5
23+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:109.2,109.16 1 5
24+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:100.34,101.58 1 4051
25+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:105.3,105.10 1 4051
26+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:101.58,103.4 1 0
27+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:112.62,113.60 1 5
28+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:117.2,118.55 2 5
29+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:122.2,122.16 1 5
30+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:113.60,115.3 1 0
31+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:118.55,120.3 1 0
32+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:125.50,127.2 1 5336
33+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:131.40,131.57 1 5
34+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:132.40,132.90 1 42228
35+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:133.40,133.67 1 2816
36+
gopkg.in/src-d/go-git.v4/formats/idxfile/encoder.go:134.40,134.56 1 5
37+
gopkg.in/src-d/go-git.v4/formats/idxfile/idxfile.go:32.67,38.2 1 33
38+
gopkg.in/src-d/go-git.v4/formats/idxfile/idxfile.go:40.36,42.31 2 6
39+
gopkg.in/src-d/go-git.v4/formats/idxfile/idxfile.go:48.2,48.13 1 6
40+
gopkg.in/src-d/go-git.v4/formats/idxfile/idxfile.go:42.31,43.21 1 1530
41+
gopkg.in/src-d/go-git.v4/formats/idxfile/idxfile.go:43.21,45.4 1 0
42+
gopkg.in/src-d/go-git.v4/formats/idxfile/idxfile.go:51.51,53.32 2 11
43+
gopkg.in/src-d/go-git.v4/formats/idxfile/idxfile.go:57.2,57.27 1 11
44+
gopkg.in/src-d/go-git.v4/formats/idxfile/idxfile.go:61.2,61.15 1 11
45+
gopkg.in/src-d/go-git.v4/formats/idxfile/idxfile.go:53.32,55.3 1 8133
46+
gopkg.in/src-d/go-git.v4/formats/idxfile/idxfile.go:57.27,59.3 1 2805
47+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:26.39,28.2 1 6
48+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:32.46,33.42 1 6
49+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:37.2,46.25 2 6
50+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:52.2,52.20 1 6
51+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:56.2,56.12 1 6
52+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:33.42,35.3 1 0
53+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:46.25,47.35 1 36
54+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:47.35,49.4 1 0
55+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:52.20,54.3 1 0
56+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:59.40,61.37 2 6
57+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:65.2,65.32 1 6
58+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:69.2,69.12 1 6
59+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:61.37,63.3 1 0
60+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:65.32,67.3 1 0
61+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:72.51,74.16 2 6
62+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:78.2,78.26 1 6
63+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:82.2,84.12 2 6
64+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:74.16,76.3 1 0
65+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:78.26,80.3 1 0
66+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:87.50,90.27 2 6
67+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:97.2,98.12 2 6
68+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:90.27,92.17 2 1530
69+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:92.17,94.4 1 0
70+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:101.55,103.25 2 6
71+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:112.2,112.12 1 6
72+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:103.25,105.43 2 4082
73+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:109.3,109.54 1 4082
74+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:105.43,107.4 1 0
75+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:115.49,117.25 2 6
76+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:123.2,123.12 1 6
77+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:117.25,118.81 1 4082
78+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:118.81,120.4 1 0
79+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:126.51,128.25 2 6
80+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:137.2,137.12 1 6
81+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:128.25,130.17 2 4082
82+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:134.3,134.36 1 4082
83+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:130.17,132.4 1 0
84+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:140.53,141.59 1 6
85+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:145.2,145.54 1 6
86+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:149.2,149.12 1 6
87+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:141.59,143.3 1 0
88+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:145.54,147.3 1 0
89+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:152.45,154.61 2 5624
90+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:158.2,158.15 1 5624
91+
gopkg.in/src-d/go-git.v4/formats/idxfile/decoder.go:154.61,156.3 1 0

‎formats/packfile/decoder.go

Lines changed: 34 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,9 @@ type Decoder struct {
4343
o core.ObjectStorage
4444
tx core.TxObjectStorage
4545

46-
offsets map[int64]core.Hash
47-
crcs map[core.Hash]uint32
46+
offsetToHash map[int64]core.Hash
47+
hashToOffset map[core.Hash]int64
48+
crcs map[core.Hash]uint32
4849
}
4950

5051
// NewDecoder returns a new Decoder that reads from r.
@@ -54,8 +55,9 @@ func NewDecoder(s *Scanner, o core.ObjectStorage) *Decoder {
5455
o: o,
5556
tx: o.Begin(),
5657

57-
offsets: make(map[int64]core.Hash, 0),
58-
crcs: make(map[core.Hash]uint32, 0),
58+
offsetToHash: make(map[int64]core.Hash, 0),
59+
hashToOffset: make(map[core.Hash]int64, 0),
60+
crcs: make(map[core.Hash]uint32, 0),
5961
}
6062
}
6163

@@ -82,11 +84,7 @@ func (d *Decoder) doDecode() error {
8284
return err
8385
}
8486

85-
if err := d.tx.Commit(); err != nil {
86-
return err
87-
}
88-
89-
return nil
87+
return d.tx.Commit()
9088
}
9189

9290
func (d *Decoder) readObjects(count uint32) error {
@@ -126,7 +124,9 @@ func (d *Decoder) ReadObject() (core.Object, error) {
126124
return obj, err
127125
}
128126

129-
d.remember(obj, h.Offset, crc)
127+
hash := obj.Hash()
128+
d.setOffset(hash, h.Offset)
129+
d.setCRC(hash, crc)
130130

131131
if _, err := d.tx.Set(obj); err != nil {
132132
return nil, err
@@ -194,34 +194,45 @@ func (d *Decoder) fillOFSDeltaObjectContent(obj core.Object, offset int64) (uint
194194
return crc, ApplyDelta(obj, base, buf.Bytes())
195195
}
196196

197-
func (d *Decoder) remember(obj core.Object, offset int64, crc uint32) {
198-
h := obj.Hash()
197+
func (d *Decoder) setOffset(h core.Hash, offset int64) {
198+
d.offsetToHash[offset] = h
199+
d.hashToOffset[h] = offset
200+
}
199201

200-
d.offsets[offset] = h
202+
func (d *Decoder) setCRC(h core.Hash, crc uint32) {
201203
d.crcs[h] = crc
202204
}
203205

204206
func (d *Decoder) recallByOffset(o int64) (core.Object, error) {
205-
h, ok := d.offsets[o]
206-
if ok {
207-
return d.recallByHash(h)
207+
if h, ok := d.offsetToHash[o]; ok {
208+
return d.tx.Get(core.AnyObject, h)
208209
}
209210

210211
return d.ReadObjectAt(o)
211212
}
212213

213214
func (d *Decoder) recallByHash(h core.Hash) (core.Object, error) {
214-
return d.tx.Get(core.AnyObject, h)
215+
obj, err := d.tx.Get(core.AnyObject, h)
216+
if err != core.ErrObjectNotFound {
217+
return obj, err
218+
}
219+
220+
if o, ok := d.hashToOffset[h]; ok {
221+
return d.ReadObjectAt(o)
222+
}
223+
224+
return nil, core.ErrObjectNotFound
225+
}
226+
227+
// SetOffsets sets the offsets, required when using the method ReadObjectAt,
228+
// without decoding the full packfile
229+
func (d *Decoder) SetOffsets(offsets map[core.Hash]int64) {
230+
d.hashToOffset = offsets
215231
}
216232

217233
// Offsets returns the objects read offset
218234
func (d *Decoder) Offsets() map[core.Hash]int64 {
219-
i := make(map[core.Hash]int64, len(d.offsets))
220-
for o, h := range d.offsets {
221-
i[h] = o
222-
}
223-
224-
return i
235+
return d.hashToOffset
225236
}
226237

227238
// CRCs returns the CRC-32 for each objected read

0 commit comments

Comments
 (0)