From 62d58489c5ddef5962e222acca4c9f0527e0aa37 Mon Sep 17 00:00:00 2001 From: Manuel Carmona Date: Wed, 16 May 2018 11:26:30 +0200 Subject: [PATCH] *: implement indexable interfaces on basic and relation tables Signed-off-by: Manuel Carmona --- blobs.go | 62 +- commit_blobs.go | 72 ++- commit_blobs_test.go | 2 +- commit_trees.go | 56 +- commits.go | 46 +- common_test.go | 2 + filters.go | 11 +- index.go | 241 ++++++++ index_test.go | 1209 +++++++++++++++++++++++++++++++++++++++ integration_test.go | 16 +- ref_commits.go | 66 ++- references.go | 79 ++- remotes.go | 30 +- remotes_test.go | 3 - repositories.go | 27 +- repository_pool.go | 19 +- repository_pool_test.go | 36 +- squash_iterator.go | 13 + table.go | 4 + tree_entries.go | 72 ++- 20 files changed, 1856 insertions(+), 210 deletions(-) create mode 100644 index.go create mode 100644 index_test.go diff --git a/blobs.go b/blobs.go index 430c9a6d1..7b68e8c4a 100644 --- a/blobs.go +++ b/blobs.go @@ -38,8 +38,11 @@ var BlobsSchema = sql.Schema{ var _ sql.PushdownProjectionAndFiltersTable = (*blobsTable)(nil) -func newBlobsTable() sql.Table { - return new(blobsTable) +func newBlobsTable() Indexable { + return &indexableTable{ + PushdownTable: new(blobsTable), + buildIterWithSelectors: blobsIterBuilder, + } } var _ Table = (*blobsTable)(nil) @@ -91,29 +94,20 @@ func (blobsTable) HandledFilters(filters []sql.Expression) []sql.Expression { return handledFilters(BlobsTableName, BlobsSchema, filters) } +func (*blobsTable) handledColumns() []string { + return []string{"blob_hash"} +} + func (r *blobsTable) WithProjectAndFilters( ctx *sql.Context, columns, filters []sql.Expression, ) (sql.RowIter, error) { span, ctx := ctx.Span("gitbase.BlobsTable") iter, err := rowIterWithSelectors( - ctx, BlobsSchema, BlobsTableName, filters, - []string{"blob_hash"}, - func(selectors selectors) (RowRepoIter, error) { - if len(selectors["blob_hash"]) == 0 { - return &blobIter{readContent: shouldReadContent(columns)}, nil - } - - hashes, err := selectors.textValues("blob_hash") - if err != nil { - return nil, err - } - - return &blobsByHashIter{ - hashes: hashes, - readContent: shouldReadContent(columns), - }, nil - }, + ctx, BlobsSchema, BlobsTableName, + filters, columns, + r.handledColumns(), + blobsIterBuilder, ) if err != nil { @@ -124,10 +118,27 @@ func (r *blobsTable) WithProjectAndFilters( return sql.NewSpanIter(span, iter), nil } +func blobsIterBuilder(_ *sql.Context, selectors selectors, columns []sql.Expression) (RowRepoIter, error) { + if len(selectors["blob_hash"]) == 0 { + return &blobIter{readContent: shouldReadContent(columns)}, nil + } + + hashes, err := selectors.textValues("blob_hash") + if err != nil { + return nil, err + } + + return &blobsByHashIter{ + hashes: hashes, + readContent: shouldReadContent(columns), + }, nil +} + type blobIter struct { repoID string iter *object.BlobIter readContent bool + lastHash string } func (i *blobIter) NewIterator(repo *Repository) (RowRepoIter, error) { @@ -139,12 +150,17 @@ func (i *blobIter) NewIterator(repo *Repository) (RowRepoIter, error) { return &blobIter{repoID: repo.ID, iter: iter, readContent: i.readContent}, nil } +func (i *blobIter) Repository() string { return i.repoID } + +func (i *blobIter) LastObject() string { return i.lastHash } + func (i *blobIter) Next() (sql.Row, error) { o, err := i.iter.Next() if err != nil { return nil, err } + i.lastHash = o.Hash.String() return blobToRow(i.repoID, o, i.readContent) } @@ -161,12 +177,17 @@ type blobsByHashIter struct { pos int hashes []string readContent bool + lastHash string } func (i *blobsByHashIter) NewIterator(repo *Repository) (RowRepoIter, error) { - return &blobsByHashIter{repo, 0, i.hashes, i.readContent}, nil + return &blobsByHashIter{repo, 0, i.hashes, i.readContent, ""}, nil } +func (i *blobsByHashIter) Repository() string { return i.repo.ID } + +func (i *blobsByHashIter) LastObject() string { return i.lastHash } + func (i *blobsByHashIter) Next() (sql.Row, error) { for { if i.pos >= len(i.hashes) { @@ -184,6 +205,7 @@ func (i *blobsByHashIter) Next() (sql.Row, error) { return nil, err } + i.lastHash = hash.String() return blobToRow(i.repo.ID, blob, i.readContent) } } diff --git a/commit_blobs.go b/commit_blobs.go index 49249fdba..690c3299a 100644 --- a/commit_blobs.go +++ b/commit_blobs.go @@ -18,8 +18,11 @@ var CommitBlobsSchema = sql.Schema{ var _ sql.PushdownProjectionAndFiltersTable = (*commitBlobsTable)(nil) -func newCommitBlobsTable() sql.Table { - return new(commitBlobsTable) +func newCommitBlobsTable() Indexable { + return &indexableTable{ + PushdownTable: new(commitBlobsTable), + buildIterWithSelectors: commitBlobsIterBuilder, + } } func (commitBlobsTable) isGitbaseTable() {} @@ -59,36 +62,18 @@ func (commitBlobsTable) HandledFilters(filters []sql.Expression) []sql.Expressio return handledFilters(CommitBlobsTableName, CommitBlobsSchema, filters) } -func (commitBlobsTable) WithProjectAndFilters( +func (commitBlobsTable) handledColumns() []string { return []string{"commit_hash", "repository_id"} } + +func (t *commitBlobsTable) WithProjectAndFilters( ctx *sql.Context, _, filters []sql.Expression, ) (sql.RowIter, error) { span, ctx := ctx.Span("gitbase.CommitBlobsTable") iter, err := rowIterWithSelectors( - ctx, CommitBlobsSchema, CommitBlobsTableName, filters, - []string{"commit_hash", "repository_id"}, - func(selectors selectors) (RowRepoIter, error) { - repos, err := selectors.textValues("repository_id") - if err != nil { - return nil, err - } - - commits, err := selectors.textValues("commit_hash") - if err != nil { - return nil, err - } - - s, ok := ctx.Session.(*Session) - if !ok { - return nil, ErrInvalidGitbaseSession.New(ctx.Session) - } - - return &commitBlobsIter{ - repos: repos, - commits: commits, - skipGitErrors: s.SkipGitErrors, - }, nil - }, + ctx, CommitBlobsSchema, CommitBlobsTableName, + filters, nil, + t.handledColumns(), + commitBlobsIterBuilder, ) if err != nil { @@ -99,8 +84,31 @@ func (commitBlobsTable) WithProjectAndFilters( return sql.NewSpanIter(span, iter), nil } +func commitBlobsIterBuilder(ctx *sql.Context, selectors selectors, columns []sql.Expression) (RowRepoIter, error) { + repos, err := selectors.textValues("repository_id") + if err != nil { + return nil, err + } + + commits, err := selectors.textValues("commit_hash") + if err != nil { + return nil, err + } + + s, ok := ctx.Session.(*Session) + if !ok { + return nil, ErrInvalidGitbaseSession.New(ctx.Session) + } + + return &commitBlobsIter{ + repos: repos, + commits: commits, + skipGitErrors: s.SkipGitErrors, + }, nil +} + type commitBlobsIter struct { - repoID string + repo *Repository iter object.CommitIter currCommit *object.Commit filesIter *object.FileIter @@ -122,13 +130,17 @@ func (i *commitBlobsIter) NewIterator(repo *Repository) (RowRepoIter, error) { } return &commitBlobsIter{ - repoID: repo.ID, + repo: repo, iter: iter, repos: i.repos, commits: i.commits, }, nil } +func (i *commitBlobsIter) Repository() string { return i.repo.ID } + +func (i *commitBlobsIter) LastObject() string { return i.currCommit.Hash.String() } + func (i *commitBlobsIter) Next() (sql.Row, error) { for { if i.iter == nil { @@ -175,7 +187,7 @@ func (i *commitBlobsIter) Next() (sql.Row, error) { } return sql.NewRow( - i.repoID, i.currCommit.Hash.String(), file.Blob.Hash.String(), + i.repo.ID, i.currCommit.Hash.String(), file.Blob.Hash.String(), ), nil } } diff --git a/commit_blobs_test.go b/commit_blobs_test.go index 28dcef2ad..c8e7b2d86 100644 --- a/commit_blobs_test.go +++ b/commit_blobs_test.go @@ -13,7 +13,7 @@ func TestCommitBlobsTableRowIter(t *testing.T) { require := require.New(t) table := newCommitBlobsTable() - require.IsType(&commitBlobsTable{}, table) + require.NotNil(table) ctx, paths, cleanup := setupRepos(t) defer cleanup() diff --git a/commit_trees.go b/commit_trees.go index 90d5f5bbc..d085a2c63 100644 --- a/commit_trees.go +++ b/commit_trees.go @@ -21,8 +21,11 @@ var CommitTreesSchema = sql.Schema{ var _ sql.PushdownProjectionAndFiltersTable = (*commitTreesTable)(nil) -func newCommitTreesTable() sql.Table { - return new(commitTreesTable) +func newCommitTreesTable() Indexable { + return &indexableTable{ + PushdownTable: new(commitTreesTable), + buildIterWithSelectors: commitTreesIterBuilder, + } } func (commitTreesTable) isGitbaseTable() {} @@ -62,31 +65,18 @@ func (commitTreesTable) HandledFilters(filters []sql.Expression) []sql.Expressio return handledFilters(CommitTreesTableName, CommitTreesSchema, filters) } -func (commitTreesTable) WithProjectAndFilters( +func (commitTreesTable) handledColumns() []string { return []string{"commit_hash", "repository_id"} } + +func (t *commitTreesTable) WithProjectAndFilters( ctx *sql.Context, _, filters []sql.Expression, ) (sql.RowIter, error) { span, ctx := ctx.Span("gitbase.CommitTreesTable") iter, err := rowIterWithSelectors( - ctx, CommitTreesSchema, CommitTreesTableName, filters, - []string{"commit_hash", "repository_id"}, - func(selectors selectors) (RowRepoIter, error) { - repos, err := selectors.textValues("repository_id") - if err != nil { - return nil, err - } - - hashes, err := selectors.textValues("commit_hash") - if err != nil { - return nil, err - } - - return &commitTreesIter{ - ctx: ctx, - commitHashes: hashes, - repos: repos, - }, nil - }, + ctx, CommitTreesSchema, CommitTreesTableName, + filters, nil, + t.handledColumns(), + commitTreesIterBuilder, ) if err != nil { @@ -97,6 +87,24 @@ func (commitTreesTable) WithProjectAndFilters( return sql.NewSpanIter(span, iter), nil } +func commitTreesIterBuilder(ctx *sql.Context, selectors selectors, columns []sql.Expression) (RowRepoIter, error) { + repos, err := selectors.textValues("repository_id") + if err != nil { + return nil, err + } + + hashes, err := selectors.textValues("commit_hash") + if err != nil { + return nil, err + } + + return &commitTreesIter{ + ctx: ctx, + commitHashes: hashes, + repos: repos, + }, nil +} + type commitTreesIter struct { ctx *sql.Context repo *Repository @@ -129,6 +137,10 @@ func (i *commitTreesIter) NewIterator(repo *Repository) (RowRepoIter, error) { }, nil } +func (i *commitTreesIter) Repository() string { return i.repo.ID } + +func (i *commitTreesIter) LastObject() string { return i.commit.Hash.String() } + func (i *commitTreesIter) Next() (sql.Row, error) { s, ok := i.ctx.Session.(*Session) if !ok { diff --git a/commits.go b/commits.go index 87a436c2e..2f161fc25 100644 --- a/commits.go +++ b/commits.go @@ -28,8 +28,11 @@ var CommitsSchema = sql.Schema{ var _ sql.PushdownProjectionAndFiltersTable = (*commitsTable)(nil) -func newCommitsTable() sql.Table { - return new(commitsTable) +func newCommitsTable() Indexable { + return &indexableTable{ + PushdownTable: new(commitsTable), + buildIterWithSelectors: commitsIterBuilder, + } } var _ Table = (*commitsTable)(nil) @@ -81,22 +84,20 @@ func (commitsTable) HandledFilters(filters []sql.Expression) []sql.Expression { return handledFilters(CommitsTableName, CommitsSchema, filters) } +func (commitsTable) handledColumns() []string { + return []string{"commit_hash"} +} + func (r *commitsTable) WithProjectAndFilters( ctx *sql.Context, _, filters []sql.Expression, ) (sql.RowIter, error) { span, ctx := ctx.Span("gitbase.CommitsTable") iter, err := rowIterWithSelectors( - ctx, CommitsSchema, CommitsTableName, filters, - []string{"commit_hash"}, - func(selectors selectors) (RowRepoIter, error) { - hashes, err := selectors.textValues("commit_hash") - if err != nil { - return nil, err - } - - return &commitIter{hashes: hashes}, nil - }, + ctx, CommitsSchema, CommitsTableName, + filters, nil, + r.handledColumns(), + commitsIterBuilder, ) if err != nil { @@ -107,10 +108,20 @@ func (r *commitsTable) WithProjectAndFilters( return sql.NewSpanIter(span, iter), nil } +func commitsIterBuilder(_ *sql.Context, selectors selectors, _ []sql.Expression) (RowRepoIter, error) { + hashes, err := selectors.textValues("commit_hash") + if err != nil { + return nil, err + } + + return &commitIter{hashes: hashes}, nil +} + type commitIter struct { - repoID string - iter object.CommitIter - hashes []string + repoID string + iter object.CommitIter + hashes []string + lastHash string } func (i *commitIter) NewIterator(repo *Repository) (RowRepoIter, error) { @@ -122,12 +133,17 @@ func (i *commitIter) NewIterator(repo *Repository) (RowRepoIter, error) { return &commitIter{repoID: repo.ID, iter: iter}, nil } +func (i *commitIter) Repository() string { return i.repoID } + +func (i *commitIter) LastObject() string { return i.lastHash } + func (i *commitIter) Next() (sql.Row, error) { o, err := i.iter.Next() if err != nil { return nil, err } + i.lastHash = o.Hash.String() return commitToRow(i.repoID, o), nil } diff --git a/common_test.go b/common_test.go index 025eee673..00703739e 100644 --- a/common_test.go +++ b/common_test.go @@ -40,6 +40,8 @@ func buildSession(t *testing.T, repos fixtures.Fixtures, path := fixture.Worktree().Root() _, err := pool.AddGit(path) if err == nil { + _, err := pool.GetRepo(path) + require.NoError(err) paths = append(paths, path) } } diff --git a/filters.go b/filters.go index 50dde14c1..1d96a40be 100644 --- a/filters.go +++ b/filters.go @@ -90,11 +90,11 @@ func canHandleEquals(schema sql.Schema, tableName string, eq *expression.Equals) switch left := eq.Left().(type) { case *expression.GetField: if _, ok := eq.Right().(*expression.Literal); ok && left.Table() == tableName { - return schema.Contains(left.Name()) + return schema.Contains(left.Name(), tableName) } case *expression.Literal: if right, ok := eq.Right().(*expression.GetField); ok && right.Table() == tableName { - return schema.Contains(right.Name()) + return schema.Contains(right.Name(), tableName) } } return false @@ -106,7 +106,7 @@ func canHandleEquals(schema sql.Schema, tableName string, eq *expression.Equals) // The GetField expr must exist in the schema and match the given table name. func canHandleIn(schema sql.Schema, tableName string, in *expression.In) bool { left, ok := in.Left().(*expression.GetField) - if !ok || !schema.Contains(left.Name()) || left.Table() != tableName { + if !ok || !schema.Contains(left.Name(), tableName) || left.Table() != tableName { return false } @@ -326,15 +326,16 @@ func rowIterWithSelectors( schema sql.Schema, tableName string, filters []sql.Expression, + columns []sql.Expression, handledCols []string, - rowIterBuilder func(selectors) (RowRepoIter, error), + iterBuild iteratorBuilder, ) (sql.RowIter, error) { selectors, filters, err := classifyFilters(schema, tableName, filters, handledCols...) if err != nil { return nil, err } - rowRepoIter, err := rowIterBuilder(selectors) + rowRepoIter, err := iterBuild(ctx, selectors, columns) if err != nil { return nil, err } diff --git a/index.go b/index.go new file mode 100644 index 000000000..887b0e20c --- /dev/null +++ b/index.go @@ -0,0 +1,241 @@ +package gitbase + +import ( + "bytes" + "encoding/gob" + + errors "gopkg.in/src-d/go-errors.v1" + "gopkg.in/src-d/go-mysql-server.v0/sql" + "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "gopkg.in/src-d/go-mysql-server.v0/sql/plan" +) + +var ( + // ErrColumnNotFound is returned when a given column is not found in the table's schema. + ErrColumnNotFound = errors.NewKind("column %s not found for table %s") + // ErrCreateIndexValue is returned if an index value can't be generated. + ErrCreateIndexValue = errors.NewKind("couldn't create index value, missing %s") + // ErrIndexValue is returned when a index value is malformed. + ErrIndexValue = errors.NewKind("wrong index value found") +) + +// Indexable represents an indexable gitbase table. +type Indexable interface { + sql.Indexable + PushdownTable +} + +// PushdownTable represents a gitbase table that is able to pushdown projections and filters. +type PushdownTable interface { + sql.PushdownProjectionAndFiltersTable + gitBase + handledColumns() []string +} + +type indexableTable struct { + PushdownTable + buildIterWithSelectors iteratorBuilder +} + +var _ sql.Indexable = (*indexableTable)(nil) + +// IndexKeyValueIter implements sql.Indexable interface. +func (i *indexableTable) IndexKeyValueIter(ctx *sql.Context, colNames []string) (sql.IndexKeyValueIter, error) { + s, ok := ctx.Session.(*Session) + if !ok || s == nil { + return nil, ErrInvalidGitbaseSession.New(ctx.Session) + } + + colIndexes := []int{} + columns := []sql.Expression{} + for _, colName := range colNames { + idx := i.Schema().IndexOf(colName, i.Name()) + if idx < 0 { + return nil, ErrColumnNotFound.New(colName, i.Name()) + } + + colIndexes = append(colIndexes, idx) + + col := expression.NewGetFieldWithTable( + idx, + i.Schema()[idx].Type, + i.Schema()[idx].Source, + i.Schema()[idx].Name, + i.Schema()[idx].Nullable, + ) + + columns = append(columns, col) + } + + rIter, err := s.Pool.RepoIter() + if err != nil { + return nil, err + } + + tableIter, err := i.buildIterWithSelectors(ctx, nil, columns) + if err != nil { + return nil, err + } + + repoIter := &rowRepoIter{ + currRepoIter: nil, + repositoryIter: rIter, + iter: tableIter, + session: s, + ctx: ctx, + } + + return &indexKVIter{ + repoIter: repoIter, + colIndexes: colIndexes, + }, nil +} + +// WithProjectFiltersAndIndex implements sql.Indexable interface. +func (i *indexableTable) WithProjectFiltersAndIndex(ctx *sql.Context, columns, filters []sql.Expression, index sql.IndexValueIter) (sql.RowIter, error) { + s, ok := ctx.Session.(*Session) + if !ok || s == nil { + return nil, ErrInvalidGitbaseSession.New(ctx.Session) + } + + selectors, filters, err := classifyFilters(i.Schema(), i.Name(), + filters, i.handledColumns()...) + if err != nil { + return nil, err + } + + rowRepoIter, err := i.buildIterWithSelectors(ctx, selectors, columns) + if err != nil { + return nil, err + } + + indexIter := &indexIter{ + iter: rowRepoIter, + idxValueIter: index, + pool: s.Pool, + } + + if len(filters) == 0 { + return indexIter, nil + } + + return plan.NewFilterIter(ctx, expression.JoinAnd(filters...), indexIter), nil +} + +type indexIter struct { + repoID string + iter RowRepoIter + currIter RowRepoIter + + pool *RepositoryPool + idxValueIter sql.IndexValueIter +} + +var _ sql.RowIter = (*indexIter)(nil) + +func (i *indexIter) Next() (sql.Row, error) { + for { + v, err := i.idxValueIter.Next() + if err != nil { + return nil, err + } + + idxVal, err := unmarshalIndexValue(v) + if err != nil || idxVal.ID == "" || idxVal.Object == "" { + return nil, ErrIndexValue.New() + } + + if i.repoID != idxVal.ID { + repo, err := i.pool.GetRepo(idxVal.ID) + if err != nil { + return nil, err + } + + iter, err := i.iter.NewIterator(repo) + if err != nil { + return nil, err + } + + i.repoID = repo.ID + i.currIter = iter + } + + return i.currIter.Next() + } +} + +func (i *indexIter) Close() error { + if i.currIter != nil { + i.currIter.Close() + } + + return nil +} + +type indexValue struct { + ID string + Object string +} + +func marshalIndexValue(value *indexValue) ([]byte, error) { + var raw bytes.Buffer + enc := gob.NewEncoder(&raw) + if err := enc.Encode(value); err != nil { + return nil, err + } + + return raw.Bytes(), nil +} + +func unmarshalIndexValue(raw []byte) (*indexValue, error) { + value := bytes.NewReader(raw) + dec := gob.NewDecoder(value) + idxValue := &indexValue{} + if err := dec.Decode(idxValue); err != nil { + return nil, err + } + + return idxValue, nil +} + +type indexKVIter struct { + repoIter *rowRepoIter + colIndexes []int +} + +var _ sql.IndexKeyValueIter = (*indexKVIter)(nil) + +func (i *indexKVIter) Next() ([]interface{}, []byte, error) { + row, err := i.repoIter.Next() + if err != nil { + return nil, nil, err + } + + repoID := i.repoIter.currRepoIter.Repository() + if repoID == "" { + return nil, nil, ErrCreateIndexValue.New("repository id") + } + + object := i.repoIter.currRepoIter.LastObject() + if object == "" { + return nil, nil, ErrCreateIndexValue.New("object") + } + + idxValue := &indexValue{repoID, object} + + colValues := []interface{}{} + for _, idx := range i.colIndexes { + colValues = append(colValues, row[idx]) + } + + value, err := marshalIndexValue(idxValue) + if err != nil { + return nil, nil, err + } + + return colValues, value, nil +} + +func (i *indexKVIter) Close() error { + return i.repoIter.Close() +} diff --git a/index_test.go b/index_test.go new file mode 100644 index 000000000..e423f386b --- /dev/null +++ b/index_test.go @@ -0,0 +1,1209 @@ +package gitbase + +import ( + "io" + "testing" + "time" + + "gopkg.in/src-d/go-mysql-server.v0/sql" + "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + + "github.com/stretchr/testify/require" +) + +type indexTest struct { + name string + node sql.Indexable + colNames []string + expectedKVs []expectedKV + filters []sql.Expression + columns []sql.Expression + expectedRows []sql.Row +} + +type expectedKV struct { + key []interface{} + idxValue *indexValue +} + +func TestIndexableTable(t *testing.T) { + ctx, paths, cleanup := setupRepos(t) + defer cleanup() + + expectedRepos := []expectedKV{} + for _, path := range paths { + expectedRepos = append(expectedRepos, expectedKV{ + key: []interface{}{path}, + idxValue: &indexValue{path, path}, + }) + } + + var tests = []*indexTest{ + { + name: "blobs indexable table", + node: newBlobsTable(), + colNames: []string{"blob_hash", "blob_size"}, + expectedKVs: []expectedKV{ + { + key: []interface{}{"32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", int64(189)}, + idxValue: &indexValue{paths[0], "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + }, + { + key: []interface{}{"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", int64(18)}, + idxValue: &indexValue{paths[0], "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + }, + { + key: []interface{}{"c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", int64(1072)}, + idxValue: &indexValue{paths[0], "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + }, + { + key: []interface{}{"7e59600739c96546163833214c36459e324bad0a", int64(9)}, + idxValue: &indexValue{paths[0], "7e59600739c96546163833214c36459e324bad0a"}, + }, + { + key: []interface{}{"d5c0f4ab811897cadf03aec358ae60d21f91c50d", int64(76110)}, + idxValue: &indexValue{paths[0], "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + }, + { + key: []interface{}{"880cd14280f4b9b6ed3986d6671f907d7cc2a198", int64(2780)}, + idxValue: &indexValue{paths[0], "880cd14280f4b9b6ed3986d6671f907d7cc2a198"}, + }, + { + key: []interface{}{"49c6bb89b17060d7b4deacb7b338fcc6ea2352a9", int64(217848)}, + idxValue: &indexValue{paths[0], "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"}, + }, + { + key: []interface{}{"c8f1d8c61f9da76f4cb49fd86322b6e685dba956", int64(706)}, + idxValue: &indexValue{paths[0], "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"}, + }, + { + key: []interface{}{"9a48f23120e880dfbe41f7c9b7b708e9ee62a492", int64(11488)}, + idxValue: &indexValue{paths[0], "9a48f23120e880dfbe41f7c9b7b708e9ee62a492"}, + }, + { + key: []interface{}{"9dea2395f5403188298c1dabe8bdafe562c491e3", int64(78)}, + idxValue: &indexValue{paths[0], "9dea2395f5403188298c1dabe8bdafe562c491e3"}, + }, + { + key: []interface{}{"278871477afb195f908155a65b5c651f1cfd02d3", int64(172)}, + idxValue: &indexValue{paths[1], "278871477afb195f908155a65b5c651f1cfd02d3"}, + }, + { + key: []interface{}{"97b013ecd2cc7f572960509f659d8068798d59ca", int64(83)}, + idxValue: &indexValue{paths[1], "97b013ecd2cc7f572960509f659d8068798d59ca"}, + }, + { + key: []interface{}{"b4f017e8c030d24aef161569b9ade3e55931ba01", int64(20)}, + idxValue: &indexValue{paths[1], "b4f017e8c030d24aef161569b9ade3e55931ba01"}, + }, + }, + columns: []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Text, BlobsTableName, "blob_hash", false), + }, + filters: []sql.Expression{ + expression.NewGreaterThanOrEqual( + expression.NewGetFieldWithTable(2, sql.Int64, BlobsTableName, "blob_size", false), + expression.NewLiteral(int64(75000), sql.Int64), + ), + }, + expectedRows: []sql.Row{ + sql.NewRow(paths[0], "d5c0f4ab811897cadf03aec358ae60d21f91c50d", int64(76110), []uint8(nil)), + sql.NewRow(paths[0], "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9", int64(217848), []uint8(nil)), + }, + }, + { + name: "tree_entries indexable table", + node: newTreeEntriesTable(), + colNames: []string{"tree_entry_name", "blob_hash"}, + expectedKVs: []expectedKV{ + { + key: []interface{}{"example.go", "880cd14280f4b9b6ed3986d6671f907d7cc2a198"}, + idxValue: &indexValue{paths[0], "a39771a7651f97faf5c72e08224d857fc35133db"}, + }, + { + key: []interface{}{"long.json", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"}, + idxValue: &indexValue{paths[0], "5a877e6a906a2743ad6e45d99c1793642aaf8eda"}, + }, + { + key: []interface{}{"short.json", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"}, + idxValue: &indexValue{paths[0], "5a877e6a906a2743ad6e45d99c1793642aaf8eda"}, + }, + { + key: []interface{}{"crappy.php", "9a48f23120e880dfbe41f7c9b7b708e9ee62a492"}, + idxValue: &indexValue{paths[0], "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"}, + }, + { + key: []interface{}{"foo.go", "9dea2395f5403188298c1dabe8bdafe562c491e3"}, + idxValue: &indexValue{paths[0], "cf4aa3b38974fb7d81f367c0830f7d78d65ab86b"}, + }, + { + key: []interface{}{".gitignore", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "a8d315b2b1c615d43042c3a62402b8a54288cf5c"}, + }, + { + key: []interface{}{"CHANGELOG", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "a8d315b2b1c615d43042c3a62402b8a54288cf5c"}, + }, + { + key: []interface{}{"LICENSE", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "a8d315b2b1c615d43042c3a62402b8a54288cf5c"}, + }, + { + key: []interface{}{"binary.jpg", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "a8d315b2b1c615d43042c3a62402b8a54288cf5c"}, + }, + { + key: []interface{}{"go/example.go", "880cd14280f4b9b6ed3986d6671f907d7cc2a198"}, + idxValue: &indexValue{paths[0], "a8d315b2b1c615d43042c3a62402b8a54288cf5c"}, + }, + { + key: []interface{}{"json/long.json", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"}, + idxValue: &indexValue{paths[0], "a8d315b2b1c615d43042c3a62402b8a54288cf5c"}, + }, + { + key: []interface{}{"json/short.json", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"}, + idxValue: &indexValue{paths[0], "a8d315b2b1c615d43042c3a62402b8a54288cf5c"}, + }, + { + key: []interface{}{"php/crappy.php", "9a48f23120e880dfbe41f7c9b7b708e9ee62a492"}, + idxValue: &indexValue{paths[0], "a8d315b2b1c615d43042c3a62402b8a54288cf5c"}, + }, + { + key: []interface{}{"vendor/foo.go", "9dea2395f5403188298c1dabe8bdafe562c491e3"}, + idxValue: &indexValue{paths[0], "a8d315b2b1c615d43042c3a62402b8a54288cf5c"}, + }, + { + key: []interface{}{".gitignore", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "fb72698cab7617ac416264415f13224dfd7a165e"}, + }, + { + key: []interface{}{"CHANGELOG", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "fb72698cab7617ac416264415f13224dfd7a165e"}, + }, + { + key: []interface{}{"LICENSE", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "fb72698cab7617ac416264415f13224dfd7a165e"}, + }, + { + key: []interface{}{"binary.jpg", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "fb72698cab7617ac416264415f13224dfd7a165e"}, + }, + { + key: []interface{}{"go/example.go", "880cd14280f4b9b6ed3986d6671f907d7cc2a198"}, + idxValue: &indexValue{paths[0], "fb72698cab7617ac416264415f13224dfd7a165e"}, + }, + { + key: []interface{}{"json/long.json", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"}, + idxValue: &indexValue{paths[0], "fb72698cab7617ac416264415f13224dfd7a165e"}, + }, + { + key: []interface{}{"json/short.json", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"}, + idxValue: &indexValue{paths[0], "fb72698cab7617ac416264415f13224dfd7a165e"}, + }, + { + key: []interface{}{"php/crappy.php", "9a48f23120e880dfbe41f7c9b7b708e9ee62a492"}, + idxValue: &indexValue{paths[0], "fb72698cab7617ac416264415f13224dfd7a165e"}, + }, + { + key: []interface{}{".gitignore", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "dbd3641b371024f44d0e469a9c8f5457b0660de1"}, + }, + { + key: []interface{}{"CHANGELOG", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "dbd3641b371024f44d0e469a9c8f5457b0660de1"}, + }, + { + key: []interface{}{"LICENSE", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "dbd3641b371024f44d0e469a9c8f5457b0660de1"}, + }, + { + key: []interface{}{"README", "7e59600739c96546163833214c36459e324bad0a"}, + idxValue: &indexValue{paths[0], "dbd3641b371024f44d0e469a9c8f5457b0660de1"}, + }, + { + key: []interface{}{"binary.jpg", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "dbd3641b371024f44d0e469a9c8f5457b0660de1"}, + }, + { + key: []interface{}{"go/example.go", "880cd14280f4b9b6ed3986d6671f907d7cc2a198"}, + idxValue: &indexValue{paths[0], "dbd3641b371024f44d0e469a9c8f5457b0660de1"}, + }, + { + key: []interface{}{"json/long.json", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"}, + idxValue: &indexValue{paths[0], "dbd3641b371024f44d0e469a9c8f5457b0660de1"}, + }, + { + key: []interface{}{"json/short.json", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"}, + idxValue: &indexValue{paths[0], "dbd3641b371024f44d0e469a9c8f5457b0660de1"}, + }, + { + key: []interface{}{"php/crappy.php", "9a48f23120e880dfbe41f7c9b7b708e9ee62a492"}, + idxValue: &indexValue{paths[0], "dbd3641b371024f44d0e469a9c8f5457b0660de1"}, + }, + { + key: []interface{}{".gitignore", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd"}, + }, + { + key: []interface{}{"CHANGELOG", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd"}, + }, + { + key: []interface{}{"LICENSE", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd"}, + }, + { + key: []interface{}{"binary.jpg", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd"}, + }, + { + key: []interface{}{"json/long.json", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"}, + idxValue: &indexValue{paths[0], "4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd"}, + }, + { + key: []interface{}{"json/short.json", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"}, + idxValue: &indexValue{paths[0], "4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd"}, + }, + { + key: []interface{}{".gitignore", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "eba74343e2f15d62adedfd8c883ee0262b5c8021"}, + }, + { + key: []interface{}{"CHANGELOG", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "eba74343e2f15d62adedfd8c883ee0262b5c8021"}, + }, + { + key: []interface{}{"LICENSE", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "eba74343e2f15d62adedfd8c883ee0262b5c8021"}, + }, + { + key: []interface{}{"binary.jpg", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "eba74343e2f15d62adedfd8c883ee0262b5c8021"}, + }, + { + key: []interface{}{".gitignore", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "c2d30fa8ef288618f65f6eed6e168e0d514886f4"}, + }, + { + key: []interface{}{"CHANGELOG", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "c2d30fa8ef288618f65f6eed6e168e0d514886f4"}, + }, + { + key: []interface{}{"LICENSE", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "c2d30fa8ef288618f65f6eed6e168e0d514886f4"}, + }, + { + key: []interface{}{".gitignore", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "8dcef98b1d52143e1e2dbc458ffe38f925786bf2"}, + }, + { + key: []interface{}{"LICENSE", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "8dcef98b1d52143e1e2dbc458ffe38f925786bf2"}, + }, + { + key: []interface{}{"binary.jpg", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "8dcef98b1d52143e1e2dbc458ffe38f925786bf2"}, + }, + { + key: []interface{}{".gitignore", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "aa9b383c260e1d05fbbf6b30a02914555e20c725"}, + }, + { + key: []interface{}{"LICENSE", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "aa9b383c260e1d05fbbf6b30a02914555e20c725"}, + }, + { + key: []interface{}{".gitmodules", "278871477afb195f908155a65b5c651f1cfd02d3"}, + idxValue: &indexValue{paths[1], "3bf5d30ad4f23cf517676fee232e3bcb8537c1d0"}, + }, + { + key: []interface{}{"README.md", "b4f017e8c030d24aef161569b9ade3e55931ba01"}, + idxValue: &indexValue{paths[1], "3bf5d30ad4f23cf517676fee232e3bcb8537c1d0"}, + }, + { + key: []interface{}{".gitmodules", "278871477afb195f908155a65b5c651f1cfd02d3"}, + idxValue: &indexValue{paths[1], "8ac3015df16d47179e903d0379b52267359c1499"}, + }, + { + key: []interface{}{".gitmodules", "278871477afb195f908155a65b5c651f1cfd02d3"}, + idxValue: &indexValue{paths[1], "c4db5d7fc75aa3bef9004122d0cf2a2679935ef8"}, + }, + { + key: []interface{}{".gitmodules", "97b013ecd2cc7f572960509f659d8068798d59ca"}, + idxValue: &indexValue{paths[1], "efe525d0f1372593df812e3f6faa4e05bb91f498"}, + }, + }, + columns: []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Text, TreeEntriesTableName, "tree_hash", false), + expression.NewGetFieldWithTable(2, sql.Text, TreeEntriesTableName, "blob_hash", false), + }, + filters: []sql.Expression{ + expression.NewEquals( + expression.NewGetFieldWithTable(4, sql.Text, TreeEntriesTableName, "tree_entry_name", false), + expression.NewLiteral("LICENSE", sql.Text), + ), + }, + expectedRows: []sql.Row{ + sql.NewRow(paths[0], "a8d315b2b1c615d43042c3a62402b8a54288cf5c", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", "100644", "LICENSE"), + sql.NewRow(paths[0], "fb72698cab7617ac416264415f13224dfd7a165e", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", "100644", "LICENSE"), + sql.NewRow(paths[0], "dbd3641b371024f44d0e469a9c8f5457b0660de1", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", "100644", "LICENSE"), + sql.NewRow(paths[0], "4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", "100644", "LICENSE"), + sql.NewRow(paths[0], "eba74343e2f15d62adedfd8c883ee0262b5c8021", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", "100644", "LICENSE"), + sql.NewRow(paths[0], "c2d30fa8ef288618f65f6eed6e168e0d514886f4", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", "100644", "LICENSE"), + sql.NewRow(paths[0], "8dcef98b1d52143e1e2dbc458ffe38f925786bf2", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", "100644", "LICENSE"), + sql.NewRow(paths[0], "aa9b383c260e1d05fbbf6b30a02914555e20c725", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", "100644", "LICENSE"), + }, + }, + { + name: "commits indexable table", + node: newCommitsTable(), + colNames: []string{"commit_hash", "committer_name"}, + expectedKVs: []expectedKV{ + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "Máximo Cuadros Ortiz"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "Máximo Cuadros Ortiz"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "Máximo Cuadros Ortiz"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "Máximo Cuadros Ortiz"}, + idxValue: &indexValue{paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + }, + { + key: []interface{}{"1669dce138d9b841a518c64b10914d88f5e488ea", "Máximo Cuadros Ortiz"}, + idxValue: &indexValue{paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea"}, + }, + { + key: []interface{}{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "Máximo Cuadros"}, + idxValue: &indexValue{paths[0], "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"}, + }, + { + key: []interface{}{"b8e471f58bcbca63b07bda20e428190409c2db47", "Daniel Ripolles"}, + idxValue: &indexValue{paths[0], "b8e471f58bcbca63b07bda20e428190409c2db47"}, + }, + { + key: []interface{}{"35e85108805c84807bc66a02d91535e1e24b38b9", "Máximo Cuadros Ortiz"}, + idxValue: &indexValue{paths[0], "35e85108805c84807bc66a02d91535e1e24b38b9"}, + }, + { + key: []interface{}{"b029517f6300c2da0f4b651b8642506cd6aaf45d", "Máximo Cuadros"}, + idxValue: &indexValue{paths[0], "b029517f6300c2da0f4b651b8642506cd6aaf45d"}, + }, + { + key: []interface{}{"47770b26e71b0f69c0ecd494b1066f8d1da4fc03", "Máximo Cuadros"}, + idxValue: &indexValue{paths[1], "47770b26e71b0f69c0ecd494b1066f8d1da4fc03"}, + }, + { + key: []interface{}{"b685400c1f9316f350965a5993d350bc746b0bf4", "Máximo Cuadros"}, + idxValue: &indexValue{paths[1], "b685400c1f9316f350965a5993d350bc746b0bf4"}, + }, + { + key: []interface{}{"c7431b5bc9d45fb64a87d4a895ce3d1073c898d2", "Máximo Cuadros"}, + idxValue: &indexValue{paths[1], "c7431b5bc9d45fb64a87d4a895ce3d1073c898d2"}, + }, + { + key: []interface{}{"f52d9c374365fec7f9962f11ebf517588b9e236e", "Máximo Cuadros"}, + idxValue: &indexValue{paths[1], "f52d9c374365fec7f9962f11ebf517588b9e236e"}, + }, + }, + columns: []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Text, CommitsTableName, "commit_hash", false), + expression.NewGetFieldWithTable(10, sql.JSON, CommitsTableName, "commit_parents", false), + }, + filters: []sql.Expression{ + expression.NewEquals( + expression.NewGetFieldWithTable(5, sql.Text, CommitsTableName, "committer_name", false), + expression.NewLiteral("Máximo Cuadros", sql.Text), + ), + }, + expectedRows: []sql.Row{ + sql.NewRow( + paths[0], + "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", + "Máximo Cuadros", "mcuadros@gmail.com", + time.Date(2015, 3, 31, 13, 47, 14, 0, time.FixedZone("", int((2*time.Hour).Seconds()))), + "Máximo Cuadros", "mcuadros@gmail.com", + time.Date(2015, 3, 31, 13, 47, 14, 0, time.FixedZone("", int((2*time.Hour).Seconds()))), + "Merge pull request #1 from dripolles/feature\n\nCreating changelog", + "c2d30fa8ef288618f65f6eed6e168e0d514886f4", + []interface{}{"b029517f6300c2da0f4b651b8642506cd6aaf45d", "b8e471f58bcbca63b07bda20e428190409c2db47"}, + ), + sql.NewRow( + paths[0], + "b029517f6300c2da0f4b651b8642506cd6aaf45d", + "Máximo Cuadros", "mcuadros@gmail.com", + time.Date(2015, 3, 31, 13, 42, 21, 0, time.FixedZone("", int((2*time.Hour).Seconds()))), + "Máximo Cuadros", "mcuadros@gmail.com", + time.Date(2015, 3, 31, 13, 42, 21, 0, time.FixedZone("", int((2*time.Hour).Seconds()))), + "Initial commit\n", + "aa9b383c260e1d05fbbf6b30a02914555e20c725", + []interface{}{}, + ), + }, + }, + { + name: "references indexable table", + node: newReferencesTable(), + colNames: []string{"commit_hash"}, + expectedKVs: []expectedKV{ + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + idxValue: &indexValue{paths[0], "HEAD"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + idxValue: &indexValue{paths[0], "refs/heads/master"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/branch"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"b685400c1f9316f350965a5993d350bc746b0bf4"}, + idxValue: &indexValue{paths[1], "HEAD"}, + }, + { + key: []interface{}{"b685400c1f9316f350965a5993d350bc746b0bf4"}, + idxValue: &indexValue{paths[1], "refs/heads/master"}, + }, + { + key: []interface{}{"b685400c1f9316f350965a5993d350bc746b0bf4"}, + idxValue: &indexValue{paths[1], "refs/remotes/origin/master"}, + }, + }, + columns: []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Text, ReferencesTableName, "ref_name", false), + expression.NewGetFieldWithTable(2, sql.Text, ReferencesTableName, "commit_hash", false), + }, + filters: []sql.Expression{ + expression.NewEquals( + expression.NewGetFieldWithTable(1, sql.Text, ReferencesTableName, "ref_name", false), + expression.NewLiteral("refs/heads/master", sql.Text), + ), + }, + expectedRows: []sql.Row{ + sql.NewRow(paths[0], "refs/heads/master", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), + }, + }, + { + name: "remotes indexable table", + node: newRemotesTable(), + colNames: []string{"remote_push_refspec", "remote_fetch_refspec"}, + expectedKVs: []expectedKV{ + { + key: []interface{}{"+refs/heads/*:refs/remotes/origin/*", "+refs/heads/*:refs/remotes/origin/*"}, + idxValue: &indexValue{paths[0], "origin"}, + }, + { + key: []interface{}{"+refs/heads/*:refs/remotes/origin/*", "+refs/heads/*:refs/remotes/origin/*"}, + idxValue: &indexValue{paths[1], "origin"}, + }, + }, + columns: []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Text, RemotesTableName, "remote_name", false), + }, + filters: []sql.Expression{ + expression.NewEquals( + expression.NewGetFieldWithTable(4, sql.Text, RemotesTableName, "remote_push_refspec", false), + expression.NewLiteral("+refs/heads/*:refs/remotes/origin/*", sql.Text), + ), + }, + expectedRows: []sql.Row{ + sql.NewRow( + paths[0], + "origin", + "git@github.com:git-fixtures/basic.git", + "git@github.com:git-fixtures/basic.git", + "+refs/heads/*:refs/remotes/origin/*", + "+refs/heads/*:refs/remotes/origin/*", + ), + }, + }, + { + name: "repositories indexable table", + node: newRepositoriesTable(), + colNames: []string{"repository_id"}, + expectedKVs: expectedRepos, + columns: []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Text, ReferencesTableName, "repository_id", false), + }, + filters: []sql.Expression{ + expression.NewEquals( + expression.NewGetFieldWithTable(0, sql.Text, ReferencesTableName, "repository_id", false), + expression.NewLiteral(paths[0], sql.Text), + ), + }, + expectedRows: []sql.Row{ + sql.NewRow(paths[0]), + }, + }, + { + name: "ref_commits indexable table", + node: newRefCommitsTable(), + colNames: []string{"commit_hash"}, + expectedKVs: []expectedKV{ + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + idxValue: &indexValue{paths[0], "HEAD"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294"}, + idxValue: &indexValue{paths[0], "HEAD"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + idxValue: &indexValue{paths[0], "HEAD"}, + }, + { + key: []interface{}{"1669dce138d9b841a518c64b10914d88f5e488ea"}, + idxValue: &indexValue{paths[0], "HEAD"}, + }, + { + key: []interface{}{"35e85108805c84807bc66a02d91535e1e24b38b9"}, + idxValue: &indexValue{paths[0], "HEAD"}, + }, + { + key: []interface{}{"b029517f6300c2da0f4b651b8642506cd6aaf45d"}, + idxValue: &indexValue{paths[0], "HEAD"}, + }, + { + key: []interface{}{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"}, + idxValue: &indexValue{paths[0], "HEAD"}, + }, + { + key: []interface{}{"b8e471f58bcbca63b07bda20e428190409c2db47"}, + idxValue: &indexValue{paths[0], "HEAD"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + idxValue: &indexValue{paths[0], "refs/heads/master"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294"}, + idxValue: &indexValue{paths[0], "refs/heads/master"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + idxValue: &indexValue{paths[0], "refs/heads/master"}, + }, + { + key: []interface{}{"1669dce138d9b841a518c64b10914d88f5e488ea"}, + idxValue: &indexValue{paths[0], "refs/heads/master"}, + }, + { + key: []interface{}{"35e85108805c84807bc66a02d91535e1e24b38b9"}, + idxValue: &indexValue{paths[0], "refs/heads/master"}, + }, + { + key: []interface{}{"b029517f6300c2da0f4b651b8642506cd6aaf45d"}, + idxValue: &indexValue{paths[0], "refs/heads/master"}, + }, + { + key: []interface{}{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"}, + idxValue: &indexValue{paths[0], "refs/heads/master"}, + }, + { + key: []interface{}{"b8e471f58bcbca63b07bda20e428190409c2db47"}, + idxValue: &indexValue{paths[0], "refs/heads/master"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/branch"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/branch"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/branch"}, + }, + { + key: []interface{}{"1669dce138d9b841a518c64b10914d88f5e488ea"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/branch"}, + }, + { + key: []interface{}{"35e85108805c84807bc66a02d91535e1e24b38b9"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/branch"}, + }, + { + key: []interface{}{"b029517f6300c2da0f4b651b8642506cd6aaf45d"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/branch"}, + }, + { + key: []interface{}{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/branch"}, + }, + { + key: []interface{}{"b8e471f58bcbca63b07bda20e428190409c2db47"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/branch"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"1669dce138d9b841a518c64b10914d88f5e488ea"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"35e85108805c84807bc66a02d91535e1e24b38b9"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"b029517f6300c2da0f4b651b8642506cd6aaf45d"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"b8e471f58bcbca63b07bda20e428190409c2db47"}, + idxValue: &indexValue{paths[0], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"b685400c1f9316f350965a5993d350bc746b0bf4"}, + idxValue: &indexValue{paths[1], "HEAD"}, + }, + { + key: []interface{}{"f52d9c374365fec7f9962f11ebf517588b9e236e"}, + idxValue: &indexValue{paths[1], "HEAD"}, + }, + { + key: []interface{}{"47770b26e71b0f69c0ecd494b1066f8d1da4fc03"}, + idxValue: &indexValue{paths[1], "HEAD"}, + }, + { + key: []interface{}{"c7431b5bc9d45fb64a87d4a895ce3d1073c898d2"}, + idxValue: &indexValue{paths[1], "HEAD"}, + }, + { + key: []interface{}{"b685400c1f9316f350965a5993d350bc746b0bf4"}, + idxValue: &indexValue{paths[1], "refs/heads/master"}, + }, + { + key: []interface{}{"f52d9c374365fec7f9962f11ebf517588b9e236e"}, + idxValue: &indexValue{paths[1], "refs/heads/master"}, + }, + { + key: []interface{}{"47770b26e71b0f69c0ecd494b1066f8d1da4fc03"}, + idxValue: &indexValue{paths[1], "refs/heads/master"}, + }, + { + key: []interface{}{"c7431b5bc9d45fb64a87d4a895ce3d1073c898d2"}, + idxValue: &indexValue{paths[1], "refs/heads/master"}, + }, + { + key: []interface{}{"b685400c1f9316f350965a5993d350bc746b0bf4"}, + idxValue: &indexValue{paths[1], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"f52d9c374365fec7f9962f11ebf517588b9e236e"}, + idxValue: &indexValue{paths[1], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"47770b26e71b0f69c0ecd494b1066f8d1da4fc03"}, + idxValue: &indexValue{paths[1], "refs/remotes/origin/master"}, + }, + { + key: []interface{}{"c7431b5bc9d45fb64a87d4a895ce3d1073c898d2"}, + idxValue: &indexValue{paths[1], "refs/remotes/origin/master"}, + }, + }, + columns: []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Text, RefCommitsTableName, "commit_hash", false), + expression.NewGetFieldWithTable(2, sql.Text, RefCommitsTableName, "ref_name", false), + expression.NewGetFieldWithTable(3, sql.Int64, RefCommitsTableName, "index", false), + }, + filters: []sql.Expression{ + expression.NewEquals( + expression.NewGetFieldWithTable(2, sql.Text, RefCommitsTableName, "ref_name", false), + expression.NewLiteral("refs/heads/master", sql.Text), + ), + }, + expectedRows: []sql.Row{ + sql.NewRow(paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "refs/heads/master", int64(0)), + sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "refs/heads/master", int64(1)), + sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "refs/heads/master", int64(2)), + sql.NewRow(paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea", "refs/heads/master", int64(3)), + sql.NewRow(paths[0], "35e85108805c84807bc66a02d91535e1e24b38b9", "refs/heads/master", int64(4)), + sql.NewRow(paths[0], "b029517f6300c2da0f4b651b8642506cd6aaf45d", "refs/heads/master", int64(5)), + sql.NewRow(paths[0], "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "refs/heads/master", int64(4)), + sql.NewRow(paths[0], "b8e471f58bcbca63b07bda20e428190409c2db47", "refs/heads/master", int64(5)), + }, + }, + { + name: "commit_trees indexable table", + node: newCommitTreesTable(), + colNames: []string{"commit_hash", "tree_hash"}, + expectedKVs: []expectedKV{ + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "dbd3641b371024f44d0e469a9c8f5457b0660de1"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "a39771a7651f97faf5c72e08224d857fc35133db"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a8d315b2b1c615d43042c3a62402b8a54288cf5c"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a39771a7651f97faf5c72e08224d857fc35133db"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "cf4aa3b38974fb7d81f367c0830f7d78d65ab86b"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "fb72698cab7617ac416264415f13224dfd7a165e"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "a39771a7651f97faf5c72e08224d857fc35133db"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd"}, + idxValue: &indexValue{paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"}, + idxValue: &indexValue{paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + }, + { + key: []interface{}{"1669dce138d9b841a518c64b10914d88f5e488ea", "eba74343e2f15d62adedfd8c883ee0262b5c8021"}, + idxValue: &indexValue{paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea"}, + }, + { + key: []interface{}{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"}, + idxValue: &indexValue{paths[0], "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"}, + }, + { + key: []interface{}{"b8e471f58bcbca63b07bda20e428190409c2db47", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"}, + idxValue: &indexValue{paths[0], "b8e471f58bcbca63b07bda20e428190409c2db47"}, + }, + { + key: []interface{}{"35e85108805c84807bc66a02d91535e1e24b38b9", "8dcef98b1d52143e1e2dbc458ffe38f925786bf2"}, + idxValue: &indexValue{paths[0], "35e85108805c84807bc66a02d91535e1e24b38b9"}, + }, + { + key: []interface{}{"b029517f6300c2da0f4b651b8642506cd6aaf45d", "aa9b383c260e1d05fbbf6b30a02914555e20c725"}, + idxValue: &indexValue{paths[0], "b029517f6300c2da0f4b651b8642506cd6aaf45d"}, + }, + { + key: []interface{}{"47770b26e71b0f69c0ecd494b1066f8d1da4fc03", "8ac3015df16d47179e903d0379b52267359c1499"}, + idxValue: &indexValue{paths[1], "47770b26e71b0f69c0ecd494b1066f8d1da4fc03"}, + }, + { + key: []interface{}{"b685400c1f9316f350965a5993d350bc746b0bf4", "3bf5d30ad4f23cf517676fee232e3bcb8537c1d0"}, + idxValue: &indexValue{paths[1], "b685400c1f9316f350965a5993d350bc746b0bf4"}, + }, + { + key: []interface{}{"c7431b5bc9d45fb64a87d4a895ce3d1073c898d2", "efe525d0f1372593df812e3f6faa4e05bb91f498"}, + idxValue: &indexValue{paths[1], "c7431b5bc9d45fb64a87d4a895ce3d1073c898d2"}, + }, + { + key: []interface{}{"f52d9c374365fec7f9962f11ebf517588b9e236e", "c4db5d7fc75aa3bef9004122d0cf2a2679935ef8"}, + idxValue: &indexValue{paths[1], "f52d9c374365fec7f9962f11ebf517588b9e236e"}, + }, + }, + columns: []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Text, CommitTreesTableName, "commit_hash", false), + expression.NewGetFieldWithTable(2, sql.Text, CommitTreesTableName, "tree_hash", false), + }, + filters: []sql.Expression{ + expression.NewEquals( + expression.NewGetFieldWithTable(1, sql.Text, CommitTreesTableName, "commit_hash", false), + expression.NewLiteral("918c48b83bd081e863dbe1b80f8998f058cd8294", sql.Text), + ), + }, + expectedRows: []sql.Row{ + sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "fb72698cab7617ac416264415f13224dfd7a165e"), + sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "a39771a7651f97faf5c72e08224d857fc35133db"), + sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"), + sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"), + }, + }, + { + name: "commit_blobs indexable table", + node: newCommitBlobsTable(), + colNames: []string{"commit_hash", "blob_hash"}, + expectedKVs: []expectedKV{ + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "7e59600739c96546163833214c36459e324bad0a"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "880cd14280f4b9b6ed3986d6671f907d7cc2a198"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "9a48f23120e880dfbe41f7c9b7b708e9ee62a492"}, + idxValue: &indexValue{paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "880cd14280f4b9b6ed3986d6671f907d7cc2a198"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "9a48f23120e880dfbe41f7c9b7b708e9ee62a492"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "9dea2395f5403188298c1dabe8bdafe562c491e3"}, + idxValue: &indexValue{paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "880cd14280f4b9b6ed3986d6671f907d7cc2a198"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"918c48b83bd081e863dbe1b80f8998f058cd8294", "9a48f23120e880dfbe41f7c9b7b708e9ee62a492"}, + idxValue: &indexValue{paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"}, + idxValue: &indexValue{paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + }, + { + key: []interface{}{"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"}, + idxValue: &indexValue{paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a"}, + }, + { + key: []interface{}{"1669dce138d9b841a518c64b10914d88f5e488ea", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea"}, + }, + { + key: []interface{}{"1669dce138d9b841a518c64b10914d88f5e488ea", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea"}, + }, + { + key: []interface{}{"1669dce138d9b841a518c64b10914d88f5e488ea", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea"}, + }, + { + key: []interface{}{"1669dce138d9b841a518c64b10914d88f5e488ea", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea"}, + }, + { + key: []interface{}{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"}, + }, + { + key: []interface{}{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"}, + }, + { + key: []interface{}{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"}, + }, + { + key: []interface{}{"b8e471f58bcbca63b07bda20e428190409c2db47", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "b8e471f58bcbca63b07bda20e428190409c2db47"}, + }, + { + key: []interface{}{"b8e471f58bcbca63b07bda20e428190409c2db47", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"}, + idxValue: &indexValue{paths[0], "b8e471f58bcbca63b07bda20e428190409c2db47"}, + }, + { + key: []interface{}{"b8e471f58bcbca63b07bda20e428190409c2db47", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "b8e471f58bcbca63b07bda20e428190409c2db47"}, + }, + { + key: []interface{}{"35e85108805c84807bc66a02d91535e1e24b38b9", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "35e85108805c84807bc66a02d91535e1e24b38b9"}, + }, + { + key: []interface{}{"35e85108805c84807bc66a02d91535e1e24b38b9", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "35e85108805c84807bc66a02d91535e1e24b38b9"}, + }, + { + key: []interface{}{"35e85108805c84807bc66a02d91535e1e24b38b9", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"}, + idxValue: &indexValue{paths[0], "35e85108805c84807bc66a02d91535e1e24b38b9"}, + }, + { + key: []interface{}{"b029517f6300c2da0f4b651b8642506cd6aaf45d", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"}, + idxValue: &indexValue{paths[0], "b029517f6300c2da0f4b651b8642506cd6aaf45d"}, + }, + { + key: []interface{}{"b029517f6300c2da0f4b651b8642506cd6aaf45d", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"}, + idxValue: &indexValue{paths[0], "b029517f6300c2da0f4b651b8642506cd6aaf45d"}, + }, + { + key: []interface{}{"47770b26e71b0f69c0ecd494b1066f8d1da4fc03", "278871477afb195f908155a65b5c651f1cfd02d3"}, + idxValue: &indexValue{paths[1], "47770b26e71b0f69c0ecd494b1066f8d1da4fc03"}, + }, + { + key: []interface{}{"b685400c1f9316f350965a5993d350bc746b0bf4", "278871477afb195f908155a65b5c651f1cfd02d3"}, + idxValue: &indexValue{paths[1], "b685400c1f9316f350965a5993d350bc746b0bf4"}, + }, + { + key: []interface{}{"b685400c1f9316f350965a5993d350bc746b0bf4", "b4f017e8c030d24aef161569b9ade3e55931ba01"}, + idxValue: &indexValue{paths[1], "b685400c1f9316f350965a5993d350bc746b0bf4"}, + }, + { + key: []interface{}{"c7431b5bc9d45fb64a87d4a895ce3d1073c898d2", "97b013ecd2cc7f572960509f659d8068798d59ca"}, + idxValue: &indexValue{paths[1], "c7431b5bc9d45fb64a87d4a895ce3d1073c898d2"}, + }, + { + key: []interface{}{"f52d9c374365fec7f9962f11ebf517588b9e236e", "278871477afb195f908155a65b5c651f1cfd02d3"}, + idxValue: &indexValue{paths[1], "f52d9c374365fec7f9962f11ebf517588b9e236e"}, + }, + }, + columns: []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Text, CommitBlobsTableName, "commit_hash", false), + expression.NewGetFieldWithTable(2, sql.Text, CommitBlobsTableName, "blob_hash", false), + }, + filters: []sql.Expression{ + expression.NewEquals( + expression.NewGetFieldWithTable(1, sql.Text, CommitBlobsTableName, "commit_hash", false), + expression.NewLiteral("1669dce138d9b841a518c64b10914d88f5e488ea", sql.Text), + ), + }, + expectedRows: []sql.Row{ + sql.NewRow(paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"), + sql.NewRow(paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"), + sql.NewRow(paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"), + sql.NewRow(paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"), + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require := require.New(t) + + idxValues := testIndexKeyValue(t, ctx, test) + require.True(len(idxValues) > 0) + + // iter only on values from one path to check that + // just that repository is used to produce rows. + filteredValues := []*indexValue{} + for _, value := range idxValues { + if value.ID == paths[0] { + filteredValues = append(filteredValues, value) + } + } + + idxValIter, err := newTestIndexValueIter(filteredValues) + require.NoError(err) + require.NotNil(idxValIter) + + testWithProjectFiltersAndIndex(t, ctx, test, idxValIter) + }) + } +} + +func testWithProjectFiltersAndIndex(t *testing.T, ctx *sql.Context, test *indexTest, idxValIter sql.IndexValueIter) { + require := require.New(t) + rowIter, err := test.node.WithProjectFiltersAndIndex( + ctx, + test.columns, + test.filters, + idxValIter, + ) + require.NoError(err) + + for _, expected := range test.expectedRows { + row, err := rowIter.Next() + require.NoError(err) + require.Exactly(expected, row) + } + + _, err = rowIter.Next() + require.EqualError(err, io.EOF.Error()) +} + +func testIndexKeyValue(t *testing.T, ctx *sql.Context, test *indexTest) []*indexValue { + require := require.New(t) + kvIter, err := test.node.IndexKeyValueIter(ctx, test.colNames) + require.NoError(err) + + idxValues := []*indexValue{} + for _, expected := range test.expectedKVs { + k, v, err := kvIter.Next() + require.NoError(err) + + require.Len(k, len(test.colNames)) + + idxValue, err := unmarshalIndexValue(v) + require.NoError(err) + + idxValues = append(idxValues, idxValue) + + require.Exactly(expected.key, k) + require.Equal(expected.idxValue, idxValue) + } + + _, _, err = kvIter.Next() + require.EqualError(err, io.EOF.Error()) + + return idxValues +} + +type testIndexValueIter struct { + values [][]byte + pos int +} + +var _ sql.IndexValueIter = (*testIndexValueIter)(nil) + +func newTestIndexValueIter(idxValues []*indexValue) (*testIndexValueIter, error) { + values := [][]byte{} + for _, v := range idxValues { + raw, err := marshalIndexValue(v) + if err != nil { + return nil, err + } + + values = append(values, raw) + } + + return &testIndexValueIter{values: values}, nil +} + +func (i *testIndexValueIter) Next() ([]byte, error) { + if i.pos >= len(i.values) { + return nil, io.EOF + } + + defer func() { i.pos++ }() + + return i.values[i.pos], nil +} + +func (i *testIndexValueIter) Close() error { return nil } diff --git a/integration_test.go b/integration_test.go index dc77d2e8a..dd48d4471 100644 --- a/integration_test.go +++ b/integration_test.go @@ -175,8 +175,8 @@ func TestUastQueries(t *testing.T) { session := gitbase.NewSession(pool) ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) _, iter, err := engine.Query(ctx, ` - SELECT uast_xpath(uast(blob_content, language(tree_entry_name, blob_content)), '//*[@roleIdentifier]') as uast, - tree_entry_name + SELECT uast_xpath(uast(blob_content, language(tree_entry_name, blob_content)), '//*[@roleIdentifier]') as uast, + tree_entry_name FROM tree_entries te INNER JOIN blobs b ON b.blob_hash = te.blob_hash @@ -334,8 +334,8 @@ func BenchmarkQueries(b *testing.B) { }{ { "simple query", - `SELECT * FROM repositories r - INNER JOIN refs rr + `SELECT * FROM repositories r + INNER JOIN refs rr ON r.repository_id = rr.repository_id`, }, { @@ -373,14 +373,14 @@ func BenchmarkQueries(b *testing.B) { }, { "join tree entries and blobs", - `SELECT * FROM tree_entries te - INNER JOIN blobs b + `SELECT * FROM tree_entries te + INNER JOIN blobs b ON te.blob_hash = b.blob_hash`, }, { "join tree entries and blobs with filters", - `SELECT * FROM tree_entries te - INNER JOIN blobs b + `SELECT * FROM tree_entries te + INNER JOIN blobs b ON te.blob_hash = b.blob_hash WHERE te.tree_entry_name = 'LICENSE'`, }, diff --git a/ref_commits.go b/ref_commits.go index 4a0f44d38..6593e4d91 100644 --- a/ref_commits.go +++ b/ref_commits.go @@ -24,8 +24,11 @@ var RefCommitsSchema = sql.Schema{ var _ sql.PushdownProjectionAndFiltersTable = (*refCommitsTable)(nil) -func newRefCommitsTable() sql.Table { - return new(refCommitsTable) +func newRefCommitsTable() Indexable { + return &indexableTable{ + PushdownTable: new(refCommitsTable), + buildIterWithSelectors: refCommitsIterBuilder, + } } func (refCommitsTable) isGitbaseTable() {} @@ -65,35 +68,18 @@ func (refCommitsTable) HandledFilters(filters []sql.Expression) []sql.Expression return handledFilters(RefCommitsTableName, RefCommitsSchema, filters) } -func (refCommitsTable) WithProjectAndFilters( +func (refCommitsTable) handledColumns() []string { return []string{"ref_name", "repository_id"} } + +func (t *refCommitsTable) WithProjectAndFilters( ctx *sql.Context, _, filters []sql.Expression, ) (sql.RowIter, error) { span, ctx := ctx.Span("gitbase.RefCommitsTable") iter, err := rowIterWithSelectors( - ctx, RefCommitsSchema, RefCommitsTableName, filters, - []string{"ref_name", "repository_id"}, - func(selectors selectors) (RowRepoIter, error) { - repos, err := selectors.textValues("repository_id") - if err != nil { - return nil, err - } - - names, err := selectors.textValues("ref_name") - if err != nil { - return nil, err - } - - for i := range names { - names[i] = strings.ToLower(names[i]) - } - - return &refCommitsIter{ - ctx: ctx, - refNames: names, - repos: repos, - }, nil - }, + ctx, RefCommitsSchema, RefCommitsTableName, + filters, nil, + t.handledColumns(), + refCommitsIterBuilder, ) if err != nil { @@ -104,6 +90,28 @@ func (refCommitsTable) WithProjectAndFilters( return sql.NewSpanIter(span, iter), nil } +func refCommitsIterBuilder(ctx *sql.Context, selectors selectors, columns []sql.Expression) (RowRepoIter, error) { + repos, err := selectors.textValues("repository_id") + if err != nil { + return nil, err + } + + names, err := selectors.textValues("ref_name") + if err != nil { + return nil, err + } + + for i := range names { + names[i] = strings.ToLower(names[i]) + } + + return &refCommitsIter{ + ctx: ctx, + refNames: names, + repos: repos, + }, nil +} + type refCommitsIter struct { ctx *sql.Context repo *Repository @@ -111,6 +119,7 @@ type refCommitsIter struct { head *plumbing.Reference commits *indexedCommitIter ref *plumbing.Reference + lastRef string // selectors for faster filtering repos []string @@ -143,6 +152,10 @@ func (i *refCommitsIter) NewIterator(repo *Repository) (RowRepoIter, error) { }, nil } +func (i *refCommitsIter) Repository() string { return i.repo.ID } + +func (i *refCommitsIter) LastObject() string { return i.lastRef } + func (i *refCommitsIter) shouldVisitRef(ref *plumbing.Reference) bool { if len(i.refNames) > 0 && !stringContains(i.refNames, strings.ToLower(ref.Name().String())) { return false @@ -213,6 +226,7 @@ func (i *refCommitsIter) Next() (sql.Row, error) { return nil, err } + i.lastRef = i.ref.Name().String() return sql.NewRow( i.repo.ID, commit.Hash.String(), diff --git a/references.go b/references.go index 705b15ac8..518a3b955 100644 --- a/references.go +++ b/references.go @@ -21,8 +21,11 @@ var RefsSchema = sql.Schema{ var _ sql.PushdownProjectionAndFiltersTable = (*referencesTable)(nil) -func newReferencesTable() sql.Table { - return new(referencesTable) +func newReferencesTable() Indexable { + return &indexableTable{ + PushdownTable: new(referencesTable), + buildIterWithSelectors: referencesIterBuilder, + } } var _ Table = (*referencesTable)(nil) @@ -74,49 +77,55 @@ func (referencesTable) HandledFilters(filters []sql.Expression) []sql.Expression return handledFilters(ReferencesTableName, RefsSchema, filters) } +func (referencesTable) handledColumns() []string { return []string{"commit_hash", "ref_name"} } + func (r *referencesTable) WithProjectAndFilters( ctx *sql.Context, _, filters []sql.Expression, ) (sql.RowIter, error) { span, ctx := ctx.Span("gitbase.ReferencesTable") iter, err := rowIterWithSelectors( - ctx, RefsSchema, ReferencesTableName, filters, - []string{"commit_hash", "ref_name"}, - func(selectors selectors) (RowRepoIter, error) { - if len(selectors["commit_hash"]) == 0 && len(selectors["ref_name"]) == 0 { - return new(referenceIter), nil - } + ctx, RefsSchema, ReferencesTableName, + filters, nil, + r.handledColumns(), + referencesIterBuilder, + ) - hashes, err := selectors.textValues("commit_hash") - if err != nil { - return nil, err - } + if err != nil { + span.Finish() + return nil, err + } - names, err := selectors.textValues("ref_name") - if err != nil { - return nil, err - } + return sql.NewSpanIter(span, iter), nil +} - for i := range names { - names[i] = strings.ToLower(names[i]) - } +func referencesIterBuilder(_ *sql.Context, selectors selectors, _ []sql.Expression) (RowRepoIter, error) { + if len(selectors["commit_hash"]) == 0 && len(selectors["ref_name"]) == 0 { + return new(referenceIter), nil + } - return &filteredReferencesIter{hashes: stringsToHashes(hashes), names: names}, nil - }, - ) + hashes, err := selectors.textValues("commit_hash") + if err != nil { + return nil, err + } + names, err := selectors.textValues("ref_name") if err != nil { - span.Finish() return nil, err } - return sql.NewSpanIter(span, iter), nil + for i := range names { + names[i] = strings.ToLower(names[i]) + } + + return &filteredReferencesIter{hashes: stringsToHashes(hashes), names: names}, nil } type referenceIter struct { head *plumbing.Reference repositoryID string iter storer.ReferenceIter + lastRef string } func (i *referenceIter) NewIterator(repo *Repository) (RowRepoIter, error) { @@ -141,11 +150,16 @@ func (i *referenceIter) NewIterator(repo *Repository) (RowRepoIter, error) { }, nil } +func (i *referenceIter) Repository() string { return i.repositoryID } + +func (i *referenceIter) LastObject() string { return i.lastRef } + func (i *referenceIter) Next() (sql.Row, error) { for { if i.head != nil { o := i.head i.head = nil + i.lastRef = "HEAD" return sql.NewRow( i.repositoryID, "HEAD", @@ -166,6 +180,7 @@ func (i *referenceIter) Next() (sql.Row, error) { continue } + i.lastRef = o.Name().String() return referenceToRow(i.repositoryID, o), nil } } @@ -179,11 +194,12 @@ func (i *referenceIter) Close() error { } type filteredReferencesIter struct { - head *plumbing.Reference - hashes []plumbing.Hash - names []string - repoID string - iter storer.ReferenceIter + head *plumbing.Reference + hashes []plumbing.Hash + names []string + repoID string + iter storer.ReferenceIter + lastRef string } func (i *filteredReferencesIter) NewIterator(repo *Repository) (RowRepoIter, error) { @@ -210,6 +226,10 @@ func (i *filteredReferencesIter) NewIterator(repo *Repository) (RowRepoIter, err }, nil } +func (i *filteredReferencesIter) Repository() string { return i.repoID } + +func (i *filteredReferencesIter) LastObject() string { return i.lastRef } + func (i *filteredReferencesIter) Next() (sql.Row, error) { for { if i.head != nil { @@ -252,6 +272,7 @@ func (i *filteredReferencesIter) Next() (sql.Row, error) { continue } + i.lastRef = o.Name().String() return referenceToRow(i.repoID, o), nil } } diff --git a/remotes.go b/remotes.go index 08cb8d63b..78cec58f4 100644 --- a/remotes.go +++ b/remotes.go @@ -21,8 +21,11 @@ var RemotesSchema = sql.Schema{ var _ sql.PushdownProjectionAndFiltersTable = (*remotesTable)(nil) -func newRemotesTable() sql.Table { - return new(remotesTable) +func newRemotesTable() Indexable { + return &indexableTable{ + PushdownTable: new(remotesTable), + buildIterWithSelectors: remotesIterBuilder, + } } var _ Table = (*remotesTable)(nil) @@ -74,17 +77,18 @@ func (remotesTable) HandledFilters(filters []sql.Expression) []sql.Expression { return handledFilters(RemotesTableName, RemotesSchema, filters) } +func (remotesTable) handledColumns() []string { return []string{} } + func (r *remotesTable) WithProjectAndFilters( ctx *sql.Context, _, filters []sql.Expression, ) (sql.RowIter, error) { span, ctx := ctx.Span("gitbase.RemotesTable") iter, err := rowIterWithSelectors( - ctx, RemotesSchema, RemotesTableName, filters, nil, - func(selectors) (RowRepoIter, error) { - // it's not worth to manually filter with the selectors - return new(remotesIter), nil - }, + ctx, RemotesSchema, RemotesTableName, + filters, nil, + r.handledColumns(), + remotesIterBuilder, ) if err != nil { @@ -95,15 +99,20 @@ func (r *remotesTable) WithProjectAndFilters( return sql.NewSpanIter(span, iter), nil } +func remotesIterBuilder(_ *sql.Context, _ selectors, _ []sql.Expression) (RowRepoIter, error) { + // it's not worth to manually filter with the selectors + return new(remotesIter), nil +} + type remotesIter struct { repositoryID string remotes []*git.Remote remotePos int urlPos int + lastRemote string } func (i *remotesIter) NewIterator(repo *Repository) (RowRepoIter, error) { - remotes, err := repo.Repo.Remotes() if err != nil { return nil, err @@ -116,6 +125,10 @@ func (i *remotesIter) NewIterator(repo *Repository) (RowRepoIter, error) { urlPos: 0}, nil } +func (i *remotesIter) Repository() string { return i.repositoryID } + +func (i *remotesIter) LastObject() string { return i.lastRemote } + func (i *remotesIter) Next() (sql.Row, error) { if i.remotePos >= len(i.remotes) { return nil, io.EOF @@ -146,6 +159,7 @@ func (i *remotesIter) Next() (sql.Row, error) { i.urlPos++ + i.lastRemote = config.Name return row, nil } diff --git a/remotes_test.go b/remotes_test.go index ac936600e..9b9882977 100644 --- a/remotes_test.go +++ b/remotes_test.go @@ -37,9 +37,6 @@ func TestRemotesTable_RowIter(t *testing.T) { table := getTable(require, RemotesTableName) - _, ok := table.(*remotesTable) - require.True(ok) - session := ctx.Session.(*Session) pool := session.Pool repository, err := pool.GetPos(0) diff --git a/repositories.go b/repositories.go index e0ca5aebc..2082649cf 100644 --- a/repositories.go +++ b/repositories.go @@ -15,8 +15,11 @@ var RepositoriesSchema = sql.Schema{ var _ sql.PushdownProjectionAndFiltersTable = (*repositoriesTable)(nil) -func newRepositoriesTable() sql.Table { - return new(repositoriesTable) +func newRepositoriesTable() Indexable { + return &indexableTable{ + PushdownTable: new(repositoriesTable), + buildIterWithSelectors: repositoriesIterBuilder, + } } var _ Table = (*repositoriesTable)(nil) @@ -68,17 +71,18 @@ func (repositoriesTable) HandledFilters(filters []sql.Expression) []sql.Expressi return handledFilters(RepositoriesTableName, RepositoriesSchema, filters) } +func (repositoriesTable) handledColumns() []string { return []string{} } + func (r *repositoriesTable) WithProjectAndFilters( ctx *sql.Context, _, filters []sql.Expression, ) (sql.RowIter, error) { span, ctx := ctx.Span("gitbase.RepositoriesTable") iter, err := rowIterWithSelectors( - ctx, RepositoriesSchema, RepositoriesTableName, filters, nil, - func(selectors) (RowRepoIter, error) { - // it's not worth to manually filter with the selectors - return new(repositoriesIter), nil - }, + ctx, RepositoriesSchema, RepositoriesTableName, + filters, nil, + r.handledColumns(), + repositoriesIterBuilder, ) if err != nil { @@ -89,6 +93,11 @@ func (r *repositoriesTable) WithProjectAndFilters( return sql.NewSpanIter(span, iter), nil } +func repositoriesIterBuilder(_ *sql.Context, _ selectors, _ []sql.Expression) (RowRepoIter, error) { + // it's not worth to manually filter with the selectors + return new(repositoriesIter), nil +} + type repositoriesIter struct { visited bool id string @@ -101,6 +110,10 @@ func (i *repositoriesIter) NewIterator(repo *Repository) (RowRepoIter, error) { }, nil } +func (i *repositoriesIter) Repository() string { return i.id } + +func (i *repositoriesIter) LastObject() string { return i.id } + func (i *repositoriesIter) Next() (sql.Row, error) { if i.visited { return nil, io.EOF diff --git a/repository_pool.go b/repository_pool.go index 1057ebb9e..409455628 100644 --- a/repository_pool.go +++ b/repository_pool.go @@ -194,7 +194,6 @@ var errInvalidRepoKind = errors.NewKind("invalid repo kind: %d") // GetPos retrieves a repository at a given position. If the position is // out of bounds it returns io.EOF. func (p *RepositoryPool) GetPos(pos int) (*Repository, error) { - if pos >= len(p.repositories) { return nil, io.EOF } @@ -204,7 +203,19 @@ func (p *RepositoryPool) GetPos(pos int) (*Repository, error) { return nil, io.EOF } - r := p.repositories[id] + return p.GetRepo(id) +} + +// ErrPoolRepoNotFound is returned when a repository id is not present in the pool. +var ErrPoolRepoNotFound = errors.NewKind("repository id %s not found in the pool") + +// GetRepo returns a repository with the given id from the pool. +func (p *RepositoryPool) GetRepo(id string) (*Repository, error) { + r, ok := p.repositories[id] + if !ok { + return nil, ErrPoolRepoNotFound.New(id) + } + var repo *Repository var err error switch r.kind { @@ -258,10 +269,14 @@ func (i *RepositoryIter) Close() error { // implementation type RowRepoIter interface { NewIterator(*Repository) (RowRepoIter, error) + Repository() string + LastObject() string Next() (sql.Row, error) Close() error } +type iteratorBuilder func(*sql.Context, selectors, []sql.Expression) (RowRepoIter, error) + // RowRepoIter is used as the base to iterate over all the repositories // in the pool type rowRepoIter struct { diff --git a/repository_pool_test.go b/repository_pool_test.go index 165b356a9..b2d6e3de4 100644 --- a/repository_pool_test.go +++ b/repository_pool_test.go @@ -39,17 +39,18 @@ func TestRepositoryPoolBasic(t *testing.T) { pool := NewRepositoryPool() - // GetPos - repo, err := pool.GetPos(0) require.Nil(repo) require.Equal(io.EOF, err) - // Add and GetPos + repo, err = pool.GetRepo("foo") + require.Nil(repo) + require.EqualError(err, ErrPoolRepoNotFound.New("foo").Error()) pool.Add("0", "/directory/should/not/exist", gitRepo) repo, err = pool.GetPos(0) - require.Error(err) + require.Nil(repo) + require.EqualError(err, git.ErrRepositoryNotExists.Error()) _, err = pool.GetPos(1) require.Equal(io.EOF, err) @@ -62,8 +63,11 @@ func TestRepositoryPoolBasic(t *testing.T) { require.Equal("1", repo.ID) require.NotNil(repo.Repo) - _, err = pool.GetPos(0) - require.Equal(git.ErrRepositoryNotExists, err) + repo, err = pool.GetRepo("1") + require.NoError(err) + require.Equal("1", repo.ID) + require.NotNil(repo.Repo) + _, err = pool.GetPos(2) require.Equal(io.EOF, err) } @@ -133,7 +137,9 @@ func TestRepositoryPoolIterator(t *testing.T) { } type testCommitIter struct { - iter object.CommitIter + iter object.CommitIter + repoID string + lastHash string } func (d *testCommitIter) NewIterator( @@ -144,11 +150,19 @@ func (d *testCommitIter) NewIterator( return nil, err } - return &testCommitIter{iter: iter}, nil + return &testCommitIter{iter: iter, repoID: repo.ID}, nil } +func (d *testCommitIter) Repository() string { return d.repoID } + +func (d *testCommitIter) LastObject() string { return d.lastHash } + func (d *testCommitIter) Next() (sql.Row, error) { - _, err := d.iter.Next() + c, err := d.iter.Next() + if err == nil { + d.lastHash = c.Hash.String() + } + return nil, err } @@ -320,6 +334,10 @@ func (d *testErrorIter) NewIterator( return nil, errIter } +func (d *testErrorIter) Repository() string { return "RepoTestError" } + +func (d *testErrorIter) LastObject() string { return "ObjectTestError" } + func (d *testErrorIter) Next() (sql.Row, error) { if d.next != nil { return d.next() diff --git a/squash_iterator.go b/squash_iterator.go index 0d185fd22..1b962c857 100644 --- a/squash_iterator.go +++ b/squash_iterator.go @@ -2456,6 +2456,19 @@ func (it *chainableRowRepoIter) NewIterator(repo *Repository) (RowRepoIter, erro return NewChainableRowRepoIter(it.ctx, i), nil } +// Repository and LastObject methods are suppossed not to be called +// since thay are used to build squashed tables ant those aren't indexable +// for the moment. +// These methods are implemented just to complain with RowRepoIter interface. + +func (it *chainableRowRepoIter) Repository() string { + panic("Repository method shouldn't be called on a chainable iterator") +} + +func (it *chainableRowRepoIter) LastObject() string { + panic("LatObject method shouldn't be called on a chainable iterator") +} + func (it *chainableRowRepoIter) Next() (sql.Row, error) { if err := it.Advance(); err != nil { return nil, err diff --git a/table.go b/table.go index 4d63aff28..842c8a668 100644 --- a/table.go +++ b/table.go @@ -9,6 +9,10 @@ import ( // Table represents a gitbase table. type Table interface { sql.Table + gitBase +} + +type gitBase interface { isGitbaseTable() } diff --git a/tree_entries.go b/tree_entries.go index 5efc77161..e4a7c1642 100644 --- a/tree_entries.go +++ b/tree_entries.go @@ -23,8 +23,11 @@ var TreeEntriesSchema = sql.Schema{ var _ sql.PushdownProjectionAndFiltersTable = (*treeEntriesTable)(nil) -func newTreeEntriesTable() sql.Table { - return new(treeEntriesTable) +func newTreeEntriesTable() Indexable { + return &indexableTable{ + PushdownTable: new(treeEntriesTable), + buildIterWithSelectors: treeEntriesIterBuilder, + } } var _ Table = (*treeEntriesTable)(nil) @@ -72,6 +75,10 @@ func (treeEntriesTable) HandledFilters(filters []sql.Expression) []sql.Expressio return handledFilters(TreeEntriesTableName, TreeEntriesSchema, filters) } +func (treeEntriesTable) handledColumns() []string { + return []string{"tree_hash"} +} + func (r *treeEntriesTable) WithProjectAndFilters( ctx *sql.Context, _, filters []sql.Expression, @@ -81,20 +88,10 @@ func (r *treeEntriesTable) WithProjectAndFilters( // projected. There would be no need to iterate files in this case, and // it would be much faster. iter, err := rowIterWithSelectors( - ctx, TreeEntriesSchema, TreeEntriesTableName, filters, - []string{"tree_hash"}, - func(selectors selectors) (RowRepoIter, error) { - if len(selectors["tree_hash"]) == 0 { - return new(treeEntryIter), nil - } - - hashes, err := selectors.textValues("tree_hash") - if err != nil { - return nil, err - } - - return &treeEntriesByHashIter{hashes: hashes}, nil - }, + ctx, TreeEntriesSchema, TreeEntriesTableName, + filters, nil, + r.handledColumns(), + treeEntriesIterBuilder, ) if err != nil { @@ -105,15 +102,29 @@ func (r *treeEntriesTable) WithProjectAndFilters( return sql.NewSpanIter(span, iter), nil } +func treeEntriesIterBuilder(_ *sql.Context, selectors selectors, _ []sql.Expression) (RowRepoIter, error) { + if len(selectors["tree_hash"]) == 0 { + return new(treeEntryIter), nil + } + + hashes, err := selectors.textValues("tree_hash") + if err != nil { + return nil, err + } + + return &treeEntriesByHashIter{hashes: hashes}, nil +} + func (r treeEntriesTable) String() string { return printTable(TreeEntriesTableName, TreeEntriesSchema) } type treeEntryIter struct { - i *object.TreeIter - tree *object.Tree - cursor int - repoID string + i *object.TreeIter + tree *object.Tree + cursor int + repoID string + lastHash string } func (i *treeEntryIter) NewIterator(repo *Repository) (RowRepoIter, error) { @@ -125,6 +136,10 @@ func (i *treeEntryIter) NewIterator(repo *Repository) (RowRepoIter, error) { return &treeEntryIter{repoID: repo.ID, i: iter}, nil } +func (i *treeEntryIter) Repository() string { return i.repoID } + +func (i *treeEntryIter) LastObject() string { return i.lastHash } + func (i *treeEntryIter) Next() (sql.Row, error) { for { if i.tree == nil { @@ -144,6 +159,7 @@ func (i *treeEntryIter) Next() (sql.Row, error) { entry := &TreeEntry{i.tree.Hash, i.tree.Entries[i.cursor]} i.cursor++ + i.lastHash = i.tree.Hash.String() return treeEntryToRow(i.repoID, entry), nil } @@ -158,17 +174,22 @@ func (i *treeEntryIter) Close() error { } type treeEntriesByHashIter struct { - hashes []string - pos int - tree *object.Tree - cursor int - repo *Repository + hashes []string + pos int + tree *object.Tree + cursor int + repo *Repository + lastHash string } func (i *treeEntriesByHashIter) NewIterator(repo *Repository) (RowRepoIter, error) { return &treeEntriesByHashIter{hashes: i.hashes, repo: repo}, nil } +func (i *treeEntriesByHashIter) Repository() string { return i.repo.ID } + +func (i *treeEntriesByHashIter) LastObject() string { return i.lastHash } + func (i *treeEntriesByHashIter) Next() (sql.Row, error) { for { if i.pos >= len(i.hashes) && i.tree == nil { @@ -196,6 +217,7 @@ func (i *treeEntriesByHashIter) Next() (sql.Row, error) { } entry := &TreeEntry{i.tree.Hash, i.tree.Entries[i.cursor]} + i.lastHash = i.tree.Hash.String() i.cursor++ return treeEntryToRow(i.repo.ID, entry), nil