Skip to content

Feature/new uast serialization #453

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 5, 2018
Merged

Feature/new uast serialization #453

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d24b2cf
internal/function: added a new way to serialize UASTs
mcarmonaa Aug 31, 2018
d14a4bf
*: add a new flag --old-uast-serialization to serialize UASTs in the …
mcarmonaa Aug 31, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cmd/gitbase/command/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ type Server struct {
DisableGit bool `long:"no-git" description:"disable the load of git standard repositories."`
DisableSiva bool `long:"no-siva" description:"disable the load of siva files."`
Verbose bool `short:"v" description:"Activates the verbose mode"`
OldUast bool `long:"old-uast-serialization" description:"serialize uast in the old format" env:"GITBASE_UAST_SERIALIZATION"`
}

type jaegerLogrus struct {
Expand Down Expand Up @@ -138,12 +139,17 @@ func (c *Server) Execute(args []string) error {
c.engine,
gitbase.NewSessionBuilder(c.pool,
gitbase.WithSkipGitErrors(c.SkipGitErrors),
gitbase.WithOldUASTSerialization(c.OldUast),
),
)
if err != nil {
return err
}

if c.OldUast {
function.UASTExpressionType = sql.Array(sql.Blob)
}

logrus.Infof("server started and listening on %s:%d", c.Host, c.Port)
return s.Start()
}
Expand Down
173 changes: 26 additions & 147 deletions internal/function/uast.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,18 @@ import (
"fmt"
"strings"

"github.com/sirupsen/logrus"
"github.com/src-d/gitbase"
bblfsh "gopkg.in/bblfsh/client-go.v2"
"gopkg.in/bblfsh/client-go.v2/tools"
"gopkg.in/bblfsh/sdk.v1/uast"

errors "gopkg.in/src-d/go-errors.v1"
"gopkg.in/src-d/go-mysql-server.v0/sql"
"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
)

var (
// ErrParseBlob is returned when the blob can't be parsed with bblfsh.
ErrParseBlob = errors.NewKind("unable to parse the given blob using bblfsh: %s")
// UASTExpressionType represents the returned SQL type by
// the functions uast, uast_mode, uast_xpath and uast_children.
UASTExpressionType sql.Type = sql.Blob
)

// UAST returns an array of UAST nodes as blobs.
Expand Down Expand Up @@ -62,7 +60,7 @@ func (f UAST) Resolved() bool {

// Type implements the Expression interface.
func (f UAST) Type() sql.Type {
return sql.Array(sql.Blob)
return UASTExpressionType
}

// Children implements the Expression interface.
Expand Down Expand Up @@ -181,7 +179,7 @@ func (f UASTMode) Resolved() bool {

// Type implements the Expression interface.
func (f UASTMode) Type() sql.Type {
return sql.Array(sql.Blob)
return UASTExpressionType
}

// Children implements the Expression interface.
Expand Down Expand Up @@ -290,7 +288,7 @@ func NewUASTXPath(uast, xpath sql.Expression) sql.Expression {

// Type implements the Expression interface.
func (UASTXPath) Type() sql.Type {
return sql.Array(sql.Blob)
return UASTExpressionType
}

// Eval implements the Expression interface.
Expand All @@ -309,15 +307,15 @@ func (f *UASTXPath) Eval(ctx *sql.Context, row sql.Row) (out interface{}, err er
return nil, err
}

if left == nil {
return nil, nil
}

nodes, err := nodesFromBlobArray(left)
nodes, err := getNodes(ctx, left)
if err != nil {
return nil, err
}

if nodes == nil {
return nil, nil
}

xpath, err := exprToString(ctx, f.Right, row)
if err != nil {
return nil, err
Expand All @@ -327,58 +325,17 @@ func (f *UASTXPath) Eval(ctx *sql.Context, row sql.Row) (out interface{}, err er
return nil, nil
}

var result []interface{}
var filtered []*uast.Node
for _, n := range nodes {
ns, err := tools.Filter(n, xpath)
if err != nil {
return nil, err
}

m, err := marshalNodes(ns)
if err != nil {
return nil, err
}

result = append(result, m...)
}

return result, nil
}

func nodesFromBlobArray(data interface{}) ([]*uast.Node, error) {
data, err := sql.Array(sql.Blob).Convert(data)
if err != nil {
return nil, err
}

arr := data.([]interface{})
var nodes = make([]*uast.Node, len(arr))
for i, n := range arr {
node := uast.NewNode()
if err := node.Unmarshal(n.([]byte)); err != nil {
return nil, err
}

nodes[i] = node
}

return nodes, nil
}

func marshalNodes(nodes []*uast.Node) ([]interface{}, error) {
m := make([]interface{}, 0, len(nodes))
for _, n := range nodes {
if n != nil {
data, err := n.Marshal()
if err != nil {
return nil, err
}

m = append(m, data)
}
filtered = append(filtered, ns...)
}

return m, nil
return marshalNodes(ctx, filtered)
}

func (f UASTXPath) String() string {
Expand All @@ -400,84 +357,6 @@ func (f UASTXPath) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error)
return fn(NewUASTXPath(left, right))
}

func exprToString(
ctx *sql.Context,
e sql.Expression,
r sql.Row,
) (string, error) {
if e == nil {
return "", nil
}

x, err := e.Eval(ctx, r)
if err != nil {
return "", err
}

if x == nil {
return "", nil
}

x, err = sql.Text.Convert(x)
if err != nil {
return "", err
}

return x.(string), nil
}

func getUAST(
ctx *sql.Context,
bytes []byte,
lang, xpath string,
mode bblfsh.Mode,
) (interface{}, error) {
session, ok := ctx.Session.(*gitbase.Session)
if !ok {
return nil, gitbase.ErrInvalidGitbaseSession.New(ctx.Session)
}

client, err := session.BblfshClient()
if err != nil {
return nil, err
}

// If we have a language we must check if it's supported. If we don't, bblfsh
// is the one that will have to identify the language.
if lang != "" {
ok, err = client.IsLanguageSupported(ctx, lang)
if err != nil {
return nil, err
}

if !ok {
return nil, nil
}
}

resp, err := client.ParseWithMode(ctx, mode, lang, bytes)
if err != nil {
logrus.Warn(ErrParseBlob.New(err))
return nil, nil
}

if len(resp.Errors) > 0 {
logrus.Warn(ErrParseBlob.New(strings.Join(resp.Errors, "\n")))
}

var nodes []*uast.Node
if xpath == "" {
nodes = []*uast.Node{resp.UAST}
} else {
nodes, err = tools.Filter(resp.UAST, xpath)
if err != nil {
return nil, err
}
}

return marshalNodes(nodes)
}

// UASTExtract extracts keys from an UAST.
type UASTExtract struct {
expression.BinaryExpression
Expand Down Expand Up @@ -514,15 +393,15 @@ func (u *UASTExtract) Eval(ctx *sql.Context, row sql.Row) (out interface{}, err
return nil, err
}

if left == nil {
return nil, nil
}

nodes, err := nodesFromBlobArray(left)
nodes, err := getNodes(ctx, left)
if err != nil {
return nil, err
}

if nodes == nil {
return nil, nil
}

key, err := exprToString(ctx, u.Right, row)
if err != nil {
return nil, err
Expand Down Expand Up @@ -609,7 +488,7 @@ func (u *UASTChildren) String() string {

// Type implements the sql.Expression interface.
func (u *UASTChildren) Type() sql.Type {
return sql.Array(sql.Blob)
return UASTExpressionType
}

// TransformUp implements the sql.Expression interface.
Expand Down Expand Up @@ -638,17 +517,17 @@ func (u *UASTChildren) Eval(ctx *sql.Context, row sql.Row) (out interface{}, err
return nil, err
}

if child == nil {
return nil, nil
}

nodes, err := nodesFromBlobArray(child)
nodes, err := getNodes(ctx, child)
if err != nil {
return nil, err
}

if nodes == nil {
return nil, nil
}

children := flattenChildren(nodes)
return marshalNodes(children)
return marshalNodes(ctx, children)
}

func flattenChildren(nodes []*uast.Node) []*uast.Node {
Expand Down
Loading