Improve support for SQLite (#61)

* Update SQLize for the new Sqlite parser

* Upgrade the build pipeline

* Add a unit test for parsing sqlite schema

* Fix the nil error in sqlite parse

* FIx the sqlite scripts

* Add a unit test for migrations generation

* Fix some parse errors, that prevented migration from being generated

* Add autoincrement keyword

* Add support for unique constraint

* Add support for check constraints

* Add mention of sqlite constraints

* Fix potential panic

* Add edge cases for default values to ensure unquote stability for sqlite

* Add some migration verification scripts

* Refactor tests, extract file read boilerplate

* Apply more CodeRabbit suggestions, fix some nitpicks

* Just a couple more finishing improvements by CodeRabbit

* Fix quote/unquote logic for default values

Author: AlexanderShelyugov

.github/workflows/go.yml

@@ -1,4 +1,4 @@
-name: Go
+name: Build pipeline
 
 on:
   push:
@@ -11,12 +11,12 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
-        go-version: 1.19
+        go-version-file: go.mod
 
     - name: Build
       run: go build -v ./...

element/column.go

@@ -5,6 +5,7 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
+	"strconv"
 	"strings"
 
 	ptypes "github.com/auxten/postgresql-parser/pkg/sql/types"
@@ -204,6 +205,40 @@ func (c Column) pkDefinition(isPrev bool) (string, bool) {
 			continue
 		}
 
+		if sql.IsSqlite() {
+			// SQLite overrides, that pingcap doesn't support
+			if opt.Tp == ast.ColumnOptionDefaultValue {
+				// Parsed StrValue may be quoted in single quotes, which breaks SQL expression.
+				// We need to unquote it and, if it's a TEXT column. quote it again with double quotes.
+				expression, err := strconv.Unquote(opt.StrValue)
+				if err != nil {
+					expression = opt.StrValue
+				}
+				if len(expression) >= 2 && expression[0] == '\'' && expression[len(expression)-1] == '\'' {
+					// remove single quotes. strconv may not detect it
+					expression = expression[1 : len(expression)-1]
+				}
+				if c.typeDefinition(isPrev) == "TEXT" {
+					expression = strconv.Quote(expression)
+				}
+				strSql += " DEFAULT " + expression
+				continue
+			}
+
+			if opt.Tp == ast.ColumnOptionAutoIncrement {
+				strSql += " AUTOINCREMENT"
+				continue
+			}
+			if opt.Tp == ast.ColumnOptionUniqKey {
+				strSql += " UNIQUE"
+				continue
+			}
+			if opt.Tp == ast.ColumnOptionCheck {
+				strSql += " CHECK (" + opt.StrValue + ")"
+				continue
+			}
+		}
+
 		if opt.Tp == ast.ColumnOptionReference && opt.Refer == nil { // manual add
 			continue
 		}

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/auxten/postgresql-parser v1.0.1
 	github.com/pingcap/parser v0.0.0-20200623164729-3a18f1e5dceb
 	github.com/pkg/errors v0.9.1
-	github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd
+	github.com/rqlite/sql v0.0.0-20241111133259-a4122fabb196
 )
 
 require (

go.sum

@@ -214,6 +214,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd h1:wW6BtayFoKaaDeIvXRE3SZVPOscSKlYD+X3bB749+zk=
 github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd/go.mod h1:ib9zVtNgRKiGuoMyUqqL5aNpk+r+++YlyiVIkclVqPg=
+github.com/rqlite/sql v0.0.0-20241111133259-a4122fabb196 h1:SjRKMwKLTEE3STO6unJlz4VlMjMv5NZgIdI9HikBeAc=
+github.com/rqlite/sql v0.0.0-20241111133259-a4122fabb196/go.mod h1:ib9zVtNgRKiGuoMyUqqL5aNpk+r+++YlyiVIkclVqPg=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=

sql-parser/sqlite.go

@@ -17,7 +17,8 @@ func (p *Parser) ParserSqlite(sql string) error {
 		return err
 	}
 
-	return sqlite.Walk(p, node)
+	_, err = sqlite.Walk(p, node)
+	return err
 }
 
 /*
@@ -31,7 +32,7 @@ AlterTableStatement
 
 sqlite does not support drop column
 */
-func (p *Parser) Visit(node sqlite.Node) (w sqlite.Visitor, err error) {
+func (p *Parser) Visit(node sqlite.Node) (w sqlite.Visitor, n sqlite.Node, err error) {
 	switch n := node.(type) {
 	case *sqlite.CreateTableStatement:
 		tbName := n.Name.String()
@@ -48,7 +49,7 @@ func (p *Parser) Visit(node sqlite.Node) (w sqlite.Visitor, err error) {
 				},
 				CurrentAttr: element.SqlAttr{
 					LiteType: n.Columns[i].Type,
-					Options:  p.parseSqliteConstrains(tbName, n.Columns[i].Constraints),
+					Options:  p.parseSqliteConstrains(tbName, n.Columns[i]),
 				},
 			}
 
@@ -122,52 +123,51 @@ func (p *Parser) Visit(node sqlite.Node) (w sqlite.Visitor, err error) {
 				},
 				CurrentAttr: element.SqlAttr{
 					LiteType: n.ColumnDef.Type,
-					Options:  p.parseSqliteConstrains(tbName, n.ColumnDef.Constraints),
+					Options:  p.parseSqliteConstrains(tbName, n.ColumnDef),
 				},
 			})
 		}
 	}
 
-	return nil, nil
+	return p, nil, nil
 }
 
-func (p *Parser) parseSqliteConstrains(tbName string, conss []sqlite.Constraint) []*ast.ColumnOption {
+func (p *Parser) parseSqliteConstrains(tbName string, columnDefinition *sqlite.ColumnDefinition) []*ast.ColumnOption {
+	// https://www.sqlite.org/syntax/column-constraint.html
+	// Also, Sqlite does not support dropping constraints, so we safely can add them here
+	conss := columnDefinition.Constraints
 	opts := []*ast.ColumnOption{}
 	for _, cons := range conss {
 		switch cons := cons.(type) {
 		case *sqlite.PrimaryKeyConstraint:
 			opts = append(opts, &ast.ColumnOption{Tp: ast.ColumnOptionPrimaryKey})
+			if cons.Autoincrement.IsValid() {
+				opts = append(opts, &ast.ColumnOption{Tp: ast.ColumnOptionAutoIncrement})
+			}
 
 		case *sqlite.NotNullConstraint:
 			opts = append(opts, &ast.ColumnOption{Tp: ast.ColumnOptionNotNull})
 
 		case *sqlite.UniqueConstraint:
-			indexCol := make([]string, len(cons.Columns))
-			for i := range cons.Columns {
-				indexCol[i] = cons.Columns[i].Collation.Name
-			}
-
-			p.Migration.AddIndex(tbName, element.Index{
-				Node: element.Node{
-					Name:   cons.Name.Name,
-					Action: element.MigrateAddAction,
-				},
-				Columns: indexCol,
-				Typ:     ast.IndexKeyTypeUnique,
-			})
+			opts = append(opts, &ast.ColumnOption{Tp: ast.ColumnOptionUniqKey})
 
 		case *sqlite.CheckConstraint:
+			opts = append(opts, &ast.ColumnOption{
+				Tp:       ast.ColumnOptionCheck,
+				StrValue: cons.Expr.String(),
+			})
 
 		case *sqlite.DefaultConstraint:
 			opts = append(opts, &ast.ColumnOption{
 				Tp:       ast.ColumnOptionDefaultValue,
-				StrValue: cons.Default.String()})
+				StrValue: cons.Expr.String(),
+			})
 		}
 	}
 
 	return opts
 }
 
-func (p Parser) VisitEnd(node sqlite.Node) error {
-	return nil
+func (p Parser) VisitEnd(node sqlite.Node) (sqlite.Node, error) {
+	return nil, nil
 }

test/README.md

@@ -0,0 +1,8 @@
+# SQLize tests
+
+This folder aims to align with the common guidelines of the [golang project layout](https://github.com/golang-standards/project-layout/tree/master/test).
+
+## How to run?
+```bash
+go test ./test/...
+```

test/sqlite/common.go

@@ -0,0 +1,27 @@
+package sqlite
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+const (
+	schemaWithOneTable  = "./testdata/schema_one_table.sql"
+	schemaWithTwoTables = "./testdata/schema_two_tables.sql"
+)
+
+func assertContains(t *testing.T, str, substr, message string) {
+	if !strings.Contains(str, substr) {
+		t.Errorf("%s: expected to find '%s' in:\n%s", message, substr, str)
+	}
+}
+
+func readFile(t *testing.T, path string) string {
+	t.Helper()
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("Failed to read file %s: %v", path, err)
+	}
+	return string(data)
+}

test/sqlite/migration_test.go

@@ -0,0 +1,47 @@
+package sqlite
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/sunary/sqlize"
+)
+
+// TestMigrationGeneratorSingleTable tests that Sqlize can generate a migration script for the simplest schema.
+func TestMigrationGeneratorSingleTable(t *testing.T) {
+	s := sqlize.NewSqlize(
+		sqlize.WithSqlite(),
+	)
+
+	schemaSql := readFile(t, schemaWithOneTable)
+	if err := s.FromString(schemaSql); err != nil {
+		t.Fatalf("failed to parse schema: %v", err)
+	}
+
+	upSQL := s.StringUp()
+	downSQL := s.StringDown()
+
+	// Validate generated migration scripts
+	assertContains(t, upSQL, "CREATE TABLE", "Up migration should create the table")
+	assertContains(t, upSQL, "AUTOINCREMENT", "Up migration should include AUTOINCREMENT")
+	assertContains(t, upSQL, "CHECK (\"age\" >= 18)", "Up migration should include CHECK constraint")
+	assertContains(t, upSQL, "UNIQUE", "Up migration should include UNIQUE values")
+	assertContains(t, upSQL, "DEFAULT", "Up migration should include DEFAULT values")
+
+	assertContains(t, downSQL, "DROP TABLE", "Down migration should drop the table")
+
+	upWithVersionSQL := s.StringUpWithVersion(0, false)
+	downWithVersionSQL := s.StringDownWithVersion(0)
+
+	// Validate versioned migration scripts
+	assertContains(t, upWithVersionSQL, "CREATE TABLE IF NOT EXISTS schema_migrations", "Initial migration should create the migrations table")
+	assertContains(t, downWithVersionSQL, "DROP TABLE IF EXISTS schema_migrations", "Down migration from before initial should drop the migrations table")
+
+	version := s.HashValue()
+	upWithVersionNumberSQL := s.StringUpWithVersion(version, false)
+
+	// Validate versioned migration scripts
+	expectedVersionInsert := fmt.Sprintf("INSERT INTO schema_migrations (version, dirty) VALUES (%d, false);", version)
+
+	assertContains(t, upWithVersionNumberSQL, expectedVersionInsert, "Versioned up migration should include version comment")
+}

test/sqlite/parser_test.go

@@ -0,0 +1,31 @@
+package sqlite
+
+import (
+	"testing"
+
+	"github.com/sunary/sqlize"
+)
+
+// TestParserSingleTable tests that Sqlize can parse a sqlite schema with one table.
+func TestParserSingleTable(t *testing.T) {
+	sqlizeCurrent := sqlize.NewSqlize(
+		sqlize.WithSqlite(),
+	)
+
+	schemaSql := readFile(t, schemaWithOneTable)
+	if err := sqlizeCurrent.FromString(schemaSql); err != nil {
+		t.Fatalf("failed to parse schema: %v", err)
+	}
+}
+
+// TestParserMultipleTables tests that Sqlize can parse a sqlite schema with foreign keys.
+func TestParserMultipleTables(t *testing.T) {
+	sqlizeCurrent := sqlize.NewSqlize(
+		sqlize.WithSqlite(),
+	)
+
+	schemaSql := readFile(t, schemaWithTwoTables)
+	if err := sqlizeCurrent.FromString(schemaSql); err != nil {
+		t.Fatalf("failed to parse schema: %v", err)
+	}
+}

test/sqlite/testdata/schema_one_table.sql

@@ -0,0 +1,33 @@
+CREATE TABLE IF NOT EXISTS table_with_all_types (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+
+    name TEXT NOT NULL,
+
+    unique_number INTEGER UNIQUE,
+    number_with_default INTEGER DEFAULT 123,
+
+    price REAL,
+    price_with_default REAL DEFAULT 0.0,
+
+    is_active BOOLEAN DEFAULT TRUE,
+
+    binary_data BLOB,
+
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+
+    age INTEGER CHECK (age >= 18),
+
+    description TEXT DEFAULT "Some description",
+
+    empty_text TEXT DEFAULT "",
+    single_char_text TEXT DEFAULT "x",
+    single_quote_escape TEXT DEFAULT "It\'s a test",
+    backslash_escape TEXT DEFAULT "C:\\Program Files",
+    newline_escape TEXT DEFAULT "Line1\nLine2",
+    tab_escape TEXT DEFAULT "Column1\tColumn2",
+    unicode_escape TEXT DEFAULT "Unicode: \u263A"
+);
+
+CREATE INDEX IF NOT EXISTS idx_name ON table_with_all_types (name);
+
+CREATE INDEX IF NOT EXISTS idx_unique_number_is_active ON table_with_all_types (unique_number, is_active);