From bf1d63372bc59dfd6df3f7e9e576889d171500ee Mon Sep 17 00:00:00 2001 From: songdemei Date: Mon, 5 Jun 2023 09:01:15 +0800 Subject: [PATCH 1/8] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=89=B9=E9=87=8F=E6=8F=92=E5=85=A5=EF=BC=8C?= =?UTF-8?q?=E6=8F=90=E9=AB=98=E5=A4=A7=E8=A1=A8=E7=9A=84=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E9=80=9F=E5=BA=A6=20=E9=BB=98=E8=AE=A4=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E4=B8=8D=E4=BD=BF=E7=94=A8use=20database=20=E8=AF=AD=E5=8F=A5?= =?UTF-8?q?=EF=BC=8C=E4=BB=A5=E9=98=B2=E6=AD=A2=E5=AF=BC=E5=85=A5=E6=97=B6?= =?UTF-8?q?=E9=80=89=E5=BA=93=E5=85=B6=E4=BB=96=E5=BA=93=E6=97=B6=EF=BC=8C?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=E9=94=99=E8=AF=AF=E7=9A=84=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mysqldump.go | 147 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 140 insertions(+), 7 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index ede4e49..c22637b 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -31,6 +31,11 @@ type dumpOption struct { isAllTable bool // 是否删除表 isDropTable bool + // 是否增加选库脚本 + isUseDb bool + + //批量插入，提高导出效率 + perDataNumber int // writer 默认为 os.Stdout writer io.Writer @@ -59,6 +64,13 @@ func WithAllDatabases() DumpOption { } } +// 是否增加指定库语句 如果多库，此设置无效 +func WithUseDb() DumpOption { + return func(option *dumpOption) { + option.isUseDb = true + } +} + // 导出指定数据库, 与 WithAllDatabases 互斥, WithAllDatabases 优先级高 func WithDBs(databases ...string) DumpOption { return func(option *dumpOption) { @@ -80,6 +92,13 @@ func WithAllTable() DumpOption { } } +// 批量insert +func WithMultyInsert(num int) DumpOption { + return func(option *dumpOption) { + option.perDataNumber = num + } +} + // 导出到指定 writer func WithWriter(writer io.Writer) DumpOption { return func(option *dumpOption) { @@ -116,7 +135,6 @@ func Dump(dns string, opts ...DumpOption) error { dbName, } } - if len(o.tables) == 0 { // 默认包含全部表 o.isAllTable = true @@ -156,7 +174,9 @@ func Dump(dns string, opts ...DumpOption) error { } else { dbs = o.dbs } - + if len(dbs) > 1 { + o.isUseDb = true + } // 2. 获取表 for _, dbStr := range dbs { _, err = db.Exec(fmt.Sprintf("USE `%s`", dbStr)) @@ -176,8 +196,10 @@ func Dump(dns string, opts ...DumpOption) error { } else { tables = o.tables } - - buf.WriteString(fmt.Sprintf("USE `%s`;\n", dbStr)) + if o.isUseDb { + //多库导出时，才会增加选库操作，否则不加选库操作 + buf.WriteString(fmt.Sprintf("USE `%s`;\n", dbStr)) + } // 3. 导出表 for _, table := range tables { @@ -195,7 +217,7 @@ func Dump(dns string, opts ...DumpOption) error { // 导出表数据 if o.isData { - err = writeTableData(db, table, buf) + err = writeTableData(db, table, buf, o.perDataNumber) if err != nil { log.Printf("[error] %v \n", err) return err @@ -284,7 +306,7 @@ func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error { return nil } -func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { +func writeTableData(db *sql.DB, table string, buf *bufio.Writer, perDataNumber int) error { // 导出表数据 buf.WriteString("-- ----------------------------\n") @@ -311,7 +333,20 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { } var values [][]interface{} + rowId := 0 + for lineRows.Next() { + ssql := "" + if rowId == 0 || perDataNumber < 2 || rowId%perDataNumber == 0 { + if rowId > 0 { + ssql = ";\n" + } + //表结构 + ssql += "INSERT INTO `" + table + "` (`" + strings.Join(columns, "`,`") + "`) VALUES \n" + } else { + buf.WriteString(",\n") + } + row := make([]interface{}, len(columns)) rowPointers := make([]interface{}, len(columns)) for i := range columns { @@ -322,9 +357,17 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { log.Printf("[error] %v \n", err) return err } + rowString, err := buildRowData(row, columnTypes) + if err != nil { + return err + } + ssql += "(" + rowString + ")" + rowId += 1 + buf.WriteString(ssql) values = append(values, row) } - + buf.WriteString(";\n\n") + return nil for _, row := range values { ssql := "INSERT INTO `" + table + "` VALUES (" @@ -418,3 +461,93 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer) error { buf.WriteString("\n\n") return nil } + +func buildRowData(row []interface{}, columnTypes []*sql.ColumnType) (ssql string, err error) { + // var ssql string + for i, col := range row { + if col == nil { + ssql += "NULL" + } else { + Type := columnTypes[i].DatabaseTypeName() + // 去除 UNSIGNED 和空格 + Type = strings.Replace(Type, "UNSIGNED", "", -1) + Type = strings.Replace(Type, " ", "", -1) + switch Type { + case "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT": + if bs, ok := col.([]byte); ok { + ssql += fmt.Sprintf("%s", string(bs)) + } else { + ssql += fmt.Sprintf("%d", col) + } + case "FLOAT", "DOUBLE": + if bs, ok := col.([]byte); ok { + ssql += fmt.Sprintf("%s", string(bs)) + } else { + ssql += fmt.Sprintf("%f", col) + } + case "DECIMAL", "DEC": + ssql += fmt.Sprintf("%s", col) + + case "DATE": + t, ok := col.(time.Time) + if !ok { + log.Println("DATE 类型转换错误") + return "", err + } + ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02")) + case "DATETIME": + t, ok := col.(time.Time) + if !ok { + log.Println("DATETIME 类型转换错误") + return "", err + } + ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) + case "TIMESTAMP": + t, ok := col.(time.Time) + if !ok { + log.Println("TIMESTAMP 类型转换错误") + return "", err + } + ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) + case "TIME": + t, ok := col.([]byte) + if !ok { + log.Println("TIME 类型转换错误") + return "", err + } + ssql += fmt.Sprintf("'%s'", string(t)) + case "YEAR": + t, ok := col.([]byte) + if !ok { + log.Println("YEAR 类型转换错误") + return "", err + } + ssql += fmt.Sprintf("%s", string(t)) + case "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT": + r := strings.NewReplacer("\n", "\\n", "'", "\\'", "\r", "\\r", "\"", "\\\"") + ssql += fmt.Sprintf("'%s'", r.Replace(fmt.Sprintf("%s", col))) + // ssql += fmt.Sprintf("'%s'", strings.Replace(fmt.Sprintf("%s", col), "'", "''", -1)) + case "BIT", "BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB": + ssql += fmt.Sprintf("0x%X", col) + case "ENUM", "SET": + ssql += fmt.Sprintf("'%s'", col) + case "BOOL", "BOOLEAN": + if col.(bool) { + ssql += "true" + } else { + ssql += "false" + } + case "JSON": + ssql += fmt.Sprintf("'%s'", col) + default: + // unsupported type + // log.Printf("unsupported type: %s", Type) + return "", fmt.Errorf("unsupported type: %s", Type) + } + } + if i < len(row)-1 { + ssql += "," + } + } + return ssql, nil +} From 2f1d1eb0708cfb79371e7d3db5bef7bd1c42c34a Mon Sep 17 00:00:00 2001 From: songdemei Date: Mon, 5 Jun 2023 09:32:47 +0800 Subject: [PATCH 2/8] note --- mysqldump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index c22637b..1486fbd 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -31,7 +31,7 @@ type dumpOption struct { isAllTable bool // 是否删除表 isDropTable bool - // 是否增加选库脚本 + // 是否增加选库脚本，多库导出时，此设置默认开启 isUseDb bool //批量插入，提高导出效率 From befa393d630780de4fe7b148e368e6e6127b733f Mon Sep 17 00:00:00 2001 From: songdemei Date: Wed, 7 Jun 2023 14:49:14 +0800 Subject: [PATCH 3/8] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E8=A7=A6=E5=8F=91=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mysqldump.go | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 1486fbd..5daf28b 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -18,6 +18,8 @@ func init() { log.SetFlags(log.Lshortfile | log.LstdFlags) } +var version string = "v0.10.0" + type dumpOption struct { // 导出表数据 isData bool @@ -40,7 +42,13 @@ type dumpOption struct { // writer 默认为 os.Stdout writer io.Writer } - +type triggerStruct struct { + Trigger string + Event string + Table string + Statement string + Timing string +} type DumpOption func(*dumpOption) // 删除表 @@ -151,10 +159,11 @@ func Dump(dns string, opts ...DumpOption) error { // 打印 Header buf.WriteString("-- ----------------------------\n") buf.WriteString("-- MySQL Database Dump\n") + buf.WriteString("-- GoMysqlDump version: " + version + "\n") buf.WriteString("-- Start Time: " + start.Format("2006-01-02 15:04:05") + "\n") buf.WriteString("-- ----------------------------\n") buf.WriteString("\n\n") - + buf.WriteString("/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;\n") // 连接数据库 db, err := sql.Open("mysql", dns) if err != nil { @@ -223,6 +232,12 @@ func Dump(dns string, opts ...DumpOption) error { return err } } + sqls, err := writeTableTrigger(db, table) + if err != nil { + log.Printf("[error] %v \n", err) + return err + } + buf.WriteString(sqls) } } @@ -312,6 +327,8 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer, perDataNumber i buf.WriteString("-- ----------------------------\n") buf.WriteString(fmt.Sprintf("-- Records of %s\n", table)) buf.WriteString("-- ----------------------------\n") + buf.WriteString(fmt.Sprintf("LOCK TABLES `%s` WRITE;\n", table)) + buf.WriteString(fmt.Sprintf("/*!40000 ALTER TABLE `%s` DISABLE KEYS */;\n", table)) lineRows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table)) if err != nil { @@ -366,7 +383,9 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer, perDataNumber i buf.WriteString(ssql) values = append(values, row) } - buf.WriteString(";\n\n") + buf.WriteString(";\n") + buf.WriteString(fmt.Sprintf("/*!40000 ALTER TABLE `%s` ENABLE KEYS */;\n", table)) + buf.WriteString("UNLOCK TABLES;\n\n") return nil for _, row := range values { ssql := "INSERT INTO `" + table + "` VALUES (" @@ -551,3 +570,61 @@ func buildRowData(row []interface{}, columnTypes []*sql.ColumnType) (ssql string } return ssql, nil } + +func writeTableTrigger(db *sql.DB, table string) (sqls string, err error) { + var sql []string + trgs, err := db.Query("SHOW TRIGGERS") + if err != nil { + log.Printf("[error] %v \n", err) + return "", err + } + defer trgs.Close() + + var columns []string + columns, err = trgs.Columns() + var triggers []triggerStruct + for trgs.Next() { + trgrow := make([]interface{}, len(columns)) + rowPointers := make([]interface{}, len(columns)) + for i := range columns { + rowPointers[i] = &trgrow[i] + } + err = trgs.Scan(rowPointers...) + if err != nil { + log.Printf("[error] %v \n", err) + return "", err + } + var trigger triggerStruct + for k, v := range trgrow { + switch columns[k] { + case "Table": + trigger.Table = fmt.Sprintf("%s", v) + case "Event": + trigger.Event = fmt.Sprintf("%s", v) + case "Trigger": + trigger.Trigger = fmt.Sprintf("%s", v) + case "Statement": + trigger.Statement = fmt.Sprintf("%s", v) + case "Timing": + trigger.Timing = fmt.Sprintf("%s", v) + } + } + if trigger.Table == table { + triggers = append(triggers, trigger) + } + } + if len(triggers) > 0 { + sql = append(sql, "-- ----------------------------") + sql = append(sql, "-- Dump table triggers --------") + sql = append(sql, "-- ----------------------------") + } + for _, v := range triggers { + sql = append(sql, "DELIMITER ;;") + sql = append(sql, "/*!50003 SET SESSION SQL_MODE=\"\" */;;") + sql = append(sql, fmt.Sprintf("/*!50003 CREATE TRIGGER `%s` %s %s ON `%s` FOR EACH ROW %s */;;", v.Trigger, v.Timing, v.Event, v.Table, v.Statement)) + sql = append(sql, "DELIMITER ;") + sql = append(sql, "/*!50003 SET SESSION SQL_MODE=@OLD_SQL_MODE */;\n") + } + + return strings.Join(sql, "\n"), nil +} From bc4bb199e540e2f63fa4d415c5d776ae60304260 Mon Sep 17 00:00:00 2001 From: songdemei Date: Wed, 7 Jun 2023 14:55:36 +0800 Subject: [PATCH 4/8] md update --- README-zh.md | 2 ++ README.md | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README-zh.md b/README-zh.md index d06f4f8..b27a40f 100644 --- a/README-zh.md +++ b/README-zh.md @@ -20,6 +20,8 @@ golang 实现的零依赖、支持所有类型、高性能、并发 mysqldump * 自定义 Writer: 如本地文件、多文件储存、远程服务器、云存储等。（默认控制台输出）。 * 支持所有 MYSQL 数据类型. * 支持 INSERT Merge, 大幅提升数据恢复性能 +* 导出支持 批量插入, 大幅提升数据恢复性能 +* 导出支持 触发器 ## QuickStart diff --git a/README.md b/README.md index feb9a7a..a711a0a 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,8 @@ A zero-dependency,all data types are supported, high-performance, concurrent mys * Supports custom Writer: data can be written to any Writer, such as local files, multiple file storage, remote servers, cloud storage, etc. (default console output). * Supports all MySQL data types QuickStart. * Support Merge Insert Option in Source Greatly improve data recovery performance - +* Support multy data in one insert +* Support dump table trigger ## QuickStart From d1b52335a1451c8acaed14b5d409d655f9d626ed Mon Sep 17 00:00:00 2001 From: songdemei Date: Wed, 7 Jun 2023 15:10:43 +0800 Subject: [PATCH 5/8] mod name --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fa0a6f3..d512b17 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/jarvanstack/mysqldump +module github.com/songdemei/mysqldump go 1.18 From 6e3ddc6962566945a1d3f7ea28427fb5af65bfa0 Mon Sep 17 00:00:00 2001 From: songdemei Date: Sun, 16 Jul 2023 23:02:22 +0800 Subject: [PATCH 6/8] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E6=9C=89=E8=A7=A6=E5=8F=91=E5=99=A8=E6=97=B6=EF=BC=8C=E6=95=88?= =?UTF-8?q?=E7=8E=87=E5=A4=AA=E6=85=A2=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mysqldump.go | 157 ++++++++++++++------------------------------------- 1 file changed, 41 insertions(+), 116 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 5daf28b..a05cb72 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -18,7 +18,7 @@ func init() { log.SetFlags(log.Lshortfile | log.LstdFlags) } -var version string = "v0.10.0" +var version string = "v0.10.2" type dumpOption struct { // 导出表数据 @@ -49,6 +49,9 @@ type triggerStruct struct { Statement string Timing string } + +var allTriggers map[string][]triggerStruct + type DumpOption func(*dumpOption) // 删除表 @@ -115,6 +118,7 @@ func WithWriter(writer io.Writer) DumpOption { } func Dump(dns string, opts ...DumpOption) error { + // 打印开始 start := time.Now() log.Printf("[info] [dump] start at %s\n", start.Format("2006-01-02 15:04:05")) @@ -223,7 +227,6 @@ func Dump(dns string, opts ...DumpOption) error { log.Printf("[error] %v \n", err) return err } - // 导出表数据 if o.isData { err = writeTableData(db, table, buf, o.perDataNumber) @@ -232,12 +235,11 @@ func Dump(dns string, opts ...DumpOption) error { return err } } - sqls, err := writeTableTrigger(db, table) + err := writeTableTrigger(db, table, buf) if err != nil { log.Printf("[error] %v \n", err) return err } - buf.WriteString(sqls) } } @@ -387,98 +389,7 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer, perDataNumber i buf.WriteString(fmt.Sprintf("/*!40000 ALTER TABLE `%s` ENABLE KEYS */;\n", table)) buf.WriteString("UNLOCK TABLES;\n\n") return nil - for _, row := range values { - ssql := "INSERT INTO `" + table + "` VALUES (" - - for i, col := range row { - if col == nil { - ssql += "NULL" - } else { - Type := columnTypes[i].DatabaseTypeName() - // 去除 UNSIGNED 和空格 - Type = strings.Replace(Type, "UNSIGNED", "", -1) - Type = strings.Replace(Type, " ", "", -1) - switch Type { - case "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT": - if bs, ok := col.([]byte); ok { - ssql += fmt.Sprintf("%s", string(bs)) - } else { - ssql += fmt.Sprintf("%d", col) - } - case "FLOAT", "DOUBLE": - if bs, ok := col.([]byte); ok { - ssql += fmt.Sprintf("%s", string(bs)) - } else { - ssql += fmt.Sprintf("%f", col) - } - case "DECIMAL", "DEC": - ssql += fmt.Sprintf("%s", col) - - case "DATE": - t, ok := col.(time.Time) - if !ok { - log.Println("DATE 类型转换错误") - return err - } - ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02")) - case "DATETIME": - t, ok := col.(time.Time) - if !ok { - log.Println("DATETIME 类型转换错误") - return err - } - ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) - case "TIMESTAMP": - t, ok := col.(time.Time) - if !ok { - log.Println("TIMESTAMP 类型转换错误") - return err - } - ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) - case "TIME": - t, ok := col.([]byte) - if !ok { - log.Println("TIME 类型转换错误") - return err - } - ssql += fmt.Sprintf("'%s'", string(t)) - case "YEAR": - t, ok := col.([]byte) - if !ok { - log.Println("YEAR 类型转换错误") - return err - } - ssql += fmt.Sprintf("%s", string(t)) - case "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT": - ssql += fmt.Sprintf("'%s'", strings.Replace(fmt.Sprintf("%s", col), "'", "''", -1)) - case "BIT", "BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB": - ssql += fmt.Sprintf("0x%X", col) - case "ENUM", "SET": - ssql += fmt.Sprintf("'%s'", col) - case "BOOL", "BOOLEAN": - if col.(bool) { - ssql += "true" - } else { - ssql += "false" - } - case "JSON": - ssql += fmt.Sprintf("'%s'", col) - default: - // unsupported type - log.Printf("unsupported type: %s", Type) - return fmt.Errorf("unsupported type: %s", Type) - } - } - if i < len(row)-1 { - ssql += "," - } - } - ssql += ");\n" - buf.WriteString(ssql) - } - buf.WriteString("\n\n") - return nil } func buildRowData(row []interface{}, columnTypes []*sql.ColumnType) (ssql string, err error) { @@ -571,18 +482,47 @@ func buildRowData(row []interface{}, columnTypes []*sql.ColumnType) (ssql string return ssql, nil } -func writeTableTrigger(db *sql.DB, table string) (sqls string, err error) { +func writeTableTrigger(db *sql.DB, table string, buf *bufio.Writer) error { var sql []string + + triggers, err := getTrigger(db, table) + if err != nil { + log.Printf("[error] %v \n", err) + return err + } + if len(triggers) > 0 { + sql = append(sql, "-- ----------------------------") + sql = append(sql, fmt.Sprintf("-- Dump table triggers of %s--------", table)) + sql = append(sql, "-- ----------------------------") + } + for _, v := range triggers { + sql = append(sql, "DELIMITER ;;") + sql = append(sql, "/*!50003 SET SESSION SQL_MODE=\"\" */;;") + sql = append(sql, fmt.Sprintf("/*!50003 CREATE TRIGGER `%s` %s %s ON `%s` FOR EACH ROW %s */;;", v.Trigger, v.Timing, v.Event, v.Table, v.Statement)) + sql = append(sql, "DELIMITER ;") + sql = append(sql, "/*!50003 SET SESSION SQL_MODE=@OLD_SQL_MODE */;\n") + } + buf.WriteString(strings.Join(sql, "\n")) + return nil +} + +func getTrigger(db *sql.DB, table string) (trigger []triggerStruct, err error) { + if allTriggers != nil { + trigger = allTriggers[table] + return trigger, nil + } else { + allTriggers = make(map[string][]triggerStruct) + } trgs, err := db.Query("SHOW TRIGGERS") if err != nil { log.Printf("[error] %v \n", err) - return "", err + return trigger, err } defer trgs.Close() var columns []string columns, err = trgs.Columns() - var triggers []triggerStruct + for trgs.Next() { trgrow := make([]interface{}, len(columns)) rowPointers := make([]interface{}, len(columns)) @@ -592,7 +532,7 @@ func writeTableTrigger(db *sql.DB, table string) (sqls string, err error) { err = trgs.Scan(rowPointers...) if err != nil { log.Printf("[error] %v \n", err) - return "", err + return trigger, err } var trigger triggerStruct for k, v := range trgrow { @@ -609,22 +549,7 @@ func writeTableTrigger(db *sql.DB, table string) (sqls string, err error) { trigger.Timing = fmt.Sprintf("%s", v) } } - if trigger.Table == table { - triggers = append(triggers, trigger) - } + allTriggers[trigger.Table] = append(allTriggers[trigger.Table], trigger) } - if len(triggers) > 0 { - sql = append(sql, "-- ----------------------------") - sql = append(sql, "-- Dump table triggers --------") - sql = append(sql, "-- ----------------------------") - } - for _, v := range triggers { - sql = append(sql, "DELIMITER ;;") - sql = append(sql, "/*!50003 SET SESSION SQL_MODE=\"\" */;;") - sql = append(sql, fmt.Sprintf("/*!50003 CREATE TRIGGER `%s` %s %s ON `%s` FOR EACH ROW %s */;;", v.Trigger, v.Timing, v.Event, v.Table, v.Statement)) - sql = append(sql, "DELIMITER ;") - sql = append(sql, "/*!50003 SET SESSION SQL_MODE=@OLD_SQL_MODE */;\n") - } - - return strings.Join(sql, "\n"), nil + return allTriggers[table], nil } From 17f546177787319f8eed8562b21320d5c5d02025 Mon Sep 17 00:00:00 2001 From: songdemei Date: Thu, 20 Jul 2023 09:44:17 +0800 Subject: [PATCH 7/8] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E8=A7=86=E5=9B=BE=E5=8A=9F=E8=83=BD=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E9=A1=B9withLogOut=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E4=B8=8D=E8=BE=93=E5=87=BA=E6=97=A5=E5=BF=97=EF=BC=8C=E5=A6=82?= =?UTF-8?q?=E6=9E=9C=E9=9C=80=E8=A6=81=EF=BC=8C=E6=89=8B=E5=8A=A8=E5=BC=80?= =?UTF-8?q?=E5=90=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mysqldump.go | 154 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 115 insertions(+), 39 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index a05cb72..1a7182d 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -41,6 +41,8 @@ type dumpOption struct { // writer 默认为 os.Stdout writer io.Writer + //是否输出日志 + logOut bool } type triggerStruct struct { Trigger string @@ -117,21 +119,33 @@ func WithWriter(writer io.Writer) DumpOption { } } +// 是否输出日志 +// @TODO: 后续增加日志的handle用于输出到其他地方 +func WithLogOut(logOut bool) DumpOption { + return func(option *dumpOption) { + option.logOut = logOut + } +} + func Dump(dns string, opts ...DumpOption) error { + var err error + + var o dumpOption // 打印开始 start := time.Now() - log.Printf("[info] [dump] start at %s\n", start.Format("2006-01-02 15:04:05")) + if o.logOut { + log.Printf("[info] [dump] start at %s\n", start.Format("2006-01-02 15:04:05")) + } + // 打印结束 defer func() { end := time.Now() - log.Printf("[info] [dump] end at %s, cost %s\n", end.Format("2006-01-02 15:04:05"), end.Sub(start)) + if o.logOut { + log.Printf("[info] [dump] end at %s, cost %s\n", end.Format("2006-01-02 15:04:05"), end.Sub(start)) + } }() - var err error - - var o dumpOption - for _, opt := range opts { opt(&o) } @@ -171,7 +185,9 @@ func Dump(dns string, opts ...DumpOption) error { // 连接数据库 db, err := sql.Open("mysql", dns) if err != nil { - log.Printf("[error] %v \n", err) + if o.logOut { + log.Printf("[error] %v \n", err) + } return err } defer db.Close() @@ -181,7 +197,9 @@ func Dump(dns string, opts ...DumpOption) error { if o.isAllDB { dbs, err = getDBs(db) if err != nil { - log.Printf("[error] %v \n", err) + if o.logOut { + log.Printf("[error] %v \n", err) + } return err } } else { @@ -194,7 +212,9 @@ func Dump(dns string, opts ...DumpOption) error { for _, dbStr := range dbs { _, err = db.Exec(fmt.Sprintf("USE `%s`", dbStr)) if err != nil { - log.Printf("[error] %v \n", err) + if o.logOut { + log.Printf("[error] %v \n", err) + } return err } @@ -202,7 +222,9 @@ func Dump(dns string, opts ...DumpOption) error { if o.isAllTable { tmp, err := getAllTables(db) if err != nil { - log.Printf("[error] %v \n", err) + if o.logOut { + log.Printf("[error] %v \n", err) + } return err } tables = tmp @@ -216,30 +238,59 @@ func Dump(dns string, opts ...DumpOption) error { // 3. 导出表 for _, table := range tables { - // 删除表 - if o.isDropTable { - buf.WriteString(fmt.Sprintf("DROP TABLE IF EXISTS `%s`;\n", table)) - } - // 导出表结构 - err = writeTableStruct(db, table, buf) + tt, err := getTableType(db, table) if err != nil { - log.Printf("[error] %v \n", err) return err } - // 导出表数据 - if o.isData { - err = writeTableData(db, table, buf, o.perDataNumber) + + if tt == "TABLE" { + // 删除表 + if o.isDropTable { + buf.WriteString(fmt.Sprintf("DROP TABLE IF EXISTS `%s`;\n", table)) + } + + // 导出表结构 + err = writeTableStruct(db, table, buf) if err != nil { - log.Printf("[error] %v \n", err) + if o.logOut { + log.Printf("[error] %v \n", err) + } + return err + } + // 导出表数据 + if o.isData { + err = writeTableData(db, table, buf, o.perDataNumber) + if err != nil { + if o.logOut { + log.Printf("[error] %v \n", err) + } + return err + } + } + err := writeTableTrigger(db, table, buf) + if err != nil { + if o.logOut { + log.Printf("[error] %v \n", err) + } return err } } - err := writeTableTrigger(db, table, buf) - if err != nil { - log.Printf("[error] %v \n", err) - return err + if tt == "VIEW" { + // 删除视图 + if o.isDropTable { + buf.WriteString(fmt.Sprintf("DROP VIEW IF EXISTS `%s`;\n", table)) + } + // 导出视图结构 + err = writeViewStruct(db, table, buf) + if err != nil { + if o.logOut { + log.Printf("[error] %v \n", err) + } + return err + } } + } } @@ -254,9 +305,27 @@ func Dump(dns string, opts ...DumpOption) error { return nil } +func getTableType(db *sql.DB, table string) (t string, err error) { + query := fmt.Sprintf("SELECT TABLE_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '%s'", table) + var tableType string + err = db.QueryRow(query).Scan(&tableType) + if err != nil { + return "", err + } + switch tableType { + case "BASE TABLE": + return "TABLE", nil + case "VIEW": + return "VIEW", nil + default: + return "", nil + } +} func getCreateTableSQL(db *sql.DB, table string) (string, error) { + var createTableSQL string + err := db.QueryRow(fmt.Sprintf("SHOW CREATE TABLE `%s`", table)).Scan(&table, &createTableSQL) if err != nil { return "", err @@ -312,7 +381,27 @@ func writeTableStruct(db *sql.DB, table string, buf *bufio.Writer) error { createTableSQL, err := getCreateTableSQL(db, table) if err != nil { - log.Printf("[error] %v \n", err) + return err + } + buf.WriteString(createTableSQL) + buf.WriteString(";") + + buf.WriteString("\n\n") + buf.WriteString("\n\n") + return nil +} + +func writeViewStruct(db *sql.DB, table string, buf *bufio.Writer) error { + // 导出视图 + buf.WriteString("-- ----------------------------\n") + buf.WriteString(fmt.Sprintf("-- View structure for %s\n", table)) + buf.WriteString("-- ----------------------------\n") + + var createTableSQL string + var charact string + var connect string + err := db.QueryRow(fmt.Sprintf("SHOW CREATE TABLE `%s`", table)).Scan(&table, &createTableSQL, &charact, &connect) + if err != nil { return err } buf.WriteString(createTableSQL) @@ -334,7 +423,6 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer, perDataNumber i lineRows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table)) if err != nil { - log.Printf("[error] %v \n", err) return err } defer lineRows.Close() @@ -342,12 +430,10 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer, perDataNumber i var columns []string columns, err = lineRows.Columns() if err != nil { - log.Printf("[error] %v \n", err) return err } columnTypes, err := lineRows.ColumnTypes() if err != nil { - log.Printf("[error] %v \n", err) return err } @@ -373,7 +459,6 @@ func writeTableData(db *sql.DB, table string, buf *bufio.Writer, perDataNumber i } err = lineRows.Scan(rowPointers...) if err != nil { - log.Printf("[error] %v \n", err) return err } rowString, err := buildRowData(row, columnTypes) @@ -421,35 +506,30 @@ func buildRowData(row []interface{}, columnTypes []*sql.ColumnType) (ssql string case "DATE": t, ok := col.(time.Time) if !ok { - log.Println("DATE 类型转换错误") return "", err } ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02")) case "DATETIME": t, ok := col.(time.Time) if !ok { - log.Println("DATETIME 类型转换错误") return "", err } ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) case "TIMESTAMP": t, ok := col.(time.Time) if !ok { - log.Println("TIMESTAMP 类型转换错误") return "", err } ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02 15:04:05")) case "TIME": t, ok := col.([]byte) if !ok { - log.Println("TIME 类型转换错误") return "", err } ssql += fmt.Sprintf("'%s'", string(t)) case "YEAR": t, ok := col.([]byte) if !ok { - log.Println("YEAR 类型转换错误") return "", err } ssql += fmt.Sprintf("%s", string(t)) @@ -471,7 +551,6 @@ func buildRowData(row []interface{}, columnTypes []*sql.ColumnType) (ssql string ssql += fmt.Sprintf("'%s'", col) default: // unsupported type - // log.Printf("unsupported type: %s", Type) return "", fmt.Errorf("unsupported type: %s", Type) } } @@ -487,7 +566,6 @@ func writeTableTrigger(db *sql.DB, table string, buf *bufio.Writer) error { triggers, err := getTrigger(db, table) if err != nil { - log.Printf("[error] %v \n", err) return err } if len(triggers) > 0 { @@ -515,7 +593,6 @@ func getTrigger(db *sql.DB, table string) (trigger []triggerStruct, err error) { } trgs, err := db.Query("SHOW TRIGGERS") if err != nil { - log.Printf("[error] %v \n", err) return trigger, err } defer trgs.Close() @@ -531,7 +608,6 @@ func getTrigger(db *sql.DB, table string) (trigger []triggerStruct, err error) { } err = trgs.Scan(rowPointers...) if err != nil { - log.Printf("[error] %v \n", err) return trigger, err } var trigger triggerStruct From 7e1ea9925a8a229c1887c8023df259e4ea543a8b Mon Sep 17 00:00:00 2001 From: songdemei Date: Thu, 20 Jul 2023 09:46:56 +0800 Subject: [PATCH 8/8] =?UTF-8?q?=E7=89=88=E6=9C=AC=E5=8D=87=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mysqldump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysqldump.go b/mysqldump.go index 1a7182d..61470a3 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -18,7 +18,7 @@ func init() { log.SetFlags(log.Lshortfile | log.LstdFlags) } -var version string = "v0.10.2" +var version string = "v0.11.0" type dumpOption struct { // 导出表数据