From 912c2524f88d43fcd51827c23d105ed9bc573ac1 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 7 May 2021 09:19:03 +0800 Subject: [PATCH 1/5] RFC: Use provided type to create the dstTable rather than inferring from the SQL types (#1872) When using dumptables to convert between dialects if a struct is provided we should use it to generate the SQL types rather than infer them by mapping from the sql types. Signed-off-by: Andrew Thornton Co-authored-by: Lunny Xiao Reviewed-on: https://gitea.com/xorm/xorm/pulls/1872 Co-authored-by: Andrew Thornton Co-committed-by: Andrew Thornton --- Makefile | 2 +- engine.go | 120 ++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 91 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 7305baa7..bf71b0f4 100644 --- a/Makefile +++ b/Makefile @@ -243,4 +243,4 @@ test-tidb\#%: go-check .PHONY: vet vet: - $(GO) vet $(shell $(GO) list ./...) \ No newline at end of file + $(GO) vet $(shell $(GO) list ./...) diff --git a/engine.go b/engine.go index 384928d6..0e498039 100644 --- a/engine.go +++ b/engine.go @@ -460,6 +460,11 @@ func formatColumnValue(dstDialect dialects.Dialect, d interface{}, col *schemas. var v = fmt.Sprintf("%s", d) return "'" + strings.Replace(v, "'", "''", -1) + "'" } else if col.SQLType.IsTime() { + if dstDialect.URI().DBType == schemas.MSSQL && col.SQLType.Name == schemas.DateTime { + if t, ok := d.(time.Time); ok { + return "'" + t.UTC().Format("2006-01-02 15:04:05") + "'" + } + } var v = fmt.Sprintf("%s", d) if strings.HasSuffix(v, " +0000 UTC") { return fmt.Sprintf("'%s'", v[0:len(v)-len(" +0000 UTC")]) @@ -537,6 +542,8 @@ func (engine *Engine) dumpTables(tables []*schemas.Table, w io.Writer, tp ...sch } dstDialect.Init(&destURI) } + cacherMgr := caches.NewManager() + dstTableCache := tags.NewParser("xorm", dstDialect, engine.GetTableMapper(), engine.GetColumnMapper(), cacherMgr) _, err := io.WriteString(w, fmt.Sprintf("/*Generated by xorm %s, from %s to %s*/\n\n", time.Now().In(engine.TZLocation).Format("2006-01-02 15:04:05"), engine.dialect.URI().DBType, dstDialect.URI().DBType)) @@ -545,9 +552,18 @@ func (engine *Engine) dumpTables(tables []*schemas.Table, w io.Writer, tp ...sch } for i, table := range tables { - tableName := table.Name + dstTable := table + if table.Type != nil { + dstTable, err = dstTableCache.Parse(reflect.New(table.Type)) + if err != nil { + engine.logger.Errorf("Unable to infer table for %s in new dialect. Error: %v", table.Name) + dstTable = table + } + } + + dstTableName := dstTable.Name if dstDialect.URI().Schema != "" { - tableName = fmt.Sprintf("%s.%s", dstDialect.URI().Schema, table.Name) + dstTableName = fmt.Sprintf("%s.%s", dstDialect.URI().Schema, dstTable.Name) } originalTableName := table.Name if engine.dialect.URI().Schema != "" { @@ -559,27 +575,30 @@ func (engine *Engine) dumpTables(tables []*schemas.Table, w io.Writer, tp ...sch return err } } - sqls, _ := dstDialect.CreateTableSQL(table, tableName) + + sqls, _ := dstDialect.CreateTableSQL(dstTable, dstTableName) for _, s := range sqls { _, err = io.WriteString(w, s+";\n") if err != nil { return err } } - if len(table.PKColumns()) > 0 && dstDialect.URI().DBType == schemas.MSSQL { - fmt.Fprintf(w, "SET IDENTITY_INSERT [%s] ON;\n", table.Name) + if len(dstTable.PKColumns()) > 0 && dstDialect.URI().DBType == schemas.MSSQL { + fmt.Fprintf(w, "SET IDENTITY_INSERT [%s] ON;\n", dstTable.Name) } - for _, index := range table.Indexes { - _, err = io.WriteString(w, dstDialect.CreateIndexSQL(table.Name, index)+";\n") + for _, index := range dstTable.Indexes { + _, err = io.WriteString(w, dstDialect.CreateIndexSQL(dstTable.Name, index)+";\n") if err != nil { return err } } cols := table.ColumnsSeq() + dstCols := dstTable.ColumnsSeq() + colNames := engine.dialect.Quoter().Join(cols, ", ") - destColNames := dstDialect.Quoter().Join(cols, ", ") + destColNames := dstDialect.Quoter().Join(dstCols, ", ") rows, err := engine.DB().QueryContext(engine.defaultContext, "SELECT "+colNames+" FROM "+engine.Quote(originalTableName)) if err != nil { @@ -587,35 +606,76 @@ func (engine *Engine) dumpTables(tables []*schemas.Table, w io.Writer, tp ...sch } defer rows.Close() - for rows.Next() { - dest := make([]interface{}, len(cols)) - err = rows.ScanSlice(&dest) - if err != nil { - return err - } - - _, err = io.WriteString(w, "INSERT INTO "+dstDialect.Quoter().Quote(tableName)+" ("+destColNames+") VALUES (") - if err != nil { - return err - } - - var temp string - for i, d := range dest { - col := table.GetColumn(cols[i]) - if col == nil { - return errors.New("unknow column error") + if table.Type != nil { + sess := engine.NewSession() + defer sess.Close() + for rows.Next() { + bean := reflect.New(table.Type) + fields, err := rows.Columns() + if err != nil { + return err + } + scanResults, err := sess.row2Slice(rows, fields, bean) + if err != nil { + return err + } + + dataStruct := utils.ReflectValue(bean.Interface()) + _, err = sess.slice2Bean(scanResults, fields, bean.Interface(), &dataStruct, table) + if err != nil { + return err + } + + _, err = io.WriteString(w, "INSERT INTO "+dstDialect.Quoter().Quote(dstTableName)+" ("+destColNames+") VALUES (") + if err != nil { + return err + } + + var temp string + for _, d := range dstCols { + col := table.GetColumn(d) + if col == nil { + return errors.New("unknown column error") + } + temp += "," + formatColumnValue(dstDialect, bean.Elem().FieldByName(col.FieldName).Interface(), col) + } + _, err = io.WriteString(w, temp[1:]+");\n") + if err != nil { + return err } - temp += "," + formatColumnValue(dstDialect, d, col) } - _, err = io.WriteString(w, temp[1:]+");\n") - if err != nil { - return err + } else { + for rows.Next() { + dest := make([]interface{}, len(cols)) + err = rows.ScanSlice(&dest) + if err != nil { + return err + } + + _, err = io.WriteString(w, "INSERT INTO "+dstDialect.Quoter().Quote(dstTableName)+" ("+destColNames+") VALUES (") + if err != nil { + return err + } + + var temp string + for i, d := range dest { + col := table.GetColumn(cols[i]) + if col == nil { + return errors.New("unknow column error") + } + + temp += "," + formatColumnValue(dstDialect, d, col) + } + _, err = io.WriteString(w, temp[1:]+");\n") + if err != nil { + return err + } } } // FIXME: Hack for postgres if dstDialect.URI().DBType == schemas.POSTGRES && table.AutoIncrColumn() != nil { - _, err = io.WriteString(w, "SELECT setval('"+tableName+"_id_seq', COALESCE((SELECT MAX("+table.AutoIncrColumn().Name+") + 1 FROM "+dstDialect.Quoter().Quote(tableName)+"), 1), false);\n") + _, err = io.WriteString(w, "SELECT setval('"+dstTableName+"_id_seq', COALESCE((SELECT MAX("+table.AutoIncrColumn().Name+") + 1 FROM "+dstDialect.Quoter().Quote(dstTableName)+"), 1), false);\n") if err != nil { return err } From 210c30a7ddec1ada4939af76aa9aa0b061cce68e Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 8 May 2021 12:27:22 +0800 Subject: [PATCH 2/5] Fix two issues with dumptables (#1903) There are two issues with #1872 which have become apparent after testing on Gitea. 1. Ensure structs which are have before processors run correctly 2. Ensure structs extending other structs work 3. Ensure that numerical enums become numeric Signed-off-by: Andrew Thornton Reviewed-on: https://gitea.com/xorm/xorm/pulls/1903 Reviewed-by: Lunny Xiao Co-authored-by: Andrew Thornton Co-committed-by: Andrew Thornton --- engine.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/engine.go b/engine.go index 0e498039..82593347 100644 --- a/engine.go +++ b/engine.go @@ -496,7 +496,7 @@ func formatColumnValue(dstDialect dialects.Dialect, d interface{}, col *schemas. } return fmt.Sprintf("%v", strconv.FormatBool(v)) } - return fmt.Sprintf("%v", d) + return fmt.Sprintf("%d", d) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: if col.SQLType.Name == schemas.Bool { v := reflect.ValueOf(d).Uint() > 0 @@ -508,7 +508,7 @@ func formatColumnValue(dstDialect dialects.Dialect, d interface{}, col *schemas. } return fmt.Sprintf("%v", strconv.FormatBool(v)) } - return fmt.Sprintf("%v", d) + return fmt.Sprintf("%d", d) default: return fmt.Sprintf("%v", d) } @@ -554,7 +554,7 @@ func (engine *Engine) dumpTables(tables []*schemas.Table, w io.Writer, tp ...sch for i, table := range tables { dstTable := table if table.Type != nil { - dstTable, err = dstTableCache.Parse(reflect.New(table.Type)) + dstTable, err = dstTableCache.Parse(reflect.New(table.Type).Elem()) if err != nil { engine.logger.Errorf("Unable to infer table for %s in new dialect. Error: %v", table.Name) dstTable = table @@ -610,7 +610,8 @@ func (engine *Engine) dumpTables(tables []*schemas.Table, w io.Writer, tp ...sch sess := engine.NewSession() defer sess.Close() for rows.Next() { - bean := reflect.New(table.Type) + beanValue := reflect.New(table.Type) + bean := beanValue.Interface() fields, err := rows.Columns() if err != nil { return err @@ -620,8 +621,8 @@ func (engine *Engine) dumpTables(tables []*schemas.Table, w io.Writer, tp ...sch return err } - dataStruct := utils.ReflectValue(bean.Interface()) - _, err = sess.slice2Bean(scanResults, fields, bean.Interface(), &dataStruct, table) + dataStruct := utils.ReflectValue(bean) + _, err = sess.slice2Bean(scanResults, fields, bean, &dataStruct, table) if err != nil { return err } @@ -637,7 +638,13 @@ func (engine *Engine) dumpTables(tables []*schemas.Table, w io.Writer, tp ...sch if col == nil { return errors.New("unknown column error") } - temp += "," + formatColumnValue(dstDialect, bean.Elem().FieldByName(col.FieldName).Interface(), col) + + fields := strings.Split(col.FieldName, ".") + field := dataStruct + for _, fieldName := range fields { + field = field.FieldByName(fieldName) + } + temp += "," + formatColumnValue(dstDialect, field.Interface(), col) } _, err = io.WriteString(w, temp[1:]+");\n") if err != nil { From dab09c73ab3b1b7a5dfd129146dd3922b6ea3a44 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 8 May 2021 21:39:03 +0800 Subject: [PATCH 3/5] Fix another bug with #1872 (#1905) Ensure that structs, arrays and slices are properly converted to strings. Signed-off-by: Andrew Thornton Reviewed-on: https://gitea.com/xorm/xorm/pulls/1905 Reviewed-by: Lunny Xiao Co-authored-by: Andrew Thornton Co-committed-by: Andrew Thornton --- engine.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/engine.go b/engine.go index 82593347..22ba6163 100644 --- a/engine.go +++ b/engine.go @@ -21,6 +21,7 @@ import ( "xorm.io/xorm/contexts" "xorm.io/xorm/core" "xorm.io/xorm/dialects" + "xorm.io/xorm/internal/json" "xorm.io/xorm/internal/utils" "xorm.io/xorm/log" "xorm.io/xorm/names" @@ -457,7 +458,19 @@ func formatColumnValue(dstDialect dialects.Dialect, d interface{}, col *schemas. } if col.SQLType.IsText() { - var v = fmt.Sprintf("%s", d) + var v string + switch reflect.TypeOf(d).Kind() { + case reflect.Struct, reflect.Array, reflect.Slice: + bytes, err := json.DefaultJSONHandler.Marshal(d) + if err != nil { + v = fmt.Sprintf("%s", d) + } else { + v = string(bytes) + } + default: + v = fmt.Sprintf("%s", d) + } + return "'" + strings.Replace(v, "'", "''", -1) + "'" } else if col.SQLType.IsTime() { if dstDialect.URI().DBType == schemas.MSSQL && col.SQLType.Name == schemas.DateTime { From d2f52eba64a3b2f89334e228905139fdd785e94a Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 9 May 2021 15:09:59 +0800 Subject: [PATCH 4/5] Byte strings in postgres aren't 0x... (#1906) Byte strings in postgres are actually E'\x...' not 0x... This is part of the follow-up to #1872 Signed-off-by: Andrew Thornton Reviewed-on: https://gitea.com/xorm/xorm/pulls/1906 Reviewed-by: Lunny Xiao Co-authored-by: Andrew Thornton Co-committed-by: Andrew Thornton --- dialects/postgres.go | 5 +++++ engine.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dialects/postgres.go b/dialects/postgres.go index 2b234c66..e76e5b7e 100644 --- a/dialects/postgres.go +++ b/dialects/postgres.go @@ -824,6 +824,11 @@ func (db *postgres) SetQuotePolicy(quotePolicy QuotePolicy) { } } +// FormatBytes formats bytes +func (db *postgres) FormatBytes(bs []byte) string { + return fmt.Sprintf("E'\\x%x'", bs) +} + func (db *postgres) SQLType(c *schemas.Column) string { var res string switch t := c.SQLType.Name; t { diff --git a/engine.go b/engine.go index 22ba6163..d49eea9a 100644 --- a/engine.go +++ b/engine.go @@ -460,7 +460,7 @@ func formatColumnValue(dstDialect dialects.Dialect, d interface{}, col *schemas. if col.SQLType.IsText() { var v string switch reflect.TypeOf(d).Kind() { - case reflect.Struct, reflect.Array, reflect.Slice: + case reflect.Struct, reflect.Array, reflect.Slice, reflect.Map: bytes, err := json.DefaultJSONHandler.Marshal(d) if err != nil { v = fmt.Sprintf("%s", d) From 711f95d80414037332d602554fab5e89e7def536 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 14 May 2021 17:36:07 +0800 Subject: [PATCH 5/5] Update Changelog for v1.1.0 --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ac229c1..13e721ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,21 @@ This changelog goes through all the changes that have been made in each release without substantial changes to our git log. +## [1.1.0](https://gitea.com/xorm/xorm/releases/tag/1.1.0) - 2021-05-14 + +* FEATURES + * Unsigned Support for mysql (#1889) + * Support modernc.org/sqlite (#1850) +* TESTING + * More tests (#1890) +* MISC + * Byte strings in postgres aren't 0x... (#1906) + * Fix another bug with #1872 (#1905) + * Fix two issues with dumptables (#1903) + * Fix comments (#1896) + * Fix comments (#1893) + * MariaDB 10.5 adds a suffix on old datatypes (#1885) + ## [1.0.7](https://gitea.com/xorm/xorm/pulls?q=&type=all&state=closed&milestone=1336) - 2021-01-21 * BUGFIXES