Merge branch 'master' into lunny/fix_859

.revive.toml

@@ -1,29 +0,0 @@
-ignoreGeneratedHeader = false
-severity = "warning"
-confidence = 0.8
-errorCode = 1
-warningCode = 1
-
-[rule.blank-imports]
-[rule.context-as-argument]
-[rule.context-keys-type]
-[rule.dot-imports]
-[rule.empty-lines]
-[rule.errorf]
-[rule.error-return]
-[rule.error-strings]
-[rule.error-naming]
-[rule.exported]
-[rule.if-return]
-[rule.increment-decrement]
-[rule.indent-error-flow]
-[rule.package-comments]
-[rule.range]
-[rule.receiver-naming]
-[rule.struct-tag]
-[rule.time-naming]
-[rule.unexported-return]
-[rule.unnecessary-stmt]
-[rule.var-declaration]
-[rule.var-naming]
-  arguments = [["ID", "UID", "UUID", "URL", "JSON"], []]

CONTRIBUTING.md

@@ -1,13 +1,13 @@
 ## Contributing to xorm
 
-`xorm` has a backlog of [pull requests](https://help.github.com/articles/using-pull-requests), but contributions are still very
+`xorm` has a backlog of [pull requests](https://gitea.com/xorm/xorm/pulls), but contributions are still very
-much welcome. You can help with patch review, submitting bug reports,
+much welcome. You can help with patch review, submitting [bug reports](https://gitea.com/xorm/xorm/issues),
 or adding new functionality. There is no formal style guide, but
 please conform to the style of existing code and general Go formatting
 conventions when submitting patches.
 
-* [fork a repo](https://help.github.com/articles/fork-a-repo)
+* [fork the repo](https://gitea.com/repo/fork/2038)
-* [creating a pull request ](https://help.github.com/articles/creating-a-pull-request)
+* [creating a pull request ](https://docs.gitea.io/en-us/pull-request/)
 
 ### Language
 
@@ -15,7 +15,7 @@ Since `xorm` is a world-wide open source project, please describe your issues or
 
 ### Sign your codes with comments
 ```
-// !<you github id>! your comments
+// !<your gitea.com id>! your comments
 
 e.g.,
 
@@ -65,7 +65,7 @@ And if your branch is related with cache, you could also enable it via `TEST_CAC
 
 ### Patch review
 
-Help review existing open [pull requests](https://help.github.com/articles/using-pull-requests) by commenting on the code or
+Help review existing open [pull requests](https://gitea.com/xorm/xorm/pulls) by commenting on the code or
 proposed functionality.
 
 ### Bug reports

Makefile

@@ -99,7 +99,6 @@ help:
 	@echo " - clean             delete integration files and build files but not css and js files"
 	@echo " - fmt               format the code"
 	@echo " - lint            	run code linter"
-	@echo " - misspell          check if a word is written wrong"
 	@echo " - test       		run default unit test"
 	@echo " - test-cockroach    run integration tests for cockroach"
 	@echo " - test-mysql        run integration tests for mysql"
@@ -131,27 +130,6 @@ golangci-lint-check:
 		curl -sfL "https://raw.githubusercontent.com/golangci/golangci-lint/v${MIN_GOLANGCI_LINT_VER_FMT}/install.sh" | sh -s -- -b $(GOPATH)/bin v$(MIN_GOLANGCI_LINT_VER_FMT); \
 	fi
 
-.PHONY: revive
-revive:
-	@hash revive > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
-		$(GO) get -u github.com/mgechev/revive; \
-	fi
-	revive -config .revive.toml -exclude=./vendor/... ./... || exit 1
-
-.PHONY: misspell
-misspell:
-	@hash misspell > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
-		$(GO) get -u github.com/client9/misspell/cmd/misspell; \
-	fi
-	misspell -w -i unknwon $(GOFILES)
-
-.PHONY: misspell-check
-misspell-check:
-	@hash misspell > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
-		$(GO) get -u github.com/client9/misspell/cmd/misspell; \
-	fi
-	misspell -error -i unknwon,destory $(GOFILES)
-
 .PHONY: test
 test: go-check
 	$(GO) test $(PACKAGES)

convert/time.go

@@ -48,6 +48,16 @@ func String2Time(s string, originalLocation *time.Location, convertedLocation *t
 		}
 		dt = dt.In(convertedLocation)
 		return &dt, nil
+	} else if len(s) == 10 && s[4] == '-' {
+		if s == "0000-00-00" || s == "0001-01-01" {
+			return &time.Time{}, nil
+		}
+		dt, err := time.ParseInLocation("2006-01-02", s, originalLocation)
+		if err != nil {
+			return nil, err
+		}
+		dt = dt.In(convertedLocation)
+		return &dt, nil
 	} else {
 		i, err := strconv.ParseInt(s, 10, 64)
 		if err == nil {

convert/time_test.go

@@ -16,6 +16,7 @@ func TestString2Time(t *testing.T) {
 	assert.NoError(t, err)
 
 	var kases = map[string]time.Time{
+		"2021-08-10":                time.Date(2021, 8, 10, 8, 0, 0, 0, expectedLoc),
 		"2021-06-06T22:58:20+08:00": time.Date(2021, 6, 6, 22, 58, 20, 0, expectedLoc),
 		"2021-07-11 10:44:00":       time.Date(2021, 7, 11, 18, 44, 0, 0, expectedLoc),
 		"2021-08-10T10:33:04Z":      time.Date(2021, 8, 10, 18, 33, 04, 0, expectedLoc),

dialects/postgres.go

@@ -1300,6 +1300,19 @@ func (db *postgres) GetIndexes(queryer core.Queryer, ctx context.Context, tableN
 			indexType = schemas.IndexType
 		}
 		colNames = getIndexColName(indexdef)
+
+		isSkip := false
+		//Oid It's a special index. You can't put it in
+		for _, element := range colNames {
+			if "oid" == element {
+				isSkip = true
+				break
+			}
+		}
+		if isSkip {
+			continue
+		}
+
 		var isRegular bool
 		if strings.HasPrefix(indexName, "IDX_"+tableName) || strings.HasPrefix(indexName, "UQE_"+tableName) {
 			newIdxName := indexName[5+len(tableName):]

engine.go

@@ -11,7 +11,9 @@ import (
 	"io"
 	"os"
 	"reflect"
+	"regexp"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -438,18 +440,18 @@ func (engine *Engine) DumpTables(tables []*schemas.Table, w io.Writer, tp ...sch
 	return engine.dumpTables(context.Background(), tables, w, tp...)
 }
 
-func formatBool(s string, dstDialect dialects.Dialect) string {
+func formatBool(s bool, dstDialect dialects.Dialect) string {
-	if dstDialect.URI().DBType == schemas.MSSQL {
+	if dstDialect.URI().DBType != schemas.POSTGRES {
-		switch s {
+		if s {
-		case "true":
 			return "1"
-		case "false":
-			return "0"
 		}
+		return "0"
 	}
-	return s
+	return strconv.FormatBool(s)
 }
 
+var controlCharactersRe = regexp.MustCompile(`[\x00-\x1f\x7f]+`)
+
 // dumpTables dump database all table structs and data to w with specify db type
 func (engine *Engine) dumpTables(ctx context.Context, tables []*schemas.Table, w io.Writer, tp ...schemas.DBType) error {
 	var dstDialect dialects.Dialect
@@ -465,7 +467,10 @@ func (engine *Engine) dumpTables(ctx context.Context, tables []*schemas.Table, w
 		destURI := dialects.URI{
 			DBType: tp[0],
 			DBName: uri.DBName,
-			Schema: uri.Schema,
+			// DO NOT SET SCHEMA HERE
+		}
+		if tp[0] == schemas.POSTGRES {
+			destURI.Schema = engine.dialect.URI().Schema
 		}
 		if err := dstDialect.Init(&destURI); err != nil {
 			return err
@@ -480,6 +485,13 @@ func (engine *Engine) dumpTables(ctx context.Context, tables []*schemas.Table, w
 		return err
 	}
 
+	if dstDialect.URI().DBType == schemas.MYSQL {
+		// For MySQL set NO_BACKLASH_ESCAPES so that strings work properly
+		if _, err := io.WriteString(w, "SET sql_mode='NO_BACKSLASH_ESCAPES';\n"); err != nil {
+			return err
+		}
+	}
+
 	for i, table := range tables {
 		dstTable := table
 		if table.Type != nil {
@@ -581,8 +593,13 @@ func (engine *Engine) dumpTables(ctx context.Context, tables []*schemas.Table, w
 						return err
 					}
 				} else {
-					if stp.IsBool() || (dstDialect.URI().DBType == schemas.MSSQL && strings.EqualFold(stp.Name, schemas.Bit)) {
+					if table.Columns()[i].SQLType.IsBool() || stp.IsBool() || (dstDialect.URI().DBType == schemas.MSSQL && strings.EqualFold(stp.Name, schemas.Bit)) {
-						if _, err = io.WriteString(w, formatBool(s.String, dstDialect)); err != nil {
+						val, err := strconv.ParseBool(s.String)
+						if err != nil {
+							return err
+						}
+
+						if _, err = io.WriteString(w, formatBool(val, dstDialect)); err != nil {
 							return err
 						}
 					} else if stp.IsNumeric() {
@@ -594,6 +611,182 @@ func (engine *Engine) dumpTables(ctx context.Context, tables []*schemas.Table, w
 						if _, err = io.WriteString(w, "'"+r+"'"); err != nil {
 							return err
 						}
+					} else if len(s.String) == 0 {
+						if _, err := io.WriteString(w, "''"); err != nil {
+							return err
+						}
+					} else if dstDialect.URI().DBType == schemas.POSTGRES {
+						if dstTable.Columns()[i].SQLType.IsBlob() {
+							// Postgres has the escape format and we should use that for bytea data
+							if _, err := fmt.Fprintf(w, "'\\x%x'", s.String); err != nil {
+								return err
+							}
+						} else {
+							// Postgres concatentates strings using || (NOTE: a NUL byte in a text segment will fail)
+							toCheck := strings.ReplaceAll(s.String, "'", "''")
+							for len(toCheck) > 0 {
+								loc := controlCharactersRe.FindStringIndex(toCheck)
+								if loc == nil {
+									if _, err := io.WriteString(w, "'"+toCheck+"'"); err != nil {
+										return err
+									}
+									break
+								}
+								if loc[0] > 0 {
+									if _, err := io.WriteString(w, "'"+toCheck[:loc[0]]+"' || "); err != nil {
+										return err
+									}
+								}
+								if _, err := io.WriteString(w, "e'"); err != nil {
+									return err
+								}
+								for i := loc[0]; i < loc[1]; i++ {
+									if _, err := fmt.Fprintf(w, "\\x%02x", toCheck[i]); err != nil {
+										return err
+									}
+								}
+								toCheck = toCheck[loc[1]:]
+								if len(toCheck) > 0 {
+									if _, err := io.WriteString(w, "' || "); err != nil {
+										return err
+									}
+								} else {
+									if _, err := io.WriteString(w, "'"); err != nil {
+										return err
+									}
+								}
+							}
+						}
+					} else if dstDialect.URI().DBType == schemas.MYSQL {
+						loc := controlCharactersRe.FindStringIndex(s.String)
+						if loc == nil {
+							if _, err := io.WriteString(w, "'"+strings.ReplaceAll(s.String, "'", "''")+"'"); err != nil {
+								return err
+							}
+						} else {
+							if _, err := io.WriteString(w, "CONCAT("); err != nil {
+								return err
+							}
+							toCheck := strings.ReplaceAll(s.String, "'", "''")
+							for len(toCheck) > 0 {
+								loc := controlCharactersRe.FindStringIndex(toCheck)
+								if loc == nil {
+									if _, err := io.WriteString(w, "'"+toCheck+"')"); err != nil {
+										return err
+									}
+									break
+								}
+								if loc[0] > 0 {
+									if _, err := io.WriteString(w, "'"+toCheck[:loc[0]]+"', "); err != nil {
+										return err
+									}
+								}
+								for i := loc[0]; i < loc[1]-1; i++ {
+									if _, err := io.WriteString(w, "CHAR("+strconv.Itoa(int(toCheck[i]))+"), "); err != nil {
+										return err
+									}
+								}
+								char := toCheck[loc[1]-1]
+								toCheck = toCheck[loc[1]:]
+								if len(toCheck) > 0 {
+									if _, err := io.WriteString(w, "CHAR("+strconv.Itoa(int(char))+"), "); err != nil {
+										return err
+									}
+								} else {
+									if _, err = io.WriteString(w, "CHAR("+strconv.Itoa(int(char))+"))"); err != nil {
+										return err
+									}
+								}
+							}
+						}
+					} else if dstDialect.URI().DBType == schemas.SQLITE {
+						if dstTable.Columns()[i].SQLType.IsBlob() {
+							// SQLite has its escape format
+							if _, err := fmt.Fprintf(w, "X'%x'", s.String); err != nil {
+								return err
+							}
+						} else {
+							// SQLite concatentates strings using || (NOTE: a NUL byte in a text segment will fail)
+							toCheck := strings.ReplaceAll(s.String, "'", "''")
+							for len(toCheck) > 0 {
+								loc := controlCharactersRe.FindStringIndex(toCheck)
+								if loc == nil {
+									if _, err := io.WriteString(w, "'"+toCheck+"'"); err != nil {
+										return err
+									}
+									break
+								}
+								if loc[0] > 0 {
+									if _, err := io.WriteString(w, "'"+toCheck[:loc[0]]+"' || "); err != nil {
+										return err
+									}
+								}
+								if _, err := fmt.Fprintf(w, "X'%x'", toCheck[loc[0]:loc[1]]); err != nil {
+									return err
+								}
+								toCheck = toCheck[loc[1]:]
+								if len(toCheck) > 0 {
+									if _, err := io.WriteString(w, " || "); err != nil {
+										return err
+									}
+								}
+							}
+						}
+					} else if dstDialect.URI().DBType == schemas.DAMENG || dstDialect.URI().DBType == schemas.ORACLE {
+						if dstTable.Columns()[i].SQLType.IsBlob() {
+							// ORACLE/DAMENG uses HEXTORAW
+							if _, err := fmt.Fprintf(w, "HEXTORAW('%x')", s.String); err != nil {
+								return err
+							}
+						} else {
+							// ORACLE/DAMENG concatentates strings in multiple ways but uses CHAR and has CONCAT
+							// (NOTE: a NUL byte in a text segment will fail)
+							if _, err := io.WriteString(w, "CONCAT("); err != nil {
+								return err
+							}
+							toCheck := strings.ReplaceAll(s.String, "'", "''")
+							for len(toCheck) > 0 {
+								loc := controlCharactersRe.FindStringIndex(toCheck)
+								if loc == nil {
+									if _, err := io.WriteString(w, "'"+toCheck+"')"); err != nil {
+										return err
+									}
+									break
+								}
+								if loc[0] > 0 {
+									if _, err := io.WriteString(w, "'"+toCheck[:loc[0]]+"', "); err != nil {
+										return err
+									}
+								}
+								for i := loc[0]; i < loc[1]-1; i++ {
+									if _, err := io.WriteString(w, "CHAR("+strconv.Itoa(int(toCheck[i]))+"), "); err != nil {
+										return err
+									}
+								}
+								char := toCheck[loc[1]-1]
+								toCheck = toCheck[loc[1]:]
+								if len(toCheck) > 0 {
+									if _, err := io.WriteString(w, "CHAR("+strconv.Itoa(int(char))+"), "); err != nil {
+										return err
+									}
+								} else {
+									if _, err = io.WriteString(w, "CHAR("+strconv.Itoa(int(char))+"))"); err != nil {
+										return err
+									}
+								}
+							}
+						}
+					} else if dstDialect.URI().DBType == schemas.MSSQL {
+						if dstTable.Columns()[i].SQLType.IsBlob() {
+							// MSSQL uses CONVERT(VARBINARY(MAX), '0xDEADBEEF', 1)
+							if _, err := fmt.Fprintf(w, "CONVERT(VARBINARY(MAX), '0x%x', 1)", s.String); err != nil {
+								return err
+							}
+						} else {
+							if _, err = io.WriteString(w, "N'"+strings.ReplaceAll(s.String, "'", "''")+"'"); err != nil {
+								return err
+							}
+						}
 					} else {
 						if _, err = io.WriteString(w, "'"+strings.ReplaceAll(s.String, "'", "''")+"'"); err != nil {
 							return err

integrations/engine_test.go

@@ -143,6 +143,7 @@ func TestDumpTables(t *testing.T) {
 
 	type TestDumpTableStruct struct {
 		Id      int64
+		Data    []byte `xorm:"BLOB"`
 		Name    string
 		IsMan   bool
 		Created time.Time `xorm:"created"`
@@ -152,10 +153,14 @@ func TestDumpTables(t *testing.T) {
 
 	_, err := testEngine.Insert([]TestDumpTableStruct{
 		{Name: "1", IsMan: true},
-		{Name: "2\n"},
+		{Name: "2\n", Data: []byte{'\000', '\001', '\002'}},
-		{Name: "3;"},
+		{Name: "3;", Data: []byte("0x000102")},
-		{Name: "4\n;\n''"},
+		{Name: "4\n;\n''", Data: []byte("Help")},
-		{Name: "5'\n"},
+		{Name: "5'\n", Data: []byte("0x48656c70")},
+		{Name: "6\\n'\n", Data: []byte("48656c70")},
+		{Name: "7\\n'\r\n", Data: []byte("7\\n'\r\n")},
+		{Name: "x0809ee"},
+		{Name: "090a10"},
 	})
 	assert.NoError(t, err)
 

integrations/session_find_test.go

@@ -711,6 +711,36 @@ func TestFindAndCountWithGroupBy(t *testing.T) {
 	assert.EqualValues(t, 2, len(results))
 }
 
+func TestFindAndCountWithDistinct(t *testing.T) {
+	assert.NoError(t, PrepareEngine())
+
+	type FindAndCountWithDistinct struct {
+		Id   int64
+		Age  int `xorm:"index"`
+		Name string
+	}
+
+	assert.NoError(t, testEngine.Sync(new(FindAndCountWithDistinct)))
+
+	_, err := testEngine.Insert([]FindAndCountWithDistinct{
+		{
+			Name: "test1",
+			Age:  10,
+		},
+		{
+			Name: "test2",
+			Age:  20,
+		},
+	})
+	assert.NoError(t, err)
+
+	var results []FindAndCountWithDistinct
+	cnt, err := testEngine.Distinct("`age`").FindAndCount(&results)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 2, cnt)
+	assert.EqualValues(t, 2, len(results))
+}
+
 type FindMapDevice struct {
 	Deviceid string `xorm:"pk"`
 	Status   int

internal/statements/query.go

@@ -334,7 +334,7 @@ func (statement *Statement) genSelectSQL(columnStr string, needLimit, needOrderB
 				fmt.Fprint(&buf, " LIMIT ", *pLimitN)
 			}
 		} else if dialect.URI().DBType == schemas.ORACLE {
-			if statement.Start != 0 && pLimitN != nil {
+			if pLimitN != nil {
 				oldString := buf.String()
 				buf.Reset()
 				rawColStr := columnStr

session_find.go

@@ -57,7 +57,7 @@ func (session *Session) FindAndCount(rowsSlicePtr interface{}, condiBean ...inte
 	if session.statement.SelectStr != "" {
 		session.statement.SelectStr = ""
 	}
-	if len(session.statement.ColumnMap) > 0 {
+	if len(session.statement.ColumnMap) > 0 && !session.statement.IsDistinct {
 		session.statement.ColumnMap = []string{}
 	}
 	if session.statement.OrderStr != "" {
@@ -254,9 +254,9 @@ func (session *Session) noCacheFind(table *schemas.Table, containerValue reflect
 
 		switch elemType.Kind() {
 		case reflect.Slice:
-			err = rows.ScanSlice(bean)
+			err = session.getSlice(rows, types, fields, bean)
 		case reflect.Map:
-			err = rows.ScanMap(bean)
+			err = session.getMap(rows, types, fields, bean)
 		default:
 			err = rows.Scan(bean)
 		}

session_insert.go

@@ -142,6 +142,13 @@ func (session *Session) insertMultipleStruct(rowsSlicePtr interface{}) (int64, e
 			if len(session.statement.ColumnMap) > 0 && !session.statement.ColumnMap.Contain(col.Name) {
 				continue
 			}
+			// !satorunooshie! set fieldValue as nil when column is nullable and zero-value
+			if _, ok := getFlagForColumn(session.statement.NullableMap, col); ok {
+				if col.Nullable && utils.IsValueZero(fieldValue) {
+					var nilValue *int
+					fieldValue = reflect.ValueOf(nilValue)
+				}
+			}
 			if (col.IsCreated || col.IsUpdated) && session.statement.UseAutoTime {
 				val, t, err := session.engine.nowTime(col)
 				if err != nil {