Add cockroach support and tests (#896)

Fix tests

fix some tests and ignore some

Add &experimental_serial_normalization=sql_sequence on test so that id will increase from 1

fix drone

Fix drone

Fix drone

Fix drone

Remove space

Add makefile

fix drone

fix drone

fix drone

update

fix tests

use sql_sequence when testing

fix postgres ci

fix circle ci for cockroach

upgrade cockroach version

add trace on error

add drone ci

run cockroach background

run cockroach backround

run cockroach backround

fix test

fix run cockroach

add cockroach test to circleci

add cockroach support

Add tests for table name (#1...
Reviewed-on: https://gitea.com/xorm/xorm/pulls/896

.drone.yml

@@ -180,6 +180,25 @@ steps:
     - push
     - pull_request
 
+- name: test-cockroach
+  pull: default
+  image: golang:1.13
+  environment:
+    GO111MODULE: "on"
+    GOPROXY: "https://goproxy.cn"
+    TEST_COCKROACH_HOST: "cockroach:26257"
+    TEST_COCKROACH_DBNAME: xorm_test
+    TEST_COCKROACH_USERNAME: root
+    TEST_COCKROACH_PASSWORD:
+  commands:
+  - sleep 10
+  - make test-cockroach
+  - TEST_CACHE_ENABLE=true make test-cockroach
+  when:
+    event:
+    - push
+    - pull_request
+
 - name: merge_coverage
   pull: default
   image: golang:1.12
@@ -196,6 +215,7 @@ steps:
   - test-postgres-schema
   - test-mssql
   - test-tidb
+  - test-cockroach
   commands:
   - make coverage
   when:
@@ -262,4 +282,15 @@ services:
     event:
     - push
     - tag
-    - pull_request
+    - pull_request
+
+- name: cockroach
+  pull: default
+  image: cockroachdb/cockroach:v19.2.4
+  commands:
+  - /cockroach/cockroach start --insecure
+  when:
+    event:
+    - push
+    - tag
+    - pull_request

Makefile

@@ -10,6 +10,12 @@ GOFILES := $(shell find . -name "*.go" -type f)
 
 PACKAGES ?= $(shell GO111MODULE=on $(GO) list ./...)
 
+TEST_COCKROACH_HOST ?= cockroach:26257
+TEST_COCKROACH_SCHEMA ?=
+TEST_COCKROACH_DBNAME ?= xorm_test
+TEST_COCKROACH_USERNAME ?= postgres
+TEST_COCKROACH_PASSWORD ?=
+
 TEST_MSSQL_HOST ?= mssql:1433
 TEST_MSSQL_DBNAME ?= gitea
 TEST_MSSQL_USERNAME ?= sa
@@ -115,6 +121,18 @@ misspell-check:
 .PHONY: test
 test: test-sqlite
 
+.PNONY: test-cockroach
+test-cockroach: go-check
+	$(GO) test -race -db=postgres -schema='$(TEST_COCKROACH_SCHEMA)' -cache=$(TEST_CACHE_ENABLE) \
+	-conn_str="postgres://$(TEST_COCKROACH_USERNAME):$(TEST_COCKROACH_PASSWORD)@$(TEST_COCKROACH_HOST)/$(TEST_COCKROACH_DBNAME)?sslmode=disable&experimental_serial_normalization=sql_sequence" \
+	-ignore_update_limit=true -coverprofile=cockroach.$(TEST_COCKROACH_SCHEMA).$(TEST_CACHE_ENABLE).coverage.out -covermode=atomic
+
+.PHONY: test-cockroach\#%
+test-cockroach\#%: go-check
+	$(GO) test -race -run $* -db=postgres -schema='$(TEST_COCKROACH_SCHEMA)' -cache=$(TEST_CACHE_ENABLE) \
+	-conn_str="postgres://$(TEST_COCKROACH_USERNAME):$(TEST_COCKROACH_PASSWORD)@$(TEST_COCKROACH_HOST)/$(TEST_COCKROACH_DBNAME)?sslmode=disable&experimental_serial_normalization=sql_sequence" \
+	-ignore_update_limit=true -coverprofile=cockroach.$(TEST_COCKROACH_SCHEMA).$(TEST_CACHE_ENABLE).coverage.out -covermode=atomic
+
 .PNONY: test-mssql
 test-mssql: go-check
 	$(GO) test -v -race -db=mssql -cache=$(TEST_CACHE_ENABLE) \

circle.yml

@@ -0,0 +1,46 @@
+dependencies:
+  override:
+    # './...' is a relative pattern which means all subdirectories
+    - go get -t -d -v ./...
+    - go get -t -d -v github.com/go-xorm/tests
+    - go get -u github.com/go-xorm/core
+    - go get -u github.com/go-xorm/builder
+    - go build -v
+
+database:
+  override:
+    - mysql -u root -e "CREATE DATABASE xorm_test DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci"
+    - mysql -u root -e "CREATE DATABASE xorm_test1 DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci"
+    - mysql -u root -e "CREATE DATABASE xorm_test2 DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci"
+    - mysql -u root -e "CREATE DATABASE xorm_test3 DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci"
+    - createdb -p 5432 -e -U postgres xorm_test
+    - createdb -p 5432 -e -U postgres xorm_test1
+    - createdb -p 5432 -e -U postgres xorm_test2
+    - createdb -p 5432 -e -U postgres xorm_test3
+    - psql xorm_test postgres -c "create schema xorm"
+
+test:
+  override:
+    # './...' is a relative pattern which means all subdirectories
+    - go get -u github.com/wadey/gocovmerge
+    - go test -v -race -db="sqlite3" -conn_str="./test.db" -coverprofile=coverage1-1.txt -covermode=atomic
+    - go test -v -race -db="sqlite3" -conn_str="./test.db" -cache=true -coverprofile=coverage1-2.txt -covermode=atomic
+    - go test -v -race -db="mysql" -conn_str="root:@/xorm_test" -coverprofile=coverage2-1.txt -covermode=atomic
+    - go test -v -race -db="mysql" -conn_str="root:@/xorm_test" -cache=true -coverprofile=coverage2-2.txt -covermode=atomic
+    - go test -v -race -db="mymysql" -conn_str="xorm_test/root/" -coverprofile=coverage3-1.txt -covermode=atomic
+    - go test -v -race -db="mymysql" -conn_str="xorm_test/root/" -cache=true -coverprofile=coverage3-2.txt -covermode=atomic
+    - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -coverprofile=coverage4-1.txt -covermode=atomic
+    - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -cache=true -coverprofile=coverage4-2.txt -covermode=atomic
+    - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -coverprofile=coverage5-1.txt -covermode=atomic
+    - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -cache=true -coverprofile=coverage5-2.txt -covermode=atomic
+    - wget -qO- https://binaries.cockroachdb.com/cockroach-v2.1.3.linux-amd64.tgz | tar  xvz
+    - ./cockroach-v2.1.3.linux-amd64/cockroach start --insecure --background --host=localhost
+    - sleep 5
+    - ./cockroach-v2.1.3.linux-amd64/cockroach sql --insecure --execute="create database xorm_test"
+    - go test -v -race -db="postgres" -conn_str="postgresql://root@localhost:26257/xorm_test?sslmode=disable" -coverprofile=coverage6-1.txt -covermode=atomic
+    - gocovmerge coverage1-1.txt coverage1-2.txt coverage2-1.txt coverage2-2.txt coverage3-1.txt coverage3-2.txt coverage4-1.txt coverage4-2.txt coverage5-1.txt coverage5-2.txt coverage6-1.txt> coverage.txt
+    - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./sqlite3.sh
+    - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./mysql.sh
+    - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./postgres.sh
+  post:
+    - bash <(curl -s https://codecov.io/bash)

dialects/postgres.go

@@ -995,7 +995,6 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att
 			return nil, nil, err
 		}
 
-		// fmt.Println(args, colName, isNullable, dataType, maxLenStr, colDefault, isPK, isUnique)
 		var maxLen int
 		if maxLenStr != nil {
 			maxLen, err = strconv.Atoi(*maxLenStr)
@@ -1007,7 +1006,22 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att
 		col.Name = strings.Trim(colName, `" `)
 
 		if colDefault != nil {
-			col.Default = *colDefault
+			var theDefault = *colDefault
+			// cockroach has type with the default value with :::
+			// and postgres with ::, we should remove them before store them
+			idx := strings.Index(theDefault, ":::")
+			if idx == -1 {
+				idx = strings.Index(theDefault, "::")
+			}
+			if idx > -1 {
+				theDefault = theDefault[:idx]
+			}
+
+			if strings.HasSuffix(theDefault, "+00:00'") {
+				theDefault = theDefault[:len(theDefault)-7] + "'"
+			}
+
+			col.Default = theDefault
 			col.DefaultIsEmpty = false
 			if strings.HasPrefix(col.Default, "nextval(") {
 				col.IsAutoIncrement = true
@@ -1022,8 +1036,8 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att
 
 		col.Nullable = (isNullable == "YES")
 
-		switch dataType {
+		switch strings.ToLower(dataType) {
-		case "character varying", "character":
+		case "character varying", "character", "string":
 			col.SQLType = schemas.SQLType{Name: schemas.Varchar, DefaultLength: 0, DefaultLength2: 0}
 		case "timestamp without time zone":
 			col.SQLType = schemas.SQLType{Name: schemas.DateTime, DefaultLength: 0, DefaultLength2: 0}
@@ -1035,13 +1049,22 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att
 			col.SQLType = schemas.SQLType{Name: schemas.Bool, DefaultLength: 0, DefaultLength2: 0}
 		case "time without time zone":
 			col.SQLType = schemas.SQLType{Name: schemas.Time, DefaultLength: 0, DefaultLength2: 0}
+		case "bytes":
+			col.SQLType = schemas.SQLType{Name: schemas.Binary, DefaultLength: 0, DefaultLength2: 0}
 		case "oid":
 			col.SQLType = schemas.SQLType{Name: schemas.BigInt, DefaultLength: 0, DefaultLength2: 0}
 		default:
-			col.SQLType = schemas.SQLType{Name: strings.ToUpper(dataType), DefaultLength: 0, DefaultLength2: 0}
+			startIdx := strings.Index(strings.ToLower(dataType), "string(")
+			if startIdx != -1 && strings.HasSuffix(dataType, ")") {
+				length := dataType[startIdx+8 : len(dataType)-1]
+				l, _ := strconv.Atoi(length)
+				col.SQLType = schemas.SQLType{Name: "STRING", DefaultLength: l, DefaultLength2: 0}
+			} else {
+				col.SQLType = schemas.SQLType{Name: strings.ToUpper(dataType), DefaultLength: 0, DefaultLength2: 0}
+			}
 		}
 		if _, ok := schemas.SqlTypes[col.SQLType.Name]; !ok {
-			return nil, nil, fmt.Errorf("Unknown colType: %v", dataType)
+			return nil, nil, fmt.Errorf("Unknown colType: %s - %s", dataType, col.SQLType.Name)
 		}
 
 		col.Length = maxLen
@@ -1128,6 +1151,10 @@ func (db *postgres) GetIndexes(ctx context.Context, tableName string) (map[strin
 		if err != nil {
 			return nil, err
 		}
+
+		if indexName == "primary" {
+			continue
+		}
 		indexName = strings.Trim(indexName, `" `)
 		if strings.HasSuffix(indexName, "_pkey") {
 			continue
@@ -1149,7 +1176,7 @@ func (db *postgres) GetIndexes(ctx context.Context, tableName string) (map[strin
 
 		index := &schemas.Index{Name: indexName, Type: indexType, Cols: make([]string, 0)}
 		for _, colName := range colNames {
-			index.Cols = append(index.Cols, strings.Trim(colName, `" `))
+			index.Cols = append(index.Cols, strings.TrimSpace(strings.Replace(colName, `"`, "", -1)))
 		}
 		index.IsRegular = isRegular
 		indexes[index.Name] = index

engine.go

@@ -284,12 +284,19 @@ func (engine *Engine) loadTableInfo(table *schemas.Table) error {
 	}
 	table.Indexes = indexes
 
+	var seq int
 	for _, index := range indexes {
 		for _, name := range index.Cols {
-			if col := table.GetColumn(name); col != nil {
+			parts := strings.Split(name, " ")
+			if len(parts) > 1 {
+				if parts[1] == "DESC" {
+					seq = 1
+				}
+			}
+			if col := table.GetColumn(parts[0]); col != nil {
 				col.Indexes[index.Name] = index.Type
 			} else {
-				return fmt.Errorf("Unknown col %s in index %v of table %v, columns %v", name, index.Name, table.Name, table.ColumnsSeq())
+				return fmt.Errorf("Unknown col %s seq %d, in index %v of table %v, columns %v", name, seq, index.Name, table.Name, table.ColumnsSeq())
 			}
 		}
 	}

session_get_test.go

@@ -335,13 +335,13 @@ func TestJSONString(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, has)
 	assert.EqualValues(t, 1, js.Id)
-	assert.EqualValues(t, `["1","2"]`, js.Content)
+	assert.True(t, `["1","2"]` == js.Content || `["1", "2"]` == js.Content)
 
 	var jss []JsonString
 	err = testEngine.Table("json_json").Find(&jss)
 	assert.NoError(t, err)
 	assert.EqualValues(t, 1, len(jss))
-	assert.EqualValues(t, `["1","2"]`, jss[0].Content)
+	assert.True(t, `["1","2"]` == jss[0].Content || `["1", "2"]` == jss[0].Content)
 }
 
 func TestGetActionMapping(t *testing.T) {

session_update_test.go

@@ -42,6 +42,11 @@ func TestUpdateMap(t *testing.T) {
 }
 
 func TestUpdateLimit(t *testing.T) {
+	if *ingoreUpdateLimit {
+		t.Skip()
+		return
+	}
+
 	assert.NoError(t, prepareEngine())
 
 	type UpdateTable2 struct {

tags_test.go

@@ -1031,6 +1031,7 @@ func TestTagDefault4(t *testing.T) {
 	}
 	assert.True(t, isDefaultExist)
 	assert.True(t, "CURRENT_TIMESTAMP" == defaultVal ||
+		"current_timestamp()" == defaultVal || // for cockroach
 		"now()" == defaultVal ||
 		"getdate" == defaultVal, defaultVal)
 }

xorm_test.go

@@ -37,9 +37,9 @@ var (
 	splitter           = flag.String("splitter", ";", "the splitter on connstr for cluster")
 	schema             = flag.String("schema", "", "specify the schema")
 	ignoreSelectUpdate = flag.Bool("ignore_select_update", false, "ignore select update if implementation difference, only for tidb")
-
+	ingoreUpdateLimit  = flag.Bool("ignore_update_limit", false, "ignore update limit if implementation difference, only for cockroach")
-	tableMapper names.Mapper
+	tableMapper        names.Mapper
-	colMapper   names.Mapper
+	colMapper          names.Mapper
 )
 
 func createEngine(dbType, connStr string) error {
@@ -59,11 +59,11 @@ func createEngine(dbType, connStr string) error {
 				db.Close()
 				*ignoreSelectUpdate = true
 			case schemas.POSTGRES:
-				db, err := sql.Open(dbType, connStr)
+				db, err := sql.Open(dbType, strings.Replace(connStr, "xorm_test", "postgres", -1))
 				if err != nil {
 					return err
 				}
-				rows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = 'xorm_test'"))
+				rows, err := db.Query("SELECT 1 FROM pg_database WHERE datname = 'xorm_test'")
 				if err != nil {
 					return fmt.Errorf("db.Query: %v", err)
 				}
@@ -75,6 +75,12 @@ func createEngine(dbType, connStr string) error {
 					}
 				}
 				if *schema != "" {
+					db.Close()
+					db, err = sql.Open(dbType, connStr)
+					if err != nil {
+						return err
+					}
+					defer db.Close()
 					if _, err = db.Exec("CREATE SCHEMA IF NOT EXISTS " + *schema); err != nil {
 						return fmt.Errorf("CREATE SCHEMA: %v", err)
 					}
@@ -178,6 +184,7 @@ func TestMain(m *testing.M) {
 
 		if err := prepareEngine(); err != nil {
 			fmt.Println(err)
+			os.Exit(1)
 			return
 		}