Add cockroach support and tests (#896)

Fix tests fix some tests and ignore some Add &experimental_serial_normalization=sql_sequence on test so that id will increase from 1 fix drone Fix drone Fix drone Fix drone Remove space Add makefile fix drone fix drone fix drone update fix tests use sql_sequence when testing fix postgres ci fix circle ci for cockroach upgrade cockroach version add trace on error add drone ci run cockroach background run cockroach backround run cockroach backround fix test fix run cockroach add cockroach test to circleci add cockroach support Add tests for table name (#1... Reviewed-on: https://gitea.com/xorm/xorm/pulls/896
2020-03-03 08:19:32 +00:00 · 2020-03-03 08:19:32 +00:00 · b079bc05ff
parent 4e59597a49
commit b079bc05ff
9 changed files with 159 additions and 17 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -180,6 +180,25 @@ steps:
    - push
    - pull_request

+- name: test-cockroach
+  pull: default
+  image: golang:1.13
+  environment:
+    GO111MODULE: "on"
+    GOPROXY: "https://goproxy.cn"
+    TEST_COCKROACH_HOST: "cockroach:26257"
+    TEST_COCKROACH_DBNAME: xorm_test
+    TEST_COCKROACH_USERNAME: root
+    TEST_COCKROACH_PASSWORD:
+  commands:
+  - sleep 10
+  - make test-cockroach
+  - TEST_CACHE_ENABLE=true make test-cockroach
+  when:
+    event:
+    - push
+    - pull_request
+
 - name: merge_coverage
  pull: default
  image: golang:1.12
@ -196,6 +215,7 @@ steps:
  - test-postgres-schema
  - test-mssql
  - test-tidb
+  - test-cockroach
  commands:
  - make coverage
  when:
@ -263,3 +283,14 @@ services:
    - push
    - tag
    - pull_request
+
+- name: cockroach
+  pull: default
+  image: cockroachdb/cockroach:v19.2.4
+  commands:
+  - /cockroach/cockroach start --insecure
+  when:
+    event:
+    - push
+    - tag
+    - pull_request
--- a/18
+++ b/18
@ -10,6 +10,12 @@ GOFILES := $(shell find . -name "*.go" -type f)

 PACKAGES ?= $(shell GO111MODULE=on $(GO) list ./...)

+TEST_COCKROACH_HOST ?= cockroach:26257
+TEST_COCKROACH_SCHEMA ?=
+TEST_COCKROACH_DBNAME ?= xorm_test
+TEST_COCKROACH_USERNAME ?= postgres
+TEST_COCKROACH_PASSWORD ?=
+
 TEST_MSSQL_HOST ?= mssql:1433
 TEST_MSSQL_DBNAME ?= gitea
 TEST_MSSQL_USERNAME ?= sa
@ -115,6 +121,18 @@ misspell-check:
 .PHONY: test
 test: test-sqlite

+.PNONY: test-cockroach
+test-cockroach: go-check
+	$(GO) test -race -db=postgres -schema='$(TEST_COCKROACH_SCHEMA)' -cache=$(TEST_CACHE_ENABLE) \
+	-conn_str="postgres://$(TEST_COCKROACH_USERNAME):$(TEST_COCKROACH_PASSWORD)@$(TEST_COCKROACH_HOST)/$(TEST_COCKROACH_DBNAME)?sslmode=disable&experimental_serial_normalization=sql_sequence" \
+	-ignore_update_limit=true -coverprofile=cockroach.$(TEST_COCKROACH_SCHEMA).$(TEST_CACHE_ENABLE).coverage.out -covermode=atomic
+
+.PHONY: test-cockroach\#%
+test-cockroach\#%: go-check
+	$(GO) test -race -run $* -db=postgres -schema='$(TEST_COCKROACH_SCHEMA)' -cache=$(TEST_CACHE_ENABLE) \
+	-conn_str="postgres://$(TEST_COCKROACH_USERNAME):$(TEST_COCKROACH_PASSWORD)@$(TEST_COCKROACH_HOST)/$(TEST_COCKROACH_DBNAME)?sslmode=disable&experimental_serial_normalization=sql_sequence" \
+	-ignore_update_limit=true -coverprofile=cockroach.$(TEST_COCKROACH_SCHEMA).$(TEST_CACHE_ENABLE).coverage.out -covermode=atomic
+
 .PNONY: test-mssql
 test-mssql: go-check
 	$(GO) test -v -race -db=mssql -cache=$(TEST_CACHE_ENABLE) \
--- a/circle.yml
+++ b/circle.yml
@ -0,0 +1,46 @@
+dependencies:
+  override:
+    # './...' is a relative pattern which means all subdirectories
+    - go get -t -d -v ./...
+    - go get -t -d -v github.com/go-xorm/tests
+    - go get -u github.com/go-xorm/core
+    - go get -u github.com/go-xorm/builder
+    - go build -v
+
+database:
+  override:
+    - mysql -u root -e "CREATE DATABASE xorm_test DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci"
+    - mysql -u root -e "CREATE DATABASE xorm_test1 DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci"
+    - mysql -u root -e "CREATE DATABASE xorm_test2 DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci"
+    - mysql -u root -e "CREATE DATABASE xorm_test3 DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci"
+    - createdb -p 5432 -e -U postgres xorm_test
+    - createdb -p 5432 -e -U postgres xorm_test1
+    - createdb -p 5432 -e -U postgres xorm_test2
+    - createdb -p 5432 -e -U postgres xorm_test3
+    - psql xorm_test postgres -c "create schema xorm"
+
+test:
+  override:
+    # './...' is a relative pattern which means all subdirectories
+    - go get -u github.com/wadey/gocovmerge
+    - go test -v -race -db="sqlite3" -conn_str="./test.db" -coverprofile=coverage1-1.txt -covermode=atomic
+    - go test -v -race -db="sqlite3" -conn_str="./test.db" -cache=true -coverprofile=coverage1-2.txt -covermode=atomic
+    - go test -v -race -db="mysql" -conn_str="root:@/xorm_test" -coverprofile=coverage2-1.txt -covermode=atomic
+    - go test -v -race -db="mysql" -conn_str="root:@/xorm_test" -cache=true -coverprofile=coverage2-2.txt -covermode=atomic
+    - go test -v -race -db="mymysql" -conn_str="xorm_test/root/" -coverprofile=coverage3-1.txt -covermode=atomic
+    - go test -v -race -db="mymysql" -conn_str="xorm_test/root/" -cache=true -coverprofile=coverage3-2.txt -covermode=atomic
+    - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -coverprofile=coverage4-1.txt -covermode=atomic
+    - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -cache=true -coverprofile=coverage4-2.txt -covermode=atomic
+    - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -coverprofile=coverage5-1.txt -covermode=atomic
+    - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -cache=true -coverprofile=coverage5-2.txt -covermode=atomic
+    - wget -qO- https://binaries.cockroachdb.com/cockroach-v2.1.3.linux-amd64.tgz | tar  xvz
+    - ./cockroach-v2.1.3.linux-amd64/cockroach start --insecure --background --host=localhost
+    - sleep 5
+    - ./cockroach-v2.1.3.linux-amd64/cockroach sql --insecure --execute="create database xorm_test"
+    - go test -v -race -db="postgres" -conn_str="postgresql://root@localhost:26257/xorm_test?sslmode=disable" -coverprofile=coverage6-1.txt -covermode=atomic
+    - gocovmerge coverage1-1.txt coverage1-2.txt coverage2-1.txt coverage2-2.txt coverage3-1.txt coverage3-2.txt coverage4-1.txt coverage4-2.txt coverage5-1.txt coverage5-2.txt coverage6-1.txt> coverage.txt
+    - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./sqlite3.sh
+    - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./mysql.sh
+    - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./postgres.sh
+  post:
+    - bash <(curl -s https://codecov.io/bash)
--- a/dialects/postgres.go
+++ b/dialects/postgres.go
@ -995,7 +995,6 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att
 			return nil, nil, err
 		}

-		// fmt.Println(args, colName, isNullable, dataType, maxLenStr, colDefault, isPK, isUnique)
 		var maxLen int
 		if maxLenStr != nil {
 			maxLen, err = strconv.Atoi(*maxLenStr)
@ -1007,7 +1006,22 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att
 		col.Name = strings.Trim(colName, `" `)

 		if colDefault != nil {
-			col.Default = *colDefault
+			var theDefault = *colDefault
+			// cockroach has type with the default value with :::
+			// and postgres with ::, we should remove them before store them
+			idx := strings.Index(theDefault, ":::")
+			if idx == -1 {
+				idx = strings.Index(theDefault, "::")
+			}
+			if idx > -1 {
+				theDefault = theDefault[:idx]
+			}
+
+			if strings.HasSuffix(theDefault, "+00:00'") {
+				theDefault = theDefault[:len(theDefault)-7] + "'"
+			}
+
+			col.Default = theDefault
 			col.DefaultIsEmpty = false
 			if strings.HasPrefix(col.Default, "nextval(") {
 				col.IsAutoIncrement = true
@ -1022,8 +1036,8 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att

 		col.Nullable = (isNullable == "YES")

-		switch dataType {
-		case "character varying", "character":
+		switch strings.ToLower(dataType) {
+		case "character varying", "character", "string":
 			col.SQLType = schemas.SQLType{Name: schemas.Varchar, DefaultLength: 0, DefaultLength2: 0}
 		case "timestamp without time zone":
 			col.SQLType = schemas.SQLType{Name: schemas.DateTime, DefaultLength: 0, DefaultLength2: 0}
@ -1035,13 +1049,22 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att
 			col.SQLType = schemas.SQLType{Name: schemas.Bool, DefaultLength: 0, DefaultLength2: 0}
 		case "time without time zone":
 			col.SQLType = schemas.SQLType{Name: schemas.Time, DefaultLength: 0, DefaultLength2: 0}
+		case "bytes":
+			col.SQLType = schemas.SQLType{Name: schemas.Binary, DefaultLength: 0, DefaultLength2: 0}
 		case "oid":
 			col.SQLType = schemas.SQLType{Name: schemas.BigInt, DefaultLength: 0, DefaultLength2: 0}
 		default:
+			startIdx := strings.Index(strings.ToLower(dataType), "string(")
+			if startIdx != -1 && strings.HasSuffix(dataType, ")") {
+				length := dataType[startIdx+8 : len(dataType)-1]
+				l, _ := strconv.Atoi(length)
+				col.SQLType = schemas.SQLType{Name: "STRING", DefaultLength: l, DefaultLength2: 0}
+			} else {
 				col.SQLType = schemas.SQLType{Name: strings.ToUpper(dataType), DefaultLength: 0, DefaultLength2: 0}
 			}
+		}
 		if _, ok := schemas.SqlTypes[col.SQLType.Name]; !ok {
-			return nil, nil, fmt.Errorf("Unknown colType: %v", dataType)
+			return nil, nil, fmt.Errorf("Unknown colType: %s - %s", dataType, col.SQLType.Name)
 		}

 		col.Length = maxLen
@ -1128,6 +1151,10 @@ func (db *postgres) GetIndexes(ctx context.Context, tableName string) (map[strin
 		if err != nil {
 			return nil, err
 		}
+
+		if indexName == "primary" {
+			continue
+		}
 		indexName = strings.Trim(indexName, `" `)
 		if strings.HasSuffix(indexName, "_pkey") {
 			continue
@ -1149,7 +1176,7 @@ func (db *postgres) GetIndexes(ctx context.Context, tableName string) (map[strin

 		index := &schemas.Index{Name: indexName, Type: indexType, Cols: make([]string, 0)}
 		for _, colName := range colNames {
-			index.Cols = append(index.Cols, strings.Trim(colName, `" `))
+			index.Cols = append(index.Cols, strings.TrimSpace(strings.Replace(colName, `"`, "", -1)))
 		}
 		index.IsRegular = isRegular
 		indexes[index.Name] = index
--- a/engine.go
+++ b/engine.go
@ -284,12 +284,19 @@ func (engine *Engine) loadTableInfo(table *schemas.Table) error {
 	}
 	table.Indexes = indexes

+	var seq int
 	for _, index := range indexes {
 		for _, name := range index.Cols {
-			if col := table.GetColumn(name); col != nil {
+			parts := strings.Split(name, " ")
+			if len(parts) > 1 {
+				if parts[1] == "DESC" {
+					seq = 1
+				}
+			}
+			if col := table.GetColumn(parts[0]); col != nil {
 				col.Indexes[index.Name] = index.Type
 			} else {
-				return fmt.Errorf("Unknown col %s in index %v of table %v, columns %v", name, index.Name, table.Name, table.ColumnsSeq())
+				return fmt.Errorf("Unknown col %s seq %d, in index %v of table %v, columns %v", name, seq, index.Name, table.Name, table.ColumnsSeq())
 			}
 		}
 	}
--- a/session_get_test.go
+++ b/session_get_test.go
@ -335,13 +335,13 @@ func TestJSONString(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, has)
 	assert.EqualValues(t, 1, js.Id)
-	assert.EqualValues(t, `["1","2"]`, js.Content)
+	assert.True(t, `["1","2"]` == js.Content || `["1", "2"]` == js.Content)

 	var jss []JsonString
 	err = testEngine.Table("json_json").Find(&jss)
 	assert.NoError(t, err)
 	assert.EqualValues(t, 1, len(jss))
-	assert.EqualValues(t, `["1","2"]`, jss[0].Content)
+	assert.True(t, `["1","2"]` == jss[0].Content || `["1", "2"]` == jss[0].Content)
 }

 func TestGetActionMapping(t *testing.T) {
--- a/session_update_test.go
+++ b/session_update_test.go
@ -42,6 +42,11 @@ func TestUpdateMap(t *testing.T) {
 }

 func TestUpdateLimit(t *testing.T) {
+	if *ingoreUpdateLimit {
+		t.Skip()
+		return
+	}
+
 	assert.NoError(t, prepareEngine())

 	type UpdateTable2 struct {
--- a/tags_test.go
+++ b/tags_test.go
@ -1031,6 +1031,7 @@ func TestTagDefault4(t *testing.T) {
 	}
 	assert.True(t, isDefaultExist)
 	assert.True(t, "CURRENT_TIMESTAMP" == defaultVal ||
+		"current_timestamp()" == defaultVal || // for cockroach
 		"now()" == defaultVal ||
 		"getdate" == defaultVal, defaultVal)
 }
--- a/xorm_test.go
+++ b/xorm_test.go
@ -37,7 +37,7 @@ var (
 	splitter           = flag.String("splitter", ";", "the splitter on connstr for cluster")
 	schema             = flag.String("schema", "", "specify the schema")
 	ignoreSelectUpdate = flag.Bool("ignore_select_update", false, "ignore select update if implementation difference, only for tidb")
-
+	ingoreUpdateLimit  = flag.Bool("ignore_update_limit", false, "ignore update limit if implementation difference, only for cockroach")
 	tableMapper        names.Mapper
 	colMapper          names.Mapper
 )
@ -59,11 +59,11 @@ func createEngine(dbType, connStr string) error {
 				db.Close()
 				*ignoreSelectUpdate = true
 			case schemas.POSTGRES:
-				db, err := sql.Open(dbType, connStr)
+				db, err := sql.Open(dbType, strings.Replace(connStr, "xorm_test", "postgres", -1))
 				if err != nil {
 					return err
 				}
-				rows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = 'xorm_test'"))
+				rows, err := db.Query("SELECT 1 FROM pg_database WHERE datname = 'xorm_test'")
 				if err != nil {
 					return fmt.Errorf("db.Query: %v", err)
 				}
@ -75,6 +75,12 @@ func createEngine(dbType, connStr string) error {
 					}
 				}
 				if *schema != "" {
+					db.Close()
+					db, err = sql.Open(dbType, connStr)
+					if err != nil {
+						return err
+					}
+					defer db.Close()
 					if _, err = db.Exec("CREATE SCHEMA IF NOT EXISTS " + *schema); err != nil {
 						return fmt.Errorf("CREATE SCHEMA: %v", err)
 					}
@ -178,6 +184,7 @@ func TestMain(m *testing.M) {

 		if err := prepareEngine(); err != nil {
 			fmt.Println(err)
+			os.Exit(1)
 			return
 		}