From b079bc05ffc299c4e6d524f440c0685dd758a6eb Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 3 Mar 2020 08:19:32 +0000 Subject: [PATCH] Add cockroach support and tests (#896) Fix tests fix some tests and ignore some Add &experimental_serial_normalization=sql_sequence on test so that id will increase from 1 fix drone Fix drone Fix drone Fix drone Remove space Add makefile fix drone fix drone fix drone update fix tests use sql_sequence when testing fix postgres ci fix circle ci for cockroach upgrade cockroach version add trace on error add drone ci run cockroach background run cockroach backround run cockroach backround fix test fix run cockroach add cockroach test to circleci add cockroach support Add tests for table name (#1... Reviewed-on: https://gitea.com/xorm/xorm/pulls/896 --- .drone.yml | 33 +++++++++++++++++++++++++++++- Makefile | 18 +++++++++++++++++ circle.yml | 46 ++++++++++++++++++++++++++++++++++++++++++ dialects/postgres.go | 41 ++++++++++++++++++++++++++++++------- engine.go | 11 ++++++++-- session_get_test.go | 4 ++-- session_update_test.go | 5 +++++ tags_test.go | 1 + xorm_test.go | 17 +++++++++++----- 9 files changed, 159 insertions(+), 17 deletions(-) create mode 100644 circle.yml diff --git a/.drone.yml b/.drone.yml index eb8ed320..dac49cdf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -180,6 +180,25 @@ steps: - push - pull_request +- name: test-cockroach + pull: default + image: golang:1.13 + environment: + GO111MODULE: "on" + GOPROXY: "https://goproxy.cn" + TEST_COCKROACH_HOST: "cockroach:26257" + TEST_COCKROACH_DBNAME: xorm_test + TEST_COCKROACH_USERNAME: root + TEST_COCKROACH_PASSWORD: + commands: + - sleep 10 + - make test-cockroach + - TEST_CACHE_ENABLE=true make test-cockroach + when: + event: + - push + - pull_request + - name: merge_coverage pull: default image: golang:1.12 @@ -196,6 +215,7 @@ steps: - test-postgres-schema - test-mssql - test-tidb + - test-cockroach commands: - make coverage when: @@ -262,4 +282,15 @@ services: event: - push - tag - - pull_request \ No newline at end of file + - pull_request + +- name: cockroach + pull: default + image: cockroachdb/cockroach:v19.2.4 + commands: + - /cockroach/cockroach start --insecure + when: + event: + - push + - tag + - pull_request diff --git a/Makefile b/Makefile index 737ca96c..faad978f 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,12 @@ GOFILES := $(shell find . -name "*.go" -type f) PACKAGES ?= $(shell GO111MODULE=on $(GO) list ./...) +TEST_COCKROACH_HOST ?= cockroach:26257 +TEST_COCKROACH_SCHEMA ?= +TEST_COCKROACH_DBNAME ?= xorm_test +TEST_COCKROACH_USERNAME ?= postgres +TEST_COCKROACH_PASSWORD ?= + TEST_MSSQL_HOST ?= mssql:1433 TEST_MSSQL_DBNAME ?= gitea TEST_MSSQL_USERNAME ?= sa @@ -115,6 +121,18 @@ misspell-check: .PHONY: test test: test-sqlite +.PNONY: test-cockroach +test-cockroach: go-check + $(GO) test -race -db=postgres -schema='$(TEST_COCKROACH_SCHEMA)' -cache=$(TEST_CACHE_ENABLE) \ + -conn_str="postgres://$(TEST_COCKROACH_USERNAME):$(TEST_COCKROACH_PASSWORD)@$(TEST_COCKROACH_HOST)/$(TEST_COCKROACH_DBNAME)?sslmode=disable&experimental_serial_normalization=sql_sequence" \ + -ignore_update_limit=true -coverprofile=cockroach.$(TEST_COCKROACH_SCHEMA).$(TEST_CACHE_ENABLE).coverage.out -covermode=atomic + +.PHONY: test-cockroach\#% +test-cockroach\#%: go-check + $(GO) test -race -run $* -db=postgres -schema='$(TEST_COCKROACH_SCHEMA)' -cache=$(TEST_CACHE_ENABLE) \ + -conn_str="postgres://$(TEST_COCKROACH_USERNAME):$(TEST_COCKROACH_PASSWORD)@$(TEST_COCKROACH_HOST)/$(TEST_COCKROACH_DBNAME)?sslmode=disable&experimental_serial_normalization=sql_sequence" \ + -ignore_update_limit=true -coverprofile=cockroach.$(TEST_COCKROACH_SCHEMA).$(TEST_CACHE_ENABLE).coverage.out -covermode=atomic + .PNONY: test-mssql test-mssql: go-check $(GO) test -v -race -db=mssql -cache=$(TEST_CACHE_ENABLE) \ diff --git a/circle.yml b/circle.yml new file mode 100644 index 00000000..8cacf5eb --- /dev/null +++ b/circle.yml @@ -0,0 +1,46 @@ +dependencies: + override: + # './...' is a relative pattern which means all subdirectories + - go get -t -d -v ./... + - go get -t -d -v github.com/go-xorm/tests + - go get -u github.com/go-xorm/core + - go get -u github.com/go-xorm/builder + - go build -v + +database: + override: + - mysql -u root -e "CREATE DATABASE xorm_test DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci" + - mysql -u root -e "CREATE DATABASE xorm_test1 DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci" + - mysql -u root -e "CREATE DATABASE xorm_test2 DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci" + - mysql -u root -e "CREATE DATABASE xorm_test3 DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci" + - createdb -p 5432 -e -U postgres xorm_test + - createdb -p 5432 -e -U postgres xorm_test1 + - createdb -p 5432 -e -U postgres xorm_test2 + - createdb -p 5432 -e -U postgres xorm_test3 + - psql xorm_test postgres -c "create schema xorm" + +test: + override: + # './...' is a relative pattern which means all subdirectories + - go get -u github.com/wadey/gocovmerge + - go test -v -race -db="sqlite3" -conn_str="./test.db" -coverprofile=coverage1-1.txt -covermode=atomic + - go test -v -race -db="sqlite3" -conn_str="./test.db" -cache=true -coverprofile=coverage1-2.txt -covermode=atomic + - go test -v -race -db="mysql" -conn_str="root:@/xorm_test" -coverprofile=coverage2-1.txt -covermode=atomic + - go test -v -race -db="mysql" -conn_str="root:@/xorm_test" -cache=true -coverprofile=coverage2-2.txt -covermode=atomic + - go test -v -race -db="mymysql" -conn_str="xorm_test/root/" -coverprofile=coverage3-1.txt -covermode=atomic + - go test -v -race -db="mymysql" -conn_str="xorm_test/root/" -cache=true -coverprofile=coverage3-2.txt -covermode=atomic + - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -coverprofile=coverage4-1.txt -covermode=atomic + - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -cache=true -coverprofile=coverage4-2.txt -covermode=atomic + - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -coverprofile=coverage5-1.txt -covermode=atomic + - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -cache=true -coverprofile=coverage5-2.txt -covermode=atomic + - wget -qO- https://binaries.cockroachdb.com/cockroach-v2.1.3.linux-amd64.tgz | tar xvz + - ./cockroach-v2.1.3.linux-amd64/cockroach start --insecure --background --host=localhost + - sleep 5 + - ./cockroach-v2.1.3.linux-amd64/cockroach sql --insecure --execute="create database xorm_test" + - go test -v -race -db="postgres" -conn_str="postgresql://root@localhost:26257/xorm_test?sslmode=disable" -coverprofile=coverage6-1.txt -covermode=atomic + - gocovmerge coverage1-1.txt coverage1-2.txt coverage2-1.txt coverage2-2.txt coverage3-1.txt coverage3-2.txt coverage4-1.txt coverage4-2.txt coverage5-1.txt coverage5-2.txt coverage6-1.txt> coverage.txt + - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./sqlite3.sh + - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./mysql.sh + - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./postgres.sh + post: + - bash <(curl -s https://codecov.io/bash) \ No newline at end of file diff --git a/dialects/postgres.go b/dialects/postgres.go index 2e314812..623b59ed 100644 --- a/dialects/postgres.go +++ b/dialects/postgres.go @@ -995,7 +995,6 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att return nil, nil, err } - // fmt.Println(args, colName, isNullable, dataType, maxLenStr, colDefault, isPK, isUnique) var maxLen int if maxLenStr != nil { maxLen, err = strconv.Atoi(*maxLenStr) @@ -1007,7 +1006,22 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att col.Name = strings.Trim(colName, `" `) if colDefault != nil { - col.Default = *colDefault + var theDefault = *colDefault + // cockroach has type with the default value with ::: + // and postgres with ::, we should remove them before store them + idx := strings.Index(theDefault, ":::") + if idx == -1 { + idx = strings.Index(theDefault, "::") + } + if idx > -1 { + theDefault = theDefault[:idx] + } + + if strings.HasSuffix(theDefault, "+00:00'") { + theDefault = theDefault[:len(theDefault)-7] + "'" + } + + col.Default = theDefault col.DefaultIsEmpty = false if strings.HasPrefix(col.Default, "nextval(") { col.IsAutoIncrement = true @@ -1022,8 +1036,8 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att col.Nullable = (isNullable == "YES") - switch dataType { - case "character varying", "character": + switch strings.ToLower(dataType) { + case "character varying", "character", "string": col.SQLType = schemas.SQLType{Name: schemas.Varchar, DefaultLength: 0, DefaultLength2: 0} case "timestamp without time zone": col.SQLType = schemas.SQLType{Name: schemas.DateTime, DefaultLength: 0, DefaultLength2: 0} @@ -1035,13 +1049,22 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att col.SQLType = schemas.SQLType{Name: schemas.Bool, DefaultLength: 0, DefaultLength2: 0} case "time without time zone": col.SQLType = schemas.SQLType{Name: schemas.Time, DefaultLength: 0, DefaultLength2: 0} + case "bytes": + col.SQLType = schemas.SQLType{Name: schemas.Binary, DefaultLength: 0, DefaultLength2: 0} case "oid": col.SQLType = schemas.SQLType{Name: schemas.BigInt, DefaultLength: 0, DefaultLength2: 0} default: - col.SQLType = schemas.SQLType{Name: strings.ToUpper(dataType), DefaultLength: 0, DefaultLength2: 0} + startIdx := strings.Index(strings.ToLower(dataType), "string(") + if startIdx != -1 && strings.HasSuffix(dataType, ")") { + length := dataType[startIdx+8 : len(dataType)-1] + l, _ := strconv.Atoi(length) + col.SQLType = schemas.SQLType{Name: "STRING", DefaultLength: l, DefaultLength2: 0} + } else { + col.SQLType = schemas.SQLType{Name: strings.ToUpper(dataType), DefaultLength: 0, DefaultLength2: 0} + } } if _, ok := schemas.SqlTypes[col.SQLType.Name]; !ok { - return nil, nil, fmt.Errorf("Unknown colType: %v", dataType) + return nil, nil, fmt.Errorf("Unknown colType: %s - %s", dataType, col.SQLType.Name) } col.Length = maxLen @@ -1128,6 +1151,10 @@ func (db *postgres) GetIndexes(ctx context.Context, tableName string) (map[strin if err != nil { return nil, err } + + if indexName == "primary" { + continue + } indexName = strings.Trim(indexName, `" `) if strings.HasSuffix(indexName, "_pkey") { continue @@ -1149,7 +1176,7 @@ func (db *postgres) GetIndexes(ctx context.Context, tableName string) (map[strin index := &schemas.Index{Name: indexName, Type: indexType, Cols: make([]string, 0)} for _, colName := range colNames { - index.Cols = append(index.Cols, strings.Trim(colName, `" `)) + index.Cols = append(index.Cols, strings.TrimSpace(strings.Replace(colName, `"`, "", -1))) } index.IsRegular = isRegular indexes[index.Name] = index diff --git a/engine.go b/engine.go index 421e89e0..7865b7cb 100644 --- a/engine.go +++ b/engine.go @@ -284,12 +284,19 @@ func (engine *Engine) loadTableInfo(table *schemas.Table) error { } table.Indexes = indexes + var seq int for _, index := range indexes { for _, name := range index.Cols { - if col := table.GetColumn(name); col != nil { + parts := strings.Split(name, " ") + if len(parts) > 1 { + if parts[1] == "DESC" { + seq = 1 + } + } + if col := table.GetColumn(parts[0]); col != nil { col.Indexes[index.Name] = index.Type } else { - return fmt.Errorf("Unknown col %s in index %v of table %v, columns %v", name, index.Name, table.Name, table.ColumnsSeq()) + return fmt.Errorf("Unknown col %s seq %d, in index %v of table %v, columns %v", name, seq, index.Name, table.Name, table.ColumnsSeq()) } } } diff --git a/session_get_test.go b/session_get_test.go index b7eac2b4..5bac9cd7 100644 --- a/session_get_test.go +++ b/session_get_test.go @@ -335,13 +335,13 @@ func TestJSONString(t *testing.T) { assert.NoError(t, err) assert.True(t, has) assert.EqualValues(t, 1, js.Id) - assert.EqualValues(t, `["1","2"]`, js.Content) + assert.True(t, `["1","2"]` == js.Content || `["1", "2"]` == js.Content) var jss []JsonString err = testEngine.Table("json_json").Find(&jss) assert.NoError(t, err) assert.EqualValues(t, 1, len(jss)) - assert.EqualValues(t, `["1","2"]`, jss[0].Content) + assert.True(t, `["1","2"]` == jss[0].Content || `["1", "2"]` == jss[0].Content) } func TestGetActionMapping(t *testing.T) { diff --git a/session_update_test.go b/session_update_test.go index 0ef59155..f76e447b 100644 --- a/session_update_test.go +++ b/session_update_test.go @@ -42,6 +42,11 @@ func TestUpdateMap(t *testing.T) { } func TestUpdateLimit(t *testing.T) { + if *ingoreUpdateLimit { + t.Skip() + return + } + assert.NoError(t, prepareEngine()) type UpdateTable2 struct { diff --git a/tags_test.go b/tags_test.go index 775fcf60..295affd8 100644 --- a/tags_test.go +++ b/tags_test.go @@ -1031,6 +1031,7 @@ func TestTagDefault4(t *testing.T) { } assert.True(t, isDefaultExist) assert.True(t, "CURRENT_TIMESTAMP" == defaultVal || + "current_timestamp()" == defaultVal || // for cockroach "now()" == defaultVal || "getdate" == defaultVal, defaultVal) } diff --git a/xorm_test.go b/xorm_test.go index 59f6c1a9..c1f38757 100644 --- a/xorm_test.go +++ b/xorm_test.go @@ -37,9 +37,9 @@ var ( splitter = flag.String("splitter", ";", "the splitter on connstr for cluster") schema = flag.String("schema", "", "specify the schema") ignoreSelectUpdate = flag.Bool("ignore_select_update", false, "ignore select update if implementation difference, only for tidb") - - tableMapper names.Mapper - colMapper names.Mapper + ingoreUpdateLimit = flag.Bool("ignore_update_limit", false, "ignore update limit if implementation difference, only for cockroach") + tableMapper names.Mapper + colMapper names.Mapper ) func createEngine(dbType, connStr string) error { @@ -59,11 +59,11 @@ func createEngine(dbType, connStr string) error { db.Close() *ignoreSelectUpdate = true case schemas.POSTGRES: - db, err := sql.Open(dbType, connStr) + db, err := sql.Open(dbType, strings.Replace(connStr, "xorm_test", "postgres", -1)) if err != nil { return err } - rows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = 'xorm_test'")) + rows, err := db.Query("SELECT 1 FROM pg_database WHERE datname = 'xorm_test'") if err != nil { return fmt.Errorf("db.Query: %v", err) } @@ -75,6 +75,12 @@ func createEngine(dbType, connStr string) error { } } if *schema != "" { + db.Close() + db, err = sql.Open(dbType, connStr) + if err != nil { + return err + } + defer db.Close() if _, err = db.Exec("CREATE SCHEMA IF NOT EXISTS " + *schema); err != nil { return fmt.Errorf("CREATE SCHEMA: %v", err) } @@ -178,6 +184,7 @@ func TestMain(m *testing.M) { if err := prepareEngine(); err != nil { fmt.Println(err) + os.Exit(1) return }