From 54f078fff2c241cb287b0212644c3678338149ed Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 16 Apr 2018 15:07:29 +0800 Subject: [PATCH 1/9] add cockroach support --- dialect_postgres.go | 30 ++++++++++++++++++++++-------- engine.go | 11 +++++++++-- test_cockroach.sh | 1 + 3 files changed, 32 insertions(+), 10 deletions(-) create mode 100755 test_cockroach.sh diff --git a/dialect_postgres.go b/dialect_postgres.go index d907c68c..d3d37918 100644 --- a/dialect_postgres.go +++ b/dialect_postgres.go @@ -952,7 +952,7 @@ func (db *postgres) IsColumnExist(tableName, colName string) (bool, error) { func (db *postgres) GetColumns(tableName string) ([]string, map[string]*core.Column, error) { args := []interface{}{tableName} - s := `SELECT column_name, column_default, is_nullable, data_type, character_maximum_length, numeric_precision, numeric_precision_radix , + s := `SELECT column_name, column_default, is_nullable, data_type, character_maximum_length, numeric_precision , CASE WHEN p.contype = 'p' THEN true ELSE false END AS primarykey, CASE WHEN p.contype = 'u' THEN true ELSE false END AS uniquekey FROM pg_attribute f @@ -987,14 +987,14 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att col.Indexes = make(map[string]int) var colName, isNullable, dataType string - var maxLenStr, colDefault, numPrecision, numRadix *string + var maxLenStr, colDefault, numPrecision *string var isPK, isUnique bool - err = rows.Scan(&colName, &colDefault, &isNullable, &dataType, &maxLenStr, &numPrecision, &numRadix, &isPK, &isUnique) + err = rows.Scan(&colName, &colDefault, &isNullable, &dataType, &maxLenStr, &numPrecision, &isPK, &isUnique) if err != nil { return nil, nil, err } - //fmt.Println(args, colName, isNullable, dataType, maxLenStr, colDefault, numPrecision, numRadix, isPK, isUnique) + //fmt.Println(args, colName, isNullable, dataType, maxLenStr, colDefault, numPrecision, isPK, isUnique) var maxLen int if maxLenStr != nil { maxLen, err = strconv.Atoi(*maxLenStr) @@ -1019,8 +1019,8 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att col.Nullable = (isNullable == "YES") - switch dataType { - case "character varying", "character": + switch strings.ToLower(dataType) { + case "character varying", "character", "string": col.SQLType = core.SQLType{Name: core.Varchar, DefaultLength: 0, DefaultLength2: 0} case "timestamp without time zone": col.SQLType = core.SQLType{Name: core.DateTime, DefaultLength: 0, DefaultLength2: 0} @@ -1032,10 +1032,19 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att col.SQLType = core.SQLType{Name: core.Bool, DefaultLength: 0, DefaultLength2: 0} case "time without time zone": col.SQLType = core.SQLType{Name: core.Time, DefaultLength: 0, DefaultLength2: 0} + case "bytes": + col.SQLType = core.SQLType{Name: core.Binary, DefaultLength: 0, DefaultLength2: 0} case "oid": col.SQLType = core.SQLType{Name: core.BigInt, DefaultLength: 0, DefaultLength2: 0} default: - col.SQLType = core.SQLType{Name: strings.ToUpper(dataType), DefaultLength: 0, DefaultLength2: 0} + startIdx := strings.Index(strings.ToLower(dataType), "string(") + if startIdx != -1 && strings.HasSuffix(dataType, ")") { + length := dataType[startIdx+8 : len(dataType)-1] + l, _ := strconv.Atoi(length) + col.SQLType = core.SQLType{Name: "STRING", DefaultLength: l, DefaultLength2: 0} + } else { + col.SQLType = core.SQLType{Name: strings.ToUpper(dataType), DefaultLength: 0, DefaultLength2: 0} + } } if _, ok := core.SqlTypes[col.SQLType.Name]; !ok { return nil, nil, fmt.Errorf("Unknown colType: %v", dataType) @@ -1113,6 +1122,10 @@ func (db *postgres) GetIndexes(tableName string) (map[string]*core.Index, error) if err != nil { return nil, err } + + if indexName == "primary" { + continue + } indexName = strings.Trim(indexName, `" `) if strings.HasSuffix(indexName, "_pkey") { continue @@ -1124,6 +1137,7 @@ func (db *postgres) GetIndexes(tableName string) (map[string]*core.Index, error) } cs := strings.Split(indexdef, "(") colNames = strings.Split(cs[1][0:len(cs[1])-1], ",") + var isRegular bool if strings.HasPrefix(indexName, "IDX_"+tableName) || strings.HasPrefix(indexName, "UQE_"+tableName) { newIdxName := indexName[5+len(tableName):] @@ -1135,7 +1149,7 @@ func (db *postgres) GetIndexes(tableName string) (map[string]*core.Index, error) index := &core.Index{Name: indexName, Type: indexType, Cols: make([]string, 0)} for _, colName := range colNames { - index.Cols = append(index.Cols, strings.Trim(colName, `" `)) + index.Cols = append(index.Cols, strings.TrimSpace(strings.Replace(colName, `"`, "", -1))) } index.IsRegular = isRegular indexes[index.Name] = index diff --git a/engine.go b/engine.go index 4984d374..ff4b1266 100644 --- a/engine.go +++ b/engine.go @@ -380,12 +380,19 @@ func (engine *Engine) DBMetas() ([]*core.Table, error) { } table.Indexes = indexes + var seq int for _, index := range indexes { for _, name := range index.Cols { - if col := table.GetColumn(name); col != nil { + parts := strings.Split(name, " ") + if len(parts) > 1 { + if parts[1] == "DESC" { + seq = 1 + } + } + if col := table.GetColumn(parts[0]); col != nil { col.Indexes[index.Name] = index.Type } else { - return nil, fmt.Errorf("Unknown col %s in index %v of table %v, columns %v", name, index.Name, table.Name, table.ColumnsSeq()) + return nil, fmt.Errorf("Unknown col %s seq %d, in index %v of table %v, columns %v", name, seq, index.Name, table.Name, table.ColumnsSeq()) } } } diff --git a/test_cockroach.sh b/test_cockroach.sh new file mode 100755 index 00000000..ea6a844b --- /dev/null +++ b/test_cockroach.sh @@ -0,0 +1 @@ +go test -db=postgres -conn_str="postgresql://root@localhost:26257/xorm_test?sslmode=disable" \ No newline at end of file From 03770c1a2bd02bf896938264028cfb89ba788012 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 17 Apr 2018 09:39:33 +0800 Subject: [PATCH 2/9] add cockroach test to circleci --- circle.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index adfd2a16..e6e3ff97 100644 --- a/circle.yml +++ b/circle.yml @@ -33,7 +33,12 @@ test: - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -cache=true -coverprofile=coverage4-2.txt -covermode=atomic - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -coverprofile=coverage5-1.txt -covermode=atomic - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -cache=true -coverprofile=coverage5-2.txt -covermode=atomic - - gocovmerge coverage1-1.txt coverage1-2.txt coverage2-1.txt coverage2-2.txt coverage3-1.txt coverage3-2.txt coverage4-1.txt coverage4-2.txt coverage5-1.txt coverage5-2.txt > coverage.txt + - wget -qO- https://binaries.cockroachdb.com/cockroach-v2.0.0.linux-amd64.tgz | tar xvz + - cp -i cockroach-v2.0.0.linux-amd64/cockroach /usr/local/bin + - cockroach start --insecure --host=localhost & + - cockroach sql --execute="create database xorm_test" + - go test -v -race -db="postgres" -conn_str="postgresql://root@localhost:26257/xorm_test?sslmode=disable" -coverprofile=coverage6-1.txt -covermode=atomic + - gocovmerge coverage1-1.txt coverage1-2.txt coverage2-1.txt coverage2-2.txt coverage3-1.txt coverage3-2.txt coverage4-1.txt coverage4-2.txt coverage5-1.txt coverage5-2.txt coverage6-1.txt> coverage.txt - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./sqlite3.sh - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./mysql.sh - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./postgres.sh From 3b9737c6a5a4d7fa82968969821d4900e1f5efa5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 17 Apr 2018 10:13:32 +0800 Subject: [PATCH 3/9] fix run cockroach --- circle.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/circle.yml b/circle.yml index e6e3ff97..80cb99f0 100644 --- a/circle.yml +++ b/circle.yml @@ -34,9 +34,8 @@ test: - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -coverprofile=coverage5-1.txt -covermode=atomic - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -cache=true -coverprofile=coverage5-2.txt -covermode=atomic - wget -qO- https://binaries.cockroachdb.com/cockroach-v2.0.0.linux-amd64.tgz | tar xvz - - cp -i cockroach-v2.0.0.linux-amd64/cockroach /usr/local/bin - - cockroach start --insecure --host=localhost & - - cockroach sql --execute="create database xorm_test" + - ./cockroach-v2.0.0.linux-amd64/cockroach start --insecure --host=localhost & + - ./cockroach-v2.0.0.linux-amd64/cockroach sql --execute="create database xorm_test" - go test -v -race -db="postgres" -conn_str="postgresql://root@localhost:26257/xorm_test?sslmode=disable" -coverprofile=coverage6-1.txt -covermode=atomic - gocovmerge coverage1-1.txt coverage1-2.txt coverage2-1.txt coverage2-2.txt coverage3-1.txt coverage3-2.txt coverage4-1.txt coverage4-2.txt coverage5-1.txt coverage5-2.txt coverage6-1.txt> coverage.txt - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./sqlite3.sh From 32ff1d0be9c8918a3a95cfc102086a4eaa48b4b7 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 17 Apr 2018 10:31:28 +0800 Subject: [PATCH 4/9] fix test --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 80cb99f0..d24117a9 100644 --- a/circle.yml +++ b/circle.yml @@ -35,7 +35,7 @@ test: - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -cache=true -coverprofile=coverage5-2.txt -covermode=atomic - wget -qO- https://binaries.cockroachdb.com/cockroach-v2.0.0.linux-amd64.tgz | tar xvz - ./cockroach-v2.0.0.linux-amd64/cockroach start --insecure --host=localhost & - - ./cockroach-v2.0.0.linux-amd64/cockroach sql --execute="create database xorm_test" + - ./cockroach-v2.0.0.linux-amd64/cockroach sql --insecure --execute="create database xorm_test" - go test -v -race -db="postgres" -conn_str="postgresql://root@localhost:26257/xorm_test?sslmode=disable" -coverprofile=coverage6-1.txt -covermode=atomic - gocovmerge coverage1-1.txt coverage1-2.txt coverage2-1.txt coverage2-2.txt coverage3-1.txt coverage3-2.txt coverage4-1.txt coverage4-2.txt coverage5-1.txt coverage5-2.txt coverage6-1.txt> coverage.txt - cd /home/ubuntu/.go_workspace/src/github.com/go-xorm/tests && ./sqlite3.sh From b7f6a6c8488626f28cc2d2054bd3492c72827f3b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 17 Apr 2018 10:54:44 +0800 Subject: [PATCH 5/9] run cockroach backround --- circle.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index d24117a9..c337843e 100644 --- a/circle.yml +++ b/circle.yml @@ -34,7 +34,9 @@ test: - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -coverprofile=coverage5-1.txt -covermode=atomic - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -cache=true -coverprofile=coverage5-2.txt -covermode=atomic - wget -qO- https://binaries.cockroachdb.com/cockroach-v2.0.0.linux-amd64.tgz | tar xvz - - ./cockroach-v2.0.0.linux-amd64/cockroach start --insecure --host=localhost & + - ./cockroach-v2.0.0.linux-amd64/cockroach start --insecure --host=localhost + background: true + - sleep 5 - ./cockroach-v2.0.0.linux-amd64/cockroach sql --insecure --execute="create database xorm_test" - go test -v -race -db="postgres" -conn_str="postgresql://root@localhost:26257/xorm_test?sslmode=disable" -coverprofile=coverage6-1.txt -covermode=atomic - gocovmerge coverage1-1.txt coverage1-2.txt coverage2-1.txt coverage2-2.txt coverage3-1.txt coverage3-2.txt coverage4-1.txt coverage4-2.txt coverage5-1.txt coverage5-2.txt coverage6-1.txt> coverage.txt From f6672741864a19f3fc31aa54f110e6b51806c83f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 17 Apr 2018 10:57:06 +0800 Subject: [PATCH 6/9] run cockroach backround --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index c337843e..d7fc0982 100644 --- a/circle.yml +++ b/circle.yml @@ -34,7 +34,7 @@ test: - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -coverprofile=coverage5-1.txt -covermode=atomic - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -cache=true -coverprofile=coverage5-2.txt -covermode=atomic - wget -qO- https://binaries.cockroachdb.com/cockroach-v2.0.0.linux-amd64.tgz | tar xvz - - ./cockroach-v2.0.0.linux-amd64/cockroach start --insecure --host=localhost + - ./cockroach-v2.0.0.linux-amd64/cockroach start --insecure --host=localhost: background: true - sleep 5 - ./cockroach-v2.0.0.linux-amd64/cockroach sql --insecure --execute="create database xorm_test" From c01b8c48d34a3d3d0655fe6024a37fb37d40f796 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 17 Apr 2018 21:29:02 +0800 Subject: [PATCH 7/9] run cockroach background --- circle.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index d7fc0982..c5885d41 100644 --- a/circle.yml +++ b/circle.yml @@ -34,8 +34,7 @@ test: - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -coverprofile=coverage5-1.txt -covermode=atomic - go test -v -race -db="postgres" -conn_str="dbname=xorm_test sslmode=disable" -schema=xorm -cache=true -coverprofile=coverage5-2.txt -covermode=atomic - wget -qO- https://binaries.cockroachdb.com/cockroach-v2.0.0.linux-amd64.tgz | tar xvz - - ./cockroach-v2.0.0.linux-amd64/cockroach start --insecure --host=localhost: - background: true + - ./cockroach-v2.0.0.linux-amd64/cockroach start --insecure --background --host=localhost - sleep 5 - ./cockroach-v2.0.0.linux-amd64/cockroach sql --insecure --execute="create database xorm_test" - go test -v -race -db="postgres" -conn_str="postgresql://root@localhost:26257/xorm_test?sslmode=disable" -coverprofile=coverage6-1.txt -covermode=atomic From e09c69d374496fa6e0fbd4b6ee2585226677349c Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Wed, 6 Jun 2018 14:23:01 -0400 Subject: [PATCH 8/9] Use available column types for cockroachdb/postgres --- dialect_postgres.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dialect_postgres.go b/dialect_postgres.go index d3d37918..c2fa3f96 100644 --- a/dialect_postgres.go +++ b/dialect_postgres.go @@ -1020,7 +1020,7 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att col.Nullable = (isNullable == "YES") switch strings.ToLower(dataType) { - case "character varying", "character", "string": + case "character varying", "character": col.SQLType = core.SQLType{Name: core.Varchar, DefaultLength: 0, DefaultLength2: 0} case "timestamp without time zone": col.SQLType = core.SQLType{Name: core.DateTime, DefaultLength: 0, DefaultLength2: 0} @@ -1037,11 +1037,11 @@ WHERE c.relkind = 'r'::char AND c.relname = $1%s AND f.attnum > 0 ORDER BY f.att case "oid": col.SQLType = core.SQLType{Name: core.BigInt, DefaultLength: 0, DefaultLength2: 0} default: - startIdx := strings.Index(strings.ToLower(dataType), "string(") + startIdx := strings.Index(strings.ToLower(dataType), "varchar(") if startIdx != -1 && strings.HasSuffix(dataType, ")") { length := dataType[startIdx+8 : len(dataType)-1] l, _ := strconv.Atoi(length) - col.SQLType = core.SQLType{Name: "STRING", DefaultLength: l, DefaultLength2: 0} + col.SQLType = core.SQLType{Name: "varchar", DefaultLength: l, DefaultLength2: 0} } else { col.SQLType = core.SQLType{Name: strings.ToUpper(dataType), DefaultLength: 0, DefaultLength2: 0} } From 48b0253bafebe52b0bed22fd06bb677e42faeaae Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Wed, 6 Jun 2018 16:52:36 -0400 Subject: [PATCH 9/9] update query --- dialect_postgres.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dialect_postgres.go b/dialect_postgres.go index c2fa3f96..ac2d0986 100644 --- a/dialect_postgres.go +++ b/dialect_postgres.go @@ -952,7 +952,13 @@ func (db *postgres) IsColumnExist(tableName, colName string) (bool, error) { func (db *postgres) GetColumns(tableName string) ([]string, map[string]*core.Column, error) { args := []interface{}{tableName} - s := `SELECT column_name, column_default, is_nullable, data_type, character_maximum_length, numeric_precision , + s := `SELECT column_name, column_default, is_nullable, replace(data_type,'STRING','VARCHAR'), + character_maximum_length, numeric_precision, + CASE + WHEN numeric_precision IS NOT NULL AND lower(data_type) = 'decimal' THEN 10 + WHEN numeric_precision IS NOT NULL AND lower(data_type) != 'decimal' THEN 2 + ELSE NULL + END AS numeric_precision_radix, CASE WHEN p.contype = 'p' THEN true ELSE false END AS primarykey, CASE WHEN p.contype = 'u' THEN true ELSE false END AS uniquekey FROM pg_attribute f