From 3556ef4b9a91f8c175905cc1b66ddc5cfcdde3db Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 26 Apr 2020 00:25:11 +0800 Subject: [PATCH] begin clickhouse support --- Makefile | 18 ++++++++ dialects/clickhouse.go | 83 +++++++++++++++++++++++++++++++++++++ dialects/clickhouse_test.go | 27 ++++++++++++ dialects/dialect.go | 21 +++++----- dialects/driver.go | 8 ++++ schemas/type.go | 11 ++--- 6 files changed, 153 insertions(+), 15 deletions(-) create mode 100644 dialects/clickhouse.go create mode 100644 dialects/clickhouse_test.go diff --git a/Makefile b/Makefile index 1bdd44c9..af98af3e 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,11 @@ TEST_TIDB_DBNAME ?= xorm_test TEST_TIDB_USERNAME ?= root TEST_TIDB_PASSWORD ?= +TEST_CLICKHOUSE_HOST ?= clickhouse:9000 +TEST_CLICKHOUSE_DBNAME ?= xorm_test +TEST_CLICKHOUSE_USERNAME ?= root +TEST_CLICKHOUSE_PASSWORD ?= + TEST_CACHE_ENABLE ?= false TEST_QUOTE_POLICY ?= always @@ -104,6 +109,7 @@ help: @echo " - test-sqlite3 run integration tests for sqlite" @echo " - test-sqlite run integration tests for pure go sqlite" @echo " - test-tidb run integration tests for tidb" + @echo " - test-clickhouse run integration tests for clickhouse" @echo " - vet examines Go source code and reports suspicious constructs" .PHONY: lint @@ -241,6 +247,18 @@ test-tidb\#%: go-check -conn_str="$(TEST_TIDB_USERNAME):$(TEST_TIDB_PASSWORD)@tcp($(TEST_TIDB_HOST))/$(TEST_TIDB_DBNAME)" \ -quote=$(TEST_QUOTE_POLICY) -coverprofile=tidb.$(TEST_QUOTE_POLICY).$(TEST_CACHE_ENABLE).coverage.out -covermode=atomic +.PNONY: test-clickhouse +test-clickhouse: go-check + $(GO) test $(INTEGRATION_PACKAGES) -v -race -db=mysql -cache=$(TEST_CACHE_ENABLE) -ignore_select_update=true \ + -conn_str="tcp://$(TEST_CLICKHOUSE_HOST)?username=$(TEST_CLICKHOUSE_USERNAME)&password=$(TEST_CLICKHOUSE_PASSWORD)&database=$(TEST_CLICKHOUSE_DBNAME)" \ + -quote=$(TEST_QUOTE_POLICY) -coverprofile=tidb.$(TEST_QUOTE_POLICY).$(TEST_CACHE_ENABLE).coverage.out -covermode=atomic + +.PHONY: test-clickhouse\#% +test-clickhouse\#%: go-check + $(GO) test $(INTEGRATION_PACKAGES) -v -race -run $* -db=mysql -cache=$(TEST_CACHE_ENABLE) -ignore_select_update=true \ + -conn_str="tcp://$(TEST_CLICKHOUSE_HOST)?username=$(TEST_CLICKHOUSE_USERNAME)&password=$(TEST_CLICKHOUSE_PASSWORD)&database=$(TEST_CLICKHOUSE_DBNAME)" \ + -quote=$(TEST_QUOTE_POLICY) -coverprofile=tidb.$(TEST_QUOTE_POLICY).$(TEST_CACHE_ENABLE).coverage.out -covermode=atomic + .PHONY: vet vet: $(GO) vet $(shell $(GO) list ./...) diff --git a/dialects/clickhouse.go b/dialects/clickhouse.go new file mode 100644 index 00000000..9ca7dcc9 --- /dev/null +++ b/dialects/clickhouse.go @@ -0,0 +1,83 @@ +// Copyright 2020 The Xorm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dialects + +import ( + "context" + "net/url" + + "xorm.io/xorm/core" + "xorm.io/xorm/schemas" +) + +type clickhouse struct { + Base +} + +func (db *clickhouse) Init(uri *URI) error { + return db.Base.Init(db, uri) +} + +func (db *clickhouse) IsReserved(name string) bool { + return false +} + +func (db *clickhouse) SQLType(c *schemas.Column) string { + return "" +} + +func (db *clickhouse) SetQuotePolicy(quotePolicy QuotePolicy) { +} + +func (*clickhouse) AutoIncrStr() string { + return "" +} + +func (*clickhouse) CreateTableSQL(t *schemas.Table, tableName string) ([]string, bool) { + return nil, false +} + +func (*clickhouse) IsTableExist(queryer core.Queryer, ctx context.Context, tableName string) (bool, error) { + return false, nil +} + +func (*clickhouse) Filters() []Filter { + return []Filter{} +} + +func (*clickhouse) GetColumns(core.Queryer, context.Context, string) ([]string, map[string]*schemas.Column, error) { + return nil, nil, nil +} + +func (db *clickhouse) GetIndexes(queryer core.Queryer, ctx context.Context, tableName string) (map[string]*schemas.Index, error) { + return nil, nil +} + +func (db *clickhouse) IndexCheckSQL(tableName, idxName string) (string, []interface{}) { + return "", nil +} + +func (db *clickhouse) GetTables(queryer core.Queryer, ctx context.Context) ([]*schemas.Table, error) { + return nil, nil +} + +// ParseClickHouse parsed clickhouse connection string +// tcp://host1:9000?username=user&password=qwerty&database=clicks&read_timeout=10&write_timeout=20&alt_hosts=host2:9000,host3:9000 +func ParseClickHouse(connStr string) (*URI, error) { + u, err := url.Parse(connStr) + if err != nil { + return nil, err + } + forms := u.Query() + return &URI{ + DBType: schemas.CLICKHOUSE, + Proto: u.Scheme, + Host: u.Hostname(), + Port: u.Port(), + DBName: forms.Get("database"), + User: forms.Get("username"), + Passwd: forms.Get("password"), + }, nil +} diff --git a/dialects/clickhouse_test.go b/dialects/clickhouse_test.go new file mode 100644 index 00000000..2e3ce87a --- /dev/null +++ b/dialects/clickhouse_test.go @@ -0,0 +1,27 @@ +// Copyright 2020 The Xorm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dialects + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "xorm.io/xorm/schemas" +) + +func TestParseClickHouse(t *testing.T) { + uri, err := ParseClickHouse("tcp://host1:9000?username=user&password=qwerty&database=clicks&read_timeout=10&write_timeout=20&alt_hosts=host2:9000,host3:9000") + assert.NoError(t, err) + + assert.EqualValues(t, &URI{ + DBType: schemas.CLICKHOUSE, + Proto: "tcp", + Host: "host1", + Port: "9000", + DBName: "clicks", + User: "user", + Passwd: "qwerty", + }, uri) +} diff --git a/dialects/dialect.go b/dialects/dialect.go index fc11eac1..30d7d136 100644 --- a/dialects/dialect.go +++ b/dialects/dialect.go @@ -211,16 +211,17 @@ func regDrvsNDialects() bool { getDriver func() Driver getDialect func() Dialect }{ - "mssql": {"mssql", func() Driver { return &odbcDriver{} }, func() Dialect { return &mssql{} }}, - "odbc": {"mssql", func() Driver { return &odbcDriver{} }, func() Dialect { return &mssql{} }}, // !nashtsai! TODO change this when supporting MS Access - "mysql": {"mysql", func() Driver { return &mysqlDriver{} }, func() Dialect { return &mysql{} }}, - "mymysql": {"mysql", func() Driver { return &mymysqlDriver{} }, func() Dialect { return &mysql{} }}, - "postgres": {"postgres", func() Driver { return &pqDriver{} }, func() Dialect { return &postgres{} }}, - "pgx": {"postgres", func() Driver { return &pqDriverPgx{} }, func() Dialect { return &postgres{} }}, - "sqlite3": {"sqlite3", func() Driver { return &sqlite3Driver{} }, func() Dialect { return &sqlite3{} }}, - "sqlite": {"sqlite3", func() Driver { return &sqlite3Driver{} }, func() Dialect { return &sqlite3{} }}, - "oci8": {"oracle", func() Driver { return &oci8Driver{} }, func() Dialect { return &oracle{} }}, - "godror": {"oracle", func() Driver { return &godrorDriver{} }, func() Dialect { return &oracle{} }}, + "mssql": {"mssql", func() Driver { return &odbcDriver{} }, func() Dialect { return &mssql{} }}, + "odbc": {"mssql", func() Driver { return &odbcDriver{} }, func() Dialect { return &mssql{} }}, // !nashtsai! TODO change this when supporting MS Access + "mysql": {"mysql", func() Driver { return &mysqlDriver{} }, func() Dialect { return &mysql{} }}, + "mymysql": {"mysql", func() Driver { return &mymysqlDriver{} }, func() Dialect { return &mysql{} }}, + "postgres": {"postgres", func() Driver { return &pqDriver{} }, func() Dialect { return &postgres{} }}, + "pgx": {"postgres", func() Driver { return &pqDriverPgx{} }, func() Dialect { return &postgres{} }}, + "sqlite3": {"sqlite3", func() Driver { return &sqlite3Driver{} }, func() Dialect { return &sqlite3{} }}, + "sqlite": {"sqlite3", func() Driver { return &sqlite3Driver{} }, func() Dialect { return &sqlite3{} }}, + "oci8": {"oracle", func() Driver { return &oci8Driver{} }, func() Dialect { return &oracle{} }}, + "godror": {"oracle", func() Driver { return &godrorDriver{} }, func() Dialect { return &oracle{} }}, + "clickhouse": {"clickhouse", func() Driver { return &driverProxy{ParseClickHouse} }, func() Dialect { return &clickhouse{} }}, } for driverName, v := range providedDrvsNDialects { diff --git a/dialects/driver.go b/dialects/driver.go index c63dbfa3..a3df773a 100644 --- a/dialects/driver.go +++ b/dialects/driver.go @@ -83,3 +83,11 @@ type baseDriver struct{} func (b *baseDriver) Scan(ctx *ScanContext, rows *core.Rows, types []*sql.ColumnType, v ...interface{}) error { return rows.Scan(v...) } + +type driverProxy struct { + parser func(connStr string) (*URI, error) +} + +func (p *driverProxy) Parse(driverName, dataSourceName string) (*URI, error) { + return p.parser(dataSourceName) +} diff --git a/schemas/type.go b/schemas/type.go index d64251bf..a29db060 100644 --- a/schemas/type.go +++ b/schemas/type.go @@ -17,11 +17,12 @@ type DBType string // enumerates all database types const ( - POSTGRES DBType = "postgres" - SQLITE DBType = "sqlite3" - MYSQL DBType = "mysql" - MSSQL DBType = "mssql" - ORACLE DBType = "oracle" + POSTGRES DBType = "postgres" + SQLITE DBType = "sqlite3" + MYSQL DBType = "mysql" + MSSQL DBType = "mssql" + ORACLE DBType = "oracle" + CLICKHOUSE DBType = "clickhouse" ) // SQLType represents SQL types