From ac84217e149aa349ec7c7d915af2adb70e2a8f3f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 7 Aug 2023 10:54:57 +0000 Subject: [PATCH 1/5] Exec support conversion data (#1970) Fix #1803 Reviewed-on: https://gitea.com/xorm/xorm/pulls/1970 --- convert/conversion.go | 14 ++++++++-- internal/statements/statement.go | 17 ++++++++++++ tests/session_raw_test.go | 47 ++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/convert/conversion.go b/convert/conversion.go index b69e345c..5577e863 100644 --- a/convert/conversion.go +++ b/convert/conversion.go @@ -16,11 +16,21 @@ import ( "time" ) +// ConversionFrom is an inteface to allow retrieve data from database +type ConversionFrom interface { + FromDB([]byte) error +} + +// ConversionTo is an interface to allow store data to database +type ConversionTo interface { + ToDB() ([]byte, error) +} + // Conversion is an interface. A type implements Conversion will according // the custom method to fill into database and retrieve from database. type Conversion interface { - FromDB([]byte) error - ToDB() ([]byte, error) + ConversionFrom + ConversionTo } // ErrNilPtr represents an error diff --git a/internal/statements/statement.go b/internal/statements/statement.go index 7ad735f5..c075ec54 100644 --- a/internal/statements/statement.go +++ b/internal/statements/statement.go @@ -644,6 +644,23 @@ func (statement *Statement) convertSQLOrArgs(sqlOrArgs ...interface{}) (string, newArgs = append(newArgs, v.In(statement.defaultTimeZone).Format("2006-01-02 15:04:05")) } else if v, ok := arg.(*time.Time); ok && v != nil { newArgs = append(newArgs, v.In(statement.defaultTimeZone).Format("2006-01-02 15:04:05")) + } else if v, ok := arg.(convert.ConversionTo); ok { + r, err := v.ToDB() + if err != nil { + return "", nil, err + } + if r != nil { + // for nvarchar column on mssql, bytes have to be converted as ucs-2 external of driver + // for binary column, a string will be converted as bytes directly. So we have to + // convert bytes as string + if statement.dialect.URI().DBType == schemas.MSSQL { + newArgs = append(newArgs, string(r)) + } else { + newArgs = append(newArgs, r) + } + } else { + newArgs = append(newArgs, nil) + } } else { newArgs = append(newArgs, arg) } diff --git a/tests/session_raw_test.go b/tests/session_raw_test.go index e6987c41..569d7bed 100644 --- a/tests/session_raw_test.go +++ b/tests/session_raw_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "xorm.io/xorm/convert" + "github.com/stretchr/testify/assert" ) @@ -65,3 +67,48 @@ func TestExecTime(t *testing.T) { assert.True(t, has) assert.EqualValues(t, now.In(testEngine.GetTZLocation()).Format("2006-01-02 15:04:05"), uet.Created.Format("2006-01-02 15:04:05")) } + +type ConversionData struct { + MyData string +} + +var _ convert.Conversion = new(ConversionData) + +func (c ConversionData) ToDB() ([]byte, error) { + return []byte(c.MyData), nil +} + +func (c *ConversionData) FromDB(bs []byte) error { + if bs != nil { + c.MyData = string(bs) + } + return nil +} + +func TestExecCustomTypes(t *testing.T) { + assert.NoError(t, PrepareEngine()) + + type UserinfoExec struct { + Uid int + Name string + Data string + } + + assert.NoError(t, testEngine.Sync2(new(UserinfoExec))) + + res, err := testEngine.Exec("INSERT INTO "+testEngine.TableName("`userinfo_exec`", true)+" (uid, name,data) VALUES (?, ?, ?)", + 1, "user", ConversionData{"data"}) + assert.NoError(t, err) + cnt, err := res.RowsAffected() + assert.NoError(t, err) + assert.EqualValues(t, 1, cnt) + + results, err := testEngine.QueryString("select * from " + testEngine.TableName("userinfo_exec", true)) + assert.NoError(t, err) + assert.EqualValues(t, 1, len(results)) + id, err := strconv.Atoi(results[0]["uid"]) + assert.NoError(t, err) + assert.EqualValues(t, 1, id) + assert.Equal(t, "user", results[0]["name"]) + assert.EqualValues(t, "data", results[0]["data"]) +} From db7c2640627d24539aa4607f50bcba7037ddd9e6 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 9 Aug 2023 03:28:52 +0000 Subject: [PATCH 2/5] Add Sync options to ignore constrains and indices (#2320) needed for https://github.com/woodpecker-ci/woodpecker/pull/2117 Reviewed-on: https://gitea.com/xorm/xorm/pulls/2320 Reviewed-by: Lunny Xiao Co-authored-by: 6543 <6543@obermui.de> Co-committed-by: 6543 <6543@obermui.de> --- interface.go | 1 + sync.go | 54 +++++++++++++++++------- tests/schema_test.go | 99 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 16 deletions(-) diff --git a/interface.go b/interface.go index d10abe9e..03dfd236 100644 --- a/interface.go +++ b/interface.go @@ -121,6 +121,7 @@ type EngineInterface interface { ShowSQL(show ...bool) Sync(...interface{}) error Sync2(...interface{}) error + SyncWithOptions(SyncOptions, ...interface{}) (*SyncResult, error) StoreEngine(storeEngine string) *Session TableInfo(bean interface{}) (*schemas.Table, error) TableName(interface{}, ...bool) string diff --git a/sync.go b/sync.go index 635a8ba9..9e1cb8c1 100644 --- a/sync.go +++ b/sync.go @@ -13,6 +13,10 @@ import ( type SyncOptions struct { WarnIfDatabaseColumnMissed bool + // IgnoreConstrains will not add, delete or update unique constrains + IgnoreConstrains bool + // IgnoreIndices will not add or delete indices + IgnoreIndices bool } type SyncResult struct{} @@ -49,6 +53,8 @@ func (session *Session) Sync2(beans ...interface{}) error { func (session *Session) Sync(beans ...interface{}) error { _, err := session.SyncWithOptions(SyncOptions{ WarnIfDatabaseColumnMissed: false, + IgnoreConstrains: false, + IgnoreIndices: false, }, beans...) return err } @@ -103,15 +109,20 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{}) return nil, err } - err = session.createUniques(bean) - if err != nil { - return nil, err + if !opts.IgnoreConstrains { + err = session.createUniques(bean) + if err != nil { + return nil, err + } } - err = session.createIndexes(bean) - if err != nil { - return nil, err + if !opts.IgnoreIndices { + err = session.createIndexes(bean) + if err != nil { + return nil, err + } } + continue } @@ -208,9 +219,12 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{}) } } + // indices found in orig table foundIndexNames := make(map[string]bool) + // indices to be added addedNames := make(map[string]*schemas.Index) + // drop indices that exist in orig and new table schema but are not equal for name, index := range table.Indexes { var oriIndex *schemas.Index for name2, index2 := range oriTable.Indexes { @@ -221,15 +235,13 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{}) } } - if oriIndex != nil { - if oriIndex.Type != index.Type { - sql := engine.dialect.DropIndexSQL(tbNameWithSchema, oriIndex) - _, err = session.exec(sql) - if err != nil { - return nil, err - } - oriIndex = nil + if oriIndex != nil && oriIndex.Type != index.Type { + sql := engine.dialect.DropIndexSQL(tbNameWithSchema, oriIndex) + _, err = session.exec(sql) + if err != nil { + return nil, err } + oriIndex = nil } if oriIndex == nil { @@ -237,8 +249,17 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{}) } } + // drop all indices that do not exist in new schema or have changed for name2, index2 := range oriTable.Indexes { if _, ok := foundIndexNames[name2]; !ok { + // ignore based on there type + if (index2.Type == schemas.IndexType && opts.IgnoreIndices) || + (index2.Type == schemas.UniqueType && opts.IgnoreConstrains) { + // make sure we do not add a index with same name later + delete(addedNames, name2) + continue + } + sql := engine.dialect.DropIndexSQL(tbNameWithSchema, index2) _, err = session.exec(sql) if err != nil { @@ -247,12 +268,13 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{}) } } + // Add new indices because either they did not exist before or were dropped to update them for name, index := range addedNames { - if index.Type == schemas.UniqueType { + if index.Type == schemas.UniqueType && !opts.IgnoreConstrains { session.statement.RefTable = table session.statement.SetTableName(tbNameWithSchema) err = session.addUnique(tbNameWithSchema, name) - } else if index.Type == schemas.IndexType { + } else if index.Type == schemas.IndexType && !opts.IgnoreIndices { session.statement.RefTable = table session.statement.SetTableName(tbNameWithSchema) err = session.addIndex(tbNameWithSchema, name) diff --git a/tests/schema_test.go b/tests/schema_test.go index c945a35c..db9f9e8f 100644 --- a/tests/schema_test.go +++ b/tests/schema_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "xorm.io/xorm" "xorm.io/xorm/schemas" ) @@ -645,3 +646,101 @@ func TestCollate(t *testing.T) { }) assert.NoError(t, err) } + +type SyncWithOpts1 struct { + Id int64 + Index int `xorm:"index"` + Unique int `xorm:"unique"` + Group1 int `xorm:"index(ttt)"` + Group2 int `xorm:"index(ttt)"` + UniGroup1 int `xorm:"unique(lll)"` + UniGroup2 int `xorm:"unique(lll)"` +} + +func (*SyncWithOpts1) TableName() string { + return "sync_with_opts" +} + +type SyncWithOpts2 struct { + Id int64 + Index int `xorm:"index"` + Unique int `xorm:""` + Group1 int `xorm:"index(ttt)"` + Group2 int `xorm:"index(ttt)"` + UniGroup1 int `xorm:""` + UniGroup2 int `xorm:"unique(lll)"` +} + +func (*SyncWithOpts2) TableName() string { + return "sync_with_opts" +} + +type SyncWithOpts3 struct { + Id int64 + Index int `xorm:""` + Unique int `xorm:"unique"` + Group1 int `xorm:""` + Group2 int `xorm:"index(ttt)"` + UniGroup1 int `xorm:"unique(lll)"` + UniGroup2 int `xorm:"unique(lll)"` +} + +func (*SyncWithOpts3) TableName() string { + return "sync_with_opts" +} + +func TestSyncWithOptions(t *testing.T) { + assert.NoError(t, PrepareEngine()) + + // ignore indices and constrains + result, err := testEngine.SyncWithOptions(xorm.SyncOptions{IgnoreIndices: true, IgnoreConstrains: true}, &SyncWithOpts1{}) + assert.NoError(t, err) + assert.NotNil(t, result) + assert.Len(t, getIndicesOfBeanFromDB(t, &SyncWithOpts1{}), 0) + + // only ignore indices + result, err = testEngine.SyncWithOptions(xorm.SyncOptions{IgnoreConstrains: true}, &SyncWithOpts2{}) + assert.NoError(t, err) + assert.NotNil(t, result) + indices := getIndicesOfBeanFromDB(t, &SyncWithOpts1{}) + assert.Len(t, indices, 2) + assert.ElementsMatch(t, []string{"ttt", "index"}, getKeysFromMap(indices)) + + // only ignore constrains + result, err = testEngine.SyncWithOptions(xorm.SyncOptions{IgnoreIndices: true}, &SyncWithOpts3{}) + assert.NoError(t, err) + assert.NotNil(t, result) + indices = getIndicesOfBeanFromDB(t, &SyncWithOpts1{}) + assert.Len(t, indices, 4) + assert.ElementsMatch(t, []string{"ttt", "index", "unique", "lll"}, getKeysFromMap(indices)) + + tableInfoFromStruct, _ := testEngine.TableInfo(&SyncWithOpts1{}) + assert.ElementsMatch(t, getKeysFromMap(tableInfoFromStruct.Indexes), getKeysFromMap(getIndicesOfBeanFromDB(t, &SyncWithOpts1{}))) + +} + +func getIndicesOfBeanFromDB(t *testing.T, bean interface{}) map[string]*schemas.Index { + dbm, err := testEngine.DBMetas() + assert.NoError(t, err) + + tName := testEngine.TableName(bean) + var tSchema *schemas.Table + for _, t := range dbm { + if t.Name == tName { + tSchema = t + break + } + } + if !assert.NotNil(t, tSchema) { + return nil + } + return tSchema.Indexes +} + +func getKeysFromMap(m map[string]*schemas.Index) []string { + var ss []string + for k := range m { + ss = append(ss, k) + } + return ss +} From eeacd22674314a0712f91033c91185a33c83cacb Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 13 Sep 2023 02:02:12 +0000 Subject: [PATCH 3/5] Fix ci (#2330) Reviewed-on: https://gitea.com/xorm/xorm/pulls/2330 --- .gitea/workflows/release-tag.yml | 4 ++-- .gitea/workflows/test-cockroach.yml | 2 +- .gitea/workflows/test-mariadb.yml | 2 +- .gitea/workflows/test-mssql.yml | 2 +- .gitea/workflows/test-mysql.yml | 2 +- .gitea/workflows/test-mysql8.yml | 2 +- .gitea/workflows/test-postgres.yml | 2 +- .gitea/workflows/test-sqlite.yml | 2 +- .gitea/workflows/test-tidb.yml | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitea/workflows/release-tag.yml b/.gitea/workflows/release-tag.yml index 10ed831e..3b788e76 100644 --- a/.gitea/workflows/release-tag.yml +++ b/.gitea/workflows/release-tag.yml @@ -13,11 +13,11 @@ jobs: with: fetch-depth: 0 - name: setup go - uses: https://github.com/actions/setup-go@v4 + uses: actions/setup-go@v4 with: go-version: '>=1.20.1' - name: Use Go Action id: use-go-action - uses: actions/release-action@main + uses: https://gitea.com/actions/release-action@main with: api_key: '${{secrets.RELEASE_TOKEN}}' \ No newline at end of file diff --git a/.gitea/workflows/test-cockroach.yml b/.gitea/workflows/test-cockroach.yml index ba966dc9..cfcda89d 100644 --- a/.gitea/workflows/test-cockroach.yml +++ b/.gitea/workflows/test-cockroach.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version: 1.20 - - uses: https://github.com/actions/checkout@v3 + - uses: actions/checkout@v3 - name: test cockroach env: TEST_COCKROACH_HOST: "cockroach:26257" diff --git a/.gitea/workflows/test-mariadb.yml b/.gitea/workflows/test-mariadb.yml index 466f3858..dbc819db 100644 --- a/.gitea/workflows/test-mariadb.yml +++ b/.gitea/workflows/test-mariadb.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version: 1.20 - - uses: https://github.com/actions/checkout@v3 + - uses: actions/checkout@v3 - name: test mariadb env: TEST_MYSQL_HOST: mariadb diff --git a/.gitea/workflows/test-mssql.yml b/.gitea/workflows/test-mssql.yml index d02e6956..04b8031a 100644 --- a/.gitea/workflows/test-mssql.yml +++ b/.gitea/workflows/test-mssql.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version: 1.20 - - uses: https://github.com/actions/checkout@v3 + - uses: actions/checkout@v3 - name: test mssql env: TEST_MSSQL_HOST: mssql diff --git a/.gitea/workflows/test-mysql.yml b/.gitea/workflows/test-mysql.yml index 03ee2725..e13354f0 100644 --- a/.gitea/workflows/test-mysql.yml +++ b/.gitea/workflows/test-mysql.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version: 1.20 - - uses: https://github.com/actions/checkout@v3 + - uses: actions/checkout@v3 - name: test mysql utf8mb4 env: TEST_MYSQL_HOST: mysql diff --git a/.gitea/workflows/test-mysql8.yml b/.gitea/workflows/test-mysql8.yml index 3fbd7c30..7362065a 100644 --- a/.gitea/workflows/test-mysql8.yml +++ b/.gitea/workflows/test-mysql8.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version: 1.20 - - uses: https://github.com/actions/checkout@v3 + - uses: actions/checkout@v3 - name: test mysql8 env: TEST_MYSQL_HOST: mysql8 diff --git a/.gitea/workflows/test-postgres.yml b/.gitea/workflows/test-postgres.yml index 89aa72c3..d4abb2ad 100644 --- a/.gitea/workflows/test-postgres.yml +++ b/.gitea/workflows/test-postgres.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version: 1.20 - - uses: https://github.com/actions/checkout@v3 + - uses: actions/checkout@v3 - name: test postgres env: TEST_PGSQL_HOST: pgsql diff --git a/.gitea/workflows/test-sqlite.yml b/.gitea/workflows/test-sqlite.yml index cca2e786..164acc10 100644 --- a/.gitea/workflows/test-sqlite.yml +++ b/.gitea/workflows/test-sqlite.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version: 1.20 - - uses: https://github.com/actions/checkout@v3 + - uses: actions/checkout@v3 - name: vet run: make vet - name: format check diff --git a/.gitea/workflows/test-tidb.yml b/.gitea/workflows/test-tidb.yml index fa6e27ad..ce898dcb 100644 --- a/.gitea/workflows/test-tidb.yml +++ b/.gitea/workflows/test-tidb.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version: 1.20 - - uses: https://github.com/actions/checkout@v3 + - uses: actions/checkout@v3 - name: test tidb env: TEST_TIDB_HOST: "tidb:4000" From 407375c9b466dc551868f95ab7feb25e07d3ffc1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 16 Sep 2023 13:48:49 +0000 Subject: [PATCH 4/5] Add test for max ( id ) (#2316) Reviewed-on: https://gitea.com/xorm/xorm/pulls/2316 --- tests/session_find_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/session_find_test.go b/tests/session_find_test.go index 2a754e2a..d991e6ba 100644 --- a/tests/session_find_test.go +++ b/tests/session_find_test.go @@ -1237,3 +1237,20 @@ func TestBuilderDialect(t *testing.T) { err := testEngine.Table("test_builder_dialect").Where(builder.Eq{"age2": 2}).Join("INNER", inner, "test_builder_dialect_foo.dialect_id = test_builder_dialect.id").Find(&result) assert.NoError(t, err) } + +func TestFindInMaxID(t *testing.T) { + assert.NoError(t, PrepareEngine()) + + type TestFindInMaxId struct { + Id int64 + Name string `xorm:"index"` + Age2 int + } + + assertSync(t, new(TestFindInMaxId)) + + var res []TestFindInMaxId + tableName := testEngine.TableName("test_find_in_max_id", true) + err := testEngine.In("id", builder.Select("max(id)").From(testEngine.Quote(tableName))).Find(&res) + assert.NoError(t, err) +} From 2885c88b77c37369c4d8edd66e87b9eed5ae21fd Mon Sep 17 00:00:00 2001 From: zzdboy <28206697@qq.com> Date: Sat, 16 Sep 2023 13:49:19 +0000 Subject: [PATCH 5/5] fix PostgreSQL version (#2332) Reviewed-on: https://gitea.com/xorm/xorm/pulls/2332 Co-authored-by: zzdboy <28206697@qq.com> Co-committed-by: zzdboy <28206697@qq.com> --- dialects/postgres.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/dialects/postgres.go b/dialects/postgres.go index f1f6a2f2..53f66184 100644 --- a/dialects/postgres.go +++ b/dialects/postgres.go @@ -821,6 +821,7 @@ func (db *postgres) Version(ctx context.Context, queryer core.Queryer) (*schemas } // Postgres: 9.5.22 on x86_64-pc-linux-gnu (Debian 9.5.22-1.pgdg90+1), compiled by gcc (Debian 6.3.0-18+deb9u1) 6.3.0 20170516, 64-bit + // Postgres: PostgreSQL 15.3, compiled by Visual C++ build 1914, 64-bit // CockroachDB CCL v19.2.4 (x86_64-unknown-linux-gnu, built if strings.HasPrefix(version, "CockroachDB") { versions := strings.Split(strings.TrimPrefix(version, "CockroachDB CCL "), " ") @@ -829,12 +830,22 @@ func (db *postgres) Version(ctx context.Context, queryer core.Queryer) (*schemas Edition: "CockroachDB", }, nil } else if strings.HasPrefix(version, "PostgreSQL") { - versions := strings.Split(strings.TrimPrefix(version, "PostgreSQL "), " on ") - return &schemas.Version{ - Number: versions[0], - Level: versions[1], - Edition: "PostgreSQL", - }, nil + if strings.Contains(version, " on ") { + versions := strings.Split(strings.TrimPrefix(version, "PostgreSQL "), " on ") + return &schemas.Version{ + Number: versions[0], + Level: versions[1], + Edition: "PostgreSQL", + }, nil + } else { + versions := strings.Split(strings.TrimPrefix(version, "PostgreSQL "), ",") + return &schemas.Version{ + Number: versions[0], + Level: versions[1], + Edition: "PostgreSQL", + }, nil + } + } return nil, errors.New("unknow database version")