From ac84217e149aa349ec7c7d915af2adb70e2a8f3f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 7 Aug 2023 10:54:57 +0000 Subject: [PATCH 1/2] Exec support conversion data (#1970) Fix #1803 Reviewed-on: https://gitea.com/xorm/xorm/pulls/1970 --- convert/conversion.go | 14 ++++++++-- internal/statements/statement.go | 17 ++++++++++++ tests/session_raw_test.go | 47 ++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/convert/conversion.go b/convert/conversion.go index b69e345c..5577e863 100644 --- a/convert/conversion.go +++ b/convert/conversion.go @@ -16,11 +16,21 @@ import ( "time" ) +// ConversionFrom is an inteface to allow retrieve data from database +type ConversionFrom interface { + FromDB([]byte) error +} + +// ConversionTo is an interface to allow store data to database +type ConversionTo interface { + ToDB() ([]byte, error) +} + // Conversion is an interface. A type implements Conversion will according // the custom method to fill into database and retrieve from database. type Conversion interface { - FromDB([]byte) error - ToDB() ([]byte, error) + ConversionFrom + ConversionTo } // ErrNilPtr represents an error diff --git a/internal/statements/statement.go b/internal/statements/statement.go index 7ad735f5..c075ec54 100644 --- a/internal/statements/statement.go +++ b/internal/statements/statement.go @@ -644,6 +644,23 @@ func (statement *Statement) convertSQLOrArgs(sqlOrArgs ...interface{}) (string, newArgs = append(newArgs, v.In(statement.defaultTimeZone).Format("2006-01-02 15:04:05")) } else if v, ok := arg.(*time.Time); ok && v != nil { newArgs = append(newArgs, v.In(statement.defaultTimeZone).Format("2006-01-02 15:04:05")) + } else if v, ok := arg.(convert.ConversionTo); ok { + r, err := v.ToDB() + if err != nil { + return "", nil, err + } + if r != nil { + // for nvarchar column on mssql, bytes have to be converted as ucs-2 external of driver + // for binary column, a string will be converted as bytes directly. So we have to + // convert bytes as string + if statement.dialect.URI().DBType == schemas.MSSQL { + newArgs = append(newArgs, string(r)) + } else { + newArgs = append(newArgs, r) + } + } else { + newArgs = append(newArgs, nil) + } } else { newArgs = append(newArgs, arg) } diff --git a/tests/session_raw_test.go b/tests/session_raw_test.go index e6987c41..569d7bed 100644 --- a/tests/session_raw_test.go +++ b/tests/session_raw_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "xorm.io/xorm/convert" + "github.com/stretchr/testify/assert" ) @@ -65,3 +67,48 @@ func TestExecTime(t *testing.T) { assert.True(t, has) assert.EqualValues(t, now.In(testEngine.GetTZLocation()).Format("2006-01-02 15:04:05"), uet.Created.Format("2006-01-02 15:04:05")) } + +type ConversionData struct { + MyData string +} + +var _ convert.Conversion = new(ConversionData) + +func (c ConversionData) ToDB() ([]byte, error) { + return []byte(c.MyData), nil +} + +func (c *ConversionData) FromDB(bs []byte) error { + if bs != nil { + c.MyData = string(bs) + } + return nil +} + +func TestExecCustomTypes(t *testing.T) { + assert.NoError(t, PrepareEngine()) + + type UserinfoExec struct { + Uid int + Name string + Data string + } + + assert.NoError(t, testEngine.Sync2(new(UserinfoExec))) + + res, err := testEngine.Exec("INSERT INTO "+testEngine.TableName("`userinfo_exec`", true)+" (uid, name,data) VALUES (?, ?, ?)", + 1, "user", ConversionData{"data"}) + assert.NoError(t, err) + cnt, err := res.RowsAffected() + assert.NoError(t, err) + assert.EqualValues(t, 1, cnt) + + results, err := testEngine.QueryString("select * from " + testEngine.TableName("userinfo_exec", true)) + assert.NoError(t, err) + assert.EqualValues(t, 1, len(results)) + id, err := strconv.Atoi(results[0]["uid"]) + assert.NoError(t, err) + assert.EqualValues(t, 1, id) + assert.Equal(t, "user", results[0]["name"]) + assert.EqualValues(t, "data", results[0]["data"]) +} From db7c2640627d24539aa4607f50bcba7037ddd9e6 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 9 Aug 2023 03:28:52 +0000 Subject: [PATCH 2/2] Add Sync options to ignore constrains and indices (#2320) needed for https://github.com/woodpecker-ci/woodpecker/pull/2117 Reviewed-on: https://gitea.com/xorm/xorm/pulls/2320 Reviewed-by: Lunny Xiao Co-authored-by: 6543 <6543@obermui.de> Co-committed-by: 6543 <6543@obermui.de> --- interface.go | 1 + sync.go | 54 +++++++++++++++++------- tests/schema_test.go | 99 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 16 deletions(-) diff --git a/interface.go b/interface.go index d10abe9e..03dfd236 100644 --- a/interface.go +++ b/interface.go @@ -121,6 +121,7 @@ type EngineInterface interface { ShowSQL(show ...bool) Sync(...interface{}) error Sync2(...interface{}) error + SyncWithOptions(SyncOptions, ...interface{}) (*SyncResult, error) StoreEngine(storeEngine string) *Session TableInfo(bean interface{}) (*schemas.Table, error) TableName(interface{}, ...bool) string diff --git a/sync.go b/sync.go index 635a8ba9..9e1cb8c1 100644 --- a/sync.go +++ b/sync.go @@ -13,6 +13,10 @@ import ( type SyncOptions struct { WarnIfDatabaseColumnMissed bool + // IgnoreConstrains will not add, delete or update unique constrains + IgnoreConstrains bool + // IgnoreIndices will not add or delete indices + IgnoreIndices bool } type SyncResult struct{} @@ -49,6 +53,8 @@ func (session *Session) Sync2(beans ...interface{}) error { func (session *Session) Sync(beans ...interface{}) error { _, err := session.SyncWithOptions(SyncOptions{ WarnIfDatabaseColumnMissed: false, + IgnoreConstrains: false, + IgnoreIndices: false, }, beans...) return err } @@ -103,15 +109,20 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{}) return nil, err } - err = session.createUniques(bean) - if err != nil { - return nil, err + if !opts.IgnoreConstrains { + err = session.createUniques(bean) + if err != nil { + return nil, err + } } - err = session.createIndexes(bean) - if err != nil { - return nil, err + if !opts.IgnoreIndices { + err = session.createIndexes(bean) + if err != nil { + return nil, err + } } + continue } @@ -208,9 +219,12 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{}) } } + // indices found in orig table foundIndexNames := make(map[string]bool) + // indices to be added addedNames := make(map[string]*schemas.Index) + // drop indices that exist in orig and new table schema but are not equal for name, index := range table.Indexes { var oriIndex *schemas.Index for name2, index2 := range oriTable.Indexes { @@ -221,15 +235,13 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{}) } } - if oriIndex != nil { - if oriIndex.Type != index.Type { - sql := engine.dialect.DropIndexSQL(tbNameWithSchema, oriIndex) - _, err = session.exec(sql) - if err != nil { - return nil, err - } - oriIndex = nil + if oriIndex != nil && oriIndex.Type != index.Type { + sql := engine.dialect.DropIndexSQL(tbNameWithSchema, oriIndex) + _, err = session.exec(sql) + if err != nil { + return nil, err } + oriIndex = nil } if oriIndex == nil { @@ -237,8 +249,17 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{}) } } + // drop all indices that do not exist in new schema or have changed for name2, index2 := range oriTable.Indexes { if _, ok := foundIndexNames[name2]; !ok { + // ignore based on there type + if (index2.Type == schemas.IndexType && opts.IgnoreIndices) || + (index2.Type == schemas.UniqueType && opts.IgnoreConstrains) { + // make sure we do not add a index with same name later + delete(addedNames, name2) + continue + } + sql := engine.dialect.DropIndexSQL(tbNameWithSchema, index2) _, err = session.exec(sql) if err != nil { @@ -247,12 +268,13 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{}) } } + // Add new indices because either they did not exist before or were dropped to update them for name, index := range addedNames { - if index.Type == schemas.UniqueType { + if index.Type == schemas.UniqueType && !opts.IgnoreConstrains { session.statement.RefTable = table session.statement.SetTableName(tbNameWithSchema) err = session.addUnique(tbNameWithSchema, name) - } else if index.Type == schemas.IndexType { + } else if index.Type == schemas.IndexType && !opts.IgnoreIndices { session.statement.RefTable = table session.statement.SetTableName(tbNameWithSchema) err = session.addIndex(tbNameWithSchema, name) diff --git a/tests/schema_test.go b/tests/schema_test.go index c945a35c..db9f9e8f 100644 --- a/tests/schema_test.go +++ b/tests/schema_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "xorm.io/xorm" "xorm.io/xorm/schemas" ) @@ -645,3 +646,101 @@ func TestCollate(t *testing.T) { }) assert.NoError(t, err) } + +type SyncWithOpts1 struct { + Id int64 + Index int `xorm:"index"` + Unique int `xorm:"unique"` + Group1 int `xorm:"index(ttt)"` + Group2 int `xorm:"index(ttt)"` + UniGroup1 int `xorm:"unique(lll)"` + UniGroup2 int `xorm:"unique(lll)"` +} + +func (*SyncWithOpts1) TableName() string { + return "sync_with_opts" +} + +type SyncWithOpts2 struct { + Id int64 + Index int `xorm:"index"` + Unique int `xorm:""` + Group1 int `xorm:"index(ttt)"` + Group2 int `xorm:"index(ttt)"` + UniGroup1 int `xorm:""` + UniGroup2 int `xorm:"unique(lll)"` +} + +func (*SyncWithOpts2) TableName() string { + return "sync_with_opts" +} + +type SyncWithOpts3 struct { + Id int64 + Index int `xorm:""` + Unique int `xorm:"unique"` + Group1 int `xorm:""` + Group2 int `xorm:"index(ttt)"` + UniGroup1 int `xorm:"unique(lll)"` + UniGroup2 int `xorm:"unique(lll)"` +} + +func (*SyncWithOpts3) TableName() string { + return "sync_with_opts" +} + +func TestSyncWithOptions(t *testing.T) { + assert.NoError(t, PrepareEngine()) + + // ignore indices and constrains + result, err := testEngine.SyncWithOptions(xorm.SyncOptions{IgnoreIndices: true, IgnoreConstrains: true}, &SyncWithOpts1{}) + assert.NoError(t, err) + assert.NotNil(t, result) + assert.Len(t, getIndicesOfBeanFromDB(t, &SyncWithOpts1{}), 0) + + // only ignore indices + result, err = testEngine.SyncWithOptions(xorm.SyncOptions{IgnoreConstrains: true}, &SyncWithOpts2{}) + assert.NoError(t, err) + assert.NotNil(t, result) + indices := getIndicesOfBeanFromDB(t, &SyncWithOpts1{}) + assert.Len(t, indices, 2) + assert.ElementsMatch(t, []string{"ttt", "index"}, getKeysFromMap(indices)) + + // only ignore constrains + result, err = testEngine.SyncWithOptions(xorm.SyncOptions{IgnoreIndices: true}, &SyncWithOpts3{}) + assert.NoError(t, err) + assert.NotNil(t, result) + indices = getIndicesOfBeanFromDB(t, &SyncWithOpts1{}) + assert.Len(t, indices, 4) + assert.ElementsMatch(t, []string{"ttt", "index", "unique", "lll"}, getKeysFromMap(indices)) + + tableInfoFromStruct, _ := testEngine.TableInfo(&SyncWithOpts1{}) + assert.ElementsMatch(t, getKeysFromMap(tableInfoFromStruct.Indexes), getKeysFromMap(getIndicesOfBeanFromDB(t, &SyncWithOpts1{}))) + +} + +func getIndicesOfBeanFromDB(t *testing.T, bean interface{}) map[string]*schemas.Index { + dbm, err := testEngine.DBMetas() + assert.NoError(t, err) + + tName := testEngine.TableName(bean) + var tSchema *schemas.Table + for _, t := range dbm { + if t.Name == tName { + tSchema = t + break + } + } + if !assert.NotNil(t, tSchema) { + return nil + } + return tSchema.Indexes +} + +func getKeysFromMap(m map[string]*schemas.Index) []string { + var ss []string + for k := range m { + ss = append(ss, k) + } + return ss +}