Merge branch 'master' into azhai-master

.gitea/workflows/release-tag.yml

@@ -13,11 +13,11 @@ jobs:
         with:
           fetch-depth: 0
       - name: setup go
-        uses: https://github.com/actions/setup-go@v4
+        uses: actions/setup-go@v4
         with:
           go-version: '>=1.20.1'
       - name: Use Go Action  
         id: use-go-action
-        uses: actions/release-action@main
+        uses: https://gitea.com/actions/release-action@main
         with:
           api_key: '${{secrets.RELEASE_TOKEN}}'

.gitea/workflows/test-cockroach.yml

@@ -36,7 +36,7 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20
-      - uses: https://github.com/actions/checkout@v3
+      - uses: actions/checkout@v3
       - name: test cockroach
         env:
           TEST_COCKROACH_HOST: "cockroach:26257"

.gitea/workflows/test-mariadb.yml

@@ -36,7 +36,7 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20
-      - uses: https://github.com/actions/checkout@v3
+      - uses: actions/checkout@v3
       - name: test mariadb
         env:
           TEST_MYSQL_HOST: mariadb

.gitea/workflows/test-mssql.yml

@@ -36,7 +36,7 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20
-      - uses: https://github.com/actions/checkout@v3
+      - uses: actions/checkout@v3
       - name: test mssql
         env:
           TEST_MSSQL_HOST: mssql

.gitea/workflows/test-mysql.yml

@@ -36,7 +36,7 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20
-      - uses: https://github.com/actions/checkout@v3
+      - uses: actions/checkout@v3
       - name: test mysql utf8mb4
         env:
           TEST_MYSQL_HOST: mysql

.gitea/workflows/test-mysql8.yml

@@ -36,7 +36,7 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20
-      - uses: https://github.com/actions/checkout@v3
+      - uses: actions/checkout@v3
       - name: test mysql8
         env:
           TEST_MYSQL_HOST: mysql8

.gitea/workflows/test-postgres.yml

@@ -36,7 +36,7 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20
-      - uses: https://github.com/actions/checkout@v3
+      - uses: actions/checkout@v3
       - name: test postgres
         env:
           TEST_PGSQL_HOST: pgsql

.gitea/workflows/test-sqlite.yml

@@ -36,7 +36,7 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20
-      - uses: https://github.com/actions/checkout@v3
+      - uses: actions/checkout@v3
       - name: vet
         run: make vet
       - name: format check

.gitea/workflows/test-tidb.yml

@@ -36,7 +36,7 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20
-      - uses: https://github.com/actions/checkout@v3
+      - uses: actions/checkout@v3
       - name: test tidb
         env:
           TEST_TIDB_HOST: "tidb:4000"

convert/conversion.go

@@ -16,11 +16,21 @@ import (
 	"time"
 )
 
+// ConversionFrom is an inteface to allow retrieve data from database
+type ConversionFrom interface {
+	FromDB([]byte) error
+}
+
+// ConversionTo is an interface to allow store data to database
+type ConversionTo interface {
+	ToDB() ([]byte, error)
+}
+
 // Conversion is an interface. A type implements Conversion will according
 // the custom method to fill into database and retrieve from database.
 type Conversion interface {
-	FromDB([]byte) error
+	ConversionFrom
-	ToDB() ([]byte, error)
+	ConversionTo
 }
 
 // ErrNilPtr represents an error

dialects/postgres.go

@@ -821,6 +821,7 @@ func (db *postgres) Version(ctx context.Context, queryer core.Queryer) (*schemas
 	}
 
 	// Postgres: 9.5.22 on x86_64-pc-linux-gnu (Debian 9.5.22-1.pgdg90+1), compiled by gcc (Debian 6.3.0-18+deb9u1) 6.3.0 20170516, 64-bit
+	// Postgres: PostgreSQL 15.3, compiled by Visual C++ build 1914, 64-bit
 	// CockroachDB CCL v19.2.4 (x86_64-unknown-linux-gnu, built
 	if strings.HasPrefix(version, "CockroachDB") {
 		versions := strings.Split(strings.TrimPrefix(version, "CockroachDB CCL "), " ")
@@ -829,12 +830,22 @@ func (db *postgres) Version(ctx context.Context, queryer core.Queryer) (*schemas
 			Edition: "CockroachDB",
 		}, nil
 	} else if strings.HasPrefix(version, "PostgreSQL") {
+		if strings.Contains(version, " on ") {
 			versions := strings.Split(strings.TrimPrefix(version, "PostgreSQL "), " on ")
 			return &schemas.Version{
 				Number:  versions[0],
 				Level:   versions[1],
 				Edition: "PostgreSQL",
 			}, nil
+		} else {
+			versions := strings.Split(strings.TrimPrefix(version, "PostgreSQL "), ",")
+			return &schemas.Version{
+				Number:  versions[0],
+				Level:   versions[1],
+				Edition: "PostgreSQL",
+			}, nil
+		}
+
 	}
 
 	return nil, errors.New("unknow database version")

interface.go

@@ -121,6 +121,7 @@ type EngineInterface interface {
 	ShowSQL(show ...bool)
 	Sync(...interface{}) error
 	Sync2(...interface{}) error
+	SyncWithOptions(SyncOptions, ...interface{}) (*SyncResult, error)
 	StoreEngine(storeEngine string) *Session
 	TableInfo(bean interface{}) (*schemas.Table, error)
 	TableName(interface{}, ...bool) string

internal/statements/statement.go

@@ -644,6 +644,23 @@ func (statement *Statement) convertSQLOrArgs(sqlOrArgs ...interface{}) (string,
 					newArgs = append(newArgs, v.In(statement.defaultTimeZone).Format("2006-01-02 15:04:05"))
 				} else if v, ok := arg.(*time.Time); ok && v != nil {
 					newArgs = append(newArgs, v.In(statement.defaultTimeZone).Format("2006-01-02 15:04:05"))
+				} else if v, ok := arg.(convert.ConversionTo); ok {
+					r, err := v.ToDB()
+					if err != nil {
+						return "", nil, err
+					}
+					if r != nil {
+						// for nvarchar column on mssql, bytes have to be converted as ucs-2 external of driver
+						// for binary column, a string will be converted as bytes directly. So we have to
+						// convert bytes as string
+						if statement.dialect.URI().DBType == schemas.MSSQL {
+							newArgs = append(newArgs, string(r))
+						} else {
+							newArgs = append(newArgs, r)
+						}
+					} else {
+						newArgs = append(newArgs, nil)
+					}
 				} else {
 					newArgs = append(newArgs, arg)
 				}

sync.go

@@ -13,6 +13,10 @@ import (
 
 type SyncOptions struct {
 	WarnIfDatabaseColumnMissed bool
+	// IgnoreConstrains will not add, delete or update unique constrains
+	IgnoreConstrains bool
+	// IgnoreIndices will not add or delete indices
+	IgnoreIndices bool
 }
 
 type SyncResult struct{}
@@ -49,6 +53,8 @@ func (session *Session) Sync2(beans ...interface{}) error {
 func (session *Session) Sync(beans ...interface{}) error {
 	_, err := session.SyncWithOptions(SyncOptions{
 		WarnIfDatabaseColumnMissed: false,
+		IgnoreConstrains:           false,
+		IgnoreIndices:              false,
 	}, beans...)
 	return err
 }
@@ -103,15 +109,20 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{})
 				return nil, err
 			}
 
+			if !opts.IgnoreConstrains {
 				err = session.createUniques(bean)
 				if err != nil {
 					return nil, err
 				}
+			}
 
+			if !opts.IgnoreIndices {
 				err = session.createIndexes(bean)
 				if err != nil {
 					return nil, err
 				}
+			}
+
 			continue
 		}
 
@@ -208,9 +219,12 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{})
 			}
 		}
 
+		// indices found in orig table
 		foundIndexNames := make(map[string]bool)
+		// indices to be added
 		addedNames := make(map[string]*schemas.Index)
 
+		// drop indices that exist in orig and new table schema but are not equal
 		for name, index := range table.Indexes {
 			var oriIndex *schemas.Index
 			for name2, index2 := range oriTable.Indexes {
@@ -221,8 +235,7 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{})
 				}
 			}
 
-			if oriIndex != nil {
+			if oriIndex != nil && oriIndex.Type != index.Type {
-				if oriIndex.Type != index.Type {
 				sql := engine.dialect.DropIndexSQL(tbNameWithSchema, oriIndex)
 				_, err = session.exec(sql)
 				if err != nil {
@@ -230,15 +243,23 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{})
 				}
 				oriIndex = nil
 			}
-			}
 
 			if oriIndex == nil {
 				addedNames[name] = index
 			}
 		}
 
+		// drop all indices that do not exist in new schema or have changed
 		for name2, index2 := range oriTable.Indexes {
 			if _, ok := foundIndexNames[name2]; !ok {
+				// ignore based on there type
+				if (index2.Type == schemas.IndexType && opts.IgnoreIndices) ||
+					(index2.Type == schemas.UniqueType && opts.IgnoreConstrains) {
+					// make sure we do not add a index with same name later
+					delete(addedNames, name2)
+					continue
+				}
+
 				sql := engine.dialect.DropIndexSQL(tbNameWithSchema, index2)
 				_, err = session.exec(sql)
 				if err != nil {
@@ -247,12 +268,13 @@ func (session *Session) SyncWithOptions(opts SyncOptions, beans ...interface{})
 			}
 		}
 
+		// Add new indices because either they did not exist before or were dropped to update them
 		for name, index := range addedNames {
-			if index.Type == schemas.UniqueType {
+			if index.Type == schemas.UniqueType && !opts.IgnoreConstrains {
 				session.statement.RefTable = table
 				session.statement.SetTableName(tbNameWithSchema)
 				err = session.addUnique(tbNameWithSchema, name)
-			} else if index.Type == schemas.IndexType {
+			} else if index.Type == schemas.IndexType && !opts.IgnoreIndices {
 				session.statement.RefTable = table
 				session.statement.SetTableName(tbNameWithSchema)
 				err = session.addIndex(tbNameWithSchema, name)

tests/schema_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"xorm.io/xorm"
 	"xorm.io/xorm/schemas"
 )
 
@@ -645,3 +646,101 @@ func TestCollate(t *testing.T) {
 	})
 	assert.NoError(t, err)
 }
+
+type SyncWithOpts1 struct {
+	Id        int64
+	Index     int `xorm:"index"`
+	Unique    int `xorm:"unique"`
+	Group1    int `xorm:"index(ttt)"`
+	Group2    int `xorm:"index(ttt)"`
+	UniGroup1 int `xorm:"unique(lll)"`
+	UniGroup2 int `xorm:"unique(lll)"`
+}
+
+func (*SyncWithOpts1) TableName() string {
+	return "sync_with_opts"
+}
+
+type SyncWithOpts2 struct {
+	Id        int64
+	Index     int `xorm:"index"`
+	Unique    int `xorm:""`
+	Group1    int `xorm:"index(ttt)"`
+	Group2    int `xorm:"index(ttt)"`
+	UniGroup1 int `xorm:""`
+	UniGroup2 int `xorm:"unique(lll)"`
+}
+
+func (*SyncWithOpts2) TableName() string {
+	return "sync_with_opts"
+}
+
+type SyncWithOpts3 struct {
+	Id        int64
+	Index     int `xorm:""`
+	Unique    int `xorm:"unique"`
+	Group1    int `xorm:""`
+	Group2    int `xorm:"index(ttt)"`
+	UniGroup1 int `xorm:"unique(lll)"`
+	UniGroup2 int `xorm:"unique(lll)"`
+}
+
+func (*SyncWithOpts3) TableName() string {
+	return "sync_with_opts"
+}
+
+func TestSyncWithOptions(t *testing.T) {
+	assert.NoError(t, PrepareEngine())
+
+	// ignore indices and constrains
+	result, err := testEngine.SyncWithOptions(xorm.SyncOptions{IgnoreIndices: true, IgnoreConstrains: true}, &SyncWithOpts1{})
+	assert.NoError(t, err)
+	assert.NotNil(t, result)
+	assert.Len(t, getIndicesOfBeanFromDB(t, &SyncWithOpts1{}), 0)
+
+	// only ignore indices
+	result, err = testEngine.SyncWithOptions(xorm.SyncOptions{IgnoreConstrains: true}, &SyncWithOpts2{})
+	assert.NoError(t, err)
+	assert.NotNil(t, result)
+	indices := getIndicesOfBeanFromDB(t, &SyncWithOpts1{})
+	assert.Len(t, indices, 2)
+	assert.ElementsMatch(t, []string{"ttt", "index"}, getKeysFromMap(indices))
+
+	// only ignore constrains
+	result, err = testEngine.SyncWithOptions(xorm.SyncOptions{IgnoreIndices: true}, &SyncWithOpts3{})
+	assert.NoError(t, err)
+	assert.NotNil(t, result)
+	indices = getIndicesOfBeanFromDB(t, &SyncWithOpts1{})
+	assert.Len(t, indices, 4)
+	assert.ElementsMatch(t, []string{"ttt", "index", "unique", "lll"}, getKeysFromMap(indices))
+
+	tableInfoFromStruct, _ := testEngine.TableInfo(&SyncWithOpts1{})
+	assert.ElementsMatch(t, getKeysFromMap(tableInfoFromStruct.Indexes), getKeysFromMap(getIndicesOfBeanFromDB(t, &SyncWithOpts1{})))
+
+}
+
+func getIndicesOfBeanFromDB(t *testing.T, bean interface{}) map[string]*schemas.Index {
+	dbm, err := testEngine.DBMetas()
+	assert.NoError(t, err)
+
+	tName := testEngine.TableName(bean)
+	var tSchema *schemas.Table
+	for _, t := range dbm {
+		if t.Name == tName {
+			tSchema = t
+			break
+		}
+	}
+	if !assert.NotNil(t, tSchema) {
+		return nil
+	}
+	return tSchema.Indexes
+}
+
+func getKeysFromMap(m map[string]*schemas.Index) []string {
+	var ss []string
+	for k := range m {
+		ss = append(ss, k)
+	}
+	return ss
+}

tests/session_find_test.go

@@ -1237,3 +1237,20 @@ func TestBuilderDialect(t *testing.T) {
 	err := testEngine.Table("test_builder_dialect").Where(builder.Eq{"age2": 2}).Join("INNER", inner, "test_builder_dialect_foo.dialect_id = test_builder_dialect.id").Find(&result)
 	assert.NoError(t, err)
 }
+
+func TestFindInMaxID(t *testing.T) {
+	assert.NoError(t, PrepareEngine())
+
+	type TestFindInMaxId struct {
+		Id   int64
+		Name string `xorm:"index"`
+		Age2 int
+	}
+
+	assertSync(t, new(TestFindInMaxId))
+
+	var res []TestFindInMaxId
+	tableName := testEngine.TableName("test_find_in_max_id", true)
+	err := testEngine.In("id", builder.Select("max(id)").From(testEngine.Quote(tableName))).Find(&res)
+	assert.NoError(t, err)
+}

tests/session_raw_test.go

@@ -9,6 +9,8 @@ import (
 	"testing"
 	"time"
 
+	"xorm.io/xorm/convert"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -65,3 +67,48 @@ func TestExecTime(t *testing.T) {
 	assert.True(t, has)
 	assert.EqualValues(t, now.In(testEngine.GetTZLocation()).Format("2006-01-02 15:04:05"), uet.Created.Format("2006-01-02 15:04:05"))
 }
+
+type ConversionData struct {
+	MyData string
+}
+
+var _ convert.Conversion = new(ConversionData)
+
+func (c ConversionData) ToDB() ([]byte, error) {
+	return []byte(c.MyData), nil
+}
+
+func (c *ConversionData) FromDB(bs []byte) error {
+	if bs != nil {
+		c.MyData = string(bs)
+	}
+	return nil
+}
+
+func TestExecCustomTypes(t *testing.T) {
+	assert.NoError(t, PrepareEngine())
+
+	type UserinfoExec struct {
+		Uid  int
+		Name string
+		Data string
+	}
+
+	assert.NoError(t, testEngine.Sync2(new(UserinfoExec)))
+
+	res, err := testEngine.Exec("INSERT INTO "+testEngine.TableName("`userinfo_exec`", true)+" (uid, name,data) VALUES (?, ?, ?)",
+		1, "user", ConversionData{"data"})
+	assert.NoError(t, err)
+	cnt, err := res.RowsAffected()
+	assert.NoError(t, err)
+	assert.EqualValues(t, 1, cnt)
+
+	results, err := testEngine.QueryString("select * from " + testEngine.TableName("userinfo_exec", true))
+	assert.NoError(t, err)
+	assert.EqualValues(t, 1, len(results))
+	id, err := strconv.Atoi(results[0]["uid"])
+	assert.NoError(t, err)
+	assert.EqualValues(t, 1, id)
+	assert.Equal(t, "user", results[0]["name"])
+	assert.EqualValues(t, "data", results[0]["data"])
+}