Merge branch 'master' into IsReserved

docs/QuickStart.md

@@ -346,7 +346,6 @@ Notice: If you want to use transaction on inserting, you should use session.Begi
 ### 5.1. Chainable APIs for Queries, Execusions and Aggregations
 
 Queries and Aggregations is basically formed by using `Get`, `Find`, `Count` methods, with conjunction of following chainable APIs to form conditions, grouping and ordering:
-查询和统计主要使用`Get`, `Find`, `Count`三个方法。在进行查询时可以使用多个方法来形成查询条件，条件函数如下：
 
 * Id([]interface{})
 Primary Key lookup
@@ -535,16 +534,16 @@ affected, err := engine.Table(new(User)).Id(id).Update(map[string]interface{}{"a
 ```
 
 
-### 6.1.乐观锁
+### 6.1.Optimistic Lock
 
-要使用乐观锁，需要使用version标记
+To enable object optimistic lock, add 'version' tag value:
 type User struct {
 	Id int64
 	Name string
 	Version int `xorm:"version"`
 }
 
-在Insert时，version标记的字段将会被设置为1，在Update时，Update的内容必须包含version原来的值。
+The version starts with 1 when inserted to DB. For updating make sure originated version value is used for optimistic lock check.
 
 ```Go
 var user User

engine.go

@@ -42,6 +42,14 @@ type Engine struct {
 
 	Logger     core.ILogger
 	TZLocation *time.Location
+
+	disableGlobalCache bool
+}
+
+func (engine *Engine) SetDisableGlobalCache(disable bool) {
+	if engine.disableGlobalCache != disable {
+		engine.disableGlobalCache = disable
+	}
 }
 
 func (engine *Engine) DriverName() string {
@@ -186,6 +194,30 @@ func (engine *Engine) logSQL(sqlStr string, sqlArgs ...interface{}) {
 	}
 }
 
+func (engine *Engine) LogSQLQueryTime(sqlStr string, args interface{}, executionBlock func() (*core.Stmt, *core.Rows, error)) (*core.Stmt, *core.Rows, error) {
+	if engine.ShowDebug {
+		b4ExecTime := time.Now()
+		stmt, res, err := executionBlock()
+		execDuration := time.Since(b4ExecTime)
+		engine.LogDebugf("sql [%s] - args [%v] - query took: %vns", sqlStr, args, execDuration.Nanoseconds())
+		return stmt, res, err
+	} else {
+		return executionBlock()
+	}
+}
+
+func (engine *Engine) LogSQLExecutionTime(sqlStr string, args interface{}, executionBlock func() (sql.Result, error)) (sql.Result, error) {
+	if engine.ShowDebug {
+		b4ExecTime := time.Now()
+		res, err := executionBlock()
+		execDuration := time.Since(b4ExecTime)
+		engine.LogDebugf("sql [%s] - args [%v] - execution took: %vns", sqlStr, args, execDuration.Nanoseconds())
+		return res, err
+	} else {
+		return executionBlock()
+	}
+}
+
 // logging error
 func (engine *Engine) overrideLogLevel(overrideLevel core.LogLevel) {
 	logLevel := engine.Logger.Level()
@@ -330,7 +362,7 @@ func (engine *Engine) DumpAll(w io.Writer) error {
 	}
 
 	for _, table := range tables {
-		_, err = io.WriteString(w, engine.dialect.CreateTableSql(table, "", "", "")+"\n\n")
+		_, err = io.WriteString(w, engine.dialect.CreateTableSql(table, "", table.StoreEngine, "")+"\n\n")
 		if err != nil {
 			return err
 		}
@@ -380,7 +412,12 @@ func (engine *Engine) DumpAll(w io.Writer) error {
 						temp += fmt.Sprintf(", '%s'", d.(string))
 					}
 				} else if col.SQLType.IsNumeric() {
-					temp += fmt.Sprintf(", %s", string(d.([]byte)))
+					switch reflect.TypeOf(d).Kind() {
+					case reflect.Slice:
+						temp += fmt.Sprintf(", %s", string(d.([]byte)))
+					default:
+						temp += fmt.Sprintf(", %v", d)
+					}
 				} else {
 					s := fmt.Sprintf("%v", d)
 					if strings.Contains(s, ":") || strings.Contains(s, "-") {
@@ -591,7 +628,7 @@ func (engine *Engine) autoMapType(v reflect.Value) *core.Table {
 	return table
 }
 
-func (engine *Engine) autoMap(bean interface{}) *core.Table {
+func (engine *Engine) TableInfo(bean interface{}) *core.Table {
 	v := rValue(bean)
 	return engine.autoMapType(v)
 }
@@ -610,7 +647,10 @@ func addIndex(indexName string, table *core.Table, col *core.Column, indexType i
 
 func (engine *Engine) newTable() *core.Table {
 	table := core.NewEmptyTable()
-	table.Cacher = engine.Cacher
+
+	if !engine.disableGlobalCache {
+		table.Cacher = engine.Cacher
+	}
 	return table
 }
 
@@ -639,6 +679,9 @@ func (engine *Engine) mapType(v reflect.Value) *core.Table {
 	var idFieldColName string
 	var err error
 
+	hasCacheTag := false
+	hasNoCacheTag := false
+
 	for i := 0; i < t.NumField(); i++ {
 		tag := t.Field(i).Tag
 
@@ -733,6 +776,14 @@ func (engine *Engine) mapType(v reflect.Value) *core.Table {
 						isUnique = true
 					case k == "NOTNULL":
 						col.Nullable = false
+					case k == "CACHE":
+						if !hasCacheTag {
+							hasCacheTag = true
+						}
+					case k == "NOCACHE":
+						if !hasNoCacheTag {
+							hasNoCacheTag = true
+						}
 					case k == "NOT":
 					default:
 						if strings.HasPrefix(k, "'") && strings.HasSuffix(k, "'") {
@@ -841,7 +892,7 @@ func (engine *Engine) mapType(v reflect.Value) *core.Table {
 		if fieldType.Kind() == reflect.Int64 && (col.FieldName == "Id" || strings.HasSuffix(col.FieldName, ".Id")) {
 			idFieldColName = col.Name
 		}
-	}
+	} // end for
 
 	if idFieldColName != "" && len(table.PrimaryKeys) == 0 {
 		col := table.GetColumn(idFieldColName)
@@ -852,6 +903,20 @@ func (engine *Engine) mapType(v reflect.Value) *core.Table {
 		table.AutoIncrement = col.Name
 	}
 
+	if hasCacheTag {
+		if engine.Cacher != nil { // !nash! use engine's cacher if provided
+			engine.Logger.Info("enable cache on table:", table.Name)
+			table.Cacher = engine.Cacher
+		} else {
+			engine.Logger.Info("enable LRU cache on table:", table.Name)
+			table.Cacher = NewLRUCacher2(NewMemoryStore(), time.Hour, 10000) // !nashtsai! HACK use LRU cacher for now
+		}
+	}
+	if hasNoCacheTag {
+		engine.Logger.Info("no cache on table:", table.Name)
+		table.Cacher = nil
+	}
+
 	return table
 }
 
@@ -900,7 +965,7 @@ func (engine *Engine) IsTableExist(bean interface{}) (bool, error) {
 }
 
 func (engine *Engine) IdOf(bean interface{}) core.PK {
-	table := engine.autoMap(bean)
+	table := engine.TableInfo(bean)
 	v := reflect.Indirect(reflect.ValueOf(bean))
 	pk := make([]interface{}, len(table.PrimaryKeys))
 	for i, col := range table.PKColumns() {
@@ -948,7 +1013,7 @@ func (engine *Engine) ClearCacheBean(bean interface{}, id string) error {
 	if t.Kind() != reflect.Struct {
 		return errors.New("error params")
 	}
-	table := engine.autoMap(bean)
+	table := engine.TableInfo(bean)
 	cacher := table.Cacher
 	if cacher == nil {
 		cacher = engine.Cacher
@@ -967,7 +1032,7 @@ func (engine *Engine) ClearCache(beans ...interface{}) error {
 		if t.Kind() != reflect.Struct {
 			return errors.New("error params")
 		}
-		table := engine.autoMap(bean)
+		table := engine.TableInfo(bean)
 		cacher := table.Cacher
 		if cacher == nil {
 			cacher = engine.Cacher
@@ -985,7 +1050,7 @@ func (engine *Engine) ClearCache(beans ...interface{}) error {
 // If you change some field, you should change the database manually.
 func (engine *Engine) Sync(beans ...interface{}) error {
 	for _, bean := range beans {
-		table := engine.autoMap(bean)
+		table := engine.TableInfo(bean)
 
 		s := engine.NewSession()
 		defer s.Close()
@@ -1084,7 +1149,7 @@ func (engine *Engine) Sync2(beans ...interface{}) error {
 	structTables := make([]*core.Table, 0)
 
 	for _, bean := range beans {
-		table := engine.autoMap(bean)
+		table := engine.TableInfo(bean)
 		structTables = append(structTables, table)
 
 		var oriTable *core.Table

helpers.go

@@ -70,28 +70,23 @@ func sliceEq(left, right []string) bool {
 	return true
 }
 
-func value2Bytes(rawValue *reflect.Value) (data []byte, err error) {
+func reflect2value(rawValue *reflect.Value) (str string, err error) {
 	aa := reflect.TypeOf((*rawValue).Interface())
 	vv := reflect.ValueOf((*rawValue).Interface())
-
-	var str string
 	switch aa.Kind() {
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
 		str = strconv.FormatInt(vv.Int(), 10)
-		data = []byte(str)
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
 		str = strconv.FormatUint(vv.Uint(), 10)
-		data = []byte(str)
 	case reflect.Float32, reflect.Float64:
 		str = strconv.FormatFloat(vv.Float(), 'f', -1, 64)
-		data = []byte(str)
 	case reflect.String:
 		str = vv.String()
-		data = []byte(str)
 	case reflect.Array, reflect.Slice:
 		switch aa.Elem().Kind() {
 		case reflect.Uint8:
-			data = rawValue.Interface().([]byte)
+			data := rawValue.Interface().([]byte)
+			str = string(data)
 		default:
 			err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
 		}
@@ -99,16 +94,13 @@ func value2Bytes(rawValue *reflect.Value) (data []byte, err error) {
 	case reflect.Struct:
 		if aa == core.TimeType {
 			str = rawValue.Interface().(time.Time).Format(time.RFC3339Nano)
-			data = []byte(str)
 		} else {
 			err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
 		}
 	case reflect.Bool:
 		str = strconv.FormatBool(vv.Bool())
-		data = []byte(str)
 	case reflect.Complex128, reflect.Complex64:
 		str = fmt.Sprintf("%v", vv.Complex())
-		data = []byte(str)
 	/* TODO: unsupported types below
 	   case reflect.Map:
 	   case reflect.Ptr:
@@ -122,6 +114,40 @@ func value2Bytes(rawValue *reflect.Value) (data []byte, err error) {
 	return
 }
 
+func value2Bytes(rawValue *reflect.Value) (data []byte, err error) {
+	var str string
+	str, err = reflect2value(rawValue)
+	if err != nil {
+		return
+	}
+	data = []byte(str)
+	return
+}
+
+func value2String(rawValue *reflect.Value) (data string, err error) {
+	data, err = reflect2value(rawValue)
+	if err != nil {
+		return
+	}
+	return
+}
+
+func rows2Strings(rows *core.Rows) (resultsSlice []map[string]string, err error) {
+	fields, err := rows.Columns()
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		result, err := row2mapStr(rows, fields)
+		if err != nil {
+			return nil, err
+		}
+		resultsSlice = append(resultsSlice, result)
+	}
+
+	return resultsSlice, nil
+}
+
 func rows2maps(rows *core.Rows) (resultsSlice []map[string][]byte, err error) {
 	fields, err := rows.Columns()
 	if err != nil {

mysql_dialect.go

@@ -418,6 +418,7 @@ func (db *mysql) GetTables() ([]*core.Table, error) {
 		}
 
 		table.Name = name
+		table.StoreEngine = engine
 		tables = append(tables, table)
 	}
 	return tables, nil

rows.go

@@ -34,7 +34,7 @@ func newRows(session *Session, bean interface{}) (*Rows, error) {
 
 	var sqlStr string
 	var args []interface{}
-	rows.session.Statement.RefTable = rows.session.Engine.autoMap(bean)
+	rows.session.Statement.RefTable = rows.session.Engine.TableInfo(bean)
 	if rows.session.Statement.RawSQL == "" {
 		sqlStr, args = rows.session.Statement.genGetSql(bean)
 	} else {

session.go

@@ -447,10 +447,12 @@ func (session *Session) exec(sqlStr string, args ...interface{}) (sql.Result, er
 
 	session.Engine.logSQL(sqlStr, args...)
 
-	if session.IsAutoCommit {
+	return session.Engine.LogSQLExecutionTime(sqlStr, args, func() (sql.Result, error) {
-		return session.innerExec(sqlStr, args...)
+		if session.IsAutoCommit {
-	}
+			return session.innerExec(sqlStr, args...)
-	return session.Tx.Exec(sqlStr, args...)
+		}
+		return session.Tx.Exec(sqlStr, args...)
+	})
 }
 
 // Exec raw sql
@@ -469,7 +471,7 @@ func (session *Session) Exec(sqlStr string, args ...interface{}) (sql.Result, er
 
 // this function create a table according a bean
 func (session *Session) CreateTable(bean interface{}) error {
-	session.Statement.RefTable = session.Engine.autoMap(bean)
+	session.Statement.RefTable = session.Engine.TableInfo(bean)
 
 	err := session.newDb()
 	if err != nil {
@@ -485,7 +487,7 @@ func (session *Session) CreateTable(bean interface{}) error {
 
 // create indexes
 func (session *Session) CreateIndexes(bean interface{}) error {
-	session.Statement.RefTable = session.Engine.autoMap(bean)
+	session.Statement.RefTable = session.Engine.TableInfo(bean)
 
 	err := session.newDb()
 	if err != nil {
@@ -508,7 +510,7 @@ func (session *Session) CreateIndexes(bean interface{}) error {
 
 // create uniques
 func (session *Session) CreateUniques(bean interface{}) error {
-	session.Statement.RefTable = session.Engine.autoMap(bean)
+	session.Statement.RefTable = session.Engine.TableInfo(bean)
 
 	err := session.newDb()
 	if err != nil {
@@ -595,7 +597,7 @@ func (session *Session) DropTable(bean interface{}) error {
 	if t.Kind() == reflect.String {
 		session.Statement.AltTableName = bean.(string)
 	} else if t.Kind() == reflect.Struct {
-		session.Statement.RefTable = session.Engine.autoMap(bean)
+		session.Statement.RefTable = session.Engine.TableInfo(bean)
 	} else {
 		return errors.New("Unsupported type")
 	}
@@ -651,6 +653,7 @@ func (session *Session) cacheGet(bean interface{}, sqlStr string, args ...interf
 			if v, ok := data[session.Statement.RefTable.PrimaryKeys[0]]; !ok {
 				return false, ErrCacheFailed
 			} else {
+				// TODO https://github.com/go-xorm/xorm/issues/144, PK may not always be int64
 				id, err = strconv.ParseInt(string(v), 10, 64)
 				if err != nil {
 					return false, err
@@ -951,7 +954,7 @@ func (session *Session) Get(bean interface{}) (bool, error) {
 	var args []interface{}
 
 	if session.Statement.RefTable == nil {
-		session.Statement.RefTable = session.Engine.autoMap(bean)
+		session.Statement.RefTable = session.Engine.TableInfo(bean)
 	}
 
 	if session.Statement.RawSQL == "" {
@@ -1393,6 +1396,34 @@ func (session *Session) dropAll() error {
 	return nil
 }
 
+func row2mapStr(rows *core.Rows, fields []string) (resultsMap map[string]string, err error) {
+	result := make(map[string]string)
+	scanResultContainers := make([]interface{}, len(fields))
+	for i := 0; i < len(fields); i++ {
+		var scanResultContainer interface{}
+		scanResultContainers[i] = &scanResultContainer
+	}
+	if err := rows.Scan(scanResultContainers...); err != nil {
+		return nil, err
+	}
+
+	for ii, key := range fields {
+		rawValue := reflect.Indirect(reflect.ValueOf(scanResultContainers[ii]))
+		//if row is null then ignore
+		if rawValue.Interface() == nil {
+			//fmt.Println("ignore ...", key, rawValue)
+			continue
+		}
+
+		if data, err := value2String(&rawValue); err == nil {
+			result[key] = data
+		} else {
+			return nil, err // !nashtsai! REVIEW, should return err or just error log?
+		}
+	}
+	return result, nil
+}
+
 func row2map(rows *core.Rows, fields []string) (resultsMap map[string][]byte, err error) {
 	result := make(map[string][]byte)
 	scanResultContainers := make([]interface{}, len(fields))
@@ -1785,15 +1816,16 @@ func (session *Session) queryPreprocess(sqlStr *string, paramStr ...interface{})
 }
 
 func (session *Session) query(sqlStr string, paramStr ...interface{}) (resultsSlice []map[string][]byte, err error) {
+
 	session.queryPreprocess(&sqlStr, paramStr...)
 
 	if session.IsAutoCommit {
-		return query(session.Db, sqlStr, paramStr...)
+		return session.innerQuery(session.Db, sqlStr, paramStr...)
 	}
-	return txQuery(session.Tx, sqlStr, paramStr...)
+	return session.txQuery(session.Tx, sqlStr, paramStr...)
 }
 
-func txQuery(tx *core.Tx, sqlStr string, params ...interface{}) (resultsSlice []map[string][]byte, err error) {
+func (session *Session) txQuery(tx *core.Tx, sqlStr string, params ...interface{}) (resultsSlice []map[string][]byte, err error) {
 	rows, err := tx.Query(sqlStr, params...)
 	if err != nil {
 		return nil, err
@@ -1803,17 +1835,26 @@ func txQuery(tx *core.Tx, sqlStr string, params ...interface{}) (resultsSlice []
 	return rows2maps(rows)
 }
 
-func query(db *core.DB, sqlStr string, params ...interface{}) (resultsSlice []map[string][]byte, err error) {
+func (session *Session) innerQuery(db *core.DB, sqlStr string, params ...interface{}) (resultsSlice []map[string][]byte, err error) {
-	s, err := db.Prepare(sqlStr)
+
+	stmt, rows, err := session.Engine.LogSQLQueryTime(sqlStr, params, func() (*core.Stmt, *core.Rows, error) {
+		stmt, err := db.Prepare(sqlStr)
+		if err != nil {
+			return stmt, nil, err
+		}
+		rows, err := stmt.Query(params...)
+
+		return stmt, rows, err
+	})
+	if rows != nil {
+		defer rows.Close()
+	}
+	if stmt != nil {
+		defer stmt.Close()
+	}
 	if err != nil {
 		return nil, err
 	}
-	defer s.Close()
-	rows, err := s.Query(params...)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
 	return rows2maps(rows)
 }
 
@@ -1831,6 +1872,62 @@ func (session *Session) Query(sqlStr string, paramStr ...interface{}) (resultsSl
 	return session.query(sqlStr, paramStr...)
 }
 
+
+
+
+
+// =============================
+// for string
+// =============================
+func (session *Session) query2(sqlStr string, paramStr ...interface{}) (resultsSlice []map[string]string, err error) {
+	session.queryPreprocess(&sqlStr, paramStr...)
+
+	if session.IsAutoCommit {
+		return query2(session.Db, sqlStr, paramStr...)
+	}
+	return txQuery2(session.Tx, sqlStr, paramStr...)
+}
+
+func txQuery2(tx *core.Tx, sqlStr string, params ...interface{}) (resultsSlice []map[string]string, err error) {
+	rows, err := tx.Query(sqlStr, params...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	return rows2Strings(rows)
+}
+
+func query2(db *core.DB, sqlStr string, params ...interface{}) (resultsSlice []map[string]string, err error) {
+	s, err := db.Prepare(sqlStr)
+	if err != nil {
+		return nil, err
+	}
+	defer s.Close()
+	rows, err := s.Query(params...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	return rows2Strings(rows)
+}
+
+// Exec a raw sql and return records as []map[string]string
+func (session *Session) Q(sqlStr string, paramStr ...interface{}) (resultsSlice []map[string]string, err error) {
+	err = session.newDb()
+	if err != nil {
+		return nil, err
+	}
+	defer session.resetStatement()
+	if session.IsAutoClose {
+		defer session.Close()
+	}
+	return session.query2(sqlStr, paramStr...)
+}
+
+
+
+
 // insert one or more beans
 func (session *Session) Insert(beans ...interface{}) (int64, error) {
 	var affected int64 = 0
@@ -1979,6 +2076,7 @@ func (session *Session) innerInsertMulti(rowsSlicePtr interface{}) (int64, error
 		strings.Join(colMultiPlaces, "),("))
 
 	res, err := session.exec(statement, args...)
+
 	if err != nil {
 		return 0, err
 	}
@@ -2635,7 +2733,7 @@ func (session *Session) value2Interface(col *core.Column, fieldValue reflect.Val
 }
 
 func (session *Session) innerInsert(bean interface{}) (int64, error) {
-	table := session.Engine.autoMap(bean)
+	table := session.Engine.TableInfo(bean)
 	session.Statement.RefTable = table
 
 	// handle BeforeInsertProcessor
@@ -3032,7 +3130,7 @@ func (session *Session) Update(bean interface{}, condiBean ...interface{}) (int6
 	// --
 
 	if t.Kind() == reflect.Struct {
-		table = session.Engine.autoMap(bean)
+		table = session.Engine.TableInfo(bean)
 		session.Statement.RefTable = table
 
 		if session.Statement.ColumnStr == "" {
@@ -3286,7 +3384,7 @@ func (session *Session) Delete(bean interface{}) (int64, error) {
 	}
 	// --
 
-	table := session.Engine.autoMap(bean)
+	table := session.Engine.TableInfo(bean)
 	session.Statement.RefTable = table
 	colNames, args := buildConditions(session.Engine, table, bean, true, true,
 		false, true, session.Statement.allUseBool, session.Statement.useAllCols,

statement.go

@@ -151,7 +151,7 @@ func (statement *Statement) Table(tableNameOrBean interface{}) *Statement {
 
 /*func (statement *Statement) genFields(bean interface{}) map[string]interface{} {
     results := make(map[string]interface{})
-    table := statement.Engine.autoMap(bean)
+    table := statement.Engine.TableInfo(bean)
     for _, col := range table.Columns {
         fieldValue := col.ValueOf(bean)
         fieldType := reflect.TypeOf(fieldValue.Interface())
@@ -1020,7 +1020,7 @@ func (s *Statement) genDropSQL() string {
 func (statement *Statement) genGetSql(bean interface{}) (string, []interface{}) {
 	var table *core.Table
 	if statement.RefTable == nil {
-		table = statement.Engine.autoMap(bean)
+		table = statement.Engine.TableInfo(bean)
 		statement.RefTable = table
 	} else {
 		table = statement.RefTable
@@ -1070,7 +1070,7 @@ func (s *Statement) genAddUniqueStr(uqeName string, cols []string) (string, []in
 }*/
 
 func (statement *Statement) genCountSql(bean interface{}) (string, []interface{}) {
-	table := statement.Engine.autoMap(bean)
+	table := statement.Engine.TableInfo(bean)
 	statement.RefTable = table
 
 	colNames, args := buildConditions(statement.Engine, table, bean, true, true, false,