From 529a264d8e18fc972ab095aae394328919e83807 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 1 Dec 2023 07:07:35 +0000 Subject: [PATCH 1/4] Fix sqlite multiline query parse (#2367) The app "DB Browser for SQLite" produces multiline queries in the `sqlite_master` table which xorm can't parse. Examples: ``` CREATE TABLE "push_mirror" ( id INTEGER not null primary key autoincrement, repo_id INTEGER, remote_name TEXT, interval INTEGER, created_unix INTEGER, last_update INTEGER, last_error TEXT, sync_on_commit INTEGER default 0 not null , `remote_address` TEXT NULL) ``` ``` CREATE INDEX "IDX_audit_event_action" ON "audit_event" ( "action" ) ``` Reviewed-on: https://gitea.com/xorm/xorm/pulls/2367 Reviewed-by: Lunny Xiao Co-authored-by: KN4CK3R Co-committed-by: KN4CK3R --- dialects/sqlite3.go | 9 ++++++--- dialects/sqlite3_test.go | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/dialects/sqlite3.go b/dialects/sqlite3.go index 62519b6b..ece360e7 100644 --- a/dialects/sqlite3.go +++ b/dialects/sqlite3.go @@ -11,6 +11,7 @@ import ( "fmt" "regexp" "strings" + "unicode" "xorm.io/xorm/core" "xorm.io/xorm/schemas" @@ -320,7 +321,7 @@ func splitColStr(colStr string) []string { var lastIdx int var hasC, hasQuote bool for i, c := range colStr { - if c == ' ' && !hasQuote { + if unicode.IsSpace(c) && !hasQuote { if hasC { results = append(results, colStr[lastIdx:i]) hasC = false @@ -350,7 +351,7 @@ func parseString(colStr string) (*schemas.Column, error) { for idx, field := range fields { if idx == 0 { - col.Name = strings.Trim(strings.Trim(field, "`[] "), `"`) + col.Name = strings.Trim(strings.TrimSpace(field), "`[]'\"") continue } else if idx == 1 { col.SQLType = schemas.SQLType{Name: field, DefaultLength: 0, DefaultLength2: 0} @@ -400,6 +401,8 @@ func (db *sqlite3) GetColumns(queryer core.Queryer, ctx context.Context, tableNa return nil, nil, errors.New("no table named " + tableName) } + name = strings.ReplaceAll(name, "\n", " ") + nStart := strings.Index(name, "(") nEnd := strings.LastIndex(name, ")") reg := regexp.MustCompile(`[^\(,\)]*(\([^\(]*\))?`) @@ -483,7 +486,7 @@ func (db *sqlite3) GetIndexes(queryer core.Queryer, ctx context.Context, tableNa if !tmpSQL.Valid { continue } - sql := tmpSQL.String + sql := strings.ReplaceAll(tmpSQL.String, "\n", " ") index := new(schemas.Index) nNStart := strings.Index(sql, "INDEX") diff --git a/dialects/sqlite3_test.go b/dialects/sqlite3_test.go index aa6c3cea..ec973ae6 100644 --- a/dialects/sqlite3_test.go +++ b/dialects/sqlite3_test.go @@ -27,6 +27,13 @@ func TestSplitColStr(t *testing.T) { "`created`", "DATETIME", "DEFAULT", "'2006-01-02 15:04:05'", "NULL", }, }, + { + colStr: ` id INTEGER not null +primary key autoincrement`, + fields: []string{ + "id", "INTEGER", "not null", "primary", "key", "autoincrement", + }, + }, } for _, kase := range kases { From b571d918586f137d4bc5819909de4f13befdbf1c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 15 Dec 2023 02:17:13 +0000 Subject: [PATCH 2/4] Add index hint support (#2375) Fix #1456 Reviewed-on: https://gitea.com/xorm/xorm/pulls/2375 --- dialects/sqlite3_test.go | 4 +-- engine.go | 7 +++++ internal/statements/index.go | 54 ++++++++++++++++++++++++++++++++ internal/statements/query.go | 1 + internal/statements/statement.go | 6 ++++ session_schema.go | 5 +++ tests/session_test.go | 11 +++++++ 7 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 internal/statements/index.go diff --git a/dialects/sqlite3_test.go b/dialects/sqlite3_test.go index ec973ae6..75115c6c 100644 --- a/dialects/sqlite3_test.go +++ b/dialects/sqlite3_test.go @@ -11,7 +11,7 @@ import ( ) func TestSplitColStr(t *testing.T) { - var kases = []struct { + kases := []struct { colStr string fields []string }{ @@ -31,7 +31,7 @@ func TestSplitColStr(t *testing.T) { colStr: ` id INTEGER not null primary key autoincrement`, fields: []string{ - "id", "INTEGER", "not null", "primary", "key", "autoincrement", + "id", "INTEGER", "not", "null", "primary", "key", "autoincrement", }, }, } diff --git a/engine.go b/engine.go index 0cbfdede..31a6b688 100644 --- a/engine.go +++ b/engine.go @@ -1433,3 +1433,10 @@ func (engine *Engine) Transaction(f func(*Session) (interface{}, error)) (interf return result, nil } + +func (engine *Engine) IndexHint(op, indexerOrColName string) *Session { + session := engine.NewSession() + session.isAutoClose = true + session.statement.LastError = session.statement.IndexHint(op, indexerOrColName) + return session +} diff --git a/internal/statements/index.go b/internal/statements/index.go new file mode 100644 index 00000000..8f54d242 --- /dev/null +++ b/internal/statements/index.go @@ -0,0 +1,54 @@ +// Copyright 2023 The Xorm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package statements + +import ( + "strings" + + "xorm.io/builder" + "xorm.io/xorm/schemas" +) + +type ErrInvalidIndexHintOperator struct { + Op string +} + +func (e ErrInvalidIndexHintOperator) Error() string { + return "invalid index hint operator: " + e.Op +} + +func (statement *Statement) IndexHint(op, indexName string) error { + op = strings.ToUpper(op) + statement.indexHints = append(statement.indexHints, indexHint{ + op: op, + indexName: indexName, + }) + return nil +} + +func (statement *Statement) writeIndexHints(w *builder.BytesWriter) error { + if len(statement.indexHints) == 0 { + return nil + } + + switch statement.dialect.URI().DBType { + case schemas.MYSQL: + return statement.writeIndexHintsMySQL(w) + default: + return ErrNotImplemented + } +} + +func (statement *Statement) writeIndexHintsMySQL(w *builder.BytesWriter) error { + for _, hint := range statement.indexHints { + if hint.op != "USE" && hint.op != "FORCE" && hint.op != "IGNORE" { + return ErrInvalidIndexHintOperator{Op: hint.op} + } + if err := statement.writeStrings(" ", hint.op, " INDEX(", hint.indexName, ")")(w); err != nil { + return err + } + } + return nil +} diff --git a/internal/statements/query.go b/internal/statements/query.go index 8a9e59e4..a36d8ad3 100644 --- a/internal/statements/query.go +++ b/internal/statements/query.go @@ -254,6 +254,7 @@ func (statement *Statement) writeSelect(buf *builder.BytesWriter, columnStr stri return statement.writeMultiple(buf, statement.writeSelectColumns(columnStr), statement.writeFrom, + statement.writeIndexHints, statement.writeWhere, statement.writeGroupBy, statement.writeHaving, diff --git a/internal/statements/statement.go b/internal/statements/statement.go index 55a3d89e..31073ad1 100644 --- a/internal/statements/statement.go +++ b/internal/statements/statement.go @@ -41,6 +41,11 @@ type join struct { args []interface{} } +type indexHint struct { + op string + indexName string +} + // Statement save all the sql info for executing SQL type Statement struct { RefTable *schemas.Table @@ -84,6 +89,7 @@ type Statement struct { BufferSize int Context contexts.ContextCache LastError error + indexHints []indexHint } // NewStatement creates a new statement diff --git a/session_schema.go b/session_schema.go index 830ba08a..f7ab2657 100644 --- a/session_schema.go +++ b/session_schema.go @@ -311,3 +311,8 @@ func (session *Session) Import(r io.Reader) ([]sql.Result, error) { return results, lastError } + +func (session *Session) IndexHint(op, indexerOrColName string) *Session { + session.statement.IndexHint(op, indexerOrColName) + return session +} diff --git a/tests/session_test.go b/tests/session_test.go index 261f2c48..aeba1141 100644 --- a/tests/session_test.go +++ b/tests/session_test.go @@ -62,3 +62,14 @@ func TestEnableSessionId(t *testing.T) { _, err := testEngine.Table("userinfo").MustLogSQL(true).Get(new(Userinfo)) assert.NoError(t, err) } + +func TestIndexHint(t *testing.T) { + assert.NoError(t, PrepareEngine()) + assertSync(t, new(Userinfo)) + if testEngine.Dialect().URI().DBType != "mysql" { + return + } + + _, err := testEngine.Table("userinfo").IndexHint("USE", "UQE_userinfo_username").Get(new(Userinfo)) + assert.NoError(t, err) +} From 7ae7474bcbf62e9b304a1807cd463e6ca0ae7ac7 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 19 Dec 2023 04:54:04 +0000 Subject: [PATCH 3/4] Add missing index hint parameter (#2378) Reviewed-on: https://gitea.com/xorm/xorm/pulls/2378 --- engine.go | 4 ++-- internal/statements/index.go | 14 ++++++++++++-- internal/statements/statement.go | 1 + session_schema.go | 4 ++-- tests/session_test.go | 8 +++++++- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/engine.go b/engine.go index 31a6b688..459c2f44 100644 --- a/engine.go +++ b/engine.go @@ -1434,9 +1434,9 @@ func (engine *Engine) Transaction(f func(*Session) (interface{}, error)) (interf return result, nil } -func (engine *Engine) IndexHint(op, indexerOrColName string) *Session { +func (engine *Engine) IndexHint(op, forType, indexerOrColName string) *Session { session := engine.NewSession() session.isAutoClose = true - session.statement.LastError = session.statement.IndexHint(op, indexerOrColName) + session.statement.LastError = session.statement.IndexHint(op, forType, indexerOrColName) return session } diff --git a/internal/statements/index.go b/internal/statements/index.go index 8f54d242..5c1420eb 100644 --- a/internal/statements/index.go +++ b/internal/statements/index.go @@ -19,10 +19,11 @@ func (e ErrInvalidIndexHintOperator) Error() string { return "invalid index hint operator: " + e.Op } -func (statement *Statement) IndexHint(op, indexName string) error { +func (statement *Statement) IndexHint(op, forType, indexName string) error { op = strings.ToUpper(op) statement.indexHints = append(statement.indexHints, indexHint{ op: op, + forType: forType, indexName: indexName, }) return nil @@ -46,7 +47,16 @@ func (statement *Statement) writeIndexHintsMySQL(w *builder.BytesWriter) error { if hint.op != "USE" && hint.op != "FORCE" && hint.op != "IGNORE" { return ErrInvalidIndexHintOperator{Op: hint.op} } - if err := statement.writeStrings(" ", hint.op, " INDEX(", hint.indexName, ")")(w); err != nil { + if err := statement.writeStrings(" ", hint.op, " INDEX")(w); err != nil { + return err + } + if hint.forType != "" { + if err := statement.writeStrings(" FOR ", hint.forType)(w); err != nil { + return err + } + } + + if err := statement.writeStrings("(", hint.indexName, ")")(w); err != nil { return err } } diff --git a/internal/statements/statement.go b/internal/statements/statement.go index 31073ad1..dd4024b5 100644 --- a/internal/statements/statement.go +++ b/internal/statements/statement.go @@ -43,6 +43,7 @@ type join struct { type indexHint struct { op string + forType string indexName string } diff --git a/session_schema.go b/session_schema.go index f7ab2657..4bb0b858 100644 --- a/session_schema.go +++ b/session_schema.go @@ -312,7 +312,7 @@ func (session *Session) Import(r io.Reader) ([]sql.Result, error) { return results, lastError } -func (session *Session) IndexHint(op, indexerOrColName string) *Session { - session.statement.IndexHint(op, indexerOrColName) +func (session *Session) IndexHint(op, forType, indexerOrColName string) *Session { + session.statement.IndexHint(op, forType, indexerOrColName) return session } diff --git a/tests/session_test.go b/tests/session_test.go index aeba1141..3286ab2e 100644 --- a/tests/session_test.go +++ b/tests/session_test.go @@ -70,6 +70,12 @@ func TestIndexHint(t *testing.T) { return } - _, err := testEngine.Table("userinfo").IndexHint("USE", "UQE_userinfo_username").Get(new(Userinfo)) + _, err := testEngine.Table("userinfo").IndexHint("USE", "", "UQE_userinfo_username").Get(new(Userinfo)) + assert.NoError(t, err) + + _, err = testEngine.Table("userinfo").IndexHint("USE", "ORDER BY", "UQE_userinfo_username").Get(new(Userinfo)) + assert.NoError(t, err) + + _, err = testEngine.Table("userinfo").IndexHint("USE", "GROUP BY", "UQE_userinfo_username").Get(new(Userinfo)) assert.NoError(t, err) } From bdd8787d84a189a7c3e8eb758622a6a926f68316 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 20 Dec 2023 07:01:32 +0000 Subject: [PATCH 4/4] Use setup go v4 actions (#2380) Reviewed-on: https://gitea.com/xorm/xorm/pulls/2380 --- .gitea/workflows/test-cockroach.yml | 2 +- .gitea/workflows/test-mariadb.yml | 2 +- .gitea/workflows/test-mssql.yml | 2 +- .gitea/workflows/test-mysql.yml | 2 +- .gitea/workflows/test-mysql8.yml | 2 +- .gitea/workflows/test-postgres.yml | 2 +- .gitea/workflows/test-sqlite.yml | 2 +- .gitea/workflows/test-tidb.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitea/workflows/test-cockroach.yml b/.gitea/workflows/test-cockroach.yml index cfcda89d..d31ae07f 100644 --- a/.gitea/workflows/test-cockroach.yml +++ b/.gitea/workflows/test-cockroach.yml @@ -33,7 +33,7 @@ jobs: # restore-keys: | # go_cache-${{ github.repository }}- # go_cache- - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: 1.20 - uses: actions/checkout@v3 diff --git a/.gitea/workflows/test-mariadb.yml b/.gitea/workflows/test-mariadb.yml index dbc819db..4fb3fc36 100644 --- a/.gitea/workflows/test-mariadb.yml +++ b/.gitea/workflows/test-mariadb.yml @@ -33,7 +33,7 @@ jobs: # restore-keys: | # go_cache-${{ github.repository }}- # go_cache- - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: 1.20 - uses: actions/checkout@v3 diff --git a/.gitea/workflows/test-mssql.yml b/.gitea/workflows/test-mssql.yml index 04b8031a..a9502b02 100644 --- a/.gitea/workflows/test-mssql.yml +++ b/.gitea/workflows/test-mssql.yml @@ -33,7 +33,7 @@ jobs: # restore-keys: | # go_cache-${{ github.repository }}- # go_cache- - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: 1.20 - uses: actions/checkout@v3 diff --git a/.gitea/workflows/test-mysql.yml b/.gitea/workflows/test-mysql.yml index e13354f0..b7d3fc21 100644 --- a/.gitea/workflows/test-mysql.yml +++ b/.gitea/workflows/test-mysql.yml @@ -33,7 +33,7 @@ jobs: # restore-keys: | # go_cache-${{ github.repository }}- # go_cache- - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: 1.20 - uses: actions/checkout@v3 diff --git a/.gitea/workflows/test-mysql8.yml b/.gitea/workflows/test-mysql8.yml index 7362065a..7b48d461 100644 --- a/.gitea/workflows/test-mysql8.yml +++ b/.gitea/workflows/test-mysql8.yml @@ -33,7 +33,7 @@ jobs: # restore-keys: | # go_cache-${{ github.repository }}- # go_cache- - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: 1.20 - uses: actions/checkout@v3 diff --git a/.gitea/workflows/test-postgres.yml b/.gitea/workflows/test-postgres.yml index d4abb2ad..deb5a77b 100644 --- a/.gitea/workflows/test-postgres.yml +++ b/.gitea/workflows/test-postgres.yml @@ -33,7 +33,7 @@ jobs: # restore-keys: | # go_cache-${{ github.repository }}- # go_cache- - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: 1.20 - uses: actions/checkout@v3 diff --git a/.gitea/workflows/test-sqlite.yml b/.gitea/workflows/test-sqlite.yml index 164acc10..473ba95d 100644 --- a/.gitea/workflows/test-sqlite.yml +++ b/.gitea/workflows/test-sqlite.yml @@ -33,7 +33,7 @@ jobs: # restore-keys: | # go_cache-${{ github.repository }}- # go_cache- - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: 1.20 - uses: actions/checkout@v3 diff --git a/.gitea/workflows/test-tidb.yml b/.gitea/workflows/test-tidb.yml index ce898dcb..f87b039a 100644 --- a/.gitea/workflows/test-tidb.yml +++ b/.gitea/workflows/test-tidb.yml @@ -33,7 +33,7 @@ jobs: # restore-keys: | # go_cache-${{ github.repository }}- # go_cache- - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: 1.20 - uses: actions/checkout@v3