Skip to content

Commit 73dc8c9

Browse files
gqcngqcn
authored
fix cache issue in Count/Value functions for gdb.Model (#2300)
* add Tag* functions to retreive most commonly used tag value from struct field for package gstructs; use description tag as default value if brief is empty for gcmd.Argument * fix cache issue in Count/Value functions for gdb.Model * add more ut case for package gdb * version updates
1 parent 576f1a7 commit 73dc8c9

File tree

8 files changed

+322
-158
lines changed

8 files changed

+322
-158
lines changed

‎contrib/drivers/mysql/mysql_model_test.go‎

Lines changed: 151 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -731,16 +731,157 @@ func Test_Model_Count(t *testing.T) {
731731
t.AssertNil(err)
732732
t.Assert(count, TableSize)
733733
})
734-
// gtest.C(t, func(t *gtest.T) {
735-
// count, err := db.Model(table).Fields("id myid").Where("id>8").Count()
736-
// t.AssertNil(err)
737-
// t.Assert(count, 2)
738-
// })
739-
// gtest.C(t, func(t *gtest.T) {
740-
// count, err := db.Model(table).As("u1").LeftJoin(table, "u2", "u2.id=u1.id").Fields("u2.id u2id").Where("u1.id>8").Count()
741-
// t.AssertNil(err)
742-
// t.Assert(count, 2)
743-
// })
734+
}
735+
736+
func Test_Model_Value_WithCache(t *testing.T) {
737+
table := createTable()
738+
defer dropTable(table)
739+
740+
gtest.C(t, func(t *gtest.T) {
741+
value, err := db.Model(table).Where("id", 1).Cache(gdb.CacheOption{
742+
Duration: time.Second * 10,
743+
Force: false,
744+
}).Value()
745+
t.AssertNil(err)
746+
t.Assert(value.Int(), 0)
747+
})
748+
gtest.C(t, func(t *gtest.T) {
749+
result, err := db.Model(table).Data(g.MapStrAny{
750+
"id": 1,
751+
"passport": fmt.Sprintf(`passport_%d`, 1),
752+
"password": fmt.Sprintf(`password_%d`, 1),
753+
"nickname": fmt.Sprintf(`nickname_%d`, 1),
754+
}).Insert()
755+
t.AssertNil(err)
756+
n, _ := result.RowsAffected()
757+
t.Assert(n, 1)
758+
})
759+
gtest.C(t, func(t *gtest.T) {
760+
value, err := db.Model(table).Where("id", 1).Cache(gdb.CacheOption{
761+
Duration: time.Second * 10,
762+
Force: false,
763+
}).Value()
764+
t.AssertNil(err)
765+
t.Assert(value.Int(), 1)
766+
})
767+
}
768+
769+
func Test_Model_Count_WithCache(t *testing.T) {
770+
table := createTable()
771+
defer dropTable(table)
772+
773+
gtest.C(t, func(t *gtest.T) {
774+
count, err := db.Model(table).Where("id", 1).Cache(gdb.CacheOption{
775+
Duration: time.Second * 10,
776+
Force: false,
777+
}).Count()
778+
t.AssertNil(err)
779+
t.Assert(count, 0)
780+
})
781+
gtest.C(t, func(t *gtest.T) {
782+
result, err := db.Model(table).Data(g.MapStrAny{
783+
"id": 1,
784+
"passport": fmt.Sprintf(`passport_%d`, 1),
785+
"password": fmt.Sprintf(`password_%d`, 1),
786+
"nickname": fmt.Sprintf(`nickname_%d`, 1),
787+
}).Insert()
788+
t.AssertNil(err)
789+
n, _ := result.RowsAffected()
790+
t.Assert(n, 1)
791+
})
792+
gtest.C(t, func(t *gtest.T) {
793+
count, err := db.Model(table).Where("id", 1).Cache(gdb.CacheOption{
794+
Duration: time.Second * 10,
795+
Force: false,
796+
}).Count()
797+
t.AssertNil(err)
798+
t.Assert(count, 1)
799+
})
800+
}
801+
802+
func Test_Model_Count_All_WithCache(t *testing.T) {
803+
table := createTable()
804+
defer dropTable(table)
805+
806+
gtest.C(t, func(t *gtest.T) {
807+
count, err := db.Model(table).Cache(gdb.CacheOption{
808+
Duration: time.Second * 10,
809+
Force: false,
810+
}).Count()
811+
t.AssertNil(err)
812+
t.Assert(count, 0)
813+
})
814+
gtest.C(t, func(t *gtest.T) {
815+
result, err := db.Model(table).Data(g.MapStrAny{
816+
"id": 1,
817+
"passport": fmt.Sprintf(`passport_%d`, 1),
818+
"password": fmt.Sprintf(`password_%d`, 1),
819+
"nickname": fmt.Sprintf(`nickname_%d`, 1),
820+
}).Insert()
821+
t.AssertNil(err)
822+
n, _ := result.RowsAffected()
823+
t.Assert(n, 1)
824+
})
825+
gtest.C(t, func(t *gtest.T) {
826+
count, err := db.Model(table).Cache(gdb.CacheOption{
827+
Duration: time.Second * 10,
828+
Force: false,
829+
}).Count()
830+
t.AssertNil(err)
831+
t.Assert(count, 1)
832+
})
833+
gtest.C(t, func(t *gtest.T) {
834+
result, err := db.Model(table).Data(g.MapStrAny{
835+
"id": 2,
836+
"passport": fmt.Sprintf(`passport_%d`, 2),
837+
"password": fmt.Sprintf(`password_%d`, 2),
838+
"nickname": fmt.Sprintf(`nickname_%d`, 2),
839+
}).Insert()
840+
t.AssertNil(err)
841+
n, _ := result.RowsAffected()
842+
t.Assert(n, 1)
843+
})
844+
gtest.C(t, func(t *gtest.T) {
845+
count, err := db.Model(table).Cache(gdb.CacheOption{
846+
Duration: time.Second * 10,
847+
Force: false,
848+
}).Count()
849+
t.AssertNil(err)
850+
t.Assert(count, 1)
851+
})
852+
}
853+
854+
func Test_Model_CountColumn_WithCache(t *testing.T) {
855+
table := createTable()
856+
defer dropTable(table)
857+
858+
gtest.C(t, func(t *gtest.T) {
859+
count, err := db.Model(table).Where("id", 1).Cache(gdb.CacheOption{
860+
Duration: time.Second * 10,
861+
Force: false,
862+
}).CountColumn("id")
863+
t.AssertNil(err)
864+
t.Assert(count, 0)
865+
})
866+
gtest.C(t, func(t *gtest.T) {
867+
result, err := db.Model(table).Data(g.MapStrAny{
868+
"id": 1,
869+
"passport": fmt.Sprintf(`passport_%d`, 1),
870+
"password": fmt.Sprintf(`password_%d`, 1),
871+
"nickname": fmt.Sprintf(`nickname_%d`, 1),
872+
}).Insert()
873+
t.AssertNil(err)
874+
n, _ := result.RowsAffected()
875+
t.Assert(n, 1)
876+
})
877+
gtest.C(t, func(t *gtest.T) {
878+
count, err := db.Model(table).Where("id", 1).Cache(gdb.CacheOption{
879+
Duration: time.Second * 10,
880+
Force: false,
881+
}).CountColumn("id")
882+
t.AssertNil(err)
883+
t.Assert(count, 1)
884+
})
744885
}
745886

746887
func Test_Model_Select(t *testing.T) {

‎database/gdb/gdb.go‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -289,20 +289,23 @@ type CatchSQLManager struct {
289289
DoCommit bool
290290
}
291291

292+
type queryType int
293+
292294
const (
293-
defaultModelSafe = false
294-
defaultCharset = `utf8`
295-
defaultProtocol = `tcp`
296-
queryTypeNormal = 0
297-
queryTypeCount = 1
298-
unionTypeNormal = 0
299-
unionTypeAll = 1
300-
defaultMaxIdleConnCount = 10 // Max idle connection count in pool.
301-
defaultMaxOpenConnCount = 0 // Max open connection count in pool. Default is no limit.
302-
defaultMaxConnLifeTime = 30 * time.Second // Max lifetime for per connection in pool in seconds.
303-
ctxTimeoutTypeExec = iota
304-
ctxTimeoutTypeQuery
305-
ctxTimeoutTypePrepare
295+
defaultModelSafe = false
296+
defaultCharset = `utf8`
297+
defaultProtocol = `tcp`
298+
queryTypeNormal queryType = 0
299+
queryTypeCount queryType = 1
300+
queryTypeValue queryType = 2
301+
unionTypeNormal = 0
302+
unionTypeAll = 1
303+
defaultMaxIdleConnCount = 10 // Max idle connection count in pool.
304+
defaultMaxOpenConnCount = 0 // Max open connection count in pool. Default is no limit.
305+
defaultMaxConnLifeTime = 30 * time.Second // Max lifetime for per connection in pool in seconds.
306+
ctxTimeoutTypeExec = 0
307+
ctxTimeoutTypeQuery = 1
308+
ctxTimeoutTypePrepare = 2
306309
cachePrefixTableFields = `TableFields:`
307310
cachePrefixSelectCache = `SelectCache:`
308311
commandEnvKeyForDryRun = "gf.gdb.dryrun"

‎database/gdb/gdb_core_underlying.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,7 @@ func (c *Core) RowsToResult(ctx context.Context, rows *sql.Rows) (Result, error)
397397
record := Record{}
398398
for i, value := range values {
399399
if value == nil {
400-
// Do not use `gvar.New(nil)` here as it creates an initialized object
400+
// DO NOT use `gvar.New(nil)` here as it creates an initialized object
401401
// which will cause struct converting issue.
402402
record[columnNames[i]] = nil
403403
} else {

‎database/gdb/gdb_model_cache.go‎

Lines changed: 37 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -85,18 +85,19 @@ func (m *Model) getSelectResultFromCache(ctx context.Context, sql string, args .
8585
if cacheItem, ok = v.Val().(*selectCacheItem); ok {
8686
// In-memory cache.
8787
return cacheItem.Result, nil
88-
} else {
89-
// Other cache, it needs conversion.
90-
if err = json.UnmarshalUseNumber(v.Bytes(), &cacheItem); err != nil {
91-
return nil, err
92-
}
93-
return cacheItem.Result, nil
9488
}
89+
// Other cache, it needs conversion.
90+
if err = json.UnmarshalUseNumber(v.Bytes(), &cacheItem); err != nil {
91+
return nil, err
92+
}
93+
return cacheItem.Result, nil
9594
}
9695
return
9796
}
9897

99-
func (m *Model) saveSelectResultToCache(ctx context.Context, result Result, sql string, args ...interface{}) (err error) {
98+
func (m *Model) saveSelectResultToCache(
99+
ctx context.Context, queryType queryType, result Result, sql string, args ...interface{},
100+
) (err error) {
100101
if !m.cacheEnabled || m.tx != nil {
101102
return
102103
}
@@ -108,22 +109,38 @@ func (m *Model) saveSelectResultToCache(ctx context.Context, result Result, sql
108109
if _, errCache := cacheObj.Remove(ctx, cacheKey); errCache != nil {
109110
intlog.Errorf(ctx, `%+v`, errCache)
110111
}
111-
} else {
112-
// In case of Cache Penetration.
113-
if result.IsEmpty() && m.cacheOption.Force {
114-
result = Result{}
115-
}
116-
var cacheItem = &selectCacheItem{
117-
Result: result,
118-
}
119-
if internalData := m.db.GetCore().GetInternalCtxDataFromCtx(ctx); internalData != nil {
120-
cacheItem.FirstResultColumn = internalData.FirstResultColumn
112+
return
113+
}
114+
// Special handler for Value/Count operations result.
115+
if len(result) > 0 {
116+
switch queryType {
117+
case queryTypeValue, queryTypeCount:
118+
if internalData := m.db.GetCore().GetInternalCtxDataFromCtx(ctx); internalData != nil {
119+
if result[0][internalData.FirstResultColumn].IsEmpty() {
120+
result = nil
121+
}
122+
}
121123
}
122-
if errCache := cacheObj.Set(ctx, cacheKey, cacheItem, m.cacheOption.Duration); errCache != nil {
123-
intlog.Errorf(ctx, `%+v`, errCache)
124+
}
125+
126+
// In case of Cache Penetration.
127+
if result.IsEmpty() {
128+
if m.cacheOption.Force {
129+
result = Result{}
130+
} else {
131+
result = nil
124132
}
125133
}
126-
return nil
134+
var cacheItem = &selectCacheItem{
135+
Result: result,
136+
}
137+
if internalData := m.db.GetCore().GetInternalCtxDataFromCtx(ctx); internalData != nil {
138+
cacheItem.FirstResultColumn = internalData.FirstResultColumn
139+
}
140+
if errCache := cacheObj.Set(ctx, cacheKey, cacheItem, m.cacheOption.Duration); errCache != nil {
141+
intlog.Errorf(ctx, `%+v`, errCache)
142+
}
143+
return
127144
}
128145

129146
func (m *Model) makeSelectCacheKey(sql string, args ...interface{}) string {

0 commit comments

Comments
 (0)