Skip to content

Commit efc966f

Browse files
asjdfasjdf
committed
fix: fix primary cache not used and finish single flight
1 parent 8158c20 commit efc966f

File tree

3 files changed

+82
-28
lines changed

3 files changed

+82
-28
lines changed

‎cache/query.go

+70-28
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"github.com/asjdf/gorm-cache/config"
77
"github.com/asjdf/gorm-cache/storage"
88
"github.com/asjdf/gorm-cache/util"
9+
"github.com/hashicorp/go-multierror"
910
"gorm.io/gorm"
1011
"gorm.io/gorm/callbacks"
1112
"reflect"
@@ -56,8 +57,17 @@ func (h *queryHandler) BeforeQuery() func(db *gorm.DB) {
5657
db.InstanceSet("gorm:cache:vars", db.Statement.Vars)
5758

5859
if util.ShouldCache(tableName, cache.Config.Tables) {
60+
hitted := false
61+
defer func() {
62+
if hitted {
63+
cache.IncrHitCount()
64+
} else {
65+
cache.IncrMissCount()
66+
}
67+
}()
68+
5969
// singleFlight Check
60-
singleFlightKey := fmt.Sprintf("%s:%s", tableName, sql)
70+
singleFlightKey := fmt.Sprintf("%s:%s", tableName, sql) // todo: key with vars
6171
h.singleFlight.mu.Lock()
6272
if h.singleFlight.m == nil {
6373
h.singleFlight.m = make(map[string]*call)
@@ -78,11 +88,11 @@ func (h *queryHandler) BeforeQuery() func(db *gorm.DB) {
7888
_ = db.AddError(err)
7989
return
8090
}
91+
hitted = true
8192
db.RowsAffected = c.rowsAffected
82-
if c.err == nil { // 为保证后续流程不走，必须设一个error
83-
db.Error = util.SingleFlightHit
84-
} else {
85-
db.Error = c.err
93+
db.Error = multierror.Append(util.SingleFlightHit) // 为保证后续流程不走，必须设一个error
94+
if c.err != nil {
95+
db.Error = multierror.Append(db.Error, c.err)
8696
}
8797
h.cache.Logger.CtxInfo(ctx, "[BeforeQuery] single flight hit for key %v", singleFlightKey)
8898
return
@@ -93,8 +103,7 @@ func (h *queryHandler) BeforeQuery() func(db *gorm.DB) {
93103
h.singleFlight.mu.Unlock()
94104
db.InstanceSet("gorm:cache:query:single_flight_call", c)
95105

96-
// try primary cache first
97-
if cache.Config.CacheLevel == config.CacheLevelAll || cache.Config.CacheLevel == config.CacheLevelOnlyPrimary {
106+
tryPrimaryCache := func() (hitted bool) {
98107
primaryKeys := getPrimaryKeysFromWhereClause(db)
99108
cache.Logger.CtxInfo(ctx, "[BeforeQuery] parse primary keys = %v", primaryKeys)
100109

@@ -141,17 +150,17 @@ func (h *queryHandler) BeforeQuery() func(db *gorm.DB) {
141150
return
142151
}
143152
db.Error = util.PrimaryCacheHit
153+
hitted = true
144154
return
145155
}
146156

147-
if cache.Config.CacheLevel == config.CacheLevelAll || cache.Config.CacheLevel == config.CacheLevelOnlySearch {
157+
trySearchCache := func() (hitted bool) {
148158
// search cache hit
149159
cacheValue, err := cache.GetSearchCache(ctx, tableName, sql, db.Statement.Vars...)
150160
if err != nil {
151161
if !errors.Is(err, storage.ErrCacheNotFound) {
152162
cache.Logger.CtxError(ctx, "[BeforeQuery] get cache value for sql %s error: %v", sql, err)
153163
}
154-
cache.IncrMissCount()
155164
db.Error = nil
156165
return
157166
}
@@ -174,8 +183,21 @@ func (h *queryHandler) BeforeQuery() func(db *gorm.DB) {
174183
return
175184
}
176185
db.Error = util.SearchCacheHit
186+
hitted = true
177187
return
178188
}
189+
190+
if cache.Config.CacheLevel == config.CacheLevelAll || cache.Config.CacheLevel == config.CacheLevelOnlyPrimary {
191+
if tryPrimaryCache() {
192+
hitted = true
193+
return
194+
}
195+
}
196+
if cache.Config.CacheLevel == config.CacheLevelAll || cache.Config.CacheLevel == config.CacheLevelOnlySearch {
197+
if !hitted && trySearchCache() {
198+
hitted = true
199+
}
200+
}
179201
}
180202
}
181203
}
@@ -196,7 +218,11 @@ func (h *queryHandler) AfterQuery() func(db *gorm.DB) {
196218
varObj, _ := db.InstanceGet("gorm:cache:vars")
197219
vars := varObj.([]interface{})
198220

199-
if db.Error == nil && util.ShouldCache(tableName, cache.Config.Tables) {
221+
if !util.ShouldCache(tableName, cache.Config.Tables) {
222+
return
223+
}
224+
225+
if db.Error == nil {
200226
// error is nil -> cache not hit, we cache newly retrieved data
201227
primaryKeys, objects := getObjectsAfterLoad(db)
202228

@@ -266,29 +292,45 @@ func (h *queryHandler) AfterQuery() func(db *gorm.DB) {
266292
return
267293
}
268294

269-
if !cache.Config.DisableCachePenetrationProtect {
270-
if errors.Is(db.Error, gorm.ErrRecordNotFound) { // 应对缓存穿透 未来可能考虑使用其他过滤器实现：如布隆过滤器
271-
cache.Logger.CtxInfo(ctx, "[AfterQuery] set cache: %v", "recordNotFound")
272-
err := cache.SetSearchCache(ctx, "recordNotFound", tableName, sql, vars...)
273-
if err != nil {
274-
cache.Logger.CtxError(ctx, "[AfterQuery] set search cache for sql: %s error: %v", sql, err)
275-
return
276-
}
277-
cache.Logger.CtxInfo(ctx, "[AfterQuery] sql %s cached", sql)
295+
// 应对缓存穿透 未来可能考虑使用其他过滤器实现：如布隆过滤器
296+
if db.Error == gorm.ErrRecordNotFound && !cache.Config.DisableCachePenetrationProtect {
297+
cache.Logger.CtxInfo(ctx, "[AfterQuery] set cache: %v", "recordNotFound")
298+
err := cache.SetSearchCache(ctx, "recordNotFound", tableName, sql, vars...)
299+
if err != nil {
300+
cache.Logger.CtxError(ctx, "[AfterQuery] set search cache for sql: %s error: %v", sql, err)
278301
return
279302
}
280-
}
281-
282-
switch db.Error {
283-
case util.RecordNotFoundCacheHit:
284-
db.Error = gorm.ErrRecordNotFound
285-
cache.IncrHitCount()
286-
case util.SearchCacheHit, util.PrimaryCacheHit, util.SingleFlightHit:
287-
db.Error = nil
288-
cache.IncrHitCount()
303+
cache.Logger.CtxInfo(ctx, "[AfterQuery] sql %s cached", sql)
304+
return
289305
}
290306
}()
307+
// 之所以将上面的部分包在一个匿名函数中是为了方便
308+
// 上面的cache完成后直接传播给其他等待中的goroutine
309+
// 上面只处理非singleflight且无错误或记录不存在的情况
291310
h.fillCallAfterQuery(db)
311+
312+
// 下面处理命中了缓存的情况
313+
// 有以下几种err是专门用来传状态的：正常的cachehit 这种情况不存在error
314+
// RecordNotFoundCacheHit 这种情况只会在notfound之后出现
315+
// SingleFlightHit 这种情况下error中除了SingleFlightHit还可能会存在其他error来自gorm的error
316+
// 且遇到任何一种hit我们都可以认为是命中了缓存 同时只可能命中至多两个hit（single+其他
317+
if merr, ok := db.Error.(*multierror.Error); ok {
318+
errs := merr.WrappedErrors()
319+
if errors.Is(errs[0], util.SingleFlightHit) {
320+
if len(errs) > 1 {
321+
db.Error = errs[1]
322+
} else {
323+
db.Error = nil
324+
}
325+
}
326+
}
327+
328+
switch db.Error {
329+
case util.RecordNotFoundCacheHit:
330+
db.Error = gorm.ErrRecordNotFound
331+
case util.SearchCacheHit, util.PrimaryCacheHit:
332+
db.Error = nil
333+
}
292334
}
293335
}
294336

‎go.mod

+4
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ require (
1919
github.com/glebarez/go-sqlite v1.20.3 // indirect
2020
github.com/google/uuid v1.3.0 // indirect
2121
github.com/gopherjs/gopherjs v1.17.2 // indirect
22+
github.com/hashicorp/errwrap v1.0.0 // indirect
23+
github.com/hashicorp/go-multierror v1.1.1 // indirect
2224
github.com/jinzhu/inflection v1.0.0 // indirect
2325
github.com/jinzhu/now v1.1.5 // indirect
2426
github.com/jtolds/gls v4.20.0+incompatible // indirect
@@ -27,6 +29,8 @@ require (
2729
github.com/modern-go/reflect2 v1.0.2 // indirect
2830
github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 // indirect
2931
github.com/smartystreets/assertions v1.13.0 // indirect
32+
go.uber.org/atomic v1.7.0 // indirect
33+
go.uber.org/multierr v1.9.0 // indirect
3034
golang.org/x/sys v0.4.0 // indirect
3135
modernc.org/libc v1.22.2 // indirect
3236
modernc.org/mathutil v1.5.0 // indirect

‎go.sum

+8
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
2222
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
2323
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
2424
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
25+
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
26+
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
27+
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
28+
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
2529
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
2630
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
2731
github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
@@ -54,6 +58,10 @@ github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3
5458
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
5559
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
5660
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
61+
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
62+
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
63+
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
64+
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
5765
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
5866
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
5967
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

0 commit comments

Comments
 (0)