Implemented cache invalidation mechanism (#11)

* Implemented cache invalidation handling, added a new required method in Cacher interface * Implemented valueToString function used to generate the query identifier taking into account pointer-case query args (#13) Co-authored-by: dennis-dko <[email protected]> * renamed queryType constants, added tests for Caches.getMutator Co-authored-by: dennis-dko <[email protected]> --------- Co-authored-by: dennis-dko <[email protected]>
go-gorm · Feb 22, 2024 · 1b3e58f · 1b3e58f
1 parent 91cf6d4
commit 1b3e58f
Show file tree

Hide file tree

Showing 8 changed files with 300 additions and 79 deletions.
diff --git a/README.md b/README.md
@@ -11,7 +11,7 @@ Gorm Caches plugin using database request reductions (easer), and response cachi
 ## Install
 
 ```bash
-go get -u github.com/go-gorm/caches/v3
+go get -u github.com/go-gorm/caches/v4
 ```
 
 ## Usage
@@ -25,7 +25,7 @@ import (
  "fmt"
  "sync"
 
- "github.com/go-gorm/caches/v3"
+ "github.com/go-gorm/caches/v4"
  "gorm.io/driver/mysql"
  "gorm.io/gorm"
 )
@@ -53,7 +53,7 @@ import (
  "sync"
  "time"
 
- "github.com/go-gorm/caches/v3"
+ "github.com/go-gorm/caches/v4"
  "gorm.io/driver/mysql"
  "gorm.io/gorm"
 )
@@ -137,7 +137,7 @@ import (
  "fmt"
  "time"
 
- "github.com/go-gorm/caches/v3"
+ "github.com/go-gorm/caches/v4"
  "github.com/redis/go-redis/v9"
  "gorm.io/driver/sqlite"
  "gorm.io/gorm"
@@ -186,6 +186,34 @@ func (c *redisCacher) Store(ctx context.Context, key string, val *caches.Query[a
  return nil
 }
 
+func (c *redisCacher) Invalidate(ctx context.Context) error {
+ var (
+ cursor uint64
+ keys []string
+ )
+ for {
+ var (
+ k []string
+ err error
+ )
+ k, cursor, err = c.rdb.Scan(ctx, cursor, fmt.Sprintf("%s*", caches.IdentifierPrefix), 0).Result()
+ if err != nil {
+ return err
+ }
+ keys = append(keys, k...)
+ if cursor == 0 {
+ break
+ }
+ }
+
+ if len(keys) > 0 {
+ if _, err := c.rdb.Del(ctx, keys...).Result(); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
 func main() {
  db, _ := gorm.Open(sqlite.Open("gorm.db"), &gorm.Config{})
 
@@ -247,7 +275,7 @@ import (
  "fmt"
  "sync"
 
- "github.com/go-gorm/caches/v3"
+ "github.com/go-gorm/caches/v4"
  "gorm.io/driver/sqlite"
  "gorm.io/gorm"
 )
@@ -299,6 +327,11 @@ func (c *memoryCacher) Store(ctx context.Context, key string, val *caches.Query[
  return nil
 }
 
+func (c *memoryCacher) Invalidate(ctx context.Context) error {
+ c.store = &sync.Map{}
+ return nil
+}
+
 func main() {
  db, _ := gorm.Open(sqlite.Open("gorm.db"), &gorm.Config{})
 

diff --git a/cacher.go b/cacher.go
@@ -8,7 +8,10 @@ type Cacher interface {
  // Get impl should check if a specific key exists in the cache and return its value
  // look at Query.Marshal
  Get(ctx context.Context, key string, q *Query[any]) (*Query[any], error)
- // Store is supposed to store a cached representation of the val param
+ // Store impl should store a cached representation of the val param
  // look at Query.Unmarshal
  Store(ctx context.Context, key string, val *Query[any]) error
+ // Invalidate impl should invalidate all cached values
+ // It will be called when INSERT / UPDATE / DELETE queries are sent to the DB
+ Invalidate(ctx context.Context) error
 }
diff --git a/cacher_test.go b/cacher_test.go
@@ -32,6 +32,10 @@ func (c *cacherMock) Store(_ context.Context, key string, val *Query[any]) error
  return nil
 }
 
+func (c *cacherMock) Invalidate(context.Context) error {
+ return nil
+}
+
 type cacherStoreErrorMock struct{}
 
 func (c *cacherStoreErrorMock) Get(context.Context, string, *Query[any]) (*Query[any], error) {
@@ -42,6 +46,10 @@ func (c *cacherStoreErrorMock) Store(context.Context, string, *Query[any]) error
  return errors.New("store-error")
 }
 
+func (c *cacherStoreErrorMock) Invalidate(context.Context) error {
+ return nil
+}
+
 type cacherGetErrorMock struct{}
 
 func (c *cacherGetErrorMock) Get(context.Context, string, *Query[any]) (*Query[any], error) {
@@ -51,3 +59,7 @@ func (c *cacherGetErrorMock) Get(context.Context, string, *Query[any]) (*Query[a
 func (c *cacherGetErrorMock) Store(context.Context, string, *Query[any]) error {
  return nil
 }
+
+func (c *cacherGetErrorMock) Invalidate(context.Context) error {
+ return nil
+}
diff --git a/caches.go b/caches.go
@@ -7,10 +7,10 @@ import (
 )
 
 type Caches struct {
- Conf *Config
+ callbacks map[queryType]func(db *gorm.DB)
+ Conf *Config
 
- queue *sync.Map
- queryCb func(*gorm.DB)
+ queue *sync.Map
 }
 
 type Config struct {
@@ -34,19 +34,37 @@ func (c *Caches) Initialize(db *gorm.DB) error {
  c.queue = &sync.Map{}
  }
 
- c.queryCb = db.Callback().Query().Get("gorm:query")
+ callbacks := make(map[queryType]func(db *gorm.DB), 4)
+ callbacks[uponQuery] = db.Callback().Query().Get("gorm:query")
+ callbacks[uponCreate] = db.Callback().Create().Get("gorm:query")
+ callbacks[uponUpdate] = db.Callback().Update().Get("gorm:query")
+ callbacks[uponDelete] = db.Callback().Delete().Get("gorm:query")
+ c.callbacks = callbacks
 
- err := db.Callback().Query().Replace("gorm:query", c.Query)
- if err != nil {
+ if err := db.Callback().Query().Replace("gorm:query", c.query); err != nil {
+ return err
+ }
+
+ if err := db.Callback().Create().Replace("gorm:query", c.getMutatorCb(uponCreate)); err != nil {
+ return err
+ }
+
+ if err := db.Callback().Update().Replace("gorm:query", c.getMutatorCb(uponUpdate)); err != nil {
+ return err
+ }
+
+ if err := db.Callback().Delete().Replace("gorm:query", c.getMutatorCb(uponDelete)); err != nil {
  return err
  }
 
  return nil
 }
 
-func (c *Caches) Query(db *gorm.DB) {
+// query is a decorator around the default "gorm:query" callback
+// it takes care to both ease database load and cache results
+func (c *Caches) query(db *gorm.DB) {
  if c.Conf.Easer == false && c.Conf.Cacher == nil {
- c.queryCb(db)
+ c.callbacks[uponQuery](db)
  return
  }
 
@@ -67,16 +85,30 @@ func (c *Caches) Query(db *gorm.DB) {
  }
 }
 
+// getMutatorCb returns a decorator which calls the Cacher's Invalidate method
+func (c *Caches) getMutatorCb(typ queryType) func(db *gorm.DB) {
+ return func(db *gorm.DB) {
+ if c.Conf.Cacher != nil {
+ if err := c.Conf.Cacher.Invalidate(db.Statement.Context); err != nil {
+ _ = db.AddError(err)
+ }
+ }
+ if cb := c.callbacks[typ]; cb != nil { // By default, gorm has no callbacks associated with mutating behaviors
+ cb(db)
+ }
+ }
+}
+
 func (c *Caches) ease(db *gorm.DB, identifier string) {
  if c.Conf.Easer == false {
- c.queryCb(db)
+ c.callbacks[uponQuery](db)
  return
  }
 
  res := ease(&queryTask{
  id: identifier,
  db: db,
- queryCb: c.queryCb,
+ queryCb: c.callbacks[uponQuery],
  }, c.queue).(*queryTask)
 
  if db.Error != nil {
@@ -123,3 +155,13 @@ func (c *Caches) storeInCache(db *gorm.DB, identifier string) {
  }
  }
 }
+
+// queryType is used to mark callbacks
+type queryType int
+
+const (
+ uponQuery queryType = iota
+ uponCreate
+ uponUpdate
+ uponDelete
+)