From 83ce20f3f4f3c8d5d3df8d897be638ef5045fd46 Mon Sep 17 00:00:00 2001 From: Travis Cline Date: Sun, 28 Aug 2022 17:35:38 -0700 Subject: [PATCH 1/2] cachefs: Port to use mailgun/groupcache which supports expiration --- README.md | 7 ++----- fs.go | 22 +++++----------------- fs_test.go | 1 + go.mod | 7 ++++++- go.sum | 17 +++++++++++++++++ package_test.go | 2 +- quantize.go | 26 -------------------------- 7 files changed, 32 insertions(+), 50 deletions(-) delete mode 100644 quantize.go diff --git a/README.md b/README.md index 01c99b5..1f6b433 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # cachefs -Package `cachefs` implements a read-only cache around a `fs.FS`, using `groupcache`. +Package `cachefs` implements a read-only cache around a `fs.FS`, using mailgun's `groupcache`. Using `cachefs` is straightforward: @@ -15,7 +15,4 @@ Using `cachefs` is straightforward: `cachefs` "wraps" the underlying file system with caching. You can specify groupcache parameters - the group name and the cache size. -`groupcache` does not support expiration, but `cachefs` supports quantizing values so that expiration happens -around the expiration duration provided. Expiration can be disabled by specifying 0 for the duration. - -See https://pkg.go.dev/github.com/golang/groupcache for more information on `groupcache`. +See https://pkg.go.dev/github.com/mailgun/groupcache for more information on `groupcache`. diff --git a/fs.go b/fs.go index 6578fdc..6670945 100644 --- a/fs.go +++ b/fs.go @@ -17,7 +17,7 @@ and the cache size. groupcache does not support expiration, but cachefs supports quantizing values so that expiration happens around the expiration duration provided. Expiration can be disabled by specifying 0 for the duration. -See https://pkg.go.dev/github.com/golang/groupcache for more information on groupcache. +See https://pkg.go.dev/github.com/mailgun/groupcache for more information on groupcache. */ package cachefs @@ -28,12 +28,10 @@ import ( "fmt" "io" "io/fs" - "net/url" - "strconv" "time" - "github.com/golang/groupcache" "github.com/google/uuid" + "github.com/mailgun/groupcache/v2" ) // Config stores the configuration settings of your cache. @@ -67,14 +65,10 @@ func (cfs *cacheFS) Open(name string) (fs.File, error) { var ( buf groupcache.ByteView - q = make(url.Values, 2) f file ) - t := quantize(time.Now(), cfs.duration, name) - q.Set("t", strconv.FormatInt(t, 10)) - q.Set("path", name) ctx := context.Background() - err := cfs.cache.Get(ctx, q.Encode(), groupcache.ByteViewSink(&buf)) + err := cfs.cache.Get(ctx, name, groupcache.ByteViewSink(&buf)) if err != nil { return nil, &fs.PathError{Op: "open", Path: name, Err: err} } @@ -105,13 +99,7 @@ func New(innerFS fs.FS, config *Config) fs.FS { duration: config.Duration, cache: groupcache.NewGroup(config.GroupName, config.SizeInBytes, groupcache.GetterFunc( func(ctx context.Context, key string, dest groupcache.Sink) error { - // Parse query which contains quantize info and path - q, err := url.ParseQuery(key) - if err != nil { - return fmt.Errorf("invalid cache key: %w", err) - } - // Open file - f, err := innerFS.Open(q.Get("path")) + f, err := innerFS.Open(key) if err != nil { return err } @@ -184,7 +172,7 @@ func New(innerFS fs.FS, config *Config) fs.FS { if n != len(data) { return fmt.Errorf("wrote incorrect number of bytes: %d of %d", n, len(data)) } - return dest.SetBytes(buf.Bytes()) + return dest.SetBytes(buf.Bytes(), time.Now().Add(config.Duration)) })), } } diff --git a/fs_test.go b/fs_test.go index ba06bdc..f6397e6 100644 --- a/fs_test.go +++ b/fs_test.go @@ -23,6 +23,7 @@ func TestFS(t *testing.T) { fs.WalkDir(fileSys, ".", func(path string, d fs.DirEntry, err error) error { if err != nil { t.Error(err) + return err } if path == "" { t.Error("Path is empty") diff --git a/go.mod b/go.mod index f7009ab..464e804 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,16 @@ module github.com/ancientlore/cachefs go 1.17 require ( - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/google/uuid v1.3.0 + github.com/mailgun/groupcache v1.3.0 + github.com/mailgun/groupcache/v2 v2.4.1 ) require ( github.com/golang/protobuf v1.5.2 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/segmentio/fasthash v1.0.3 // indirect + github.com/sirupsen/logrus v1.6.0 // indirect + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect google.golang.org/protobuf v1.26.0 // indirect ) diff --git a/go.sum b/go.sum index 7318762..09b0920 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -7,6 +9,21 @@ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/mailgun/groupcache v1.3.0 h1:qie8iED3OIo2ICCbYx9vZybsVUGBJdZkroidGfao7q4= +github.com/mailgun/groupcache v1.3.0/go.mod h1:IC2jAVGyQ4t9S8D1Hsul0zMWkMDuVR8N/Cex7bgCvNg= +github.com/mailgun/groupcache/v2 v2.4.1 h1:E2WWpvUTfuBJINX698P1YBWtw8Y+0iHFAAkaYNORYkA= +github.com/mailgun/groupcache/v2 v2.4.1/go.mod h1:L+wDfh8BrXRyYH1VHBtjv/UxYhjIFYvnk9kmpF8/ij4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= +github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= diff --git a/package_test.go b/package_test.go index 37889e8..7132f2b 100644 --- a/package_test.go +++ b/package_test.go @@ -4,7 +4,7 @@ import ( "log" "testing" - "github.com/golang/groupcache" + "github.com/mailgun/groupcache/v2" ) func TestMain(t *testing.M) { diff --git a/quantize.go b/quantize.go deleted file mode 100644 index a12f749..0000000 --- a/quantize.go +++ /dev/null @@ -1,26 +0,0 @@ -package cachefs - -import ( - "hash/adler32" - "time" -) - -// quantize returns an integer to use as part of a cache key so that -// time is "quantized" for a certain interval. Also, the checksum of -// a string as a second offset. This prevents all the items from -// expiring at once, but does create variability in how long they -// can be cached. This function allows us to use a cache like -// golang/groupcache, which has no expiry mechanism of its own. -func quantize(t time.Time, d time.Duration, s string) int64 { - if d == 0 { - return 0 - } - sum := adler32.Checksum([]byte(s)) - offset := time.Duration(float64(sum) * float64(d) / float64(2<<31) / 4.0) - return t.UnixNano() / int64(d.Nanoseconds()+offset.Nanoseconds()) -} - -// quantizeOffset returns the maximum cache duration for a given sum. -func quantizeOffset(sum int32, d time.Duration) time.Duration { - return time.Duration(float64(sum) * float64(d) / float64(2<<31) / 4.0) -} From 475c3674e0058c3c2fe31b87bf0184706eded90d Mon Sep 17 00:00:00 2001 From: Travis Cline Date: Sun, 28 Aug 2022 17:44:10 -0700 Subject: [PATCH 2/2] cachefs: Add InvalidateCacheFS interface and extend cacheFS to implement it --- fs.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fs.go b/fs.go index 6670945..373376d 100644 --- a/fs.go +++ b/fs.go @@ -34,6 +34,14 @@ import ( "github.com/mailgun/groupcache/v2" ) +// A InvalidateCacheFS is a file system with a InvalidateCache method. +type InvalidateCacheFS interface { + fs.FS + + // InvalidateCache invalidates the cache for a path on the FS. + InvalidateCache(ctx context.Context, path string) error +} + // Config stores the configuration settings of your cache. type Config struct { GroupName string // Name of the groupcache group @@ -49,6 +57,9 @@ type cacheFS struct { cache *groupcache.Group } +// Statically declare that cacheFS satisfies InvalidateCacheFS. +var _ InvalidateCacheFS = (*cacheFS)(nil) + // Open opens the named file. // // When Open returns an error, it should be of type *fs.PathError @@ -176,3 +187,8 @@ func New(innerFS fs.FS, config *Config) fs.FS { })), } } + +// InvalidateCache invalidates the cache for a path in the filesystem. +func (cfs *cacheFS) InvalidateCache(ctx context.Context, path string) error { + return cfs.cache.Remove(ctx, path) +}