From 29a7b520dfa8ece2f3b648ea955611fbc20c6d6a Mon Sep 17 00:00:00 2001 From: Evgenii Klimov Date: Wed, 24 Jan 2024 13:45:27 +0300 Subject: [PATCH] feat(storage): implement cached storage --- internal/storage/cachedstorage.go | 80 ++++++++ internal/storage/cachedstorage_test.go | 211 +++++++++++++++++++ internal/storage/storage_mock_test.go | 269 +++++++++++++++++++++++++ test/mockery.yaml | 3 + 4 files changed, 563 insertions(+) create mode 100644 internal/storage/cachedstorage.go create mode 100644 internal/storage/cachedstorage_test.go create mode 100644 internal/storage/storage_mock_test.go diff --git a/internal/storage/cachedstorage.go b/internal/storage/cachedstorage.go new file mode 100644 index 0000000..cfbe4f0 --- /dev/null +++ b/internal/storage/cachedstorage.go @@ -0,0 +1,80 @@ +package storage + +import ( + "log/slog" + "time" + + "github.com/eklmv/pdfcertificates/internal/cache" +) + +type CachedStorage struct { + Storage + c cache.Cache[uint32, certFile] +} + +type certFile struct { + file []byte + timestamp time.Time + size uint64 +} + +func (cf certFile) Size() uint64 { + return cf.size +} + +func NewCachedStorage(storage Storage) *CachedStorage { + c := cache.NewSafeCache(cache.NewLRUCache[uint32, certFile](0)) + return &CachedStorage{Storage: storage, c: c} +} + +func (cs *CachedStorage) Add(id string, cert []byte, timestamp time.Time) error { + err := cs.Storage.Add(id, cert, timestamp) + if err == nil { + hash := cache.HashString(id) + cf := certFile{ + file: cert, + timestamp: timestamp, + } + cf.size = cache.SizeOf(cf) + cs.c.Add(hash, cf) + } + return err +} + +func (cs *CachedStorage) Get(id string, timestamp time.Time) (cert []byte, err error) { + hash := cache.HashString(id) + cf, ok := cs.c.Peek(hash) + if ok && (cf.timestamp.Equal(timestamp) || cf.timestamp.After(timestamp)) { + cs.c.Touch(hash) + slog.Info("sucessful storage cache hit", slog.String("id", id), slog.Time("timestamp", timestamp)) + return cf.file, nil + } + cert, err = cs.Storage.Get(id, timestamp) + if err == nil { + cf := certFile{ + file: cert, + timestamp: timestamp, + } + cf.size = cache.SizeOf(cf) + cs.c.Add(hash, cf) + } + return +} + +func (cs *CachedStorage) Delete(id string, timestamp time.Time) { + hash := cache.HashString(id) + cf, ok := cs.c.Peek(hash) + if ok && (cf.timestamp.Equal(timestamp) || cf.timestamp.Before(timestamp)) { + cs.c.Remove(hash) + } + cs.Storage.Delete(id, timestamp) +} + +func (cs *CachedStorage) Exists(id string, timestamp time.Time) bool { + hash := cache.HashString(id) + cf, ok := cs.c.Peek(hash) + if ok && (cf.timestamp.Equal(timestamp) || cf.timestamp.After(timestamp)) { + return true + } + return cs.Storage.Exists(id, timestamp) +} diff --git a/internal/storage/cachedstorage_test.go b/internal/storage/cachedstorage_test.go new file mode 100644 index 0000000..6068310 --- /dev/null +++ b/internal/storage/cachedstorage_test.go @@ -0,0 +1,211 @@ +package storage + +import ( + "fmt" + "testing" + "time" + + "github.com/eklmv/pdfcertificates/internal/cache" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCachedStorageImplementsInterface(t *testing.T) { + assert.Implements(t, (*Storage)(nil), &CachedStorage{}) +} + +func TestCachedStorageAdd(t *testing.T) { + t.Run("should cache certificate if underlying Add was successful", func(t *testing.T) { + id := "00000000" + cert := []byte("Hello, world!") + timestamp := time.Now() + m := NewMockStorage(t) + cs := NewCachedStorage(m) + + m.EXPECT().Add(id, cert, timestamp).Return(nil).Once() + + err := cs.Add(id, cert, timestamp) + + require.NoError(t, err) + assert.Equal(t, uint64(1), cs.c.Len()) + assert.Equal(t, cache.HashString(id), cs.c.Keys()[0]) + assert.Equal(t, cert, cs.c.Values()[0].file) + m.AssertExpectations(t) + }) + t.Run("should not affect cache if underlying Add failed", func(t *testing.T) { + id := "00000000" + cert := []byte("Hello, world!") + timestamp := time.Now() + m := NewMockStorage(t) + cs := NewCachedStorage(m) + + m.EXPECT().Add(id, cert, timestamp).Return(fmt.Errorf("failed")).Once() + + err := cs.Add(id, cert, timestamp) + + require.Error(t, err) + assert.Empty(t, cs.c.Len()) + assert.Empty(t, cs.c.Size()) + assert.Empty(t, cs.c.Keys()) + m.AssertExpectations(t) + }) +} + +func TestCachedStorageGet(t *testing.T) { + t.Run("return cached file if requested timestamp same or older, avoid underlying call", func(t *testing.T) { + id := "00000000" + exp := []byte("Hello, world!") + timestamp := time.Now() + older_timestamp := timestamp.Add(-1 * time.Hour) + m := NewMockStorage(t) + cs := NewCachedStorage(m) + m.EXPECT().Add(id, exp, timestamp).Return(nil).Once() + err := cs.Add(id, exp, timestamp) + require.NoError(t, err) + + got, err := cs.Get(id, timestamp) + + require.NoError(t, err) + assert.Equal(t, exp, got) + + got, err = cs.Get(id, older_timestamp) + + require.NoError(t, err) + assert.Equal(t, exp, got) + m.AssertExpectations(t) + }) + t.Run("if requested timestamp is newer than stored make underlying call and cache result", func(t *testing.T) { + id := "00000000" + cert := []byte("Hello, world!") + exp := []byte("Newer certificate") + timestamp := time.Now() + newer_timestamp := timestamp.Add(1 * time.Hour) + m := NewMockStorage(t) + cs := NewCachedStorage(m) + m.EXPECT().Add(id, cert, timestamp).Return(nil).Once() + err := cs.Add(id, cert, timestamp) + require.NoError(t, err) + + m.EXPECT().Get(id, newer_timestamp).Return(exp, nil).Once() + got, err := cs.Get(id, newer_timestamp) + + require.NoError(t, err) + assert.Equal(t, exp, got) + assert.Equal(t, uint64(1), cs.c.Len()) + assert.Equal(t, cache.HashString(id), cs.c.Keys()[0]) + assert.Equal(t, exp, cs.c.Values()[0].file) + m.AssertExpectations(t) + }) +} + +func TestCachedStorageDelete(t *testing.T) { + t.Run("remove cached certificate and make underlying call", func(t *testing.T) { + id := "00000000" + cert := []byte("Hello, world!") + timestamp := time.Now() + m := NewMockStorage(t) + cs := NewCachedStorage(m) + m.EXPECT().Add(id, cert, timestamp).Return(nil).Once() + err := cs.Add(id, cert, timestamp) + require.NoError(t, err) + + m.EXPECT().Delete(id, timestamp).Once() + cs.Delete(id, timestamp) + + assert.Empty(t, cs.c.Len()) + assert.Empty(t, cs.c.Size()) + assert.Empty(t, cs.c.Keys()) + assert.Empty(t, cs.c.Values()) + m.AssertExpectations(t) + }) + t.Run("remove cached certificate if requested timestamp newer and make underlying call", func(t *testing.T) { + id := "00000000" + cert := []byte("Hello, world!") + timestamp := time.Now() + newer_timestamp := timestamp.Add(1 * time.Hour) + m := NewMockStorage(t) + cs := NewCachedStorage(m) + m.EXPECT().Add(id, cert, timestamp).Return(nil).Once() + err := cs.Add(id, cert, timestamp) + require.NoError(t, err) + + m.EXPECT().Delete(id, newer_timestamp).Once() + cs.Delete(id, newer_timestamp) + + assert.Empty(t, cs.c.Len()) + assert.Empty(t, cs.c.Size()) + assert.Empty(t, cs.c.Keys()) + assert.Empty(t, cs.c.Values()) + m.AssertExpectations(t) + }) + t.Run("do not affect cache if requested timestamp older and make underlying call", func(t *testing.T) { + id := "00000000" + cert := []byte("Hello, world!") + timestamp := time.Now() + older_timestamp := timestamp.Add(-1 * time.Hour) + m := NewMockStorage(t) + cs := NewCachedStorage(m) + m.EXPECT().Add(id, cert, timestamp).Return(nil).Once() + err := cs.Add(id, cert, timestamp) + require.NoError(t, err) + + m.EXPECT().Delete(id, older_timestamp).Once() + cs.Delete(id, older_timestamp) + + assert.Equal(t, uint64(1), cs.c.Len()) + assert.Equal(t, cache.HashString(id), cs.c.Keys()[0]) + assert.Equal(t, cert, cs.c.Values()[0].file) + m.AssertExpectations(t) + }) +} + +func TestCachedStorageExists(t *testing.T) { + t.Run("return true if certificate cached with same or newer timestamp, avoid underlying call", func(t *testing.T) { + id := "00000000" + cert := []byte("Hello, world!") + timestamp := time.Now() + older_timestamp := timestamp.Add(-1 * time.Hour) + m := NewMockStorage(t) + cs := NewCachedStorage(m) + m.EXPECT().Add(id, cert, timestamp).Return(nil).Once() + err := cs.Add(id, cert, timestamp) + require.NoError(t, err) + + got := cs.Exists(id, timestamp) + assert.True(t, got) + + got = cs.Exists(id, older_timestamp) + assert.True(t, got) + + m.AssertExpectations(t) + }) + t.Run("if certificate cached with older timestamp make underlying call", func(t *testing.T) { + id := "00000000" + cert := []byte("Hello, world!") + timestamp := time.Now() + newer_timestamp := timestamp.Add(1 * time.Hour) + m := NewMockStorage(t) + cs := NewCachedStorage(m) + m.EXPECT().Add(id, cert, timestamp).Return(nil).Once() + err := cs.Add(id, cert, timestamp) + require.NoError(t, err) + + m.EXPECT().Exists(id, newer_timestamp).Return(false).Once() + got := cs.Exists(id, newer_timestamp) + + assert.False(t, got) + m.AssertExpectations(t) + }) + t.Run("if certificate not cached make underlying call", func(t *testing.T) { + id := "00000000" + timestamp := time.Now() + m := NewMockStorage(t) + cs := NewCachedStorage(m) + + m.EXPECT().Exists(id, timestamp).Return(true).Once() + got := cs.Exists(id, timestamp) + + assert.True(t, got) + m.AssertExpectations(t) + }) +} diff --git a/internal/storage/storage_mock_test.go b/internal/storage/storage_mock_test.go new file mode 100644 index 0000000..b124c45 --- /dev/null +++ b/internal/storage/storage_mock_test.go @@ -0,0 +1,269 @@ +// Code generated by mockery v2.38.0. DO NOT EDIT. + +package storage + +import ( + time "time" + + mock "github.com/stretchr/testify/mock" +) + +// MockStorage is an autogenerated mock type for the Storage type +type MockStorage struct { + mock.Mock +} + +type MockStorage_Expecter struct { + mock *mock.Mock +} + +func (_m *MockStorage) EXPECT() *MockStorage_Expecter { + return &MockStorage_Expecter{mock: &_m.Mock} +} + +// Add provides a mock function with given fields: id, cert, timestamp +func (_m *MockStorage) Add(id string, cert []byte, timestamp time.Time) error { + ret := _m.Called(id, cert, timestamp) + + if len(ret) == 0 { + panic("no return value specified for Add") + } + + var r0 error + if rf, ok := ret.Get(0).(func(string, []byte, time.Time) error); ok { + r0 = rf(id, cert, timestamp) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStorage_Add_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Add' +type MockStorage_Add_Call struct { + *mock.Call +} + +// Add is a helper method to define mock.On call +// - id string +// - cert []byte +// - timestamp time.Time +func (_e *MockStorage_Expecter) Add(id interface{}, cert interface{}, timestamp interface{}) *MockStorage_Add_Call { + return &MockStorage_Add_Call{Call: _e.mock.On("Add", id, cert, timestamp)} +} + +func (_c *MockStorage_Add_Call) Run(run func(id string, cert []byte, timestamp time.Time)) *MockStorage_Add_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].([]byte), args[2].(time.Time)) + }) + return _c +} + +func (_c *MockStorage_Add_Call) Return(_a0 error) *MockStorage_Add_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockStorage_Add_Call) RunAndReturn(run func(string, []byte, time.Time) error) *MockStorage_Add_Call { + _c.Call.Return(run) + return _c +} + +// Delete provides a mock function with given fields: id, timestamp +func (_m *MockStorage) Delete(id string, timestamp time.Time) { + _m.Called(id, timestamp) +} + +// MockStorage_Delete_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Delete' +type MockStorage_Delete_Call struct { + *mock.Call +} + +// Delete is a helper method to define mock.On call +// - id string +// - timestamp time.Time +func (_e *MockStorage_Expecter) Delete(id interface{}, timestamp interface{}) *MockStorage_Delete_Call { + return &MockStorage_Delete_Call{Call: _e.mock.On("Delete", id, timestamp)} +} + +func (_c *MockStorage_Delete_Call) Run(run func(id string, timestamp time.Time)) *MockStorage_Delete_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(time.Time)) + }) + return _c +} + +func (_c *MockStorage_Delete_Call) Return() *MockStorage_Delete_Call { + _c.Call.Return() + return _c +} + +func (_c *MockStorage_Delete_Call) RunAndReturn(run func(string, time.Time)) *MockStorage_Delete_Call { + _c.Call.Return(run) + return _c +} + +// Exists provides a mock function with given fields: id, timestamp +func (_m *MockStorage) Exists(id string, timestamp time.Time) bool { + ret := _m.Called(id, timestamp) + + if len(ret) == 0 { + panic("no return value specified for Exists") + } + + var r0 bool + if rf, ok := ret.Get(0).(func(string, time.Time) bool); ok { + r0 = rf(id, timestamp) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// MockStorage_Exists_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Exists' +type MockStorage_Exists_Call struct { + *mock.Call +} + +// Exists is a helper method to define mock.On call +// - id string +// - timestamp time.Time +func (_e *MockStorage_Expecter) Exists(id interface{}, timestamp interface{}) *MockStorage_Exists_Call { + return &MockStorage_Exists_Call{Call: _e.mock.On("Exists", id, timestamp)} +} + +func (_c *MockStorage_Exists_Call) Run(run func(id string, timestamp time.Time)) *MockStorage_Exists_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(time.Time)) + }) + return _c +} + +func (_c *MockStorage_Exists_Call) Return(_a0 bool) *MockStorage_Exists_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockStorage_Exists_Call) RunAndReturn(run func(string, time.Time) bool) *MockStorage_Exists_Call { + _c.Call.Return(run) + return _c +} + +// Get provides a mock function with given fields: id, timestamp +func (_m *MockStorage) Get(id string, timestamp time.Time) ([]byte, error) { + ret := _m.Called(id, timestamp) + + if len(ret) == 0 { + panic("no return value specified for Get") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(string, time.Time) ([]byte, error)); ok { + return rf(id, timestamp) + } + if rf, ok := ret.Get(0).(func(string, time.Time) []byte); ok { + r0 = rf(id, timestamp) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(string, time.Time) error); ok { + r1 = rf(id, timestamp) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockStorage_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type MockStorage_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - id string +// - timestamp time.Time +func (_e *MockStorage_Expecter) Get(id interface{}, timestamp interface{}) *MockStorage_Get_Call { + return &MockStorage_Get_Call{Call: _e.mock.On("Get", id, timestamp)} +} + +func (_c *MockStorage_Get_Call) Run(run func(id string, timestamp time.Time)) *MockStorage_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(time.Time)) + }) + return _c +} + +func (_c *MockStorage_Get_Call) Return(cert []byte, err error) *MockStorage_Get_Call { + _c.Call.Return(cert, err) + return _c +} + +func (_c *MockStorage_Get_Call) RunAndReturn(run func(string, time.Time) ([]byte, error)) *MockStorage_Get_Call { + _c.Call.Return(run) + return _c +} + +// Load provides a mock function with given fields: +func (_m *MockStorage) Load() error { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Load") + } + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStorage_Load_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Load' +type MockStorage_Load_Call struct { + *mock.Call +} + +// Load is a helper method to define mock.On call +func (_e *MockStorage_Expecter) Load() *MockStorage_Load_Call { + return &MockStorage_Load_Call{Call: _e.mock.On("Load")} +} + +func (_c *MockStorage_Load_Call) Run(run func()) *MockStorage_Load_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockStorage_Load_Call) Return(_a0 error) *MockStorage_Load_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockStorage_Load_Call) RunAndReturn(run func() error) *MockStorage_Load_Call { + _c.Call.Return(run) + return _c +} + +// NewMockStorage creates a new instance of MockStorage. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockStorage(t interface { + mock.TestingT + Cleanup(func()) +}) *MockStorage { + mock := &MockStorage{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/test/mockery.yaml b/test/mockery.yaml index bce2fac..3215e89 100644 --- a/test/mockery.yaml +++ b/test/mockery.yaml @@ -7,3 +7,6 @@ packages: github.com/eklmv/pdfcertificates/internal/db: interfaces: Querier: + github.com/eklmv/pdfcertificates/internal/storage: + interfaces: + Storage: