From e577788171be2976ae1e23de780b03b348c26018 Mon Sep 17 00:00:00 2001 From: movciari Date: Sun, 17 Jan 2016 13:58:16 +0100 Subject: [PATCH 1/5] Cache handles stale values --- pycrest/eve.py | 40 +++++++++++++++++++++++++++------------- tests/test_api.py | 6 +++--- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/pycrest/eve.py b/pycrest/eve.py index a1cf2b3..7f4064c 100644 --- a/pycrest/eve.py +++ b/pycrest/eve.py @@ -30,7 +30,7 @@ class APICache(object): - def put(self, key, value): + def put(self, key, value, ex): raise NotImplementedError def get(self, key): @@ -50,9 +50,9 @@ def __init__(self, path): def _getpath(self, key): return os.path.join(self.path, str(hash(key)) + '.cache') - def put(self, key, value): + def put(self, key, value, ex): with open(self._getpath(key), 'wb') as f: - f.write(zlib.compress(pickle.dumps(value, -1))) + f.write(zlib.compress(pickle.dumps((value, ex), -1))) self._cache[key] = value def get(self, key): @@ -61,7 +61,14 @@ def get(self, key): try: with open(self._getpath(key), 'rb') as f: - return pickle.loads(zlib.decompress(f.read())) + ret = pickle.loads(zlib.decompress(f.read())) + if ret[1] > time.time(): + return ret[0] + + # if we are here, we got stale cache, so we can invalidate it + self.invalidate(key) + return None + except IOError as ex: if ex.errno == 2: # file does not exist (yet) return None @@ -85,10 +92,20 @@ def __init__(self): self._dict = {} def get(self, key): - return self._dict.get(key, None) + ret = self._dict.get(key, None) + if ret is not None and ret[1] > time.time(): + # cache hit + return ret[0] + elif ret is None: + # cache miss + return None + else: + # stale, delete from cache + self.invalidate(key) + return None - def put(self, key, value): - self._dict[key] = value + def put(self, key, value, ex): + self._dict[key] = value, ex def invalidate(self, key): self._dict.pop(key, None) @@ -141,12 +158,9 @@ def get(self, resource, params=None): # check cache key = (resource, frozenset(self._session.headers.items()), frozenset(prms.items())) cached = self.cache.get(key) - if cached and cached['expires'] > time.time(): + if cached: logger.debug('Cache hit for resource %s (params=%s)', resource, prms) - return cached['payload'] - elif cached: - logger.debug('Cache stale for resource %s (params=%s)', resource, prms) - self.cache.invalidate(key) + return cached else: logger.debug('Cache miss for resource %s (params=%s', resource, prms) @@ -161,7 +175,7 @@ def get(self, resource, params=None): key = (resource, frozenset(self._session.headers.items()), frozenset(prms.items())) expires = self._get_expires(res) if expires > 0: - self.cache.put(key, {'expires': time.time() + expires, 'payload': ret}) + self.cache.put(key, ret, expires) return ret diff --git a/tests/test_api.py b/tests/test_api.py index 64b074e..14e2e80 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -462,7 +462,7 @@ def test_apicache(self, mock_open, mock_unlink, mock_mkdir, mock_isdir): # Just because pragma: no cover is ugly cache = pycrest.eve.APICache() self.assertRaises(NotImplementedError, lambda: cache.get("foo")) - self.assertRaises(NotImplementedError, lambda: cache.put("foo", "bar")) + self.assertRaises(NotImplementedError, lambda: cache.put("foo", "bar", time.time())) self.assertRaises(NotImplementedError, lambda: cache.invalidate("foo")) # Test default DictCache @@ -470,7 +470,7 @@ def test_apicache(self, mock_open, mock_unlink, mock_mkdir, mock_isdir): self.assertEqual(type(crest.cache).__name__, "DictCache") crest.cache.invalidate('nxkey') self.assertEqual(crest.cache.get('nxkey'), None) - crest.cache.put('key', 'value') + crest.cache.put('key', 'value', time.time() + 1) self.assertEqual(crest.cache.get('key'), 'value') @@ -490,7 +490,7 @@ def test_apicache(self, mock_open, mock_unlink, mock_mkdir, mock_isdir): self.assertEqual(crest.cache.get('nxkey'), None) # cache (key, value) pair and retrieve it - crest.cache.put('key', 'value') + crest.cache.put('key', 'value', time.time() + 1) self.assertEqual(crest.cache.get('key'), 'value') # retrieve from disk From 7f8094e05fbe1c683b794d65104b12f8abfd6cbf Mon Sep 17 00:00:00 2001 From: movciari Date: Sun, 17 Jan 2016 13:58:16 +0100 Subject: [PATCH 2/5] Cache handles stale values --- pycrest/eve.py | 40 +++++++++++++++++++++++++++------------- tests/test_api.py | 8 ++++---- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/pycrest/eve.py b/pycrest/eve.py index a1cf2b3..7f4064c 100644 --- a/pycrest/eve.py +++ b/pycrest/eve.py @@ -30,7 +30,7 @@ class APICache(object): - def put(self, key, value): + def put(self, key, value, ex): raise NotImplementedError def get(self, key): @@ -50,9 +50,9 @@ def __init__(self, path): def _getpath(self, key): return os.path.join(self.path, str(hash(key)) + '.cache') - def put(self, key, value): + def put(self, key, value, ex): with open(self._getpath(key), 'wb') as f: - f.write(zlib.compress(pickle.dumps(value, -1))) + f.write(zlib.compress(pickle.dumps((value, ex), -1))) self._cache[key] = value def get(self, key): @@ -61,7 +61,14 @@ def get(self, key): try: with open(self._getpath(key), 'rb') as f: - return pickle.loads(zlib.decompress(f.read())) + ret = pickle.loads(zlib.decompress(f.read())) + if ret[1] > time.time(): + return ret[0] + + # if we are here, we got stale cache, so we can invalidate it + self.invalidate(key) + return None + except IOError as ex: if ex.errno == 2: # file does not exist (yet) return None @@ -85,10 +92,20 @@ def __init__(self): self._dict = {} def get(self, key): - return self._dict.get(key, None) + ret = self._dict.get(key, None) + if ret is not None and ret[1] > time.time(): + # cache hit + return ret[0] + elif ret is None: + # cache miss + return None + else: + # stale, delete from cache + self.invalidate(key) + return None - def put(self, key, value): - self._dict[key] = value + def put(self, key, value, ex): + self._dict[key] = value, ex def invalidate(self, key): self._dict.pop(key, None) @@ -141,12 +158,9 @@ def get(self, resource, params=None): # check cache key = (resource, frozenset(self._session.headers.items()), frozenset(prms.items())) cached = self.cache.get(key) - if cached and cached['expires'] > time.time(): + if cached: logger.debug('Cache hit for resource %s (params=%s)', resource, prms) - return cached['payload'] - elif cached: - logger.debug('Cache stale for resource %s (params=%s)', resource, prms) - self.cache.invalidate(key) + return cached else: logger.debug('Cache miss for resource %s (params=%s', resource, prms) @@ -161,7 +175,7 @@ def get(self, resource, params=None): key = (resource, frozenset(self._session.headers.items()), frozenset(prms.items())) expires = self._get_expires(res) if expires > 0: - self.cache.put(key, {'expires': time.time() + expires, 'payload': ret}) + self.cache.put(key, ret, expires) return ret diff --git a/tests/test_api.py b/tests/test_api.py index 64b074e..cd92bc7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -436,7 +436,7 @@ def token_mock(url, request): recf = open(path, 'r') rec = pickle.loads(zlib.decompress(recf.read())) recf.close() - rec['expires'] = 1 + rec = rec[0], 1 recf = open(path, 'w') recf.write(zlib.compress(pickle.dumps(rec))) @@ -462,7 +462,7 @@ def test_apicache(self, mock_open, mock_unlink, mock_mkdir, mock_isdir): # Just because pragma: no cover is ugly cache = pycrest.eve.APICache() self.assertRaises(NotImplementedError, lambda: cache.get("foo")) - self.assertRaises(NotImplementedError, lambda: cache.put("foo", "bar")) + self.assertRaises(NotImplementedError, lambda: cache.put("foo", "bar", time.time())) self.assertRaises(NotImplementedError, lambda: cache.invalidate("foo")) # Test default DictCache @@ -470,7 +470,7 @@ def test_apicache(self, mock_open, mock_unlink, mock_mkdir, mock_isdir): self.assertEqual(type(crest.cache).__name__, "DictCache") crest.cache.invalidate('nxkey') self.assertEqual(crest.cache.get('nxkey'), None) - crest.cache.put('key', 'value') + crest.cache.put('key', 'value', time.time() + 1) self.assertEqual(crest.cache.get('key'), 'value') @@ -490,7 +490,7 @@ def test_apicache(self, mock_open, mock_unlink, mock_mkdir, mock_isdir): self.assertEqual(crest.cache.get('nxkey'), None) # cache (key, value) pair and retrieve it - crest.cache.put('key', 'value') + crest.cache.put('key', 'value', time.time() + 1) self.assertEqual(crest.cache.get('key'), 'value') # retrieve from disk From 92ba5961ba3fae8493b6f8e52b4730c75496860b Mon Sep 17 00:00:00 2001 From: movciari Date: Sun, 17 Jan 2016 14:34:15 +0100 Subject: [PATCH 3/5] Cache handles stale values --- tests/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index cd92bc7..4006c6e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -240,7 +240,7 @@ def test_pagination(url, request): recf = open(path, 'r') rec = pickle.loads(zlib.decompress(recf.read())) recf.close() - rec['expires'] = 1 + rec = rec[0], 1 recf = open(path, 'w') recf.write(zlib.compress(pickle.dumps(rec))) @@ -633,4 +633,4 @@ def brokenInt(url, request): with mock.patch('time.time') as mock_time: mock_time.return_value = 0 eve.shouldCache() - self.assertEqual(list(eve.cache._dict.items())[0][1]['expires'], 300) + self.assertEqual(list(eve.cache._dict.items())[0][1][1], 300) From d7c354175fb9f82553e0e85110adfcbcab246848 Mon Sep 17 00:00:00 2001 From: movciari Date: Sun, 17 Jan 2016 14:37:43 +0100 Subject: [PATCH 4/5] Cache handles stale values --- pycrest/eve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycrest/eve.py b/pycrest/eve.py index 7f4064c..de07447 100644 --- a/pycrest/eve.py +++ b/pycrest/eve.py @@ -105,7 +105,7 @@ def get(self, key): return None def put(self, key, value, ex): - self._dict[key] = value, ex + self._dict[key] = value, ex + time.time() def invalidate(self, key): self._dict.pop(key, None) From 3ea350d58abd6621f796a35a6a620c2831bce3ef Mon Sep 17 00:00:00 2001 From: movciari Date: Sun, 17 Jan 2016 14:38:22 +0100 Subject: [PATCH 5/5] Cache handles stale values --- pycrest/eve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycrest/eve.py b/pycrest/eve.py index de07447..725da9a 100644 --- a/pycrest/eve.py +++ b/pycrest/eve.py @@ -52,7 +52,7 @@ def _getpath(self, key): def put(self, key, value, ex): with open(self._getpath(key), 'wb') as f: - f.write(zlib.compress(pickle.dumps((value, ex), -1))) + f.write(zlib.compress(pickle.dumps((value, ex + time.time()), -1))) self._cache[key] = value def get(self, key):