diff --git a/pycrest/eve.py b/pycrest/eve.py index a1cf2b3..725da9a 100644 --- a/pycrest/eve.py +++ b/pycrest/eve.py @@ -30,7 +30,7 @@ class APICache(object): - def put(self, key, value): + def put(self, key, value, ex): raise NotImplementedError def get(self, key): @@ -50,9 +50,9 @@ def __init__(self, path): def _getpath(self, key): return os.path.join(self.path, str(hash(key)) + '.cache') - def put(self, key, value): + def put(self, key, value, ex): with open(self._getpath(key), 'wb') as f: - f.write(zlib.compress(pickle.dumps(value, -1))) + f.write(zlib.compress(pickle.dumps((value, ex + time.time()), -1))) self._cache[key] = value def get(self, key): @@ -61,7 +61,14 @@ def get(self, key): try: with open(self._getpath(key), 'rb') as f: - return pickle.loads(zlib.decompress(f.read())) + ret = pickle.loads(zlib.decompress(f.read())) + if ret[1] > time.time(): + return ret[0] + + # if we are here, we got stale cache, so we can invalidate it + self.invalidate(key) + return None + except IOError as ex: if ex.errno == 2: # file does not exist (yet) return None @@ -85,10 +92,20 @@ def __init__(self): self._dict = {} def get(self, key): - return self._dict.get(key, None) + ret = self._dict.get(key, None) + if ret is not None and ret[1] > time.time(): + # cache hit + return ret[0] + elif ret is None: + # cache miss + return None + else: + # stale, delete from cache + self.invalidate(key) + return None - def put(self, key, value): - self._dict[key] = value + def put(self, key, value, ex): + self._dict[key] = value, ex + time.time() def invalidate(self, key): self._dict.pop(key, None) @@ -141,12 +158,9 @@ def get(self, resource, params=None): # check cache key = (resource, frozenset(self._session.headers.items()), frozenset(prms.items())) cached = self.cache.get(key) - if cached and cached['expires'] > time.time(): + if cached: logger.debug('Cache hit for resource %s (params=%s)', resource, prms) - return cached['payload'] - elif cached: - logger.debug('Cache stale for resource %s (params=%s)', resource, prms) - self.cache.invalidate(key) + return cached else: logger.debug('Cache miss for resource %s (params=%s', resource, prms) @@ -161,7 +175,7 @@ def get(self, resource, params=None): key = (resource, frozenset(self._session.headers.items()), frozenset(prms.items())) expires = self._get_expires(res) if expires > 0: - self.cache.put(key, {'expires': time.time() + expires, 'payload': ret}) + self.cache.put(key, ret, expires) return ret diff --git a/tests/test_api.py b/tests/test_api.py index 64b074e..4006c6e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -240,7 +240,7 @@ def test_pagination(url, request): recf = open(path, 'r') rec = pickle.loads(zlib.decompress(recf.read())) recf.close() - rec['expires'] = 1 + rec = rec[0], 1 recf = open(path, 'w') recf.write(zlib.compress(pickle.dumps(rec))) @@ -436,7 +436,7 @@ def token_mock(url, request): recf = open(path, 'r') rec = pickle.loads(zlib.decompress(recf.read())) recf.close() - rec['expires'] = 1 + rec = rec[0], 1 recf = open(path, 'w') recf.write(zlib.compress(pickle.dumps(rec))) @@ -462,7 +462,7 @@ def test_apicache(self, mock_open, mock_unlink, mock_mkdir, mock_isdir): # Just because pragma: no cover is ugly cache = pycrest.eve.APICache() self.assertRaises(NotImplementedError, lambda: cache.get("foo")) - self.assertRaises(NotImplementedError, lambda: cache.put("foo", "bar")) + self.assertRaises(NotImplementedError, lambda: cache.put("foo", "bar", time.time())) self.assertRaises(NotImplementedError, lambda: cache.invalidate("foo")) # Test default DictCache @@ -470,7 +470,7 @@ def test_apicache(self, mock_open, mock_unlink, mock_mkdir, mock_isdir): self.assertEqual(type(crest.cache).__name__, "DictCache") crest.cache.invalidate('nxkey') self.assertEqual(crest.cache.get('nxkey'), None) - crest.cache.put('key', 'value') + crest.cache.put('key', 'value', time.time() + 1) self.assertEqual(crest.cache.get('key'), 'value') @@ -490,7 +490,7 @@ def test_apicache(self, mock_open, mock_unlink, mock_mkdir, mock_isdir): self.assertEqual(crest.cache.get('nxkey'), None) # cache (key, value) pair and retrieve it - crest.cache.put('key', 'value') + crest.cache.put('key', 'value', time.time() + 1) self.assertEqual(crest.cache.get('key'), 'value') # retrieve from disk @@ -633,4 +633,4 @@ def brokenInt(url, request): with mock.patch('time.time') as mock_time: mock_time.return_value = 0 eve.shouldCache() - self.assertEqual(list(eve.cache._dict.items())[0][1]['expires'], 300) + self.assertEqual(list(eve.cache._dict.items())[0][1][1], 300)