Change in vdsm[master]: cache: Add caching decorator with invalidation

Friday, 31 October 2014

Nir Soffer has uploaded a new change for review.

Change subject: cache: Add caching decorator with invalidation
......................................................................

cache: Add caching decorator with invalidation

The new cache.memoized extends utils.memoized, adding invalidation
support.

Features added:

- An optional "validate" argument. This is a callable invoked each time
  the memoized function is called. When the callable returns False, the
  cache is invalidated.

- Memoized functions have an "invalidate" method, used to invalidate the
  cache during testing.

- file_validator - invalidates the cache when a file changes.

Example usage:

    from vdsm.cache import memoized, file_validator

    @memoized(file_validator('/bigfile'))
    def parse_bigfile():
        # Expensive code processing '/bigfile' contents

Change-Id: I6dd8fb29d94286e3e3a3e29b8218501cbdc5c018
Signed-off-by: Nir Soffer <nsoffer(a)redhat.com&gt;
---
M lib/vdsm/Makefile.am
A lib/vdsm/cache.py
M tests/Makefile.am
A tests/cacheTests.py
4 files changed, 366 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/09/34709/1

diff --git a/lib/vdsm/Makefile.am b/lib/vdsm/Makefile.am
index b862e71..6f0040d 100644
--- a/lib/vdsm/Makefile.am
+++ b/lib/vdsm/Makefile.am
@@ -23,6 +23,7 @@
 
 dist_vdsmpylib_PYTHON = \
 	__init__.py \
+	cache.py \
 	compat.py \
 	define.py \
 	exception.py \
diff --git a/lib/vdsm/cache.py b/lib/vdsm/cache.py
new file mode 100644
index 0000000..9806e40
--- /dev/null
+++ b/lib/vdsm/cache.py
@@ -0,0 +1,98 @@
+#
+# Copyright 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+
+import errno
+import os
+import functools
+
+
+def memoized(validate=None):
+    """
+    Return a caching decorator supporting invalidation.
+
+    The decorator accepts an optional validate callable, called each time the
+    memoized function is called. If the validate callable return True, the
+    memoized function will use the cache. If the validate callable return
+    False, the memoized cache is cleared.
+
+    The memoized function may accept multiple positional arguments. The
+    cache store the result for each combination of arguments. Functions with
+    kwargs are not supported.
+
+    Memoized functions have an "invalidate" method, used to invalidate the
+    memoized cache during testing.
+
+    To invalidate the cache when a file changes, use the file_validator from
+    this module.
+
+    Example usage:
+
+        from vdsm.cache import memoized, file_validator
+
+        @memoized(file_validator('/bigfile'))
+        def parse_bigfile():
+            # Expensive code processing '/bigfile' contents
+
+    """
+    def decorator(f):
+        cache = {}
+
+        @functools.wraps(f)
+        def wrapper(*args):
+            if validate is not None and not validate():
+                cache.clear()
+            try:
+                value = cache[args]
+            except KeyError:
+                value = cache[args] = f(*args)
+            return value
+
+        wrapper.invalidate = cache.clear
+        return wrapper
+
+    return decorator
+
+
+class file_validator(object):
+    """
+    I'm a validator returning False when a file has changed since the last
+    validation.
+    """
+
+    UNKNOWN = 0
+    MISSING = 1
+
+    def __init__(self, path):
+        self.path = path
+        self.stats = self.UNKNOWN
+
+    def __call__(self):
+        try:
+            stats = os.stat(self.path)
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            stats = self.MISSING
+        else:
+            stats = stats.st_ino, stats.st_size, stats.st_mtime
+        if stats != self.stats:
+            self.stats = stats
+            return False
+        return True
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 36a1cdd..6fa7e64 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -26,6 +26,7 @@
 	alignmentScanTests.py \
 	blocksdTests.py \
 	bridgeTests.py \
+	cacheTests.py \
 	cPopenTests.py \
 	capsTests.py \
 	clientifTests.py \
diff --git a/tests/cacheTests.py b/tests/cacheTests.py
new file mode 100644
index 0000000..8927b39
--- /dev/null
+++ b/tests/cacheTests.py
@@ -0,0 +1,266 @@
+#
+# Copyright 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301  USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+
+import os
+from vdsm.cache import memoized
+from vdsm.cache import file_validator
+from testlib import VdsmTestCase
+from testlib import namedTemporaryDir
+
+
+class Validator(object):
+    """ I'm a callable returning a boolean value (self.valid)
"""
+
+    def __init__(self):
+        self.valid = True
+        self.count = 0
+
+    def __call__(self):
+        self.count += 1
+        return self.valid
+
+
+class Accessor(object):
+    """ I'm recording how many times a dict was accessed.
"""
+
+    def __init__(self, d):
+        self.d = d
+        self.count = 0
+
+    def get(self, key):
+        self.count += 1
+        return self.d[key]
+
+
+class MemoizedTests(VdsmTestCase):
+
+    def setUp(self):
+        self.values = {'a': 0, 'b': 10, ('a',): 20, ('a',
'b'): 30}
+
+    def test_no_args(self):
+        accessor = Accessor(self.values)
+
+        @memoized()
+        def func(key):
+            return accessor.get(key)
+
+        # Fill the cache
+        self.assertEqual(func('a'), self.values['a'])
+        self.assertEqual(func('b'), self.values['b'])
+        self.assertEqual(accessor.count, 2)
+
+        # Values served now from the cache
+        self.assertEqual(func('a'), self.values['a'])
+        self.assertEqual(func('b'), self.values['b'])
+        self.assertEqual(accessor.count, 2)
+
+    def test_validation(self):
+        accessor = Accessor(self.values)
+        validator = Validator()
+
+        @memoized(validator)
+        def func(key):
+            return accessor.get(key)
+
+        # Fill the cache
+        self.assertEqual(func('a'), self.values['a'])
+        self.assertEqual(func('b'), self.values['b'])
+        self.assertEqual(accessor.count, 2)
+        self.assertEqual(validator.count, 2)
+
+        # Values served now from the cache
+        self.assertEqual(func('a'), self.values['a'])
+        self.assertEqual(func('b'), self.values['b'])
+        self.assertEqual(accessor.count, 2)
+        self.assertEqual(validator.count, 4)
+
+        # Values has changed
+        self.values['a'] += 1
+        self.values['b'] += 1
+
+        # Next call should clear the cache
+        validator.valid = False
+        self.assertEqual(func('a'), self.values['a'])
+        self.assertEqual(accessor.count, 3)
+        self.assertEqual(validator.count, 5)
+
+        # Next call should add next value to cache
+        validator.valid = True
+        self.assertEqual(func('b'), self.values['b'])
+        self.assertEqual(accessor.count, 4)
+        self.assertEqual(validator.count, 6)
+
+        # Values served now from the cache
+        self.assertEqual(func('a'), self.values['a'])
+        self.assertEqual(func('b'), self.values['b'])
+        self.assertEqual(accessor.count, 4)
+        self.assertEqual(validator.count, 8)
+
+    def test_raise_errors_in_memoized_func(self):
+        accessor = Accessor(self.values)
+        validator = Validator()
+
+        @memoized(validator)
+        def func(key):
+            return accessor.get(key)
+
+        # First run should fail, second shold fill the cache
+        self.assertRaises(KeyError, func, 'no such key')
+        self.assertEqual(func('a'), self.values['a'])
+        self.assertEqual(accessor.count, 2)
+        self.assertEqual(validator.count, 2)
+
+    def test_multiple_args(self):
+        accessor = Accessor(self.values)
+
+        @memoized()
+        def func(*args):
+            return accessor.get(args)
+
+        # Fill the cache
+        self.assertEqual(func('a'), self.values[('a',)])
+        self.assertEqual(func('a', 'b'), self.values[('a',
'b')])
+        self.assertEqual(accessor.count, 2)
+
+        # Values served now from the cache
+        self.assertEqual(func('a'), self.values[('a',)])
+        self.assertEqual(func('a', 'b'), self.values[('a',
'b')])
+        self.assertEqual(accessor.count, 2)
+
+    def test_kwargs_not_supported(self):
+        @memoized()
+        def func(a=None, b=None):
+            pass
+        self.assertRaises(TypeError, func, a=1, b=2)
+
+    def test_invalidate(self):
+        accessor = Accessor(self.values)
+
+        @memoized()
+        def func(key):
+            return accessor.get(key)
+
+        self.assertEqual(func('a'), self.values['a'])
+        self.assertEqual(accessor.count, 1)
+
+        func.invalidate()
+
+        self.assertEqual(func('a'), self.values['a'])
+        self.assertEqual(accessor.count, 2)
+
+
+class FileValidatorTests(VdsmTestCase):
+
+    def test_no_file(self):
+        with namedTemporaryDir() as tempdir:
+            path = os.path.join(tempdir, 'data')
+            validator = file_validator(path)
+
+            # Must be False so memoise call the decorated function
+            self.assertEqual(validator(), False)
+
+            # Since file state did not change, must remain True
+            self.assertEqual(validator(), True)
+
+    def test_file_created(self):
+        with namedTemporaryDir() as tempdir:
+            path = os.path.join(tempdir, 'data')
+            validator = file_validator(path)
+
+            self.assertEqual(validator(), False)
+            self.assertEqual(validator(), True)
+
+            with open(path, 'w') as f:
+                f.write('data')
+
+            self.assertEqual(validator(), False)
+            self.assertEqual(validator(), True)
+
+    def test_file_removed(self):
+        with namedTemporaryDir() as tempdir:
+            path = os.path.join(tempdir, 'data')
+            validator = file_validator(path)
+
+            with open(path, 'w') as f:
+                f.write('data')
+
+            self.assertEqual(validator(), False)
+            self.assertEqual(validator(), True)
+
+            os.unlink(path)
+
+            self.assertEqual(validator(), False)
+            self.assertEqual(validator(), True)
+
+    def test_size_changed(self):
+        with namedTemporaryDir() as tempdir:
+            path = os.path.join(tempdir, 'data')
+            validator = file_validator(path)
+            data = 'old data'
+            with open(path, 'w') as f:
+                f.write(data)
+
+            self.assertEqual(validator(), False)
+            self.assertEqual(validator(), True)
+
+            with open(path, 'w') as f:
+                f.write(data + ' new data')
+
+            self.assertEqual(validator(), False)
+            self.assertEqual(validator(), True)
+
+    def test_mtime_changed(self):
+        with namedTemporaryDir() as tempdir:
+            path = os.path.join(tempdir, 'data')
+            validator = file_validator(path)
+            data = 'old data'
+            with open(path, 'w') as f:
+                f.write(data)
+
+            self.assertEqual(validator(), False)
+            self.assertEqual(validator(), True)
+
+            # Fake timestamp change, as timestamp resolution may not be good
+            # enough when comparing changes during the test.
+            atime = mtime = os.path.getmtime(path) + 1
+            os.utime(path, (atime, mtime))
+
+            self.assertEqual(validator(), False)
+            self.assertEqual(validator(), True)
+
+    def test_ino_changed(self):
+        with namedTemporaryDir() as tempdir:
+            path = os.path.join(tempdir, 'data')
+            validator = file_validator(path)
+            data = 'old data'
+            with open(path, 'w') as f:
+                f.write(data)
+
+            self.assertEqual(validator(), False)
+            self.assertEqual(validator(), True)
+
+            tmp = path + '.tmp'
+            with open(tmp, 'w') as f:
+                f.write(data)
+            os.rename(tmp, path)
+
+            self.assertEqual(validator(), False)
+            self.assertEqual(validator(), True)


-- 
To view, visit http://gerrit.ovirt.org/34709
To unsubscribe, visit http://gerrit.ovirt.org/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I6dd8fb29d94286e3e3a3e29b8218501cbdc5c018
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Nir Soffer <nsoffer(a)redhat.com&gt;

    

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011