vdsm-patches July 2015

vdsm-patches@lists.fedorahosted.org

44 participants
919 discussions

Change in vdsm[master]: block: add blkdiscard on zero image
by Federico Simoncelli 15 Nov '15

15 Nov '15

Federico Simoncelli has uploaded a new change for review. Change subject: block: add blkdiscard on zero image ...................................................................... block: add blkdiscard on zero image Change-Id: I4e059c556c550440727b36b8af8e5dfc29ce2ccb Signed-off-by: Federico Simoncelli <fsimonce(a)redhat.com> --- M vdsm/storage/blockSD.py 1 file changed, 2 insertions(+), 0 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/31/35631/1 diff --git a/vdsm/storage/blockSD.py b/vdsm/storage/blockSD.py index daaa94f..a2370b8 100644 --- a/vdsm/storage/blockSD.py +++ b/vdsm/storage/blockSD.py @@ -33,6 +33,7 @@ from vdsm.config import config from vdsm import constants from vdsm import utils +from vdsm import utillinux import misc import fileUtils import sd @@ -219,6 +220,7 @@ try: misc.ddWatchCopy("/dev/zero", path, aborting, size) + utillinux.blkdiscard(path) except Exception as e: log.exception('zeroing operation failed') raise se.VolumesZeroingError(path) -- To view, visit http://gerrit.ovirt.org/35631 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I4e059c556c550440727b36b8af8e5dfc29ce2ccb Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Federico Simoncelli <fsimonce(a)redhat.com>

6 10

Change in vdsm[master]: rwlock: Replace misc.RWLock
by Nir Soffer 15 Nov '15

15 Nov '15

Nir Soffer has uploaded a new change for review. Change subject: rwlock: Replace misc.RWLock ...................................................................... rwlock: Replace misc.RWLock This patch removes misc.RWLock and replace it with simpler rwlock.RWLock. The new lock does not support recursive locking or lock demotion, but I think they are not used in by current code - not tested yet. Change-Id: I9ae6064e8e031339303e64606a70673807c4083a Signed-off-by: Nir Soffer <nsoffer(a)redhat.com> --- M vdsm/storage/misc.py M vdsm/storage/resourceManager.py 2 files changed, 4 insertions(+), 112 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/23/43423/1 diff --git a/vdsm/storage/misc.py b/vdsm/storage/misc.py index b14fd26..e45fad8 100644 --- a/vdsm/storage/misc.py +++ b/vdsm/storage/misc.py @@ -530,115 +530,6 @@ return 0 -class RWLock(object): - """ - A simple ReadWriteLock implementation. - - The lock must be released by the thread that acquired it. Once a thread - has acquired a lock, the same thread may acquire it again without blocking; - the thread must release it once for each time it has acquired it. Note that - lock promotion (acquiring an exclusive lock under a shared lock is - forbidden and will raise an exception. - - The lock puts all requests in a queue. The request is granted when The - previous one is released. - - Each request is represented by a :class:`threading.Event` object. When the - Event is set the request is granted. This enables multiple callers to wait - for a request thus implementing a shared lock. - """ - class _contextLock(object): - def __init__(self, owner, exclusive): - self._owner = owner - self._exclusive = exclusive - - def __enter__(self): - self._owner.acquire(self._exclusive) - - def __exit__(self, exc_type, exc_value, traceback): - self._owner.release() - - def __init__(self): - self._syncRoot = threading.Lock() - self._queue = Queue.Queue() - self._currentSharedLock = None - self._currentState = None - self._holdingThreads = {} - - self.shared = self._contextLock(self, False) - self.exclusive = self._contextLock(self, True) - - def acquireRead(self): - return self.acquire(False) - - def acquireWrite(self): - return self.acquire(True) - - def acquire(self, exclusive): - currentEvent = None - currentThread = threading.currentThread() - - # Handle reacquiring lock in the same thread - if currentThread in self._holdingThreads: - if self._currentState is False and exclusive: - raise RuntimeError("Lock promotion is forbidden.") - - self._holdingThreads[currentThread] += 1 - return - - with self._syncRoot: - # Handle regular acquisition - if exclusive: - currentEvent = threading.Event() - self._currentSharedLock = None - else: - if self._currentSharedLock is None: - self._currentSharedLock = threading.Event() - - currentEvent = self._currentSharedLock - - try: - self._queue.put_nowait((currentEvent, exclusive)) - except Queue.Full: - raise RuntimeError("There are too many objects waiting for " - "this lock") - - if self._queue.unfinished_tasks == 1: - # Bootstrap the process if needed. A lock is released the when - # the next request is granted. When there is no one to grant - # the request you have to grant it yourself. - event, self._currentState = self._queue.get_nowait() - event.set() - - currentEvent.wait() - - self._holdingThreads[currentThread] = 0 - - def release(self): - currentThread = threading.currentThread() - - if currentThread not in self._holdingThreads: - raise RuntimeError("Releasing an lock without acquiring it first") - - # If in nested lock don't really release - if self._holdingThreads[currentThread] > 0: - self._holdingThreads[currentThread] -= 1 - return - - del self._holdingThreads[currentThread] - - with self._syncRoot: - self._queue.task_done() - - if self._queue.empty(): - self._currentState = None - return - - nextRequest, self._currentState = self._queue.get_nowait() - - nextRequest.set() - - class DynamicBarrier(object): def __init__(self): self._cond = threading.Condition() diff --git a/vdsm/storage/resourceManager.py b/vdsm/storage/resourceManager.py index b1b0dc7..a7e67a2 100644 --- a/vdsm/storage/resourceManager.py +++ b/vdsm/storage/resourceManager.py @@ -29,6 +29,7 @@ import storage_exception as se import misc from logUtils import SimpleLogAdapter +from vdsm import rwlock from vdsm import utils @@ -287,7 +288,7 @@ self.autoRelease = True self._isValid = True - self._syncRoot = misc.RWLock() + self._syncRoot = rwlock.RWLock() def __wrapObj(self): for attr in dir(self.__wrappedObject): @@ -380,11 +381,11 @@ """ def __init__(self, factory): self.resources = {} - self.lock = threading.Lock() # misc.RWLock() + self.lock = threading.Lock() # rwlock.RWLock() self.factory = factory def __init__(self): - self._syncRoot = misc.RWLock() + self._syncRoot = rwlock.RWLock() self._namespaces = {} @classmethod -- To view, visit https://gerrit.ovirt.org/43423 To unsubscribe, visit https://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I9ae6064e8e031339303e64606a70673807c4083a Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Nir Soffer <nsoffer(a)redhat.com>

5 24

Change in vdsm[master]: rwlock: Support non-blocking acquire
by Nir Soffer 15 Nov '15

15 Nov '15

Nir Soffer has uploaded a new change for review. Change subject: rwlock: Support non-blocking acquire ...................................................................... rwlock: Support non-blocking acquire This patch adds non-blocking acquire suggested in https://gerrit.ovirt.org/42773. This is a simpler alternative to timed acquire, suggested in https://gerrit.ovirt.org/42909. Change-Id: Iec721a07087349050bfe9aa11aacf3be9695fb85 Signed-off-by: Nir Soffer <nsoffer(a)redhat.com> --- M lib/vdsm/rwlock.py M tests/rwlock_test.py 2 files changed, 30 insertions(+), 48 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/28/42928/1 diff --git a/lib/vdsm/rwlock.py b/lib/vdsm/rwlock.py index c6ce1ea..2281851 100644 --- a/lib/vdsm/rwlock.py +++ b/lib/vdsm/rwlock.py @@ -32,17 +32,23 @@ self._readers = set() self._writer = None - def acquireWrite(self): + def acquireWrite(self, block=True): with self._lock: if self._writer or self._readers or self._waiters: + if not block: + return False self._wait(True) self._writer = threading.current_thread() + return True - def acquireRead(self): + def acquireRead(self, block=True): with self._lock: if self._writer or self._waiters: + if not block: + return False self._wait(False) self._readers.add(threading.current_thread()) + return True def release(self): me = threading.current_thread() diff --git a/tests/rwlock_test.py b/tests/rwlock_test.py index 1c28f26..7ef5aa6 100644 --- a/tests/rwlock_test.py +++ b/tests/rwlock_test.py @@ -121,54 +121,30 @@ for t in threads: t.stop() - @slowtest - def test_shared_context_blocks_writer(self): - lock = RWLock() - writer = RWThread(lock.exclusive) - try: - with lock.shared: - writer.start() - if not writer.ready.wait(2): - raise RuntimeError("Timeout waiting for writer thread") - # Writer must block - self.assertFalse(writer.acquired.wait(1)) - finally: - writer.stop() - - def test_shared_context_allows_reader(self): - lock = RWLock() - with lock.shared: - reader = RWThread(lock.shared) - with utils.running(reader): - self.assertTrue(reader.acquired.wait(1)) - - @slowtest - def test_exclusive_context_blocks_writer(self): - lock = RWLock() - writer = RWThread(lock.exclusive) - try: - with lock.exclusive: - writer.start() - if not writer.ready.wait(2): - raise RuntimeError("Timeout waiting for writer thread") - # Reader must block - self.assertFalse(writer.acquired.wait(1)) - finally: - writer.stop() - - @slowtest - def test_exclusive_context_blocks_reader(self): + def test_reader_blocks_writer(self): lock = RWLock() reader = RWThread(lock.shared) - try: - with lock.exclusive: - reader.start() - if not reader.ready.wait(2): - raise RuntimeError("Timeout waiting for reader thread") - # Reader must block - self.assertFalse(reader.acquired.wait(1)) - finally: - reader.stop() + with utils.running(reader): + if not reader.acquired.wait(2): + raise RuntimeError("Timeout waiting for reader thread") + self.assertFalse(lock.acquireWrite(block=False)) + + def test_writer_blocks_writer(self): + lock = RWLock() + writer = RWThread(lock.exclusive) + with utils.running(writer): + if not writer.acquired.wait(2): + raise RuntimeError("Timeout waiting for writer thread") + self.assertFalse(lock.acquireWrite(block=False)) + + + def test_writer_blocks_reader(self): + lock = RWLock() + writer = RWThread(lock.exclusive) + with utils.running(writer): + if not writer.acquired.wait(2): + raise RuntimeError("Timeout waiting for writer thread") + self.assertFalse(lock.acquireRead(block=False)) @expandPermutations -- To view, visit https://gerrit.ovirt.org/42928 To unsubscribe, visit https://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Iec721a07087349050bfe9aa11aacf3be9695fb85 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Nir Soffer <nsoffer(a)redhat.com>

4 43

Change in vdsm[master]: rwlock: Support timed acquire
by Nir Soffer 15 Nov '15

15 Nov '15

Nir Soffer has uploaded a new change for review. Change subject: rwlock: Support timed acquire ...................................................................... rwlock: Support timed acquire Add acquire timeout - with timeout=0, can be used to implement non-blocking acquire. With bigger timeout, make it easy to fail a request after a timeout, instead of leaving a blocked thread forever. Change-Id: Idd9b11452d74b566b8989f41244f2d0534327214 Signed-off-by: Nir Soffer <nsoffer(a)redhat.com> --- M lib/vdsm/rwlock.py M tests/rwlock_test.py 2 files changed, 65 insertions(+), 21 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/09/42909/1 diff --git a/lib/vdsm/rwlock.py b/lib/vdsm/rwlock.py index 900d91d..72f962c 100644 --- a/lib/vdsm/rwlock.py +++ b/lib/vdsm/rwlock.py @@ -22,6 +22,10 @@ import threading +class Timeout(Exception): + pass + + class RWLock(object): def __init__(self): @@ -30,16 +34,16 @@ self._readers = set() self._writer = None - def acquireWrite(self): + def acquireWrite(self, timeout=None): with self._lock: if self._writer or self._readers or self._waiters: - self._wait(True) + self._wait(True, timeout) self._writer = threading.current_thread() - def acquireRead(self): + def acquireRead(self, timeout=None): with self._lock: if self._writer or self._waiters: - self._wait(False) + self._wait(False, timeout) self._readers.add(threading.current_thread()) def release(self): @@ -60,13 +64,13 @@ if self._waiters: self._wakeup_waiter() - def _wait(self, wants_write): + def _wait(self, wants_write, timeout): waiter = Waiter(wants_write) self._waiters.append(waiter) try: self._lock.release() try: - waiter.wait() + waiter.wait(timeout) finally: self._lock.acquire() finally: @@ -84,8 +88,10 @@ self.wants_write = wants_write self._event = threading.Event() - def wait(self): - self._event.wait() + def wait(self, timeout): + if not self._event.wait(timeout): + raise Timeout("Timeout acquiring lock for %s" % + "writing" if self.wants_write else "reading") def wakeup(self): self._event.set() diff --git a/tests/rwlock_test.py b/tests/rwlock_test.py index e67e27a..5ddf066 100644 --- a/tests/rwlock_test.py +++ b/tests/rwlock_test.py @@ -25,10 +25,40 @@ from testlib import VdsmTestCase from testValidation import slowtest, stresstest -from vdsm.rwlock import RWLock +from vdsm.rwlock import RWLock, Timeout class RWLockTests(VdsmTestCase): + + def test_writer_blocks_other_writer(self): + lock = RWLock() + lock.acquireWrite() + try: + log = [] + start_thread(writer, lock, log, timeout=0).join() + self.assertEqual(log, ["writer timeout"]) + finally: + lock.release() + + def test_writer_blocks_other_reader(self): + lock = RWLock() + lock.acquireWrite() + try: + log = [] + start_thread(reader, lock, log, timeout=0).join() + self.assertEqual(log, ["reader timeout"]) + finally: + lock.release() + + def test_reader_blocks_other_writer(self): + lock = RWLock() + lock.acquireRead() + try: + log = [] + start_thread(writer, lock, log, timeout=0).join() + self.assertEqual(log, ["writer timeout"]) + finally: + lock.release() @slowtest def test_concurrent_readers(self): @@ -194,22 +224,30 @@ self.assertEqual(log, ["reader acquired"]) -def writer(lock, log, hold=0): - lock.acquireWrite() +def writer(lock, log, timeout=1, hold=0): try: - log.append("writer acquired") - time.sleep(hold) - finally: - lock.release() + lock.acquireWrite(timeout) + except Timeout: + log.append("writer timeout") + else: + try: + log.append("writer acquired") + time.sleep(hold) + finally: + lock.release() -def reader(lock, log, hold=0): - lock.acquireRead() +def reader(lock, log, timeout=1, hold=0): try: - log.append("reader acquired") - time.sleep(hold) - finally: - lock.release() + lock.acquireRead(timeout) + except Timeout: + log.append("reader timeout") + else: + try: + log.append("reader acquired") + time.sleep(hold) + finally: + lock.release() def start_thread(func, *args, **kwargs): -- To view, visit https://gerrit.ovirt.org/42909 To unsubscribe, visit https://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Idd9b11452d74b566b8989f41244f2d0534327214 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Nir Soffer <nsoffer(a)redhat.com>

6 45

Change in vdsm[master]: rwlock: Add simpler RWLock supporting acquire timeout
by Nir Soffer 15 Nov '15

15 Nov '15

Nir Soffer has uploaded a new change for review. Change subject: rwlock: Add simpler RWLock supporting acquire timeout ...................................................................... rwlock: Add simpler RWLock supporting acquire timeout We have RWLock implementation in storage.misc, used for by the resource manager. We want to simplify the resource manager, which is way too complex to understand or maintain. Patch https://gerrit.ovirt.org/42773 suggests to add a non-blocking acquire to the current RWLock, needed for the new simple lock manager. However, adding more code to the current RWLock is not a good idea. Instead, this patch replace the current implementation with a simpler one. This implementation does not support recursive locking; I'm not sure we need this, and usually having a recursive lock is a design smell. We will add it later only if required. Change-Id: I2466c137c89598772fb46347eb02195916883cac Signed-off-by: Nir Soffer <nsoffer(a)redhat.com> --- M debian/vdsm-python.install M lib/vdsm/Makefile.am A lib/vdsm/rwlock.py M tests/rwlock_test.py M vdsm.spec.in 5 files changed, 95 insertions(+), 1 deletion(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/08/42908/1 diff --git a/debian/vdsm-python.install b/debian/vdsm-python.install index 57d5033..3047d9f 100644 --- a/debian/vdsm-python.install +++ b/debian/vdsm-python.install @@ -28,6 +28,7 @@ ./usr/lib/python2.7/dist-packages/vdsm/pthread.py ./usr/lib/python2.7/dist-packages/vdsm/qemuimg.py ./usr/lib/python2.7/dist-packages/vdsm/response.py +./usr/lib/python2.7/dist-packages/vdsm/rwlock.py ./usr/lib/python2.7/dist-packages/vdsm/schedule.py ./usr/lib/python2.7/dist-packages/vdsm/sslutils.py ./usr/lib/python2.7/dist-packages/vdsm/tool/__init__.py diff --git a/lib/vdsm/Makefile.am b/lib/vdsm/Makefile.am index 95e236f..45cef02 100644 --- a/lib/vdsm/Makefile.am +++ b/lib/vdsm/Makefile.am @@ -38,6 +38,7 @@ pthread.py \ qemuimg.py \ response.py \ + rwlock.py \ schedule.py \ sslutils.py \ sysctl.py \ diff --git a/lib/vdsm/rwlock.py b/lib/vdsm/rwlock.py new file mode 100644 index 0000000..900d91d --- /dev/null +++ b/lib/vdsm/rwlock.py @@ -0,0 +1,91 @@ +# +# Copyright 2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Refer to the README and COPYING files for full details of the license +# + +from __future__ import absolute_import +import threading + + +class RWLock(object): + + def __init__(self): + self._lock = threading.Lock() + self._waiters = [] + self._readers = set() + self._writer = None + + def acquireWrite(self): + with self._lock: + if self._writer or self._readers or self._waiters: + self._wait(True) + self._writer = threading.current_thread() + + def acquireRead(self): + with self._lock: + if self._writer or self._waiters: + self._wait(False) + self._readers.add(threading.current_thread()) + + def release(self): + me = threading.current_thread() + with self._lock: + if self._writer: + if self._writer is not me: + raise RuntimeError("Thread %s attempted to release a " + "write lock held by thread %s" + % (me, self._writer)) + self._writer = None + else: + if me not in self._readers: + raise RuntimeError("Thread %s attempted to release a " + "read lock it does not hold" + % (me,)) + self._readers.remove(me) + if self._waiters: + self._wakeup_waiter() + + def _wait(self, wants_write): + waiter = Waiter(wants_write) + self._waiters.append(waiter) + try: + self._lock.release() + try: + waiter.wait() + finally: + self._lock.acquire() + finally: + self._waiters.remove(waiter) + + def _wakeup_waiter(self): + if self._readers and self._waiters[0].wants_write: + return + self._waiters[0].wakeup() + + +class Waiter(object): + + def __init__(self, wants_write): + self.wants_write = wants_write + self._event = threading.Event() + + def wait(self): + self._event.wait() + + def wakeup(self): + self._event.set() diff --git a/tests/rwlock_test.py b/tests/rwlock_test.py index 80d04f1..e67e27a 100644 --- a/tests/rwlock_test.py +++ b/tests/rwlock_test.py @@ -25,7 +25,7 @@ from testlib import VdsmTestCase from testValidation import slowtest, stresstest -from storage.misc import RWLock +from vdsm.rwlock import RWLock class RWLockTests(VdsmTestCase): diff --git a/vdsm.spec.in b/vdsm.spec.in index 2b41835..35a9670 100644 --- a/vdsm.spec.in +++ b/vdsm.spec.in @@ -1263,6 +1263,7 @@ %{python_sitelib}/%{vdsm_name}/pthread.py* %{python_sitelib}/%{vdsm_name}/qemuimg.py* %{python_sitelib}/%{vdsm_name}/response.py* +%{python_sitelib}/%{vdsm_name}/rwlock.py* %{python_sitelib}/%{vdsm_name}/netconfpersistence.py* %{python_sitelib}/%{vdsm_name}/schedule.py* %{python_sitelib}/%{vdsm_name}/sslutils.py* -- To view, visit https://gerrit.ovirt.org/42908 To unsubscribe, visit https://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I2466c137c89598772fb46347eb02195916883cac Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Nir Soffer <nsoffer(a)redhat.com>

6 53

Change in vdsm[master]: lvm: Exclude faulty devices from lvm long filter
by Nir Soffer 15 Nov '15

15 Nov '15

Nir Soffer has uploaded a new change for review. Change subject: lvm: Exclude faulty devices from lvm long filter ...................................................................... lvm: Exclude faulty devices from lvm long filter lvm commands use filter to limit access to relevant devices. When the filter includes a faulty device, lvm commands may block for several minutes (stuck in D state). We have seen getDevicesList command stuck for up to 10 minutes because of faulty devices in the long filter. We used to build the filter from all multipath devices. Now we build the filter only from devices which have at least one active paths. # multipath -ll 360060160f4a0300038ed7058b5e9e311 dm-0 DGC ,VRAID size=15G features='0' hwhandler='1 emc' wp=rw |-+- policy='service-time 0' prio=0 status=enabled | `- 4:0:3:0 sdd 8:48 failed faulty running `-+- policy='service-time 0' prio=0 status=enabled `- 4:0:2:0 sdb 8:16 failed faulty running 360060160f4a030003268ab211002e411 dm-1 DGC ,VRAID size=30G features='1 queue_if_no_path' hwhandler='1 emc' wp=rw |-+- policy='service-time 0' prio=4 status=active | `- 4:0:3:1 sde 8:64 active ready running `-+- policy='service-time 0' prio=1 status=enabled `- 4:0:2:1 sdc 8:32 active ready running Previously, both devices were included in the filter, now only 360060160f4a030003268ab211002e411 will be included in lvm filter. A faulty device which became active again will be included in lvm filter after the next refresh (every 5 minutes), or after trying edit or create a new storage domain. lvm uses also short filter, including devices used by the certain vg or lv. It is possible that we also have to exclude such devices from the short filter. This will be handled later if needed. Change-Id: I6d7a973bcefa95813fdc289847760c0955aca30c Bug-Url: https://bugzilla.redhat.com/880738 Signed-off-by: Nir Soffer <nsoffer(a)redhat.com> --- M vdsm/storage/lvm.py M vdsm/storage/multipath.py 2 files changed, 13 insertions(+), 1 deletion(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/75/31875/1 diff --git a/vdsm/storage/lvm.py b/vdsm/storage/lvm.py index 86edf55..9cfad01 100644 --- a/vdsm/storage/lvm.py +++ b/vdsm/storage/lvm.py @@ -244,7 +244,7 @@ if not self._filterStale: return self._extraCfg - self._extraCfg = _buildConfig(multipath.getMPDevNamesIter()) + self._extraCfg = _buildConfig(multipath.getActiveMPDevNamesIter()) _updateLvmConf(self._extraCfg) self._filterStale = False diff --git a/vdsm/storage/multipath.py b/vdsm/storage/multipath.py index ba98866..2b30995 100644 --- a/vdsm/storage/multipath.py +++ b/vdsm/storage/multipath.py @@ -382,6 +382,18 @@ yield os.path.join(devicemapper.DMPATH_PREFIX, name) +def getActiveMPDevNamesIter(): + status = devicemapper.getPathsStatus() + for dmId, guid in getMPDevsIter(): + active = [slave for slave in devicemapper.getSlaves(dmId) + if status.get(slave) == "active"] + if not active: + log.warning("Skipping device %s - no active slave", guid) + continue + log.debug("Found device %s %s", guid, active) + yield os.path.join(devicemapper.DMPATH_PREFIX, guid) + + def getMPDevsIter(): """ Collect the list of all the multipath block devices. -- To view, visit http://gerrit.ovirt.org/31875 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I6d7a973bcefa95813fdc289847760c0955aca30c Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Nir Soffer <nsoffer(a)redhat.com>

5 18

Change in vdsm[master]: utils: allow retry of multiple functions
by ykaplan＠redhat.com 14 Nov '15

14 Nov '15

Yeela Kaplan has uploaded a new change for review. Change subject: utils: allow retry of multiple functions ...................................................................... utils: allow retry of multiple functions Change-Id: I6c805ef33b710fbbe674ac62ae96c3fdf47423e1 Signed-off-by: Yeela Kaplan <ykaplan(a)redhat.com> --- M lib/vdsm/libvirtconnection.py M lib/vdsm/utils.py M tests/functional/vmRecoveryTests.py M tests/miscTests.py M tests/remoteFileHandlerTests.py M tests/utilsTests.py M vdsm/clientIF.py M vdsm/storage/hsm.py M vdsm/supervdsm.py 9 files changed, 18 insertions(+), 13 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/36/36336/1 diff --git a/lib/vdsm/libvirtconnection.py b/lib/vdsm/libvirtconnection.py index 492a02e..55af991 100644 --- a/lib/vdsm/libvirtconnection.py +++ b/lib/vdsm/libvirtconnection.py @@ -99,7 +99,7 @@ libvirtOpen = functools.partial( libvirt.openAuth, uri, auth, 0) - return utils.retry(libvirtOpen, timeout=10, sleep=0.2) + return utils.retry((libvirtOpen,), timeout=10, sleep=0.2) def get(target=None, killOnFailure=True): diff --git a/lib/vdsm/utils.py b/lib/vdsm/utils.py index db38e0c..ed4dec6 100644 --- a/lib/vdsm/utils.py +++ b/lib/vdsm/utils.py @@ -903,13 +903,13 @@ return unicode(self.cmd) -def retry(func, expectedException=Exception, tries=None, +def retry(funcs, expectedException=Exception, tries=None, timeout=None, sleep=1, stopCallback=None): """ - Retry a function. Wraps the retry logic so you don't have to + Retry functions. Wraps the retry logic so you don't have to implement it each time you need it. - :param func: The callable to run. + :param funcs: a tuple of the callables to run. :param expectedException: The exception you expect to receive when the function fails. :param tries: The number of times to try. None\0,-1 means infinite. @@ -932,7 +932,9 @@ while True: tries -= 1 try: - return func() + for func in funcs: + res = func() + return res except expectedException: if tries == 0: raise diff --git a/tests/functional/vmRecoveryTests.py b/tests/functional/vmRecoveryTests.py index 37a96f3..0028f62 100644 --- a/tests/functional/vmRecoveryTests.py +++ b/tests/functional/vmRecoveryTests.py @@ -96,7 +96,8 @@ def ensure_vdsm_started(self): vdsm.utils.retry( - self.setUp, expectedException=(socket.error, KeyError), tries=10) + (self.setUp,), expectedException=(socket.error, KeyError), + tries=10) @ValidateRunningAsRoot def test_vm_recovery(self): diff --git a/tests/miscTests.py b/tests/miscTests.py index a955979..eb8b11d 100644 --- a/tests/miscTests.py +++ b/tests/miscTests.py @@ -1047,7 +1047,7 @@ nice = utils.pidStat(proc.pid).nice self.assertEquals(nice, 10) - utils.retry(AssertionError, test, tries=10, sleep=0.1) + utils.retry((test,), AssertionError, tries=10, sleep=0.1) proc.kill() proc.wait() diff --git a/tests/remoteFileHandlerTests.py b/tests/remoteFileHandlerTests.py index 0c62f60..c093c2d 100644 --- a/tests/remoteFileHandlerTests.py +++ b/tests/remoteFileHandlerTests.py @@ -70,7 +70,7 @@ p.stop() test = lambda: self.assertFalse(os.path.exists(procPath)) - utils.retry(test, AssertionError, timeout=4, sleep=0.1) + utils.retry((test,), AssertionError, timeout=4, sleep=0.1) class RemoteFileHandlerTruncateTests(TestCaseBase): diff --git a/tests/utilsTests.py b/tests/utilsTests.py index cb3da85..75119e8 100644 --- a/tests/utilsTests.py +++ b/tests/utilsTests.py @@ -57,7 +57,8 @@ "fool about it.") # W. C. Fields - self.assertRaises(RuntimeError, utils.retry, foo, tries=(limit + 10), + self.assertRaises(RuntimeError, utils.retry, (foo,), + tries=(limit + 10), sleep=0, stopCallback=stopCallback) # Make sure we had the proper amount of iterations before failing self.assertEquals(counter[0], limit) @@ -111,7 +112,7 @@ try: test = lambda: self.assertEquals(utils.getCmdArgs(sproc.pid), tuple()) - utils.retry(AssertionError, test, tries=10, sleep=0.1) + utils.retry((test,), AssertionError, tries=10, sleep=0.1) finally: sproc.wait() diff --git a/vdsm/clientIF.py b/vdsm/clientIF.py index 971d5ff..673975a 100644 --- a/vdsm/clientIF.py +++ b/vdsm/clientIF.py @@ -393,7 +393,7 @@ # Trying to run recover process until it works. During that time vdsm # stays in recovery mode (_recover=True), means all api requests # returns with "vdsm is in initializing process" message. - utils.retry(self._recoverExistingVms, sleep=5) + utils.retry((self._recoverExistingVms,), sleep=5) def _recoverExistingVms(self): try: diff --git a/vdsm/storage/hsm.py b/vdsm/storage/hsm.py index c8aaf93..fa6eb5a 100644 --- a/vdsm/storage/hsm.py +++ b/vdsm/storage/hsm.py @@ -3155,7 +3155,7 @@ supervdsm.getProxy().appropriateDevice(guid, thiefId) supervdsm.getProxy().udevTrigger(guid) devPath = os.path.join(devicemapper.DMPATH_PREFIX, guid) - utils.retry(partial(fileUtils.validateQemuReadable, devPath), + utils.retry((partial(fileUtils.validateQemuReadable, devPath),), expectedException=OSError, timeout=QEMU_READABLE_TIMEOUT) diff --git a/vdsm/supervdsm.py b/vdsm/supervdsm.py index 8e42af2..c29a1f2 100644 --- a/vdsm/supervdsm.py +++ b/vdsm/supervdsm.py @@ -75,7 +75,8 @@ self._manager.register('open') self._log.debug("Trying to connect to Super Vdsm") try: - utils.retry(self._manager.connect, Exception, timeout=60, tries=3) + utils.retry((self._manager.connect,), Exception, + timeout=60, tries=3) except Exception as ex: msg = "Connect to supervdsm service failed: %s" % ex utils.panic(msg) -- To view, visit http://gerrit.ovirt.org/36336 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I6c805ef33b710fbbe674ac62ae96c3fdf47423e1 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Yeela Kaplan <ykaplan(a)redhat.com>

4 6

Change in vdsm[master]: lib: add support for utillinux commands
by Federico Simoncelli 14 Nov '15

14 Nov '15

Federico Simoncelli has uploaded a new change for review. Change subject: lib: add support for utillinux commands ...................................................................... lib: add support for utillinux commands Change-Id: I2ea7dd19fadc600b8fe78fb436ae430d35f52165 Signed-off-by: Federico Simoncelli <fsimonce(a)redhat.com> --- M lib/vdsm/Makefile.am A lib/vdsm/utillinux.py M tests/Makefile.am A tests/utillinuxTests.py M vdsm.spec.in 5 files changed, 132 insertions(+), 0 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/29/35629/1 diff --git a/lib/vdsm/Makefile.am b/lib/vdsm/Makefile.am index b862e71..4e0868a 100644 --- a/lib/vdsm/Makefile.am +++ b/lib/vdsm/Makefile.am @@ -33,6 +33,7 @@ profile.py \ qemuimg.py \ sslutils.py \ + utillinux.py \ utils.py \ vdscli.py \ virtsparsify.py \ diff --git a/lib/vdsm/utillinux.py b/lib/vdsm/utillinux.py new file mode 100644 index 0000000..7fb42cf --- /dev/null +++ b/lib/vdsm/utillinux.py @@ -0,0 +1,47 @@ +# +# Copyright 2014 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Refer to the README and COPYING files for full details of the license +# + +import os +import re +import signal + +from . import utils + +_blkdiscard = utils.CommandPath("blkdiscard", + "/sbin/blkdiscard",) # Fedora, EL6 + + +def blkdiscard(device, offset=None, length=None, secure=False): + cmd = [_blkdiscard.cmd] + + if offset: + cmd.extend(("-o", str(offset))) + + if length: + cmd.extend(("-l", str(length))) + + if secure: + cmd.append("-s") + + cmd.append(device) + rc, out, err = utils.execCmd(cmd, deathSignal=signal.SIGKILL) + + if rc != 0: + raise QImgError(rc, out, err) diff --git a/tests/Makefile.am b/tests/Makefile.am index b365921..5aa9fc3 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -79,6 +79,7 @@ testlibTests.py \ toolTests.py \ transportWrapperTests.py \ + utillinuxTests.py \ utilsTests.py \ vdsClientTests.py \ vmApiTests.py \ diff --git a/tests/utillinuxTests.py b/tests/utillinuxTests.py new file mode 100644 index 0000000..dbbb50e --- /dev/null +++ b/tests/utillinuxTests.py @@ -0,0 +1,82 @@ +# +# Copyright 2014 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Refer to the README and COPYING files for full details of the license +# + +from testlib import VdsmTestCase as TestCaseBase +import monkeypatch +from vdsm import utillinux +from vdsm import utils + +BLKDISCARD = utillinux._blkdiscard.cmd + + +class FakeCmd(object): + + def __init__(self, module, name, *calls): + self.patch = monkeypatch.Patch([(module, name, self)]) + self.calls = list(calls) + + def __call__(self, cmd, **kw): + call = self.calls.pop(0) + return call(cmd, **kw) + + def __enter__(self): + self.patch.apply() + + def __exit__(self, t=None, v=None, tb=None): + self.patch.revert() + + +class BlkDiscardTests(TestCaseBase): + + def test_no_options(self): + def discard(cmd, **kw): + expected = [BLKDISCARD, 'device'] + self.assertEqual(cmd, expected) + return 0, '', '' + + with FakeCmd(utils, 'execCmd', discard): + utillinux.blkdiscard('device') + + def test_offset(self): + def discard(cmd, **kw): + expected = [BLKDISCARD, '-o', '1024', 'device'] + self.assertEqual(cmd, expected) + return 0, '', '' + + with FakeCmd(utils, 'execCmd', discard): + utillinux.blkdiscard('device', offset=1024) + + def test_length(self): + def discard(cmd, **kw): + expected = [BLKDISCARD, '-l', '2048', 'device'] + self.assertEqual(cmd, expected) + return 0, '', '' + + with FakeCmd(utils, 'execCmd', discard): + utillinux.blkdiscard('device', length=2048) + + def test_offset_and_length(self): + def discard(cmd, **kw): + expected = [BLKDISCARD, '-o', '1024', '-l', '2048', 'device'] + self.assertEqual(cmd, expected) + return 0, '', '' + + with FakeCmd(utils, 'execCmd', discard): + utillinux.blkdiscard('device', offset=1024, length=2048) diff --git a/vdsm.spec.in b/vdsm.spec.in index d15f5c2..befe0e0 100644 --- a/vdsm.spec.in +++ b/vdsm.spec.in @@ -1252,6 +1252,7 @@ %{python_sitelib}/%{vdsm_name}/qemuimg.py* %{python_sitelib}/%{vdsm_name}/netconfpersistence.py* %{python_sitelib}/%{vdsm_name}/sslutils.py* +%{python_sitelib}/%{vdsm_name}/utillinux.py* %{python_sitelib}/%{vdsm_name}/utils.py* %{python_sitelib}/%{vdsm_name}/vdscli.py* %{python_sitelib}/%{vdsm_name}/virtsparsify.py* -- To view, visit http://gerrit.ovirt.org/35629 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I2ea7dd19fadc600b8fe78fb436ae430d35f52165 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Federico Simoncelli <fsimonce(a)redhat.com>

7 10

Change in vdsm[master]: lvm: Decrease number of retries before failing
by Nir Soffer 14 Nov '15

14 Nov '15

Nir Soffer has uploaded a new change for review. Change subject: lvm: Decrease number of retries before failing ...................................................................... lvm: Decrease number of retries before failing Previously we configured lvm to stop accessing a device during an lvm opoeration after 3 errors. This can cause lvm to block 3 times when trying to access inaccessible device. With current iscsi settings, each block can be 120 seconds, total 360 seconds. We have seen lvm block for couple of minutes in such cases. The retries seems uneeded when working with multiple paths, as multipath already retry all available paths after SCSI errors on one path. However when working with single path, multipath should fail after one try. According to lvm developer this may decrease the time lvm is blocked when devices are not accesible. (Not tested yet) Change-Id: I5d11abaaff45ce86e88c6589264e162318ac1f1d Relates-To: https://bugzilla.redhat.com/880738 Signed-off-by: Nir Soffer <nsoffer(a)redhat.com> --- M vdsm/storage/lvm.py 1 file changed, 1 insertion(+), 1 deletion(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/56/32356/1 diff --git a/vdsm/storage/lvm.py b/vdsm/storage/lvm.py index 86edf55..f760f7b 100644 --- a/vdsm/storage/lvm.py +++ b/vdsm/storage/lvm.py @@ -106,7 +106,7 @@ preferred_names = ["^/dev/mapper/"] ignore_suspended_devices=1 write_cache_state=0 -disable_after_error_count=3 +disable_after_error_count=1 obtain_device_list_from_udev=0 %s } -- To view, visit http://gerrit.ovirt.org/32356 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I5d11abaaff45ce86e88c6589264e162318ac1f1d Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Nir Soffer <nsoffer(a)redhat.com>

2 4

Change in vdsm[master]: clusterlock: Remove uneeded workaround
by Nir Soffer 13 Nov '15

13 Nov '15

Nir Soffer has uploaded a new change for review. Change subject: clusterlock: Remove uneeded workaround ...................................................................... clusterlock: Remove uneeded workaround Sanlock version XXXX had a off-by-one bug when calling get_hosts with a host id, returning info for the next host. This bug is fixed in version XXX. Now we can use the hostId parameter, making the call more efficient and simpligying clusterlock code. Change-Id: Ide75e749fbc2916540c2b526b78fedc247b5c6f9 Signed-off-by: Nir Soffer <nsoffer(a)redhat.com> --- M vdsm.spec.in M vdsm/storage/clusterlock.py 2 files changed, 5 insertions(+), 16 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/62/31162/1 diff --git a/vdsm.spec.in b/vdsm.spec.in index 5ba6fc6..fc39eb9 100644 --- a/vdsm.spec.in +++ b/vdsm.spec.in @@ -167,7 +167,7 @@ Requires: iscsi-initiator-utils >= 6.2.0.873-21 %endif -Requires: sanlock >= 2.8, sanlock-python +Requires: sanlock >= XXX, sanlock-python %if 0%{?rhel} Requires: python-ethtool >= 0.6-3 diff --git a/vdsm/storage/clusterlock.py b/vdsm/storage/clusterlock.py index 24a5d81..17cdd53 100644 --- a/vdsm/storage/clusterlock.py +++ b/vdsm/storage/clusterlock.py @@ -265,26 +265,15 @@ return False def getHostStatus(self, hostId): - # Note: get_hosts has off-by-one bug when asking for particular host - # id, so get all hosts info and filter. - # See https://bugzilla.redhat.com/1111210 try: - hosts = sanlock.get_hosts(self._sdUUID) + hosts = sanlock.get_hosts(self._sdUUID, hostId) except sanlock.SanlockException as e: self.log.debug("Unable to get host %d status in lockspace %s: %s", hostId, self._sdUUID, e) return HOST_STATUS_UNAVAILABLE - - for info in hosts: - if info['host_id'] == hostId: - status = info['flags'] - return self.STATUS_NAME[status] - - # get_hosts with host_id=0 returns only hosts with timestamp != 0, - # which means that no host is using this host id now. If there a - # timestamp, sanlock will return HOST_UNKNOWN and then HOST_LIVE or - # HOST_FAIL. - return HOST_STATUS_FREE + else: + status = hosts[0]['flags'] + return self.STATUS_NAME[status] # The hostId parameter is maintained here only for compatibility with # ClusterLock. We could consider to remove it in the future but keeping it -- To view, visit http://gerrit.ovirt.org/31162 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ide75e749fbc2916540c2b526b78fedc247b5c6f9 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Nir Soffer <nsoffer(a)redhat.com>

7 21

← Newer
1
...
4
5
6
7
8
9
10
...
92
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

vdsm-patches July 2015