Francesco Romani has uploaded a new change for review.
Change subject: WIP: virt: better timeout for periodic operations ......................................................................
WIP: virt: better timeout for periodic operations
Change-Id: I735d314532c6852889e3ac188651acff044888ed WIP: TODO: Documentation pending Signed-off-by: Francesco Romani fromani@redhat.com --- M tests/periodicTests.py M vdsm/virt/periodic.py M vdsm/virt/utils.py M vdsm/virt/vm.py 4 files changed, 52 insertions(+), 20 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/33/39333/1
diff --git a/tests/periodicTests.py b/tests/periodicTests.py index 80ef09e..d22f5ce 100644 --- a/tests/periodicTests.py +++ b/tests/periodicTests.py @@ -204,8 +204,9 @@
VMS = defaultdict(int)
- def __init__(self, vm): + def __init__(self, vm, timeout): self._vm = vm + self._timeout = timeout # unused
self.required = True self.runnable = True diff --git a/vdsm/virt/periodic.py b/vdsm/virt/periodic.py index fe8bbc8..3159c04 100644 --- a/vdsm/virt/periodic.py +++ b/vdsm/virt/periodic.py @@ -33,6 +33,7 @@ from vdsm.utils import monotonic_time
from . import sampling +from .errors import TimeoutError
# just a made up number. Maybe should be equal to number of cores? @@ -229,7 +230,7 @@ skipped = []
for vm_id, vm_obj in vms.iteritems(): - op = self._create(vm_obj) + op = self._create(vm_obj, self._timeout)
if not op.required: continue @@ -253,8 +254,9 @@
class UpdateVolumes(object): - def __init__(self, vm): + def __init__(self, vm, timeout): self._vm = vm + self._timeout = timeout
@property def required(self): @@ -267,15 +269,16 @@
def __call__(self): for drive in self._vm.getDiskDevices(): - # TODO: If this block (it is actually possible?) - # we must make sure we don't overwrite good data - # with stale old data. - self._vm.updateDriveVolume(drive) + try: + self._vm.updateDriveVolume(drive, timeout=self._timeout) + except TimeoutError: + pass # TODO log it?
class NumaInfoMonitor(object): - def __init__(self, vm): + def __init__(self, vm, timeout): self._vm = vm + self._timeout = timeout
@property def required(self): @@ -286,12 +289,16 @@ return self._vm.isDomainReadyForCommands()
def __call__(self): - self._vm.updateNumaInfo() + try: + self._vm.updateNumaInfo(timeout=self._timeout) + except TimeoutError: + pass # TODO log it?
class BlockjobMonitor(object): - def __init__(self, vm): + def __init__(self, vm, timeout): self._vm = vm + self._timeout = timeout
@property def required(self): @@ -307,7 +314,10 @@ return self._vm.isDomainReadyForCommands()
def __call__(self): - self._vm.updateVmJobs() + try: + self._vm.updateVmJobs(timeout=self._timeout) + except TimeoutError: + pass # TODO log it?
class DriveWatermarkMonitor(object): diff --git a/vdsm/virt/utils.py b/vdsm/virt/utils.py index 60becfa..385c893 100644 --- a/vdsm/virt/utils.py +++ b/vdsm/virt/utils.py @@ -25,6 +25,7 @@ import threading
from vdsm.utils import monotonic_time +from .errors import TimeoutError
def isVdsmImage(drive): @@ -110,3 +111,14 @@ raise ItemExpired
return value + + +def discard_if_blocked(timeout, func, *args): + start = monotonic_time() + res = func(*args) + elapsed = monotonic_time() - start + + if timeout is None or elapsed <= timeout: + return res + + raise TimeoutError diff --git a/vdsm/virt/vm.py b/vdsm/virt/vm.py index 460d45a..eb56ed6 100644 --- a/vdsm/virt/vm.py +++ b/vdsm/virt/vm.py @@ -72,7 +72,7 @@ from . import vmxml
from .errors import TimeoutError -from .utils import isVdsmImage +from .utils import isVdsmImage, discard_if_blocked from vmpowerdown import VmShutdown, VmReboot
_VMCHANNEL_DEVICE_NAME = 'com.redhat.rhevm.vdsm' @@ -2652,13 +2652,16 @@
raise LookupError("No such drive: '%s'" % drive)
- def updateDriveVolume(self, vmDrive): + # CAN_BLOCK + # PERIODIC_OPERATION + def updateDriveVolume(self, vmDrive, timeout=None): if not vmDrive.device == 'disk' or not isVdsmImage(vmDrive): return
- volSize = self.cif.irs.getVolumeSize( - vmDrive.domainID, vmDrive.poolID, vmDrive.imageID, - vmDrive.volumeID) + volSize = discard_if_blocked( + timeout, self.cif.irs.getVolumeSize, + vmDrive.domainID, vmDrive.poolID, + vmDrive.imageID, vmDrive.volumeID)
if volSize['status']['code'] != 0: self.log.error( @@ -4248,8 +4251,11 @@ with self._jobsLock: return bool(self.conf['_blockJobs'])
- def updateVmJobs(self): - self._vmJobs = self.queryBlockJobs() + # CAN_BLOCK + # PERIODIC_OPERATION + def updateVmJobs(self, timeout=None): + self._vmJobs = discard_if_blocked( + timeout, self.queryBlockJobs)
def queryBlockJobs(self): def startCleanup(job, drive, needPivot): @@ -4576,8 +4582,11 @@ statsAge) stats['monitorResponse'] = '-1'
- def updateNumaInfo(self): - self._numaInfo = numaUtils.getVmNumaNodeRuntimeInfo(self) + # CAN_BLOCK + # PERIODIC_OPERATION + def updateNumaInfo(self, timeout=None): + self._numaInfo = discard_if_blocked( + timeout, numaUtils.getVmNumaNodeRuntimeInfo, self)
@property def hasGuestNumaNode(self):