vdsm-patches November 2013

vdsm-patches@lists.fedorahosted.org

60 participants
580 discussions

Change in vdsm[master]: Avoid redundant volume produces.
by ewarszaw＠redhat.com 21 Jul '14

21 Jul '14

Eduardo has uploaded a new change for review. Change subject: Avoid redundant volume produces. ...................................................................... Avoid redundant volume produces. Add sd.getVolumePath() returns the volume path without produce it. Deprecating hsm.getVolumePath() and hsm.prepareVolume(). When removed, remove API.prepare(), BindingXMLRPC.volumePrepare(), API.getPath, BindingXMLRPC.volumeGetPath(), etc. Change-Id: I3ad53a7e8a66d7f9bdd62048f2bf1f722a490c5c Signed-off-by: Eduardo <ewarszaw(a)redhat.com> --- M vdsm/storage/fileSD.py M vdsm/storage/hsm.py M vdsm/storage/sd.py 3 files changed, 11 insertions(+), 6 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/91/17991/1 diff --git a/vdsm/storage/fileSD.py b/vdsm/storage/fileSD.py index 9d1493d..8cbea23 100644 --- a/vdsm/storage/fileSD.py +++ b/vdsm/storage/fileSD.py @@ -302,8 +302,7 @@ Return the volume lease (leasePath, leaseOffset) """ if self.hasVolumeLeases(): - vol = self.produceVolume(imgUUID, volUUID) - volumePath = vol.getVolumePath() + volumePath = self.getVolumePath(imgUUID, volUUID) leasePath = volumePath + fileVolume.LEASE_FILEEXT return leasePath, fileVolume.LEASE_FILEOFFSET return None, None @@ -426,8 +425,9 @@ # NFS volumes. In theory it is necessary to fix the permission # of the leaf only but to not introduce an additional requirement # (ordered volUUIDs) we fix them all. - for vol in [self.produceVolume(imgUUID, x) for x in volUUIDs]: - self.oop.fileUtils.copyUserModeToGroup(vol.getVolumePath()) + for volUUID in volUUIDs: + volPath = self.getVolumePath(imgUUID, volUUID) + self.oop.fileUtils.copyUserModeToGroup(volPath) @classmethod def format(cls, sdUUID): diff --git a/vdsm/storage/hsm.py b/vdsm/storage/hsm.py index c754ee8..3545677 100644 --- a/vdsm/storage/hsm.py +++ b/vdsm/storage/hsm.py @@ -3076,6 +3076,7 @@ volUUID=volUUID).getInfo() return dict(info=info) + @deprecated @public def getVolumePath(self, sdUUID, spUUID, imgUUID, volUUID, options=None): """ @@ -3100,8 +3101,7 @@ """ vars.task.getSharedLock(STORAGE, sdUUID) path = sdCache.produce( - sdUUID=sdUUID).produceVolume(imgUUID=imgUUID, - volUUID=volUUID).getVolumePath() + sdUUID=sdUUID).getVolumePath(imgUUID, volUUID) return dict(path=path) @public @@ -3127,6 +3127,7 @@ if fails: self.log.error("Failed to remove the following rules: %s", fails) + @deprecated @public def prepareVolume(self, sdUUID, spUUID, imgUUID, volUUID, rw=True, options=None): diff --git a/vdsm/storage/sd.py b/vdsm/storage/sd.py index 36c4877..dde7832 100644 --- a/vdsm/storage/sd.py +++ b/vdsm/storage/sd.py @@ -640,6 +640,10 @@ # If it has a repo we don't have multiple domains. Assume single pool return os.path.join(self.storage_repository, self.getPools()[0]) + def getVolumePath(self, imgUUID, volUUID): + return os.path.join(self.mountpoint, self.sdUUID, 'images', imgUUID, + volUUID) + def getIsoDomainImagesDir(self): """ Get 'images' directory from Iso domain -- To view, visit http://gerrit.ovirt.org/17991 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3ad53a7e8a66d7f9bdd62048f2bf1f722a490c5c Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Eduardo <ewarszaw(a)redhat.com>

5 7

Change in vdsm[master]: [WIP] Towards a more (block) secure HSM.
by ewarszaw＠redhat.com 21 Jul '14

21 Jul '14

Eduardo has uploaded a new change for review. Change subject: [WIP] Towards a more (block) secure HSM. ...................................................................... [WIP] Towards a more (block) secure HSM. Change-Id: I30df4ee5cdb6b44cf14d8cb155436aac7442a07d --- M vdsm/storage/hsm.py M vdsm/storage/lvm.py M vdsm/storage/sp.py 3 files changed, 25 insertions(+), 5 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/18/2218/1 -- To view, visit http://gerrit.ovirt.org/2218 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I30df4ee5cdb6b44cf14d8cb155436aac7442a07d Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Eduardo <ewarszaw(a)redhat.com>

6 28

Change in vdsm[master]: vm: small refactor for _normalizeVdsmImg
by Federico Simoncelli 14 Jul '14

14 Jul '14

Federico Simoncelli has uploaded a new change for review. Change subject: vm: small refactor for _normalizeVdsmImg ...................................................................... vm: small refactor for _normalizeVdsmImg Change-Id: Ie68292eee4b82fbe8527e3960739979cfe117dfa Signed-off-by: Federico Simoncelli <fsimonce(a)redhat.com> --- M vdsm/vm.py 1 file changed, 18 insertions(+), 17 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/57/19157/1 diff --git a/vdsm/vm.py b/vdsm/vm.py index 92d274e..2605f24 100644 --- a/vdsm/vm.py +++ b/vdsm/vm.py @@ -1779,25 +1779,26 @@ break return str(idx) - def _normalizeVdsmImg(self, drv): - drv['reqsize'] = drv.get('reqsize', '0') # Backward compatible - if 'device' not in drv: - drv['device'] = 'disk' + def _normalizeVdsmImg(self, drive): + drive['device'] = drive.get('device', 'disk') # Disk by default + drive['reqsize'] = drive.get('reqsize', '0') # Backward compatible - if drv['device'] == 'disk': - res = self.cif.irs.getVolumeSize(drv['domainID'], drv['poolID'], - drv['imageID'], drv['volumeID']) - try: - drv['truesize'] = res['truesize'] - drv['apparentsize'] = res['apparentsize'] - except KeyError: - self.log.error("Unable to get volume size for %s", - drv['volumeID'], exc_info=True) - raise RuntimeError("Volume %s is corrupted or missing" % - drv['volumeID']) + if drive['device'] == 'disk': + volInfo = self.cif.irs.getVolumeInfo( + drive['domainID'], drive['poolID'], drive['imageID'], + drive['volumeID']) + + if volInfo.get('status', {}).get('code', -1): + self.log.error( + "Unable to get volume info for %s", drive['volumeID']) + raise RuntimeError( + "Volume %s is corrupted or missing" % drive['volumeID']) + + drive['truesize'] = volInfo['info']['truesize'] + drive['apparentsize'] = volInfo['info']['apparentsize'] else: - drv['truesize'] = 0 - drv['apparentsize'] = 0 + drive['truesize'] = 0 + drive['apparentsize'] = 0 @classmethod def _normalizeDriveSharedAttribute(self, drive): -- To view, visit http://gerrit.ovirt.org/19157 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ie68292eee4b82fbe8527e3960739979cfe117dfa Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Federico Simoncelli <fsimonce(a)redhat.com>

5 9

Change in vdsm[master]: vmDevices: introduce VmDeviceContainer
by mpoledni＠redhat.com 14 Jul '14

14 Jul '14

Martin Polednik has uploaded a new change for review. Change subject: vmDevices: introduce VmDeviceContainer ...................................................................... vmDevices: introduce VmDeviceContainer EARLY WORK IN PROGRESS: VmDeviceContainer is structure that will allow us to store devices as a class instances while keeping backwards compatibility with old self.conf['devices'] Change-Id: I65debd35115da078df0c0cb6f50c57feb984c5a3 Signed-off-by: Martin Polednik <mpoledni(a)redhat.com> --- M vdsm/vm.py 1 file changed, 33 insertions(+), 0 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/38/21138/1 diff --git a/vdsm/vm.py b/vdsm/vm.py index 5ae54d7..36cdbb1 100644 --- a/vdsm/vm.py +++ b/vdsm/vm.py @@ -1727,6 +1727,39 @@ return m +class VmDeviceContainer(dict): + @property + def legacy(self): + """ + Return list of device dicts that represents backwards-compatible + self.conf['devices'] + + [..., {..., 'type': 'disk', ...}, ...] + """ + deviceList = [] + for key in self.keys(): + for device in self[key]: + # loop through devices __slots__ and return all set attributes + deviceList.append(dict((attr, getattr(device, attr)) + for attr in device.__slots__ + if hasattr(device, attr))) + + return deviceList + + def restoreLegacy(self, state): + """ + Reconstruct container using old self.conf['devices'] structure of + + [..., {..., 'type': 'disk', ...}, ...] + to + + VmDeviceContainer[DISK_DEVICES] = + [..., {..., 'type': 'disk', ...}, ...] + """ + for device in state: + self[device['type']] = device + + class Vm(object): """ Used for abstracting communication between various parts of the -- To view, visit http://gerrit.ovirt.org/21138 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I65debd35115da078df0c0cb6f50c57feb984c5a3 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Martin Polednik <mpoledni(a)redhat.com>

4 12

Change in vdsm[master]: vmDevices: add mechanism to persist vmDevice defaults
by mpoledni＠redhat.com 14 Jul '14

14 Jul '14

Martin Polednik has uploaded a new change for review. Change subject: vmDevices: add mechanism to persist vmDevice defaults ...................................................................... vmDevices: add mechanism to persist vmDevice defaults Multiple vmDevices such as BalloonDevice or watchdogDevice currently do not persist their defaults in class attributes but rather use them in XML directly, hiding their existence. This patch aims to change this behavior by implementing vmDevice._defaults(), which adds the default attributes to instance directly to enable future persistence of these classes. Change-Id: Idc8383cbce78490c8dfab1c253883a06459f1547 Signed-off-by: Martin Polednik <mpoledni(a)redhat.com> --- M tests/vmTests.py M vdsm/vm.py 2 files changed, 45 insertions(+), 29 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/66/21066/1 diff --git a/tests/vmTests.py b/tests/vmTests.py index 1f69f0a..5c813fc 100644 --- a/tests/vmTests.py +++ b/tests/vmTests.py @@ -261,7 +261,7 @@ def testWatchdogXML(self): watchdogXML = '<watchdog action="none" model="i6300esb"/>' - dev = {'device': 'watchdog', 'type': 'watchdog', + dev = {'device': 'watchdog', 'specParams': {'model': 'i6300esb', 'action': 'none'}} watchdog = vm.WatchdogDevice(self.conf, self.log, **dev) self.assertXML(watchdog.getXML(), watchdogXML) diff --git a/vdsm/vm.py b/vdsm/vm.py index 796735f..36b9598 100644 --- a/vdsm/vm.py +++ b/vdsm/vm.py @@ -1167,6 +1167,7 @@ class VmDevice(object): def __init__(self, conf, log, **kwargs): + self.specParams = {} for attr, value in kwargs.iteritems(): try: setattr(self, attr, value) @@ -1175,6 +1176,10 @@ self.conf = conf self.log = log self._deviceXML = None + self._defaults() + + def _defaults(self): + pass def __str__(self): attrs = [':'.join((a, str(getattr(self, a)))) for a in dir(self) @@ -1192,10 +1197,13 @@ elemAttrs['type'] = deviceType for attrName in attributes: - if not hasattr(self, attrName): + if attrName in self.specParams: + attr = self.specParams[attrName] + elif hasattr(self, attrName): + attr = getattr(self, attrName) + else: continue - attr = getattr(self, attrName) if isinstance(attr, dict): element.appendChildWithArgs(attrName, **attr) else: @@ -1215,37 +1223,44 @@ class ControllerDevice(VmDevice): + def _defaults(self): + if self.device == 'virtio-serial': + if 'index' not in self.specParams: + self.index = '0' + if 'ports' not in self.specParams: + self.ports = '16' def getXML(self): """ Create domxml for controller device """ ctrl = self.createXmlElem('controller', self.device, - ['index', 'model', 'master', 'address']) - if self.device == 'virtio-serial': - ctrl.setAttrs(index='0', ports='16') + ['index', 'model', 'master', 'address', + 'ports']) return ctrl class VideoDevice(VmDevice): + def _defaults(self): + if 'vram' not in self.specParams: + self.specParams['vram'] = '32768' + if 'heads' not in self.specParams: + self.specParams['heads'] = '1' def getXML(self): """ Create domxml for video device """ video = self.createXmlElem('video', None, ['address']) - sourceAttrs = {'vram': self.specParams.get('vram', '32768'), - 'heads': self.specParams.get('heads', '1')} - if 'ram' in self.specParams: - sourceAttrs['ram'] = self.specParams['ram'] + model = self.createXmlElem('model', self.device, + ['vram', 'heads', 'ram']) - video.appendChildWithArgs('model', type=self.device, **sourceAttrs) + video.appendChild(model) return video class SoundDevice(VmDevice): - def getXML(self): """ Create domxml for sound device @@ -1256,7 +1271,6 @@ class NetworkInterfaceDevice(VmDevice): - def __init__(self, conf, log, **kwargs): # pyLint can't tell that the Device.__init__() will # set a nicModel attribute, so modify the kwarg list @@ -1646,7 +1660,6 @@ class BalloonDevice(VmDevice): - def getXML(self): """ Create domxml for a memory balloon device. @@ -1662,11 +1675,11 @@ class WatchdogDevice(VmDevice): - def __init__(self, *args, **kwargs): - super(WatchdogDevice, self).__init__(*args, **kwargs) - - if not hasattr(self, 'specParams'): - self.specParams = {} + def _defaults(self): + if 'model' not in self.specParams: + self.specParams['model'] = 'i6300esb' + if 'action' not in self.specParams: + self.specParams['action'] = 'none' def getXML(self): """ @@ -1677,9 +1690,8 @@ function='0x0'/> </watchdog> """ - m = self.createXmlElem(self.type, None, ['address']) - m.setAttrs(model=self.specParams.get('model', 'i6300esb'), - action=self.specParams.get('action', 'none')) + m = self.createXmlElem(self.device, None, ['address', 'model', + 'action']) return m @@ -1692,11 +1704,8 @@ <address ... /> </smartcard> """ - card = self.createXmlElem(self.device, None, ['address']) - sourceAttrs = {'mode': self.specParams['mode']} - if sourceAttrs['mode'] != 'host': - sourceAttrs['type'] = self.specParams['type'] - card.setAttrs(**sourceAttrs) + card = self.createXmlElem(self.device, None, ['address', 'mode', + 'type']) return card @@ -1713,6 +1722,12 @@ class ConsoleDevice(VmDevice): + def _defaults(self): + self.type = 'pty' + self.port = '0' + self.specParams['type'] = 'virtio' + self.specParams['port'] = '0' + def getXML(self): """ Create domxml for a console device. @@ -1721,8 +1736,9 @@ <target type='virtio' port='0'/> </console> """ - m = self.createXmlElem('console', 'pty') - m.appendChildWithArgs('target', type='virtio', port='0') + m = self.createXmlElem(self.device, self.type) + m.appendChildWithArgs('target', None, type=self.specParams['type'], + port=self.specParams['port']) return m -- To view, visit http://gerrit.ovirt.org/21066 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Idc8383cbce78490c8dfab1c253883a06459f1547 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Martin Polednik <mpoledni(a)redhat.com>

5 9

Change in vdsm[master]: gluster: [WIP]Get size information of gluster volume
by avishwan＠redhat.com 14 Jul '14

14 Jul '14

Aravinda VK has uploaded a new change for review. Change subject: gluster: [WIP]Get size information of gluster volume ...................................................................... gluster: [WIP]Get size information of gluster volume Change-Id: I358d4f3bf793ecc1a01e0592d68919d1405f6e19 Signed-off-by: Aravinda VK <avishwan(a)redhat.com> --- M client/vdsClientGluster.py M vdsm.spec.in M vdsm/gluster/Makefile.am M vdsm/gluster/__init__.py M vdsm/gluster/api.py A vdsm/gluster/gfapi.py 6 files changed, 100 insertions(+), 1 deletion(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/22/17822/1 diff --git a/client/vdsClientGluster.py b/client/vdsClientGluster.py index 90af83e..644bfe3 100644 --- a/client/vdsClientGluster.py +++ b/client/vdsClientGluster.py @@ -424,6 +424,17 @@ pp.pprint(status) return status['status']['code'], status['status']['message'] + def do_glusterVolumeSize(self, args): + params = self._eqSplit(args) + try: + volumeName = params.get('volumeName', '') + except: + raise ValueError + + status = self.s.glusterVolumeSize(volumeName) + pp.pprint(status) + return status['status']['code'], status['status']['message'] + def getGlusterCmdDict(serv): return \ @@ -705,4 +716,9 @@ 'not set' '(swift, glusterd, smb, memcached)' )), + 'glusterVolumeSize': ( + serv.do_glusterVolumeSize, + ('volumeName=<volume name>', + 'Returns total, available and used space status of gluster volume' + )), } diff --git a/vdsm.spec.in b/vdsm.spec.in index 21b3565..e49e102 100644 --- a/vdsm.spec.in +++ b/vdsm.spec.in @@ -1271,6 +1271,7 @@ %doc COPYING %{_datadir}/%{vdsm_name}/gluster/api.py* %{_datadir}/%{vdsm_name}/gluster/vdsmapi-gluster-schema.json +%{_datadir}/%{vdsm_name}/gluster/gfapi.py* %{_datadir}/%{vdsm_name}/gluster/hooks.py* %{_datadir}/%{vdsm_name}/gluster/services.py* %endif diff --git a/vdsm/gluster/Makefile.am b/vdsm/gluster/Makefile.am index dd5434d..9c2989f 100644 --- a/vdsm/gluster/Makefile.am +++ b/vdsm/gluster/Makefile.am @@ -26,6 +26,7 @@ api.py \ cli.py \ exception.py \ + gfapi.py \ hooks.py \ hostname.py \ services.py \ diff --git a/vdsm/gluster/__init__.py b/vdsm/gluster/__init__.py index bec70ea..e5a2fd6 100644 --- a/vdsm/gluster/__init__.py +++ b/vdsm/gluster/__init__.py @@ -22,7 +22,7 @@ import tempfile from functools import wraps -MODULE_LIST = ('cli', 'hooks', 'services') +MODULE_LIST = ('cli', 'hooks', 'services', 'gfapi') def makePublic(func): diff --git a/vdsm/gluster/api.py b/vdsm/gluster/api.py index 4bd8308..89546c1 100644 --- a/vdsm/gluster/api.py +++ b/vdsm/gluster/api.py @@ -287,6 +287,22 @@ status = self.svdsmProxy.glusterServicesGet(serviceNames) return {'services': status} + @exportAsVerb + def volumeSize(self, volumeName, options=None): + data = self.svdsmProxy.glusterVolumeStatvfs(volumeName) + # f_blocks = Total number of blocks + # f_bfree = Total number of blocks free + # f_bavail = Total number of blocks available for non root user + # total blocks available = f_blocks - (f_bfree - f_bavail) + total_blocks_available = data['f_blocks'] - \ + (data['f_bfree'] - data['f_bavail']) + return { + 'total': total_blocks_available * data.f_bsize / 1024, + 'free': data['f_bavail'] * data['f_bsize'] / 1024, + 'used': (total_blocks_available - data['f_bavail']) * \ + data['f_bsize'] / 1024 + } + def getGlusterMethods(gluster): l = [] diff --git a/vdsm/gluster/gfapi.py b/vdsm/gluster/gfapi.py new file mode 100644 index 0000000..abfdabd --- /dev/null +++ b/vdsm/gluster/gfapi.py @@ -0,0 +1,65 @@ +# +# Copyright 2013 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Refer to the README and COPYING files for full details of the license +# +from ctypes import * + +# import exception as ge +from . import makePublic + +GLUSTER_VOL_PROTOCAL='tcp' +GLUSTER_VOL_HOST='localhost' +GLUSTER_VOL_PORT=24007 + +class Stat (Structure): + _fields_ = [ + ('f_bsize', c_ulong), + ('f_frsize', c_ulong), + ('f_blocks', c_ulong), + ('f_bfree', c_ulong), + ('f_bavail', c_ulong), + ('f_files', c_ulong), + ('f_ffree', c_ulong), + ('f_favail', c_ulong), + ('f_fsid', c_ulong), + ('f_flag', c_ulong), + ('f_namemax', c_ulong), + ('__f_spare', c_int * 6), + ] + + +api = CDLL("libgfapi.so",RTLD_GLOBAL) +api.glfs_statvfs.restype = c_int +api.glfs_statvfs.argtypes = [c_void_p, c_char_p, POINTER(Stat)] + +@makePublic +def volumeStatvfs(volumeId): + path = "/" + fs = api.glfs_new(volumeId) + api.glfs_set_volfile_server(fs, + GLUSTER_VOL_PROTOCAL, + GLUSTER_VOL_HOST, + GLUSTER_VOL_PORT) + api.glfs_init(fs) + + x = Stat() + rc = api.glfs_statvfs(fs, path, byref(x)) + statvfsData = {} + for k in x._fields_: + statvfsData[k[0]] = getattr(x, k[0]) + return statvfsData -- To view, visit http://gerrit.ovirt.org/17822 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I358d4f3bf793ecc1a01e0592d68919d1405f6e19 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Aravinda VK <avishwan(a)redhat.com>

8 42

Change in vdsm[master]: Add hostusbdirect hook to vdsm_hooks.
by lyarwood＠redhat.com 23 Jun '14

23 Jun '14

Lee Yarwood has uploaded a new change for review. Change subject: Add hostusbdirect hook to vdsm_hooks. ...................................................................... Add hostusbdirect hook to vdsm_hooks. This is a slight variation of the hostusb hook. This version uses the bus and device IDs to attach host USB device to the guest using the following libvirt XML : <hostdev mode='subsystem' type='usb' managed='yes'> <source> <address bus='001' device='003'/> </source> </hostdev> In the future this could be merged with the hostusb hook. Change-Id: I00b4b03993cd7290462b0e17457892b23dd41b31 Signed-off-by: Lee Yarwood <lyarwood(a)redhat.com> --- M configure.ac M vdsm.spec.in M vdsm_hooks/Makefile.am A vdsm_hooks/hostusbdirect/Makefile.am A vdsm_hooks/hostusbdirect/README A vdsm_hooks/hostusbdirect/after_vm_destroy.py A vdsm_hooks/hostusbdirect/before_vm_migrate_source.py A vdsm_hooks/hostusbdirect/before_vm_start.py A vdsm_hooks/hostusbdirect/sudoers.vdsm_hook_hostusbdirect 9 files changed, 262 insertions(+), 0 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/28/17428/1 diff --git a/configure.ac b/configure.ac index 0ccca95..39e31f0 100644 --- a/configure.ac +++ b/configure.ac @@ -225,6 +225,7 @@ vdsm_hooks/fileinject/Makefile vdsm_hooks/floppy/Makefile vdsm_hooks/hostusb/Makefile + vdsm_hooks/hostusbdirect/Makefile vdsm_hooks/hugepages/Makefile vdsm_hooks/isolatedprivatevlan/Makefile vdsm_hooks/macspoof/Makefile diff --git a/vdsm.spec.in b/vdsm.spec.in index 235d1db..c1b9431 100644 --- a/vdsm.spec.in +++ b/vdsm.spec.in @@ -389,6 +389,15 @@ Hook is getting vendor and product id of USB device disconnect it from host and attach it to VM +%package hook-hostusb-direct +Summary: Allow attaching USB device from host by bus and device ids. +BuildArch: noarch +Requires: usbutils + +%description hook-hostusb-direct +Hook is getting bus and device id of USB device +disconnect it from host and attach it to VM + %package hook-hugepages Summary: Huge pages enable user to handle VM with 2048KB page files. BuildArch: noarch diff --git a/vdsm_hooks/Makefile.am b/vdsm_hooks/Makefile.am index 8a8d594..fb4052b 100644 --- a/vdsm_hooks/Makefile.am +++ b/vdsm_hooks/Makefile.am @@ -30,6 +30,7 @@ fileinject \ floppy \ hostusb \ + hostusbdirect \ hugepages \ isolatedprivatevlan \ macspoof \ diff --git a/vdsm_hooks/hostusbdirect/Makefile.am b/vdsm_hooks/hostusbdirect/Makefile.am new file mode 100644 index 0000000..9d0cdbc --- /dev/null +++ b/vdsm_hooks/hostusbdirect/Makefile.am @@ -0,0 +1,52 @@ +# +# Copyright 2013 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Refer to the README and COPYING files for full details of the license +# + +EXTRA_DIST = \ + after_vm_destroy.py \ + before_vm_migrate_source.py \ + before_vm_start.py \ + sudoers.vdsm_hook_hostusbdirect + +install-data-hook: + chmod 440 $(DESTDIR)$(sysconfdir)/sudoers.d/50_vdsm_hook_hostusbdirect + +install-data-local: install-data-sudoers + $(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/before_vm_start + $(INSTALL_SCRIPT) $(srcdir)/before_vm_start.py \ + $(DESTDIR)$(vdsmhooksdir)/before_vm_start/50_hostusbdirect + $(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/after_vm_destroy + $(INSTALL_SCRIPT) $(srcdir)/after_vm_destroy.py \ + $(DESTDIR)$(vdsmhooksdir)/after_vm_destroy/50_hostusbdirect + $(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/before_vm_migrate_source + $(INSTALL_SCRIPT) $(srcdir)/before_vm_migrate_source.py \ + $(DESTDIR)$(vdsmhooksdir)/before_vm_migrate_source/50_hostusbdirect + +uninstall-local: uninstall-data-sudoers + $(RM) $(DESTDIR)$(vdsmhooksdir)/before_vm_start/50_hostusbdirect + $(RM) $(DESTDIR)$(vdsmhooksdir)/after_vm_destroy/50_hostusbdirect + $(RM) $(DESTDIR)$(vdsmhooksdir)/before_vm_migrate_source/50_hostusbdirect + +install-data-sudoers: + $(MKDIR_P) $(DESTDIR)$(sysconfdir)/sudoers.d + $(INSTALL_DATA) $(srcdir)/sudoers.vdsm_hook_hostusbdirect \ + $(DESTDIR)$(sysconfdir)/sudoers.d/50_vdsm_hook_hostusbdirect + +uninstall-data-sudoers: + $(RM) $(DESTDIR)$(sysconfdir)/sudoers.d/50_vdsm_hook_hostusbdirect diff --git a/vdsm_hooks/hostusbdirect/README b/vdsm_hooks/hostusbdirect/README new file mode 100644 index 0000000..297c559 --- /dev/null +++ b/vdsm_hooks/hostusbdirect/README @@ -0,0 +1,18 @@ +host usb hook direct +==================== +add hosts usb device/s to VM using bus and device ids: + +<hostdev mode='subsystem' type='usb'> + <source> + <address bus='001' device='003'/> + </source> +</hostdev> + +syntax: + hostusbdirect=001:003&001:002 + i.e. + hostusb=bus:device (can add more then one with '&' separator) + +Note: + The VM must be pinned to host and this hook will + fail any migration attempt. diff --git a/vdsm_hooks/hostusbdirect/after_vm_destroy.py b/vdsm_hooks/hostusbdirect/after_vm_destroy.py new file mode 100755 index 0000000..3882347 --- /dev/null +++ b/vdsm_hooks/hostusbdirect/after_vm_destroy.py @@ -0,0 +1,64 @@ +#!/usr/bin/python + +import os +import sys +import traceback + +import hooking + +''' +after_vm_destroy: +return the original owner of the usb device +''' + +HOOK_HOSTUSB_PATH = '/var/run/vdsm/hooks/hostusbdirect-permissions' + + +def get_owner(devpath): + uid = pid = -1 + content = '' + + if not os.path.isfile(HOOK_HOSTUSB_PATH): + return uid, pid + + f = file(HOOK_HOSTUSB_PATH, 'r') + for line in f: + if len(line) > 0 and line.split(':')[0] == devpath: + entry = line.split(':') + uid = entry[1] + pid = entry[2] + elif len(line) > 0: + content += line + '\n' + + f.close() + if uid != -1: + f = file(HOOK_HOSTUSB_PATH, 'w') + f.writelines(content) + f.close() + + return uid, pid + + +def chown(bus, device): + devpath = '/dev/bus/usb/' + bus + '/' + device + uid, gid = get_owner(devpath) + + owner = str(uid) + ':' + str(gid) + command = ['/bin/chown', owner, devpath] + retcode, out, err = hooking.execCmd(command, sudo=True, raw=True) + if retcode != 0: + sys.stderr.write('hostusbdirect: error chown %s to %s, err = %s\n' % + (devpath, owner, err)) + sys.exit(2) + + +if 'hostusbdirect' in os.environ: + try: + for usb in os.environ['hostusbdirect'].split('&'): + bus, device = usb.split(':') + chown(bus, device) + + except: + sys.stderr.write('hostusb after_vm_destroy: [unexpected error]: %s\n' % + traceback.format_exc()) + sys.exit(2) diff --git a/vdsm_hooks/hostusbdirect/before_vm_migrate_source.py b/vdsm_hooks/hostusbdirect/before_vm_migrate_source.py new file mode 100755 index 0000000..43a2eca --- /dev/null +++ b/vdsm_hooks/hostusbdirect/before_vm_migrate_source.py @@ -0,0 +1,8 @@ +#!/usr/bin/python + +import os +import sys + +if 'hostusbdirect' in os.environ: + sys.stderr.write("hostusbdirect: can't migrate VM with host usb devices\n") + sys.exit(2) diff --git a/vdsm_hooks/hostusbdirect/before_vm_start.py b/vdsm_hooks/hostusbdirect/before_vm_start.py new file mode 100755 index 0000000..57307aa --- /dev/null +++ b/vdsm_hooks/hostusbdirect/before_vm_start.py @@ -0,0 +1,108 @@ +#!/usr/bin/python + +import os +import sys +import grp +import pwd +import traceback + +import hooking + +''' +host usb hook direct +==================== + +syntax: + hostusbdirect=001:001&001:002 + i.e. + hostusbdirect=bus:device (can add more then one with '&' separator) + +Note: + The VM must be pinned to host and this hook will + fail any migration attempt. +''' + +HOOK_HOSTUSB_PATH = '/var/run/vdsm/hooks/hostusbdirect-permissions' + + +def log_dev_owner(devpath, user, group): + entry = devpath + ":" + str(user) + ":" + str(group) + + if not os.path.isdir(os.path.dirname(HOOK_HOSTUSB_PATH)): + os.mkdir(os.path.dirname(HOOK_HOSTUSB_PATH)) + + if os.path.isfile(HOOK_HOSTUSB_PATH): + f = file(HOOK_HOSTUSB_PATH, 'r') + for line in f: + if entry == line: + f.close() + return + + f = file(HOOK_HOSTUSB_PATH, 'a') + f.writelines(entry) + f.close() + + +def chown(bus, device): + + devpath = '/dev/bus/usb/' + bus + '/' + device + stat = os.stat(devpath) + + group = grp.getgrnam('qemu') + gid = group.gr_gid + user = pwd.getpwnam('qemu') + uid = user.pw_uid + + owner = str(uid) + ':' + str(gid) + command = ['/bin/chown', owner, devpath] + retcode, out, err = hooking.execCmd(command, sudo=True, raw=True) + if retcode != 0: + sys.stderr.write('hostusbdirect: error chown %s to %s, err = %s\n' % + (devpath, owner, err)) + sys.exit(2) + + log_dev_owner(devpath, stat.st_uid, stat.st_gid) + + +def add_devices(domxml, devices): + domain = domxml.getElementsByTagName('devices')[0] + + for device in devices: + # Add the following for each device : + #<hostdev mode='subsystem' type='usb' managed='yes'> + # <source> + # <address bus='001' device='003'/> + # </source> + #</hostdev> + + hostdev = domxml.createElement('hostdev') + hostdev.setAttribute('mode', 'subsystem') + hostdev.setAttribute('type', 'usb') + hostdev.setAttribute('managed', 'yes') + + source = domxml.createElement('source') + hostdev.appendChild(source) + + address = domxml.createElement('address') + address.setAttribute('bus', device['bus']) + address.setAttribute('device', device['device']) + source.appendChild(address) + + domain.appendChild(hostdev) + +if 'usbhostdirect' in os.environ: + try: + domxml = hooking.read_domxml() + devices = [] + + for usb in os.environ['usbhostdirect'].split('&'): + bus, device = usb.split(':') + chown(bus, device) + devices.append({'bus': bus, 'device': device}) + + add_devices(domxml, devices) + hooking.write_domxml(domxml) + except: + sys.stderr.write('hostusbdirect: [unexpected error]: %s\n' % + traceback.format_exc()) + sys.exit(2) diff --git a/vdsm_hooks/hostusbdirect/sudoers.vdsm_hook_hostusbdirect b/vdsm_hooks/hostusbdirect/sudoers.vdsm_hook_hostusbdirect new file mode 100644 index 0000000..cba601d --- /dev/null +++ b/vdsm_hooks/hostusbdirect/sudoers.vdsm_hook_hostusbdirect @@ -0,0 +1 @@ +vdsm ALL=(ALL) NOPASSWD: /bin/chown -- To view, visit http://gerrit.ovirt.org/17428 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I00b4b03993cd7290462b0e17457892b23dd41b31 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Lee Yarwood <lyarwood(a)redhat.com>

7 22

Change in vdsm[master]: [WIP] Implement a process to do dangerous IO in C
by smizrahi＠redhat.com 23 Jun '14

23 Jun '14

Saggi Mizrahi has uploaded a new change for review. Change subject: [WIP] Implement a process to do dangerous IO in C ...................................................................... [WIP] Implement a process to do dangerous IO in C This replaces the process pool with a process that can serve multiple requests written in C. This implementation is much more scalable and lightweight. Should solve bugs related to running out of helpers, logging getting suck, python forking deadlocking, running out of memory and other things as well. The communication between VDSM and the IOProcess is done with json objects. The IOProcess starts with 3 thread: 1. requestReader - reads requests from the pipe, builds a DOM representation of it and queues it up for handling 2. responseWriter - gets response DOMs from the queue converts them to a JSON string and send it over the pipe 3. requestHandler - pops requests from the queue and provisions threads for handling them. Currently we I just allocate a new thread per request. If there is ever a need to have a thread pool this is where the load balancing is going to sit. Each request gets the are as a JsonNode and returns a response that is a JsonNode as well. Most exported functions are pretty trivial and are a good example on how to write new ones. Unlink the ProcessPoolHelper, high level commands sit of the OopWrapper and are run from the client side instead of being implemented in C on the IOProcess side. Change-Id: Ie4664d5330debbe38ba33b74ebb586ac42913b4a Signed-off-by: Saggi Mizrahi <smizrahi(a)redhat.com> --- M configure.ac M tests/Makefile.am A tests/ioprocessTests.py A tests/outOfProcessTests.py D tests/processPoolTests.py M vdsm.spec.in M vdsm/constants.py.in M vdsm/storage/Makefile.am M vdsm/storage/fileSD.py M vdsm/storage/fileUtils.py M vdsm/storage/fileVolume.py A vdsm/storage/ioprocess.py A vdsm/storage/ioprocess/.gitignore A vdsm/storage/ioprocess/Makefile.am A vdsm/storage/ioprocess/exported-functions.c A vdsm/storage/ioprocess/exported-functions.h A vdsm/storage/ioprocess/ioprocess.c A vdsm/storage/ioprocess/json-dom-generator.c A vdsm/storage/ioprocess/json-dom-generator.h A vdsm/storage/ioprocess/json-dom-parser.c A vdsm/storage/ioprocess/json-dom-parser.h A vdsm/storage/ioprocess/json-dom.c A vdsm/storage/ioprocess/json-dom.h M vdsm/storage/misc.py M vdsm/storage/nfsSD.py M vdsm/storage/outOfProcess.py D vdsm/storage/processPool.py M vdsm/storage/sd.py M vdsm/storage/sp.py M vdsm/storage/task.py 30 files changed, 3,018 insertions(+), 666 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/46/3946/1 -- To view, visit http://gerrit.ovirt.org/3946 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ie4664d5330debbe38ba33b74ebb586ac42913b4a Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Saggi Mizrahi <smizrahi(a)redhat.com>

4 11

Change in vdsm[master]: generate quemu-kvm coredumps using abrt
by mtayer＠redhat.com 23 Jun '14

23 Jun '14

mooli tayer has uploaded a new change for review. Change subject: generate quemu-kvm coredumps using abrt ...................................................................... generate quemu-kvm coredumps using abrt Change-Id: I1acb287cdfcc9b8c05569fe1fc01e88957309ff9 Signed-off-by: Mooli Tayer <mtayer(a)redhat.com> --- M init/vdsmd_init_common.sh.in M vdsm.spec.in 2 files changed, 1 insertion(+), 5 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/51/20751/1 diff --git a/init/vdsmd_init_common.sh.in b/init/vdsmd_init_common.sh.in index 43a86ef..6a55818 100644 --- a/init/vdsmd_init_common.sh.in +++ b/init/vdsmd_init_common.sh.in @@ -25,11 +25,6 @@ task_configure_coredump() { local conf_file="@CONFDIR@/vdsm.conf" local getconfitem="@VDSMDIR@/get-conf-item" - - if "${getconfitem}" "${conf_file}" vars core_dump_enable false | - tr A-Z a-z | grep -q true; then - echo "/var/log/core/core.%p.%t.dump" > /proc/sys/kernel/core_pattern - fi } task_run_init_hooks(){ diff --git a/vdsm.spec.in b/vdsm.spec.in index 34bdc2d..fd8023c 100644 --- a/vdsm.spec.in +++ b/vdsm.spec.in @@ -107,6 +107,7 @@ Requires: libguestfs-tools-c Requires: %{name}-xmlrpc = %{version}-%{release} Requires: mom >= 0.3.2-3 +Requires: abrt-addon-ccpp %ifarch x86_64 Requires: python-dmidecode -- To view, visit http://gerrit.ovirt.org/20751 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I1acb287cdfcc9b8c05569fe1fc01e88957309ff9 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: mooli tayer <mtayer(a)redhat.com>

6 20

Change in vdsm[master]: Rescan FC when rescanning for new multipath devices
by smizrahi＠redhat.com 23 Jun '14

23 Jun '14

Saggi Mizrahi has uploaded a new change for review. Change subject: Rescan FC when rescanning for new multipath devices ...................................................................... Rescan FC when rescanning for new multipath devices Change-Id: Idec939222676a24452e8825b36db68839bfd2bbc Signed-off-by: Saggi Mizrahi <smizrahi(a)redhat.com> --- M lib/vdsm/utils.py M tests/utilsTests.py M vdsm/storage/Makefile.am A vdsm/storage/fc.py M vdsm/storage/multipath.py 5 files changed, 61 insertions(+), 2 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/39/19539/1 diff --git a/lib/vdsm/utils.py b/lib/vdsm/utils.py index f06cad2..e273514 100644 --- a/lib/vdsm/utils.py +++ b/lib/vdsm/utils.py @@ -793,6 +793,24 @@ return functools.partial(self.__call__, obj) +def tee_async(data, outputFiles=[], append=False): + cmd = ["tee"] + if append: + cmd.append("--append") + + cmd.extend(outputFiles) + proc = execCmd(cmd, sync=False) + proc.stdin.write(data) + + def parser(rc, out, err): + if rc != 0: + raise OSError(-1, err) + + return out + + return AsyncProcessOperation(proc, parser) + + def validateMinimalKeySet(dictionary, reqParams): if not all(key in dictionary for key in reqParams): raise ValueError diff --git a/tests/utilsTests.py b/tests/utilsTests.py index 656c121..5b83ec0 100644 --- a/tests/utilsTests.py +++ b/tests/utilsTests.py @@ -18,11 +18,14 @@ # Refer to the README and COPYING files for full details of the license # import errno +import time +import tempfile +import shutil +import os from testrunner import VdsmTestCase as TestCaseBase from vdsm import utils from storage import misc -import time class RetryTests(TestCaseBase): @@ -63,6 +66,33 @@ sproc.wait() +class TeeTests(TestCaseBase): + def setUp(self): + self.dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.dir) + + def test(self): + d = self.dir + n = 10 + files = [] + for i in range(n): + files.append(os.path.join(d, "%d.tmp" % i)) + + data = "hello" + res = utils.tee_async(data, files).result() + self.assertEquals((data, None), res) + for fname in files: + with open(fname, "r") as f: + self.assertEquals(data, f.read()) + + def testError(self): + res, err = utils.tee_async("hello", ["/"]).result() + self.assertEquals(res, None) + self.assertNotEquals(err, None) + + class CommandPathTests(TestCaseBase): def testExisting(self): cp = utils.CommandPath('sh', 'utter nonsense', '/bin/sh') diff --git a/vdsm/storage/Makefile.am b/vdsm/storage/Makefile.am index 2d3e9a6..a367c9b 100644 --- a/vdsm/storage/Makefile.am +++ b/vdsm/storage/Makefile.am @@ -33,6 +33,7 @@ fileSD.py \ fileUtils.py \ fileVolume.py \ + fc.py \ fuser.py \ glusterSD.py \ glusterVolume.py \ diff --git a/vdsm/storage/fc.py b/vdsm/storage/fc.py new file mode 100644 index 0000000..96aa710 --- /dev/null +++ b/vdsm/storage/fc.py @@ -0,0 +1,8 @@ +from glob import glob +from vdsm import utils + + +def rescan(): + """Issue command to discover new ports if the fc driver supports it""" + # Use tee to make sure we don't do that IO directly + utils.tee_async("1", glob("/sys/class/fc_host/host*/issue_lip")) diff --git a/vdsm/storage/multipath.py b/vdsm/storage/multipath.py index 6d6fcbd..076bfca 100644 --- a/vdsm/storage/multipath.py +++ b/vdsm/storage/multipath.py @@ -36,6 +36,7 @@ import iscsi import supervdsm import devicemapper +import fc import storage_exception as se @@ -106,7 +107,8 @@ Should only be called from hsm._rescanDevices() """ - # First ask iSCSI to rescan all its sessions + # First ask fc and iSCSI to rescan all of their sessions + fc.rescan() iscsi.rescan() # Now let multipath daemon pick up new devices -- To view, visit http://gerrit.ovirt.org/19539 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Idec939222676a24452e8825b36db68839bfd2bbc Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Saggi Mizrahi <smizrahi(a)redhat.com>

5 16

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

vdsm-patches November 2013