Adam Litke has uploaded a new change for review.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
sdm: Add find_domain_manifest helpers
Change-Id: Ia5d139bde2349c8231141fe842d9ada96fa279fe Signed-off-by: Adam Litke alitke@redhat.com --- M tests/Makefile.am A tests/domainmanifestfactory_tests.py M tests/manifest_tests.py M tests/storagefakelib.py M tests/storagetestutils.py M vdsm.spec.in M vdsm/storage/Makefile.am M vdsm/storage/blockSD.py A vdsm/storage/domainManifestFactory.py M vdsm/storage/glusterSD.py M vdsm/storage/localFsSD.py M vdsm/storage/nfsSD.py 12 files changed, 214 insertions(+), 3 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/55/43555/1
diff --git a/tests/Makefile.am b/tests/Makefile.am index 9cc3692..f116fc4 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -43,6 +43,7 @@ cpuProfileTests.py \ deviceTests.py \ domainDescriptorTests.py \ + domainmanifestfactory_tests.py \ encodingTests.py \ executorTests.py \ fileSDTests.py \ diff --git a/tests/domainmanifestfactory_tests.py b/tests/domainmanifestfactory_tests.py new file mode 100644 index 0000000..4c3f63e --- /dev/null +++ b/tests/domainmanifestfactory_tests.py @@ -0,0 +1,124 @@ +# Copyright 2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Refer to the README and COPYING files for full details of the license +# + +import os +import uuid + +from testlib import VdsmTestCase, namedTemporaryDir +from monkeypatch import MonkeyPatchScope +from storagefakelib import FakeLVM +from storagetestutils import make_blocksd_manifest, make_vg, \ + make_filesd_manifest + +from storage import sd, blockSD, fileSD, localFsSD, mount +from storage import storage_exception as se +from storage import domainManifestFactory + + +class DomainManifestFactoryTests(VdsmTestCase): + def get_patches(self, tmpdir, lvm): + return [(blockSD, 'lvm', lvm), + (blockSD, 'selectMetadata', lambda x: {}), + (sd.StorageDomain, 'storage_repository', tmpdir), + (mount, 'isMounted', lambda x: True)] + + def make_mnt(self, tmpdir, link_name): + mntdir = os.path.join(tmpdir, sd.DOMAIN_MNT_POINT) + os.mkdir(mntdir) + os.symlink(tmpdir, os.path.join(mntdir, link_name)) + + def test_localfs_layout(self): + with namedTemporaryDir() as tmpdir: + self.make_mnt(tmpdir, '_localfssd') + lvm = FakeLVM(tmpdir) + with MonkeyPatchScope(self.get_patches(tmpdir, lvm)): + manifest = make_filesd_manifest(tmpdir) + found_manifest = domainManifestFactory.produce( + manifest.sdUUID, manifest._metadata) + self.assertEquals(manifest.sdUUID, found_manifest.sdUUID) + self.assertIsInstance(found_manifest, + fileSD.FileStorageDomainManifest) + + def test_nfs_layout(self): + with namedTemporaryDir() as tmpdir: + self.make_mnt(tmpdir, '192.168.1.1:_localfssd') + lvm = FakeLVM(tmpdir) + with MonkeyPatchScope(self.get_patches(tmpdir, lvm)): + manifest = make_filesd_manifest(tmpdir) + found_manifest = domainManifestFactory.produce( + manifest.sdUUID, manifest._metadata) + self.assertEquals(manifest.sdUUID, found_manifest.sdUUID) + self.assertIsInstance(found_manifest, + fileSD.FileStorageDomainManifest) + + def test_glusterfs_layout(self): + with namedTemporaryDir() as tmpdir: + mntdir = os.path.join(tmpdir, sd.DOMAIN_MNT_POINT) + glusterdir = os.path.join(mntdir, sd.GLUSTERSD_DIR) + os.makedirs(glusterdir) + os.symlink(tmpdir, os.path.join(glusterdir, 'host:_glustersd')) + + lvm = FakeLVM(tmpdir) + with MonkeyPatchScope(self.get_patches(tmpdir, lvm)): + manifest = make_filesd_manifest(tmpdir) + found_manifest = domainManifestFactory.produce( + manifest.sdUUID, manifest._metadata) + self.assertEquals(manifest.sdUUID, found_manifest.sdUUID) + self.assertEquals(manifest.sdUUID, found_manifest.sdUUID) + self.assertIsInstance(found_manifest, + fileSD.FileStorageDomainManifest) + + def test_block_layout(self): + with namedTemporaryDir() as tmpdir: + lvm = FakeLVM(tmpdir) + with MonkeyPatchScope(self.get_patches(tmpdir, lvm)): + manifest = make_blocksd_manifest() + vg_name = make_vg(lvm, manifest) + lvm.vgmd[vg_name]['tags'] = [blockSD.STORAGE_DOMAIN_TAG] + found_manifest = domainManifestFactory.produce( + manifest.sdUUID, manifest._metadata) + self.assertEquals(manifest.sdUUID, found_manifest.sdUUID) + self.assertIsInstance(found_manifest, + blockSD.BlockStorageDomainManifest) + + def test_cached_finder(self): + def _trap(unused): + raise RuntimeError() + + with namedTemporaryDir() as tmpdir: + self.make_mnt(tmpdir, '_localfssd') + lvm = FakeLVM(tmpdir) + patches = self.get_patches(tmpdir, lvm) + with MonkeyPatchScope(patches): + manifest = make_filesd_manifest(tmpdir) + res1 = domainManifestFactory.produce( + manifest.sdUUID, manifest._metadata) + patches.append((localFsSD, 'findDomainManifest', _trap)) + with MonkeyPatchScope(patches): + res2 = domainManifestFactory.produce( + manifest.sdUUID, manifest._metadata) + self.assertEquals(res1.sdUUID, res2.sdUUID) + + def test_not_found(self): + sdUUID = str(uuid.uuid4()) + with namedTemporaryDir() as tmpdir: + lvm = FakeLVM(tmpdir) + with MonkeyPatchScope(self.get_patches(tmpdir, lvm)): + self.assertRaises(se.StorageDomainDoesNotExist, + domainManifestFactory.produce, sdUUID) \ No newline at end of file diff --git a/tests/manifest_tests.py b/tests/manifest_tests.py index a22d7cf..967b663 100644 --- a/tests/manifest_tests.py +++ b/tests/manifest_tests.py @@ -161,7 +161,6 @@
imguuid = str(uuid.uuid4()) imagepath = manifest.getImagePath(imguuid) - os.makedirs(os.path.dirname(imagepath)) with open(imagepath, 'w') as f: f.truncate(0) self.assertEquals(set(), manifest.getAllImages()) diff --git a/tests/storagefakelib.py b/tests/storagefakelib.py index cb40fbb..bf79d91 100644 --- a/tests/storagefakelib.py +++ b/tests/storagefakelib.py @@ -23,6 +23,7 @@ from copy import deepcopy
from storage import lvm as real_lvm +from storage import storage_exception as se
class FakeLVM(object): @@ -154,7 +155,11 @@ return real_lvm.PV(**md)
def getVG(self, vgName): - vg_md = deepcopy(self.vgmd[vgName]) + try: + vg_md = deepcopy(self.vgmd[vgName]) + except KeyError: + raise se.VolumeGroupDoesNotExist(vgName) + vg_attr = real_lvm.VG_ATTR(**vg_md['attr']) vg_md['attr'] = vg_attr return real_lvm.VG(**vg_md) diff --git a/tests/storagetestutils.py b/tests/storagetestutils.py index ed052af..eae803a 100644 --- a/tests/storagetestutils.py +++ b/tests/storagetestutils.py @@ -79,7 +79,7 @@ if metadata is None: metadata = {sd.DMDK_VERSION: 3} manifest = fileSD.FileStorageDomainManifest(domain_path, metadata) - os.makedirs(os.path.join(manifest.domaindir, sduuid, sd.DOMAIN_IMAGES)) + os.makedirs(os.path.join(manifest.domaindir, sd.DOMAIN_IMAGES)) return manifest
diff --git a/vdsm.spec.in b/vdsm.spec.in index 4ba9812..57e673e 100644 --- a/vdsm.spec.in +++ b/vdsm.spec.in @@ -906,6 +906,7 @@ %{_datadir}/%{vdsm_name}/storage/curlImgWrap.py* %{_datadir}/%{vdsm_name}/storage/devicemapper.py* %{_datadir}/%{vdsm_name}/storage/dispatcher.py* +%{_datadir}/%{vdsm_name}/storage/domainManifestFactory.py %{_datadir}/%{vdsm_name}/storage/monitor.py* %{_datadir}/%{vdsm_name}/storage/fileSD.py* %{_datadir}/%{vdsm_name}/storage/fileUtils.py* diff --git a/vdsm/storage/Makefile.am b/vdsm/storage/Makefile.am index a4ec93a..e460518 100644 --- a/vdsm/storage/Makefile.am +++ b/vdsm/storage/Makefile.am @@ -31,6 +31,7 @@ curlImgWrap.py \ devicemapper.py \ dispatcher.py \ + domainManifestFactory.py \ fileSD.py \ fileUtils.py \ fileVolume.py \ diff --git a/vdsm/storage/blockSD.py b/vdsm/storage/blockSD.py index 121cc6b..6de2737 100644 --- a/vdsm/storage/blockSD.py +++ b/vdsm/storage/blockSD.py @@ -1426,5 +1426,10 @@ return BlockStorageDomain(BlockStorageDomain.findDomainPath(sdUUID))
+def findDomainManifest(sdUUID, metadata=None): + return BlockStorageDomainManifest( + BlockStorageDomain.findDomainPath(sdUUID), metadata) + + def getStorageDomainsList(): return [vg.name for vg in lvm.getAllVGs() if _isSD(vg)] diff --git a/vdsm/storage/domainManifestFactory.py b/vdsm/storage/domainManifestFactory.py new file mode 100644 index 0000000..fced9de --- /dev/null +++ b/vdsm/storage/domainManifestFactory.py @@ -0,0 +1,60 @@ +# +# Copyright 2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Refer to the README and COPYING files for full details of the license +# + +import logging + +import blockSD, glusterSD, localFsSD, nfsSD +import storage_exception as se + + +log = logging.getLogger('Storage.domainManifestFactory') + +domain_manifest_producers = {} + + +def produce(sdUUID, metadata=None): + log.error("looking for domain manifest %s", sdUUID) + + # Check our cache of finder functions first. Since a domain manifest + # cannot change its type we don't have to worry about invalidation. + try: + producer = domain_manifest_producers[sdUUID] + except KeyError: + pass + else: + return producer(sdUUID, metadata) + + # The order is somewhat important, it's ordered + # by how quickly get can find the domain. For instance + # if an nfs mount is unavailable we will get stuck + # until it times out, this should affect fetching + # of block\local domains. If for any case in the future + # this changes, please update the order. + for mod in (blockSD, glusterSD, localFsSD, nfsSD): + try: + manifest = mod.findDomainManifest(sdUUID, metadata) + domain_manifest_producers[sdUUID] = mod.findDomainManifest + return manifest + except se.StorageDomainDoesNotExist: + pass + except Exception: + log.exception("Error while looking for domain `%s`", sdUUID) + + raise se.StorageDomainDoesNotExist(sdUUID) diff --git a/vdsm/storage/glusterSD.py b/vdsm/storage/glusterSD.py index 2931083..a62e242 100644 --- a/vdsm/storage/glusterSD.py +++ b/vdsm/storage/glusterSD.py @@ -30,3 +30,8 @@
def findDomain(sdUUID): return GlusterStorageDomain(GlusterStorageDomain.findDomainPath(sdUUID)) + + +def findDomainManifest(sdUUID, metadata=None): + return fileSD.FileStorageDomainManifest( + GlusterStorageDomain.findDomainPath(sdUUID), metadata) diff --git a/vdsm/storage/localFsSD.py b/vdsm/storage/localFsSD.py index 7e11330..b4e0ce0 100644 --- a/vdsm/storage/localFsSD.py +++ b/vdsm/storage/localFsSD.py @@ -119,3 +119,8 @@
def findDomain(sdUUID): return LocalFsStorageDomain(LocalFsStorageDomain.findDomainPath(sdUUID)) + + +def findDomainManifest(sdUUID, metadata=None): + return fileSD.FileStorageDomainManifest( + LocalFsStorageDomain.findDomainPath(sdUUID), metadata) \ No newline at end of file diff --git a/vdsm/storage/nfsSD.py b/vdsm/storage/nfsSD.py index 1f94439..739efc8 100644 --- a/vdsm/storage/nfsSD.py +++ b/vdsm/storage/nfsSD.py @@ -120,3 +120,8 @@
def findDomain(sdUUID): return NfsStorageDomain(NfsStorageDomain.findDomainPath(sdUUID)) + + +def findDomainManifest(sdUUID, metadata=None): + return fileSD.FileStorageDomainManifest( + NfsStorageDomain.findDomainPath(sdUUID), metadata)
automation@ovirt.org has posted comments on this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Patch Set 1:
* Update tracker::IGNORE, no Bug-Url found * Check Bug-Url::WARN, no bug url found, make sure header matches 'Bug-Url: ' and is a valid url. * Check merged to previous::IGNORE, Not in stable branch (['ovirt-3.5', 'ovirt-3.4', 'ovirt-3.3'])
Nir Soffer has posted comments on this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Patch Set 1:
(1 comment)
https://gerrit.ovirt.org/#/c/43555/1/vdsm/storage/domainManifestFactory.py File vdsm/storage/domainManifestFactory.py:
Please use lowercase module names. Also this name is too long. Can we find shorter name like sdc? Line 1: # Line 2: # Copyright 2015 Red Hat, Inc. Line 3: # Line 4: # This program is free software; you can redistribute it and/or modify
automation@ovirt.org has posted comments on this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Patch Set 2:
* Update tracker::IGNORE, no Bug-Url found * Check Bug-Url::WARN, no bug url found, make sure header matches 'Bug-Url: ' and is a valid url. * Check merged to previous::IGNORE, Not in stable branch (['ovirt-3.5', 'ovirt-3.4', 'ovirt-3.3'])
Nir Soffer has posted comments on this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Patch Set 2: Code-Review-1
(1 comment)
Partial review
https://gerrit.ovirt.org/#/c/43555/2/vdsm/storage/nfsSD.py File vdsm/storage/nfsSD.py:
Line 123: Line 124: Line 125: def findDomainManifest(sdUUID, metadata=None): Line 126: return fileSD.FileStorageDomainManifest( Line 127: NfsStorageDomain.findDomainPath(sdUUID), metadata) This formating is not clear, lets use 2 short lines instead.
path = NfsStorageDomain.findDomainPath(sdUUID) return fileSD.FileStorageDomainManifest(path, metadata)
automation@ovirt.org has posted comments on this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Patch Set 3:
* Update tracker::IGNORE, no Bug-Url found * Check Bug-Url::WARN, no bug url found, make sure header matches 'Bug-Url: ' and is a valid url. * Check merged to previous::IGNORE, Not in stable branch (['ovirt-3.5', 'ovirt-3.4', 'ovirt-3.3'])
automation@ovirt.org has posted comments on this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Patch Set 4:
* Update tracker::IGNORE, no Bug-Url found * Check Bug-Url::WARN, no bug url found, make sure header matches 'Bug-Url: ' and is a valid url. * Check merged to previous::IGNORE, Not in stable branch (['ovirt-3.5', 'ovirt-3.4', 'ovirt-3.3'])
automation@ovirt.org has posted comments on this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Patch Set 5:
* Update tracker::IGNORE, no Bug-Url found * Check Bug-Url::WARN, no bug url found, make sure header matches 'Bug-Url: ' and is a valid url. * Check merged to previous::IGNORE, Not in stable branch (['ovirt-3.5', 'ovirt-3.4', 'ovirt-3.3'])
automation@ovirt.org has posted comments on this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Patch Set 6:
* Update tracker::IGNORE, no Bug-Url found * Check Bug-Url::WARN, no bug url found, make sure header matches 'Bug-Url: ' and is a valid url. * Check merged to previous::IGNORE, Not in stable branch (['ovirt-3.5', 'ovirt-3.4', 'ovirt-3.3'])
automation@ovirt.org has posted comments on this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Patch Set 7:
* Update tracker::IGNORE, no Bug-Url found * Check Bug-Url::WARN, no bug url found, make sure header matches 'Bug-Url: ' and is a valid url. * Check merged to previous::IGNORE, Not in stable branch (['ovirt-3.5', 'ovirt-3.4', 'ovirt-3.3'])
automation@ovirt.org has posted comments on this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Patch Set 8:
* Update tracker::IGNORE, no Bug-Url found * Check Bug-Url::WARN, no bug url found, make sure header matches 'Bug-Url: ' and is a valid url. * Check merged to previous::IGNORE, Not in stable branch (['ovirt-3.5', 'ovirt-3.4', 'ovirt-3.3'])
Jenkins CI RO has abandoned this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Abandoned
Abandoned due to no activity - please restore if still relevant
gerrit-hooks has posted comments on this change.
Change subject: sdm: Add find_domain_manifest helpers ......................................................................
Patch Set 8:
* Update tracker: IGNORE, no Bug-Url found
vdsm-patches@lists.fedorahosted.org