[NEW PATCH] BZ#????? - one process pool to rule them all (via gerrit-bot)
by smizrahi@redhat.com
New patch submitted by Saggi Mizrahi (smizrahi(a)redhat.com)
You can review this change at: http://gerrit.usersys.redhat.com/744
commit 6978ada2d810ac73e2fda4c67af52e388bdc1c75
Author: Saggi Mizrahi <smizrahi(a)redhat.com>
Date: Mon Jul 25 16:18:36 2011 +0300
BZ#????? - one process pool to rule them all
Instead of having a process pool per domain use one pool but limit the amount of
workers a domain can hold at any given time.
Change-Id: Ia69071a049761b271a982c600cf0520a782c7eac
diff --git a/vdsm/config.py b/vdsm/config.py
index 488aa7b..dbed034 100644
--- a/vdsm/config.py
+++ b/vdsm/config.py
@@ -146,9 +146,10 @@ config.set('irs', 'gc_blocker_force_collect_interval', '60')
config.set('irs', 'maximum_domains_in_pool', '100')
# Process Pool Configuration
-config.set('irs', 'process_pool_size', '20')
+config.set('irs', 'process_pool_size', '100')
config.set('irs', 'process_pool_timeout', '60')
config.set('irs', 'process_pool_grace_period', '2')
+config.set("irs", "process_pool_max_slots_per_domain", '10')
#####################################################################
config.add_section('addresses')
diff --git a/vdsm/storage/outOfProcess.py b/vdsm/storage/outOfProcess.py
index 48a556a..e1a4de7 100644
--- a/vdsm/storage/outOfProcess.py
+++ b/vdsm/storage/outOfProcess.py
@@ -2,15 +2,18 @@ import os as mod_os
import glob as mod_glob
import types
from config import config
+import threading
+from functools import wraps
from fileUtils import open_ex
import fileUtils as mod_fileUtils
-from processPool import ProcessPool
+from processPool import ProcessPool, NoFreeHelpersError
MAX_HELPERS = config.getint("irs", "process_pool_size")
GRACE_PERIOD = config.getint("irs", "process_pool_grace_period")
DEFAULT_TIMEOUT = config.getint("irs", "process_pool_timeout")
+HELPERS_PER_DOMAIN = config.getint("irs", "process_pool_max_slots_per_domain")
_globalPool = ProcessPool(MAX_HELPERS, GRACE_PERIOD, DEFAULT_TIMEOUT)
@@ -67,6 +70,32 @@ setattr(os, 'path', _ModuleWrapper(mod_os.path))
fileUtils = _ModuleWrapper(mod_fileUtils)
+class ProcessPoolLimiter(object):
+ def __init__(self, procPool, limit):
+ self._procPool = procPool
+ self._limit = limit
+ self._lock = threading.Lock()
+ self._counter = 0
+
+ def wrapFunction(self, func):
+ @wraps(func)
+ def wrapper(*args, **kwds):
+ return self.runExternally(func, *args, **kwds)
+ return wrapper
+
+ def runExternally(self, *args, **kwargs):
+ with self._lock:
+ if self._counter >= self._limit:
+ raise NoFreeHelpersError("You reached the process limit")
+
+ self._counter += 1
+ try:
+ return self._procPool.runExternally(*args, **kwargs)
+ finally:
+ with self._lock:
+ self._counter -= 1
+
+
class OopWrapper(object):
def __init__(self, procPool):
self._processPool = procPool
diff --git a/vdsm/storage/processPool.py b/vdsm/storage/processPool.py
index f8ef4af..98a32e2 100644
--- a/vdsm/storage/processPool.py
+++ b/vdsm/storage/processPool.py
@@ -23,7 +23,7 @@ class ProcessPool(object):
self._maxSubProcess = maxSubProcess
self._gracePeriod = gracePeriod
self.timeout = timeout
- self._helperPool = [None] * self._maxSubProcess
+ self._helperPool = [Helper() for i in range(self._maxSubProcess)]
self._lockPool = [Lock() for i in range(self._maxSubProcess)]
self._closed = False
diff --git a/vdsm/storage/sd.py b/vdsm/storage/sd.py
index 4422487..344d578 100644
--- a/vdsm/storage/sd.py
+++ b/vdsm/storage/sd.py
@@ -224,6 +224,7 @@ class ProcessPoolDict(dict):
def __init__(self):
dict.__init__(self)
self._lock = threading.Lock()
+ self._pool = None
def __getitem__(self, key):
try:
@@ -235,8 +236,14 @@ class ProcessPoolDict(dict):
return dict.__getitem__(self, key)
def _createProcessPool(self, key):
- _domainPool = ProcessPool(oop.MAX_HELPERS, oop.GRACE_PERIOD, oop.DEFAULT_TIMEOUT)
- return oop.OopWrapper(_domainPool)
+ # I initialize the pool dict on first call
+ # because it's created on import and generating
+ # hundreds of subprocess on import is not recommended
+ if self._pool is None:
+ with self._lock:
+ if self._pool is None:
+ self._pool = ProcessPool(oop.MAX_HELPERS, oop.GRACE_PERIOD, oop.DEFAULT_TIMEOUT)
+ return oop.OopWrapper(oop.ProcessPoolLimiter(self._pool, oop.HELPERS_PER_DOMAIN))
# Dictionary for process pools per sdUUID
processPoolDict = ProcessPoolDict()
12 years, 7 months
[NEW PATCH] BZ#732914 VDSM must exit if libvirt is not running (via gerrit-bot)
by Federico Simoncelli
New patch submitted by Federico Simoncelli (fsimonce(a)redhat.com)
You can review this change at: http://gerrit.usersys.redhat.com/861
commit 067f769de4df00cf4015e82acd16c1319938a14f
Author: Federico Simoncelli <fsimonce(a)redhat.com>
Date: Mon Aug 29 11:01:16 2011 +0000
BZ#732914 VDSM must exit if libvirt is not running
Change-Id: I673184b8e5d765a9397f3fc14a70f7c31b907b3e
diff --git a/vdsm/libvirtconnection.py b/vdsm/libvirtconnection.py
index 1585b87..4eee356 100644
--- a/vdsm/libvirtconnection.py
+++ b/vdsm/libvirtconnection.py
@@ -103,7 +103,12 @@ def get(cif=None):
conn = __connections.get(id(cif))
if not conn:
- conn = libvirt.openAuth('qemu:///system', auth, 0)
+ try:
+ conn = libvirt.openAuth('qemu:///system', auth, 0)
+ except libvirt.libvirtError:
+ cif.log.error('Unable to connect to libvirt, taking vdsm down.',
+ exc_info=True)
+ cif.prepareForShutdown()
__connections[id(cif)] = conn
if cif != None:
for ev in (libvirt.VIR_DOMAIN_EVENT_ID_LIFECYCLE,
12 years, 7 months
[NEW PATCH] BZ#733910 - Clean links in directories in __cleanStorageRepository (via gerrit-bot)
by Yotam Oron
New patch submitted by Yotam Oron (yoron(a)redhat.com)
You can review this change at: http://gerrit.usersys.redhat.com/858
commit aaff596ebf15d43be91d2e87ead6b9b6664c575a
Author: Yotam Oron <yoron(a)redhat.com>
Date: Sun Aug 28 17:23:02 2011 +0300
BZ#733910 - Clean links in directories in __cleanStorageRepository
os.walk() sees links to directories as directories, so the code
tries to delete the links with rmdir(), fails, and then the consequent
rmdir() of the parent dir also fails since the directory is not empty.
Instead of rmdir(), unlink() a link.
Change-Id: I0bd54dcca2b73f0f371aae4d6952a0da137c25db
diff --git a/vdsm/storage/hsm.py b/vdsm/storage/hsm.py
index f83316c..7975a72 100644
--- a/vdsm/storage/hsm.py
+++ b/vdsm/storage/hsm.py
@@ -277,7 +277,11 @@ class HSM:
for directory in rmDirList:
try:
- os.rmdir(directory)
+ # os.walk() can see a link to a directory as a directory
+ if os.path.islink(directory):
+ os.unlink(directory)
+ else:
+ os.rmdir(directory)
except Exception, ex:
self.log.warn("Could not delete dir '%s' (%s: %s)." % (fullPath, ex.__class__.__name__, str(ex)))
12 years, 7 months
[NEW PATCH] BZ#734136 - Stricter partition detection (via gerrit-bot)
by smizrahi@redhat.com
New patch submitted by Saggi Mizrahi (smizrahi(a)redhat.com)
You can review this change at: http://gerrit.usersys.redhat.com/866
commit 8a883f5629c356e739ee0690a90f13db71c156a4
Author: Saggi Mizrahi <smizrahi(a)redhat.com>
Date: Tue Aug 30 11:41:34 2011 +0300
BZ#734136 - Stricter partition detection
Change-Id: Ibc242c7a3810efbc89083d0f0ea6fb193a63e331
diff --git a/vdsm/storage/devicemapper.py b/vdsm/storage/devicemapper.py
index 08cd533..8000928 100644
--- a/vdsm/storage/devicemapper.py
+++ b/vdsm/storage/devicemapper.py
@@ -29,6 +29,8 @@ from constants import EXT_DMSETUP
DMPATH_FORMAT = "/dev/mapper/%s"
+DM_PREFIX_PARTITION = "Part"
+
def getDmId(deviceMultipathName):
devlinkPath = DMPATH_FORMAT % deviceMultipathName
try:
@@ -99,8 +101,18 @@ def isBlockDevice(devName):
return False
def isPartitioned(devName):
- devName = resolveDevName(devName)
- return (len(getHolders(devName)) > 0)
+ dmName = resolveDevName(devName)
+ #Sometimes partitions are not managed by device mapper
+ possiblePartPath = os.path.join("/sys/block/", devName, devName + "1")
+ if os.path.exists(possiblePartPath):
+ return True
+
+ mpathName = getDevName(dmName)
+ for holder in getHolders(dmName):
+ if getDevName(holder).startswith(mpathName):
+ return True
+
+ return False
def getAllSlaves():
deps = {}
12 years, 7 months
[NEW PATCH] Strip gerrit headers from changelog (via gerrit-bot)
by Federico Simoncelli
New patch submitted by Federico Simoncelli (fsimonce(a)redhat.com)
You can review this change at: http://gerrit.usersys.redhat.com/846
commit 7834a7efbc7e4b77b2061009640bad8f814a5dae
Author: Federico Simoncelli <fsimonce(a)redhat.com>
Date: Tue Aug 23 08:52:54 2011 +0000
Strip gerrit headers from changelog
Change-Id: Ie3c09ee32e85a2f8044a4b96570d08557bfd8728
diff --git a/build-aux/gitlog-to-changelog b/build-aux/gitlog-to-changelog
index 4559704..b0db305 100755
--- a/build-aux/gitlog-to-changelog
+++ b/build-aux/gitlog-to-changelog
@@ -152,6 +152,9 @@ sub quoted_cmd(@)
# Omit "Signed-off-by..." lines.
@line = grep !/^Signed-off-by: .*>$/, @line;
+ # Omit gerrit lines.
+ @line = grep !/^(Change-Id|Reviewed-(on|by)|Tested-by): .*$/, @line;
+
# If there were any lines
if (@line == 0)
{
12 years, 7 months