New subject: Change in vdsm[master]: Using CrabRPC to communicate between vdsm and super vdsm

Monday, 11 February 2013

Yaniv Bronhaim has uploaded a new change for review.

Change subject: [WIP] Changing startup and rpc for vdsm and supervdsm
......................................................................

[WIP] Changing startup and rpc for vdsm and supervdsm

This patch runs vdsm as root, start external process for supervdsm and
drop privileges for vdsm.

Still in progress:
 - need to add timestamp for supervdsm process
 - define crash senerios
 - remove Process and use fork instead

Change-Id: I4f1053e7d1264003fa265e44899d8b02f98bd68a
Signed-off-by: Yaniv Bronhaim <ybronhei(a)redhat.com&gt;
---
M NEWS
M tests/superVdsmTests.py
M tests/testrunner.py
M vdsm.spec.in
M vdsm/supervdsm.py
M vdsm/supervdsmServer.py
M vdsm/vdsm
M vdsm/vdsmd.init.in
8 files changed, 245 insertions(+), 378 deletions(-)


  git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/10/11910/1

diff --git a/NEWS b/NEWS
old mode 100644
new mode 100755
diff --git a/tests/superVdsmTests.py b/tests/superVdsmTests.py
index 87de66d..13164ac 100644
--- a/tests/superVdsmTests.py
+++ b/tests/superVdsmTests.py
@@ -1,73 +1,37 @@
 from testrunner import VdsmTestCase as TestCaseBase
 import supervdsm
-import testValidation
-import tempfile
-from vdsm import utils
 import os
-import uuid
-from vdsm import constants
-from storage import misc
-from monkeypatch import MonkeyPatch
-from time import sleep
-
-
-(a)utils.memoized
-def getNeededPythonPath():
-    testDir = os.path.dirname(__file__)
-    base = os.path.dirname(testDir)
-    vdsmPath = os.path.join(base, 'vdsm')
-    cliPath = os.path.join(base, 'vdsm_cli')
-    pyPath = "PYTHONPATH=" + ':'.join([base, vdsmPath, cliPath])
-    return pyPath
-
-
-def monkeyStart(self):
-    self._authkey = str(uuid.uuid4())
-    self._log.debug("Launching Super Vdsm")
-
-    superVdsmCmd = [getNeededPythonPath(), constants.EXT_PYTHON,
-                    supervdsm.SUPERVDSM,
-                    self._authkey, str(os.getpid()),
-                    self.pidfile, self.timestamp, self.address,
-                    str(os.getuid())]
-    misc.execCmd(superVdsmCmd, sync=False, sudo=True)
-    sleep(2)
+from multiprocessing import Process
+from supervdsmServer import SuperVdsmManager
 
 
 class TestSuperVdsm(TestCaseBase):
+
     def setUp(self):
-        testValidation.checkSudo(['python', supervdsm.SUPERVDSM])
-        self._proxy = supervdsm.getProxy()
-
-        # temporary values to run temporary svdsm
-        self.pidfd, pidfile = tempfile.mkstemp()
-        self.timefd, timestamp = tempfile.mkstemp()
-        self.addfd, address = tempfile.mkstemp()
-
-        self._proxy.setIPCPaths(pidfile, timestamp, address)
+        self._proxy = None
+        self.svdsmProc = Process(target=SuperVdsmManager.
+                                 getInstance().restartSvdsm,
+                                 args=(os.getpid(), ))
+        self.svdsmProc.start()
+        # wait until next process starts
+        self._proxy = supervdsm.SuperVdsmProxy()
 
     def tearDown(self):
-        supervdsm.extraPythonPathList = []
-        for fd in (self.pidfd, self.timefd, self.addfd):
-            os.close(fd)
-        self._proxy.kill()  # cleanning old temp files
+        try:
+            self._proxy.stopSuperVdsm()
+        # Throwing exception when the server is closed
+        except:
+            pass
 
-    @MonkeyPatch(supervdsm.SuperVdsmProxy, '_start', monkeyStart)
-    def testIsSuperUp(self):
-        self._proxy.ping()  # this call initiate svdsm
-        self.assertTrue(self._proxy.isRunning())
+    def testPing(self):
+        if self._proxy.ping():
+            return True
+        else:
+            return False
 
-    @MonkeyPatch(supervdsm.SuperVdsmProxy, '_start', monkeyStart)
-    def testKillSuper(self):
-        self._proxy.ping()
-        self._proxy.kill()
-        self.assertFalse(self._proxy.isRunning())
-        self._proxy.ping()  # Launching vdsm after kill
-        self.assertTrue(self._proxy.isRunning())
+    def testStartAndClose(self):
+        return True
 
-    @MonkeyPatch(supervdsm.SuperVdsmProxy, '_start', monkeyStart)
-    def testNoPidFile(self):
-        self._proxy.ping()  # svdsm is up
-        self.assertTrue(self._proxy.isRunning())
-        utils.rmFile(self._proxy.timestamp)
-        self.assertRaises(IOError, self._proxy.isRunning)
+    def testGetDeviceInfo(self):
+        print self._proxy.getLsBlk()
+        return True
diff --git a/tests/testrunner.py b/tests/testrunner.py
index dfe9885..17d150b 100644
--- a/tests/testrunner.py
+++ b/tests/testrunner.py
@@ -28,13 +28,6 @@
 from nose import core
 from nose import result
 
-# Monkey patch pthreading in necessary
-if sys.version_info[0] == 2:
-    # as long as we work with Python 2, we need to monkey-patch threading
-    # module before it is ever used.
-    import pthreading
-    pthreading.monkey_patch()
-
 
 from testValidation import SlowTestsPlugin
 
diff --git a/vdsm.spec.in b/vdsm.spec.in
index a6bd24c..01c0c9f 100644
--- a/vdsm.spec.in
+++ b/vdsm.spec.in
@@ -40,6 +40,7 @@
 # BuildRequires needed by the tests during the build
 BuildRequires: python-ethtool
 BuildRequires: python-pthreading
+BuildRequires: python-rpyc
 BuildRequires: libselinux-python
 BuildRequires: libvirt-python
 BuildRequires: genisoimage
@@ -76,6 +77,7 @@
 Requires: rpm-python
 Requires: nfs-utils
 Requires: python-pthreading
+Requires: python-rpyc
 Requires: m2crypto
 Requires: %{name}-xmlrpc = %{version}-%{release}
 
diff --git a/vdsm/supervdsm.py b/vdsm/supervdsm.py
index 10abae0..51dea2f 100644
--- a/vdsm/supervdsm.py
+++ b/vdsm/supervdsm.py
@@ -21,43 +21,18 @@
 
 import os
 from multiprocessing import AuthenticationError
-from multiprocessing.managers import BaseManager
 import logging
 import threading
-import uuid
-from time import sleep
-from errno import ENOENT, ESRCH
-
-import storage.misc as misc
+import socket
 from vdsm import constants, utils
+import rpyc
 
 _g_singletonSupervdsmInstance = None
 _g_singletonSupervdsmInstance_lock = threading.Lock()
 
 
-def __supervdsmServerPath():
-    base = os.path.dirname(__file__)
-
-    # serverFile can be both the py or pyc file. In oVirt node we don't keep
-    # py files. this method looks for one of the two to calculate the absolute
-    # path of supervdsmServer
-    for serverFile in ("supervdsmServer.py", "supervdsmServer.pyc"):
-        serverPath = os.path.join(base, serverFile)
-        if os.path.exists(serverPath):
-            return os.path.abspath(serverPath)
-
-    raise RuntimeError("SuperVDSM Server not found")
-
-PIDFILE = os.path.join(constants.P_VDSM_RUN, "svdsm.pid")
-TIMESTAMP = os.path.join(constants.P_VDSM_RUN, "svdsm.time")
 ADDRESS = os.path.join(constants.P_VDSM_RUN, "svdsm.sock")
-SUPERVDSM = __supervdsmServerPath()
-
-extraPythonPathList = []
-
-
-class _SuperVdsmManager(BaseManager):
-    pass
+PORT = 669
 
 
 class ProxyCaller(object):
@@ -70,22 +45,12 @@
         callMethod = lambda: \
             getattr(self._supervdsmProxy._svdsm, self._funcName)(*args,
                                                                  **kwargs)
-        if not self._supervdsmProxy.isRunning():
-            # getting inside only when svdsm is down. its rare case so we
-            # don't care that isRunning will run twice
-            with self._supervdsmProxy.proxyLock:
-                if not self._supervdsmProxy.isRunning():
-                    self._supervdsmProxy.launch()
-
         try:
             return callMethod()
-        # handling internal exception that we raise to identify supervdsm
-        # validation. only this exception can cause kill!
-        except AuthenticationError:
-            with self._supervdsmProxy.proxyLock:
-                self._supervdsmProxy.kill()
-                self._supervdsmProxy.launch()
-            return callMethod()
+        except(IOError, socket.error, AuthenticationError):
+            #os.kill(os.getpid(), signal.SIGTERM)
+            print "method couldn't be called\n"
+            raise
 
 
 class SuperVdsmProxy(object):
@@ -95,106 +60,19 @@
     _log = logging.getLogger("SuperVdsmProxy")
 
     def __init__(self):
-        self.proxyLock = threading.Lock()
-        self._firstLaunch = True
-
-        # Declaration of public variables that keep files' names that svdsm
-        # uses. We need to be able to change these variables so that running
-        # tests doesn't disturb and already running VDSM on the host.
-        self.setIPCPaths(PIDFILE, TIMESTAMP, ADDRESS)
-
-    def setIPCPaths(self, pidfile, timestamp, address):
-        self.pidfile = pidfile
-        self.timestamp = timestamp
-        self.address = address
-
-    def open(self, *args, **kwargs):
-        return self._manager.open(*args, **kwargs)
-
-    def _cleanOldFiles(self):
-        self._log.debug("Cleanning svdsm old files: %s, %s, %s",
-                        self.pidfile, self.timestamp, self.address)
-        for f in (self.pidfile, self.timestamp, self.address):
-            utils.rmFile(f)
-
-    def _start(self):
-        self._authkey = str(uuid.uuid4())
-        self._log.debug("Launching Super Vdsm")
-
-        # we pass to svdsm filenames and uid. Svdsm will use those filenames
-        # to create its internal files and give to the passed uid the
-        # permissions to read those files.
-        superVdsmCmd = [constants.EXT_PYTHON, SUPERVDSM,
-                        self._authkey, str(os.getpid()),
-                        self.pidfile, self.timestamp, self.address,
-                        str(os.getuid())]
-
-        misc.execCmd(superVdsmCmd, sync=False, sudo=True)
-        sleep(2)
-
-    def kill(self):
         try:
-            with open(self.pidfile, "r") as f:
-                pid = int(f.read().strip())
-            misc.execCmd([constants.EXT_KILL, "-9", str(pid)], sudo=True)
-        except Exception:
-            self._log.error("Could not kill old Super Vdsm %s",
-                            exc_info=True)
-
-        self._cleanOldFiles()
-        self._authkey = None
-        self._manager = None
-        self._svdsm = None
-        self._firstLaunch = True
-
-    def isRunning(self):
-        if self._firstLaunch or self._svdsm is None:
-            return False
-
-        try:
-            with open(self.pidfile, "r") as f:
-                spid = f.read().strip()
-            with open(self.timestamp, "r") as f:
-                createdTime = f.read().strip()
-        except IOError as e:
-            # pid file and timestamp file must be exist after first launch,
-            # otherwise excpetion will be raised to svdsm caller
-            if e.errno == ENOENT and self._firstLaunch:
-                return False
-            else:
-                raise
-
-        try:
-            pTime = str(misc.getProcCtime(spid))
-        except OSError as e:
-            if e.errno == ESRCH:
-                # Means pid is not exist, svdsm was killed
-                return False
-            else:
-                raise
-
-        if pTime == createdTime:
-            return True
-        else:
-            return False
+            utils.retry(self._connect, Exception, tries=3, sleep=3)
+            self._log.debug("Connected to Super Vdsm")
+        except:
+            print("Could not init super vdsm proxy")
 
     def _connect(self):
-        self._manager = _SuperVdsmManager(address=self.address,
-                                          authkey=self._authkey)
-        self._manager.register('instance')
-        self._manager.register('open')
-        self._log.debug("Trying to connect to Super Vdsm")
         try:
-            self._manager.connect()
-        except Exception as ex:
-            self._log.warn("Connect to svdsm failed %s", ex)
+            self._manager = rpyc.connect('localhost', PORT)
+        except Exception, ex:
+            print("Connect failed %s", ex)
             raise
-        self._svdsm = self._manager.instance()
-
-    def launch(self):
-        self._firstLaunch = False
-        self._start()
-        utils.retry(self._connect, Exception, timeout=60)
+        self._svdsm = self._manager.root
 
     def __getattr__(self, name):
         return ProxyCaller(self, name)
diff --git a/vdsm/supervdsmServer.py b/vdsm/supervdsmServer.py
index 83a5803..1924e2a 100755
--- a/vdsm/supervdsmServer.py
+++ b/vdsm/supervdsmServer.py
@@ -24,10 +24,11 @@
 import os
 import stat
 import errno
-import threading
 import re
 from time import sleep
 import signal
+import threading
+from supervdsm import PORT
 from multiprocessing import Pipe, Process
 from gluster import cli as gcli
 import storage.misc as misc
@@ -38,13 +39,14 @@
 from lsblk import getLsBlk as _getLsBlk
 from storage.multipath import getScsiSerial as _getScsiSerial
 from storage.iscsi import forceIScsiScan as _forceIScsiScan
-from storage.iscsi import getDevIscsiInfo as _getdeviSCSIinfo
 from storage.iscsi import readSessionInfo as _readSessionInfo
-from supervdsm import _SuperVdsmManager
-from storage.fileUtils import chown, resolveGid, resolveUid
+from storage.iscsi import getDevIscsiInfo as _getdeviSCSIinfo
+from storage.fileUtils import resolveGid, resolveUid
 from storage.fileUtils import validateAccess as _validateAccess
-from vdsm.constants import METADATA_GROUP, EXT_UDEVADM, \
+from vdsm.constants import EXT_UDEVADM, \
     DISKIMAGE_USER, DISKIMAGE_GROUP, P_LIBVIRT_VMCHANNELS
+#    P_VDSM_RUN
+
 from storage.devicemapper import _removeMapping, _getPathsStatus
 import configNetwork
 from vdsm.config import config
@@ -53,12 +55,17 @@
 import mkimage
 from storage.multipath import MPATH_CONF
 import zombieReaper
+import rpyc
+from rpyc.utils.server import ThreadedServer
+
 
 _UDEV_RULE_FILE_DIR = "/etc/udev/rules.d/"
 _UDEV_RULE_FILE_PREFIX = "99-vdsm-"
 _UDEV_RULE_FILE_EXT = ".rules"
 _UDEV_RULE_FILE_NAME = _UDEV_RULE_FILE_DIR + _UDEV_RULE_FILE_PREFIX + \
     "%s-%s" + _UDEV_RULE_FILE_EXT
+#PIDFILE = P_VDSM_RUN + 'svdsm.pid'
+PIDFILE = '/tmp/file'
 
 RUN_AS_TIMEOUT = config.getint("irs", "process_pool_timeout")
 
@@ -83,19 +90,18 @@
 LOG_CONF_PATH = "/etc/vdsm/logger.conf"
 
 
-class _SuperVdsm(object):
+class _SuperVdsmService(rpyc.Service):
 
     UDEV_WITH_RELOAD_VERSION = 181
-
     log = logging.getLogger("SuperVdsm.ServerCallback")
 
     @logDecorator
-    def ping(self, *args, **kwargs):
+    def exposed_ping(self, *args, **kwargs):
         # This method exists for testing purposes
         return True
 
     @logDecorator
-    def getHardwareInfo(self, *args, **kwargs):
+    def exposed_getHardwareInfo(self, *args, **kwargs):
         if platform.machine() in ('x86_64', 'i686'):
             from dmidecodeUtil import getHardwareInfoStructure
             return getHardwareInfoStructure()
@@ -104,53 +110,56 @@
             return {}
 
     @logDecorator
-    def getDevicePartedInfo(self, *args, **kwargs):
+    def exposed_getDevicePartedInfo(self, *args, **kwargs):
         return _getDevicePartedInfo(*args, **kwargs)
 
     @logDecorator
-    def getMdDeviceUuidMap(self, *args, **kwargs):
+    def exposed_getMdDeviceUuidMap(self, *args, **kwargs):
         return _getMdDeviceUuidMap(*args, **kwargs)
 
     @logDecorator
-    def getLsBlk(self, *args, **kwargs):
+    def exposed_getLsBlk(self, *args, **kwargs):
         return _getLsBlk(*args, **kwargs)
 
     @logDecorator
-    def readMultipathConf(self):
+    def exposed_stopSuperVdsm(self):
+        SuperVdsmManager.getInstance().closeServer()
+
+    @logDecorator
+    def exposed_readMultipathConf(self):
         with open(MPATH_CONF) as f:
             return [x.strip("\n") for x in f.readlines()]
 
     @logDecorator
-    def getScsiSerial(self, *args, **kwargs):
+    def exposed_getScsiSerial(self, *args, **kwargs):
         return _getScsiSerial(*args, **kwargs)
 
     @logDecorator
-    def forceIScsiScan(self, *args, **kwargs):
+    def exposed_forceIScsiScan(self, *args, **kwargs):
         return _forceIScsiScan(*args, **kwargs)
 
     @logDecorator
-    def removeDeviceMapping(self, devName):
+    def exposed_removeDeviceMapping(self, devName):
         return _removeMapping(devName)
 
     @logDecorator
-    def getdeviSCSIinfo(self, *args, **kwargs):
+    def exposed_getdeviSCSIinfo(self, *args, **kwargs):
         return _getdeviSCSIinfo(*args, **kwargs)
 
     @logDecorator
-    def readSessionInfo(self, sessionID):
+    def exposed_readSessionInfo(self, sessionID):
         return _readSessionInfo(sessionID)
 
-    @logDecorator
-    def getPathsStatus(self):
+    def exposed_getPathsStatus(self):
         return _getPathsStatus()
 
     @logDecorator
-    def getVmPid(self, vmName):
+    def exposed_getVmPid(self, vmName):
         pidFile = "/var/run/libvirt/qemu/%s.pid" % vmName
         return open(pidFile).read()
 
     @logDecorator
-    def prepareVmChannel(self, socketFile):
+    def exposed_prepareVmChannel(self, socketFile):
         if socketFile.startswith(P_LIBVIRT_VMCHANNELS):
             mode = os.stat(socketFile).st_mode | stat.S_IWGRP
             os.chmod(socketFile, mode)
@@ -158,19 +167,19 @@
             raise Exception("Incorporate socketFile")
 
     @logDecorator
-    def addNetwork(self, bridge, options):
+    def exposed_addNetwork(self, bridge, options):
         return configNetwork.addNetwork(bridge, **options)
 
     @logDecorator
-    def delNetwork(self, bridge, options):
+    def exposed_delNetwork(self, bridge, options):
         return configNetwork.delNetwork(bridge, **options)
 
     @logDecorator
-    def editNetwork(self, oldBridge, newBridge, options):
+    def exposed_editNetwork(self, oldBridge, newBridge, options):
         return configNetwork.editNetwork(oldBridge, newBridge, **options)
 
     @logDecorator
-    def setupNetworks(self, networks={}, bondings={}, options={}):
+    def exposed_setupNetworks(self, networks={}, bondings={}, options={}):
         return configNetwork.setupNetworks(networks, bondings, **options)
 
     def _runAs(self, user, groups, func, args=(), kwargs={}):
@@ -216,17 +225,16 @@
         return res
 
     @logDecorator
-    def validateAccess(self, user, groups, *args, **kwargs):
+    def exposed_validateAccess(self, user, groups, *args, **kwargs):
         return self._runAs(user, groups, _validateAccess, args=args,
                            kwargs=kwargs)
 
     @logDecorator
-    def setSafeNetworkConfig(self):
+    def exposed_setSafeNetworkConfig(self):
         return configNetwork.setSafeNetworkConfig()
 
     @logDecorator
-    def udevTrigger(self, guid):
-        self.__udevReloadRules(guid)
+    def exposed_udevTrigger(self, guid):
         cmd = [EXT_UDEVADM, 'trigger', '--verbose', '--action',
'change',
                '--property-match=DM_NAME=%s' % guid]
         rc, out, err = misc.execCmd(cmd, sudo=False)
@@ -235,7 +243,7 @@
                           %s, out %s\nerr %s" % (guid, out, err))
 
     @logDecorator
-    def appropriateDevice(self, guid, thiefId):
+    def exposed_appropriateDevice(self, guid, thiefId):
         ruleFile = _UDEV_RULE_FILE_NAME % (guid, thiefId)
         rule = 'SYMLINK=="mapper/%s", OWNER="%s",
GROUP="%s"\n' % (guid,
                DISKIMAGE_USER, DISKIMAGE_GROUP)
@@ -243,7 +251,7 @@
             rf.write(rule)
 
     @logDecorator
-    def rmAppropriateRules(self, thiefId):
+    def exposed_rmAppropriateRules(self, thiefId):
         re_apprDevRule = "^" + _UDEV_RULE_FILE_PREFIX + ".*?-" +
thiefId + \
                          _UDEV_RULE_FILE_EXT + "$"
         rules = [os.path.join(_UDEV_RULE_FILE_DIR, r) for r in
@@ -258,7 +266,7 @@
         return fails
 
     @logDecorator
-    def ksmTune(self, tuningParams):
+    def exposed_ksmTune(self, tuningParams):
         '''
         Set KSM tuning parameters for MOM, which runs without root privilege
         when it's lauched by vdsm. So it needs supervdsm's assistance to tune
@@ -267,7 +275,7 @@
         return ksm.tune(tuningParams)
 
     @logDecorator
-    def setPortMirroring(self, networkName, ifaceName):
+    def exposed_setPortMirroring(self, networkName, ifaceName):
         '''
         Copy networkName traffic of a bridge to an interface
 
@@ -282,7 +290,7 @@
         tc.setPortMirroring(networkName, ifaceName)
 
     @logDecorator
-    def unsetPortMirroring(self, networkName, target):
+    def exposed_unsetPortMirroring(self, networkName, target):
         '''
         Release captured mirror networkName traffic from networkName bridge
 
@@ -294,15 +302,15 @@
         tc.unsetPortMirroring(networkName, target)
 
     @logDecorator
-    def mkFloppyFs(self, vmId, files):
+    def exposed_mkFloppyFs(self, vmId, files):
         return mkimage.mkFloppyFs(vmId, files)
 
     @logDecorator
-    def mkIsoFs(self, vmId, files):
+    def exposed_mkIsoFs(self, vmId, files):
         return mkimage.mkIsoFs(vmId, files)
 
     @logDecorator
-    def removeFs(self, path):
+    def exposed_removeFs(self, path):
         return mkimage.removeFs(path)
 
     def __udevReloadRules(self, guid):
@@ -332,95 +340,100 @@
         return self.__udevVersion() > self.UDEV_WITH_RELOAD_VERSION
 
 
-def __pokeParent(parentPid, address, log):
-    try:
-        while True:
-            os.kill(parentPid, 0)
-            sleep(2)
-    except Exception:
-        utils.rmFile(address)
-        log.debug("Killing SuperVdsm Process")
-        os.kill(os.getpid(), signal.SIGTERM)
-
-
-def main():
+def bound_gluster():
     def bind(func):
-        def wrapper(_SuperVdsm, *args, **kwargs):
+        def wrapper(_SuperVdsmService, *args, **kwargs):
             return func(*args, **kwargs)
         return wrapper
 
     for name in dir(gcli):
         func = getattr(gcli, name)
         if getattr(func, 'superVdsm', False):
-            setattr(_SuperVdsm,
+            setattr(_SuperVdsmService,
                     'gluster%s%s' % (name[0].upper(), name[1:]),
                     logDecorator(bind(func)))
 
-    try:
-        logging.config.fileConfig(LOG_CONF_PATH)
-    except:
-        logging.basicConfig(filename='/dev/stdout', filemode='w+',
-                            level=logging.DEBUG)
-        log = logging.getLogger("SuperVdsm.Server")
-        log.warn("Could not init proper logging", exc_info=True)
 
-    log = logging.getLogger("SuperVdsm.Server")
+class SuperVdsmManager():
 
-    try:
-        log.debug("Making sure I'm root")
-        if os.geteuid() != 0:
-            sys.exit(errno.EPERM)
+    _instance = None
+    _server = None
 
-        log.debug("Parsing cmd args")
-        authkey, parentPid, pidfile, timestamp, address, uid = sys.argv[1:]
+    @classmethod
+    def getInstance(cls):
+        if not cls._instance:
+            cls._instance = SuperVdsmManager()
+        return cls._instance
 
-        log.debug("Creating PID and TIMESTAMP files: %s, %s",
-                  pidfile, timestamp)
-        spid = os.getpid()
-        with open(pidfile, "w") as f:
-            f.write(str(spid) + "\n")
-        with open(timestamp, "w") as f:
-            f.write(str(misc.getProcCtime(spid) + "\n"))
-
-        log.debug("Cleaning old socket %s", address)
-        if os.path.exists(address):
-            os.unlink(address)
-
-        zombieReaper.registerSignalHandler()
-
-        log.debug("Setting up keep alive thread")
-
-        monThread = threading.Thread(target=__pokeParent,
-                                     args=[int(parentPid), address, log])
-        monThread.setDaemon(True)
-        monThread.start()
-
+    def __pokeParent(self, parentPid):
         try:
-            log.debug("Creating remote object manager")
-            manager = _SuperVdsmManager(address=address, authkey=authkey)
-            manager.register('instance', callable=_SuperVdsm)
+            while True:
+                os.kill(parentPid, 0)
+                sleep(2)
+        except Exception:
+            os.kill(os.getpid(), signal.SIGKILL)
 
-            server = manager.get_server()
-            servThread = threading.Thread(target=server.serve_forever)
-            servThread.setDaemon(True)
-            servThread.start()
+    def initLog(self):
+        try:
+            logging.config.fileConfig(LOG_CONF_PATH)
+        except:
+            logging.basicConfig(filename='/dev/stdout', filemode='w+',
+                                level=logging.DEBUG)
 
-            for f in (address, timestamp, pidfile):
-                chown(f, int(uid), METADATA_GROUP)
+        log = logging.getLogger("SuperVdsm.Server")
+        return log
 
-            log.debug("Started serving super vdsm object")
+    def closeServer(self):
+        if self._server is not None:
+            self._server.close()
+            self._server = None
 
-            # Python bug of thread.join() will block signal
-            # http://bugs.python.org/issue1167930
-            while servThread.isAlive():
-                servThread.join(5)
+    def initServer(self):
+        try:
+            self._server = ThreadedServer(_SuperVdsmService,
+                                          port=PORT)
+        except Exception:
+            raise
+
+    def _killSupervdsm(self):
+        try:
+            with open(PIDFILE, "r") as f:
+                pid = int(f.read().strip())
+            # TODO: should check timestamp?
+            os.kill(pid, signal.SIGKILL)
+        except:
+            pass
+
+    def restartSvdsm(self, vdsmPid):
+        bound_gluster()
+        self.log = self.initLog()
+        try:
+            self.log.debug("closing old instance of supervdsm")
+
+            # kill old supervdsm
+            self._killSupervdsm()
+
+            self.log.debug("starting supervdsm server")
+            utils.retry(self.initServer, Exception, sleep=3, tries=3)
+            self.log.debug("Creating PID file %d", os.getpid())
+
+            # TODO: need to save timestamp before cleanning the old instance
+            with open(PIDFILE, "w") as f:
+                f.write(str(os.getpid()) + "\n")
+
+            self.log.debug("Cleaning old socket")
+
+            self.log.debug("Setting up keep alive thread")
+            monThread = threading.Thread(target=self.__pokeParent,
+                                         args=[int(vdsmPid)])
+            monThread.setDaemon(True)
+            monThread.start()
+
+            self.log.debug("Started serving super vdsm object")
+            self._server.start()
+
+        except Exception:
+            self.log.error("Could not start Super Vdsm", exc_info=True)
+            sys.exit(1)
         finally:
-            if os.path.exists(address):
-                utils.rmFile(address)
-
-    except Exception:
-        log.error("Could not start Super Vdsm", exc_info=True)
-        sys.exit(1)
-
-if __name__ == '__main__':
-    main()
+            self.log.debug("super vdsm server is closed")
diff --git a/vdsm/vdsm b/vdsm/vdsm
index 4746a0f..dc9b679 100755
--- a/vdsm/vdsm
+++ b/vdsm/vdsm
@@ -10,31 +10,46 @@
 
 import os
 import sys
-import getopt
 import signal
-import getpass
-import pwd
-import grp
 import threading
 import logging
-import syslog
 from logging import config as lconfig
+from pwd import getpwnam
+from multiprocessing import Process
 
-from vdsm import constants
-import zombieReaper
-import dsaversion
 if sys.version_info[0] == 2:
     # as long as we work with Python 2, we need to monkey-patch threading
     # module before it is ever used.
     import pthreading
     pthreading.monkey_patch()
 
-loggerConfFile = constants.P_VDSM_CONF + 'logger.conf'
+from vdsm.constants import VDSM_USER, VDSM_GROUP, \
+    P_VDSM_RUN, P_VDSM_CONF
+import dsaversion
+from supervdsmServer import SuperVdsmManager
+from storage.fileUtils import chown
+from supervdsm import getProxy
+from clientIF import clientIF
+import zombieReaper
 
 
 def usage():
     print "Usage:  vdsm [OPTIONS]"
     print "     -h  - Display this help message"
+
+loggerConfFile = P_VDSM_CONF + 'logger.conf'
+LOGFILE = '/var/log/vdsm/vdsm.log'
+
+
+def vdsmExit():
+    try:
+        cif = clientIF.getInstance()
+        cif.prepareForShutdown()
+        pass
+    except:
+        pass
+    finally:
+        getProxy().stopSuperVdsm()
 
 
 def serve_clients(log):
@@ -43,6 +58,7 @@
     def sigtermHandler(signum, frame):
         if cif:
             cif.prepareForShutdown()
+        vdsmExit()
 
     def sigusr1Handler(signum, frame):
         if cif and cif.irs:
@@ -58,7 +74,7 @@
     cif.serve()
 
 
-def run():
+def initLogger():
     lconfig.fileConfig(loggerConfFile)
     logging.addLevelName(5, 'TRACE')
     logging.TRACE = 5  # impolite but helpful
@@ -78,8 +94,26 @@
     try:
         logging.root.handlers.append(logging.StreamHandler())
         log.handlers.append(logging.StreamHandler())
+        chown(LOGFILE, VDSM_USER, VDSM_GROUP)
+    except:
+        log.error("Exception raised", exc_info=True)
+    return log
 
-        pidfile = constants.P_VDSM_RUN + 'vdsmd.pid'
+
+def startVdsm(log):
+    def dropPrivileges():
+        if os.getuid() != 0:
+            log.error("should start vdsm with root")
+            sys.exit(1)
+        vdsm_uid, vdsm_gid = getpwnam(VDSM_USER)[2:4:]
+
+        os.setgroups([])
+        os.setgid(vdsm_gid)
+        os.setuid(vdsm_uid)
+
+    try:
+        dropPrivileges()
+        pidfile = P_VDSM_RUN + 'vdsmd.pid'
         file(pidfile, 'w').write(str(os.getpid()) + "\n")
         os.chmod(pidfile, 0664)
 
@@ -89,59 +123,42 @@
                  nodename, release)
         serve_clients(log)
     except:
-        log.error("Exception raised", exc_info=True)
-
-    log.info("VDSM main thread ended. Waiting for %d other threads..." %
-             (threading.activeCount() - 1))
-    for t in threading.enumerate():
-        if hasattr(t, 'stop'):
-            t.stop()
-        log.info(str(t))
+        log.error("start vdsm failed", exc_info=True)
+        raise
+    finally:
+        log.info("VDSM main thread ended. Waiting for %d other threads..."
+                 % (threading.activeCount() - 1))
+        for t in threading.enumerate():
+            if hasattr(t, 'stop'):
+                t.stop()
+            log.info(str(t))
 
 
-def parse_args():
-    opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
-    for o, v in opts:
-        o = o.lower()
-        if o == "-h" or o == "--help":
-            usage()
-            sys.exit(0)
-
-    if len(args) >= 1:
-        usage()
-        sys.exit(1)
-
-
-def __assertLogPermission():
-    if not os.access(constants.P_VDSM_LOG, os.W_OK):
-        syslog.syslog("vdsm log directory is not accessible")
-        sys.exit(1)
-
-    logfile = constants.P_VDSM_LOG + "/vdsm.log"
-    if not os.path.exists(logfile):
-        # if file not exist, and vdsm has an access to log directory- continue
-        return
-
-    if not os.access(logfile, os.W_OK):
-        syslog.syslog("error in accessing vdsm log file")
-        sys.exit(1)
-
-
-def __assertVdsmUser():
-    username = getpass.getuser()
-    if username != constants.VDSM_USER:
-        syslog.syslog("VDSM failed to start: running user is not %s, trying "
-                      "to run from user %s" % (constants.VDSM_USER, username))
-        sys.exit(1)
-    group = grp.getgrnam(constants.VDSM_GROUP)
-    if (constants.VDSM_USER not in group.gr_mem) and \
-       (pwd.getpwnam(constants.VDSM_USER).pw_gid != group.gr_gid):
-        syslog.syslog("VDSM failed to start: vdsm user is not in KVM group")
-        sys.exit(1)
+def __waitSvdsm(svdsmProc):
+    logger.info("wait for svdsm start")
+    try:
+        svdsmProc.join()
+    except Exception:
+        logger.error("error occurs when waiting svdsm")
+    finally:
+        logger.info("supervdsm exited, prepare to restart")
+        vdsmExit()
 
 if __name__ == '__main__':
-    __assertVdsmUser()
-    __assertLogPermission()
     os.setpgrp()
-    parse_args()
-    run()
+    logger = initLogger()
+    cif = None
+
+    # When key is None, connections are restricted to child processes
+    vdsmPid = os.getpid()
+    svdsmProc = Process(target=SuperVdsmManager.
+                        getInstance().restartSvdsm, args=(vdsmPid, ))
+    try:
+        svdsmProc.start()
+        waitThread = threading.Thread(target=__waitSvdsm,
+                                      args=[svdsmProc])
+        waitThread.setDaemon(True)
+        waitThread.start()
+        startVdsm(logger)
+    except:
+        vdsmExit()
diff --git a/vdsm/vdsmd.init.in b/vdsm/vdsmd.init.in
index 1f845c6..f15264c 100755
--- a/vdsm/vdsmd.init.in
+++ b/vdsm/vdsmd.init.in
@@ -499,7 +499,7 @@
 
     LIBVIRT_LOG_FILTERS=`$GETCONFITEM $CONF_FILE vars libvirt_log_filters "1:libvirt
1:remote"` \
     LIBVIRT_LOG_OUTPUTS=`$GETCONFITEM $CONF_FILE vars libvirt_log_outputs
"1:file:/var/log/vdsm/libvirt.log"` \
-    LC_ALL=C NICELEVEL=$vdsm_nice daemon --user=vdsm @VDSMDIR@/respawn --minlifetime 10
--daemon --masterpid $RESPAWNPIDFILE $VDSM_BIN
+    LC_ALL=C NICELEVEL=$vdsm_nice daemon @VDSMDIR@/respawn --minlifetime 10 --daemon
--masterpid $RESPAWNPIDFILE $VDSM_BIN
     RETVAL=$?
     [ "$RETVAL" -eq 0 ] && log_success_msg $"$prog start" ||
log_failure_msg $"$prog start"
     [ "$RETVAL" -eq 0 ] && touch /var/lock/subsys/vdsmd


--
To view, visit http://gerrit.ovirt.org/11910
To unsubscribe, visit http://gerrit.ovirt.org/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I4f1053e7d1264003fa265e44899d8b02f98bd68a
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Yaniv Bronhaim <ybronhei(a)redhat.com&gt;

    

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

Change in vdsm[master]: [WIP] Changing startup and rpc for vdsm and supervdsm