New subject: Change in vdsm[master]: net: ovs: better rollback

Thursday, 1 October 2015

Petr Horáček has uploaded a new change for review.

Change subject: net: ovs: draft of better rollback
......................................................................

net: ovs: draft of better rollback

now it's in a piggy state but working!

Change-Id: I8f6b63d03bb9579e260bfad1686047a431f69543
Signed-off-by: Petr Horáček <phoracek(a)redhat.com&gt;
---
M debian/vdsm-hook-ovs.install
M tests/functional/networkTestsOVS.py
M vdsm.spec.in
M vdsm_hooks/ovs/Makefile.am
M vdsm_hooks/ovs/README
A vdsm_hooks/ovs/ovs_after_network_setup.py
A vdsm_hooks/ovs/ovs_after_network_setup_failture.py
M vdsm_hooks/ovs/ovs_before_network_setup.py
M vdsm_hooks/ovs/ovs_utils.py
9 files changed, 238 insertions(+), 60 deletions(-)


  git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/07/46907/1

diff --git a/debian/vdsm-hook-ovs.install b/debian/vdsm-hook-ovs.install
index 36e1f0f..c7b6692 100644
--- a/debian/vdsm-hook-ovs.install
+++ b/debian/vdsm-hook-ovs.install
@@ -10,3 +10,7 @@
 usr/libexec/vdsm/hooks/before_device_create/ovs_utils.py
 usr/libexec/vdsm/hooks/before_network_setup/50_ovs
 usr/libexec/vdsm/hooks/before_network_setup/ovs_utils.py
+usr/libexec/vdsm/hooks/after_network_setup/50_ovs
+usr/libexec/vdsm/hooks/after_network_setup/ovs_utils.py
+usr/libexec/vdsm/hooks/after_network_setup_failture/50_ovs
+usr/libexec/vdsm/hooks/after_network_setup_failture/ovs_utils.py
diff --git a/tests/functional/networkTestsOVS.py b/tests/functional/networkTestsOVS.py
index b26105d..d1fe95e 100644
--- a/tests/functional/networkTestsOVS.py
+++ b/tests/functional/networkTestsOVS.py
@@ -41,6 +41,7 @@
 NetworkTest.__test__ = False
 
 BRIDGE_NAME = 'ovsbr0'
+ERR_HOOK_ERROR = 78
 
 # Tests which are not supported by OVS hook (because of OVS hook or because of
 # tests themselves). Some of these tests should be inherited and 'repaired'
diff --git a/vdsm.spec.in b/vdsm.spec.in
index fe70e24..bdbd704 100644
--- a/vdsm.spec.in
+++ b/vdsm.spec.in
@@ -1180,6 +1180,10 @@
 %{_libexecdir}/%{vdsm_name}/hooks/before_network_setup/ovs_setup_mtu.py*
 %{_libexecdir}/%{vdsm_name}/hooks/before_network_setup/ovs_setup_ovs.py*
 %{_libexecdir}/%{vdsm_name}/hooks/before_network_setup/ovs_utils.py*
+%{_libexecdir}/%{vdsm_name}/hooks/after_network_setup/50_ovs
+%{_libexecdir}/%{vdsm_name}/hooks/after_network_setup/ovs_utils.py*
+%{_libexecdir}/%{vdsm_name}/hooks/after_network_setup_failture/50_ovs
+%{_libexecdir}/%{vdsm_name}/hooks/after_network_setup_failture/ovs_utils.py*
 
 %files hook-macspoof
 %defattr(-, root, root, -)
diff --git a/vdsm_hooks/ovs/Makefile.am b/vdsm_hooks/ovs/Makefile.am
index 4699ef5..48f1820 100644
--- a/vdsm_hooks/ovs/Makefile.am
+++ b/vdsm_hooks/ovs/Makefile.am
@@ -36,6 +36,8 @@
 	ovs_before_network_setup_ovs.py \
 	ovs_before_network_setup_ip.py \
 	ovs_before_network_setup_mtu.py \
+	ovs_after_network_setup.py \
+	ovs_after_network_setup_failture.py \
 	$(utilsfile) \
 	sudoers.in
 
@@ -59,6 +61,12 @@
 		$(DESTDIR)$(vdsmhooksdir)/before_network_setup/ovs_setup_ip.py
 	$(INSTALL_SCRIPT) $(srcdir)/ovs_before_network_setup_mtu.py \
 		$(DESTDIR)$(vdsmhooksdir)/before_network_setup/ovs_setup_mtu.py
+	$(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/after_network_setup
+	$(INSTALL_SCRIPT) $(srcdir)/ovs_after_network_setup.py \
+		$(DESTDIR)$(vdsmhooksdir)/after_network_setup/50_ovs
+	$(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture
+	$(INSTALL_SCRIPT) $(srcdir)/ovs_after_network_setup_failture.py \
+		$(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture/50_ovs
 
 uninstall-local: uninstall-data-utils uninstall-data-sudoers
 	$(RM) $(DESTDIR)$(vdsmhooksdir)/after_get_caps/50_ovs
@@ -68,6 +76,8 @@
 	$(RM) $(DESTDIR)$(vdsmhooksdir)/before_network_setup/ovs_setup_ovs.py
 	$(RM) $(DESTDIR)$(vdsmhooksdir)/before_network_setup/ovs_setup_ip.py
 	$(RM) $(DESTDIR)$(vdsmhooksdir)/before_network_setup/ovs_setup_mtu.py
+	$(RM) $(DESTDIR)$(vdsmhooksdir)/after_network_setup/50_ovs
+	$(RM) $(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture/50_ovs
 
 install-data-utils:
 	$(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/after_get_caps
@@ -82,12 +92,20 @@
 	$(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/before_network_setup
 	$(INSTALL_SCRIPT) $(srcdir)/$(utilsfile) \
 		$(DESTDIR)$(vdsmhooksdir)/before_network_setup/$(utilsfile)
+	$(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/after_network_setup
+	$(INSTALL_SCRIPT) $(srcdir)/$(utilsfile) \
+		$(DESTDIR)$(vdsmhooksdir)/after_network_setup/$(utilsfile)
+	$(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture
+	$(INSTALL_SCRIPT) $(srcdir)/$(utilsfile) \
+		$(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture/$(utilsfile)
 
 uninstall-data-utils:
 	$(RM) $(DESTDIR)$(vdsmhooksdir)/after_get_caps/$(utilsfile)
 	$(RM) $(DESTDIR)$(vdsmhooksdir)/after_get_stats/$(utilsfile)
 	$(RM) $(DESTDIR)$(vdsmhooksdir)/before_device_create/$(utilsfile)
 	$(RM) $(DESTDIR)$(vdsmhooksdir)/before_network_setup/$(utilsfile)
+	$(RM) $(DESTDIR)$(vdsmhooksdir)/after_network_setup/$(utilsfile)
+	$(RM) $(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture/$(utilsfile)
 
 install-data-sudoers:
 	$(MKDIR_P) $(DESTDIR)$(sysconfdir)/sudoers.d
diff --git a/vdsm_hooks/ovs/README b/vdsm_hooks/ovs/README
index 78e4fcb..8b7bba0 100644
--- a/vdsm_hooks/ovs/README
+++ b/vdsm_hooks/ovs/README
@@ -53,6 +53,22 @@
 - All networks are bridged
 
 
+Rollback
+--------
+
+OVS rollback is a bit complex and *zaslouzisi* some explanation:
+
+1) After ovs_before_network_setup.py:configure() calls prepare_ovs() (which
+   is the last safe moment before we touch any system configuration) we save
+   initial network configuration into a temporary file.
+2) If an error occurs during setupNetworks(),
+   ovs_after_network_setup_failture.py is executed. If there is an temporary
+   file containing initial network configuration, ovs_utils.py:rollback()
+   is called.
+3) If there was no error, ovs_after_network_setup.py is executed and just
+   removes temporary initial config file.
+
+
 Backporting
 -----------
 
diff --git a/vdsm_hooks/ovs/ovs_after_network_setup.py
b/vdsm_hooks/ovs/ovs_after_network_setup.py
new file mode 100644
index 0000000..7f99db1
--- /dev/null
+++ b/vdsm_hooks/ovs/ovs_after_network_setup.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# Copyright 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+import traceback
+
+import hooking
+
+from ovs_utils import remove_init_config, log
+
+
+def main():
+    setup_nets_config = hooking.read_json()
+
+    inRollback =
setup_nets_config['request']['options'].get('_inRollback')
+    inOVSRollback = setup_nets_config['request']['options'].get(
+        '_inOVSRollback')
+
+    if inRollback and not inOVSRollback:
+        log('Non-OVS rollback is done. Leaving OVS init_config for OVS '
+            'rollback.')
+    elif inRollback and inOVSRollback:
+        log('OVS rollback is done. Removing OVS init_config backup.')
+        remove_init_config()
+    else:
+        log('Network setup was successfull. Removing OVS init_config backup.')
+        remove_init_config()
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except:
+        hooking.exit_hook(traceback.format_exc())
diff --git a/vdsm_hooks/ovs/ovs_after_network_setup_failture.py
b/vdsm_hooks/ovs/ovs_after_network_setup_failture.py
new file mode 100755
index 0000000..9935a13
--- /dev/null
+++ b/vdsm_hooks/ovs/ovs_after_network_setup_failture.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# Copyright 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+import traceback
+
+from vdsm import supervdsm
+
+import hooking
+
+from ovs_utils import log
+
+
+def main():
+    setup_nets_config = hooking.read_json()
+    in_rollback =
setup_nets_config['request']['options'].get('_inRollback',
+                                                              False)
+    if in_rollback:
+        log('Failed while trying to rollback.')
+    else:
+        log('Configuration failed. In this point, non-OVS rollback should be '
+            'done. Executing OVS rollback.')
+        supervdsm.getProxy().setupNetworks(
+            {}, {}, {'connectivityCheck': False, '_inRollback': True,
+                     '_inOVSRollback': True})
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except:
+        hooking.exit_hook(traceback.format_exc())
diff --git a/vdsm_hooks/ovs/ovs_before_network_setup.py
b/vdsm_hooks/ovs/ovs_before_network_setup.py
index 54a941e..64a52c1 100755
--- a/vdsm_hooks/ovs/ovs_before_network_setup.py
+++ b/vdsm_hooks/ovs/ovs_before_network_setup.py
@@ -17,21 +17,27 @@
 #
 # Refer to the README and COPYING files for full details of the license
 #
-from contextlib import contextmanager
 from copy import deepcopy
 import sys
 import traceback
+
+from libvirt import libvirtError
 
 from vdsm.netconfpersistence import RunningConfig
 
 from hooking import execCmd
 import hooking
 
-from ovs_utils import (is_ovs_network, is_ovs_bond, rollback, log, EXT_IP,
-                       EXT_OVS_VSCTL)
+from ovs_utils import (is_ovs_network, is_ovs_bond, load_init_config, log,
+                       save_init_config, iter_ovs_nets, suppress,
+                       destroy_ovs_bridge, EXT_IP, EXT_OVS_VSCTL)
 from ovs_setup_ovs import configure_ovs, prepare_ovs
 from ovs_setup_ip import configure_ip
 from ovs_setup_mtu import configure_mtu
+
+# TODO: move required modules into vdsm/lib
+sys.path.append('/usr/share/vdsm')
+from network.configurators import libvirt
 
 
 def _separate_ovs_nets_bonds(nets, bonds, running_config):
@@ -59,29 +65,50 @@
     return ovs_nets, non_ovs_nets, ovs_bonds, non_ovs_bonds
 
 
-@contextmanager
-def _rollback(running_config, initial_config, in_rollback):
-    try:
-        yield
-    except:
-        if in_rollback:
-            log('Failed while trying to rollback:')
-        else:
-            log('Configuration failed. Entering rollback.')
-            rollback(running_config, initial_config)
-            log('Rollback finished. Initial error:')
-        raise
+def _remove_ovs_nets(initial_config, running_config):
+    log('Remove OVS networks: %s %s' % (initial_config, running_config))
+    for libvirt_ovs_nets in (iter_ovs_nets(running_config.networks),
+                             iter_ovs_nets(initial_config.networks)):
+        for net, attrs in libvirt_ovs_nets:
+            with suppress(libvirtError):  # network not found
+                libvirt.removeNetwork(net)
+
+    destroy_ovs_bridge()
+    for net, attrs in running_config.networks.items():
+        if is_ovs_network(attrs):
+            running_config.networks.pop(net)
+    for bond, attrs in running_config.bonds.items():
+        if is_ovs_bond(attrs):
+            running_config.bonds.pop(bond)
+    running_config.save()
 
 
-def configure(nets, bonds, running_config, in_rollback):
+def _rollback(running_config):
+    initial_config = load_init_config()
+    if initial_config is None:
+        log('No needed OVS changes to be done.')
+    else:
+        log('Remove OVS networks.')
+        _remove_ovs_nets(initial_config, running_config)
+        log('Reconfigure OVS networks according to initial_config.')
+        _configure(initial_config.networks, initial_config.bonds,
+                   running_config, save_init=False)
+
+
+def _configure(nets, bonds, running_config, save_init=True):
     initial_config = deepcopy(running_config)
 
     commands, libvirt_create, libvirt_remove = prepare_ovs(
         nets, bonds, running_config)
-    with _rollback(running_config, initial_config, in_rollback):
-        configure_ovs(commands, libvirt_create, libvirt_remove, running_config)
-        configure_mtu(running_config)
-        configure_ip(nets, initial_config.networks)
+
+    if save_init:
+        log('Saving initial configuration for optional rollback: %s' %
+            initial_config)
+        save_init_config(initial_config)
+
+    configure_ovs(commands, libvirt_create, libvirt_remove, running_config)
+    configure_mtu(running_config)
+    configure_ip(nets, initial_config.networks)
 
     log('Saving running configuration: %s %s' % (running_config.networks,
                                                  running_config.bonds))
@@ -95,16 +122,29 @@
     running_config = RunningConfig()
     networks = setup_nets_config['request']['networks']
     bondings = setup_nets_config['request']['bondings']
-    inRollback =
setup_nets_config['request']['options'].get('_inRollback',
-                                                             False)
-    ovs_nets, non_ovs_nets, ovs_bonds, non_ovs_bonds = \
-        _separate_ovs_nets_bonds(networks, bondings, running_config)
-    configure(ovs_nets, ovs_bonds, running_config, inRollback)
 
-    setup_nets_config['request']['bondings'] = non_ovs_bonds
-    setup_nets_config['request']['networks'] = non_ovs_nets
-    log('Hook finished, returning non-OVS networks and bondings back '
-                'to VDSM: %s' % setup_nets_config)
+    inRollback =
setup_nets_config['request']['options'].get('_inRollback')
+    inOVSRollback = setup_nets_config['request']['options'].get(
+        '_inOVSRollback')
+
+    if inRollback and not inOVSRollback:
+        log('Non-OVS rollback is to be done. Returning nets_config unchanged')
+    elif inOVSRollback:
+        log('OVS rollback is to be done.')
+        _rollback(running_config)
+        setup_nets_config['request']['bondings'] = {}
+        setup_nets_config['request']['networks'] = {}
+        log('OVS rollback finished, returning empty networks and bondings '
+            'configuration back to VDSM.')
+    else:
+        ovs_nets, non_ovs_nets, ovs_bonds, non_ovs_bonds = \
+            _separate_ovs_nets_bonds(networks, bondings, running_config)
+        _configure(ovs_nets, ovs_bonds, running_config)
+        setup_nets_config['request']['bondings'] = non_ovs_bonds
+        setup_nets_config['request']['networks'] = non_ovs_nets
+        log('Hook finished, returning non-OVS networks and bondings back to '
+            'VDSM: %s' % setup_nets_config)
+
     hooking.write_json(setup_nets_config)
 
 
diff --git a/vdsm_hooks/ovs/ovs_utils.py b/vdsm_hooks/ovs/ovs_utils.py
index 7375d20..d592b8f 100644
--- a/vdsm_hooks/ovs/ovs_utils.py
+++ b/vdsm_hooks/ovs/ovs_utils.py
@@ -18,19 +18,14 @@
 # Refer to the README and COPYING files for full details of the license
 #
 from contextlib import contextmanager
-import sys
-
-from libvirt import libvirtError
+import errno
+import os
+import pickle
 
 from hooking import execCmd
 import hooking
 
 from vdsm.utils import CommandPath
-from vdsm import supervdsm
-
-# TODO: move required modules into vdsm/lib
-sys.path.append('/usr/share/vdsm')
-from network.configurators import libvirt
 
 EXT_IP = CommandPath('ip', '/sbin/ip').cmd
 EXT_OVS_VSCTL = CommandPath('ovs-vsctl',
@@ -40,6 +35,8 @@
                              '/usr/sbin/ovs-appctl',
                              '/usr/bin/ovs-appctl').cmd
 BRIDGE_NAME = 'ovsbr0'
+
+INIT_CONFIG_FILE = '/tmp/ovs_init_config'  # TODO: VDSM tmp folder
 
 
 def rget(dict, keys, default=None):
@@ -123,28 +120,30 @@
         raise Exception('\n'.join(err))
 
 
-def rollback(running_config, initial_config):
-    diff = running_config.diffFrom(initial_config)
-    if diff:
-        for libvirt_ovs_nets in (iter_ovs_nets(running_config.networks),
-                                 iter_ovs_nets(initial_config.networks)):
-            for net, attrs in libvirt_ovs_nets:
-                with suppress(libvirtError):  # network not found
-                    libvirt.removeNetwork(net)
-
-        destroy_ovs_bridge()
-        for net, attrs in running_config.networks.items():
-            if is_ovs_network(attrs):
-                running_config.networks.pop(net)
-        for bond, attrs in running_config.bonds.items():
-            if is_ovs_bond(attrs):
-                running_config.bonds.pop(bond)
-        running_config.save()
-
-        supervdsm.getProxy().setupNetworks(
-            initial_config.networks, initial_config.bonds,
-            {'connectivityCheck': False, '_inRollback': True})
-
-
 def log(message):
     hooking.log('OVS: %s' % message)
+
+
+def load_init_config():
+    try:
+        with open(INIT_CONFIG_FILE) as f:
+            init_config = pickle.load(f)
+    except IOError as e:
+        if e.errno != errno.ENOENT:
+            raise
+        return None
+    else:
+        return init_config
+
+
+def save_init_config(init_config):
+    with open(INIT_CONFIG_FILE, 'w') as f:
+        pickle.dump(init_config, f)
+
+
+def remove_init_config():
+    try:
+        os.remove(INIT_CONFIG_FILE)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise


-- 
To view, visit https://gerrit.ovirt.org/46907
To unsubscribe, visit https://gerrit.ovirt.org/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I8f6b63d03bb9579e260bfad1686047a431f69543
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Petr Horáček <phoracek(a)redhat.com&gt;

    

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

Change in vdsm[master]: net: ovs: draft of better rollback