Petr Horáček has uploaded a new change for review.
Change subject: net: ovs: draft of better rollback ......................................................................
net: ovs: draft of better rollback
now it's in a piggy state but working!
Change-Id: I8f6b63d03bb9579e260bfad1686047a431f69543 Signed-off-by: Petr Horáček phoracek@redhat.com --- M debian/vdsm-hook-ovs.install M tests/functional/networkTestsOVS.py M vdsm.spec.in M vdsm_hooks/ovs/Makefile.am M vdsm_hooks/ovs/README A vdsm_hooks/ovs/ovs_after_network_setup.py A vdsm_hooks/ovs/ovs_after_network_setup_failture.py M vdsm_hooks/ovs/ovs_before_network_setup.py M vdsm_hooks/ovs/ovs_utils.py 9 files changed, 238 insertions(+), 60 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/07/46907/1
diff --git a/debian/vdsm-hook-ovs.install b/debian/vdsm-hook-ovs.install index 36e1f0f..c7b6692 100644 --- a/debian/vdsm-hook-ovs.install +++ b/debian/vdsm-hook-ovs.install @@ -10,3 +10,7 @@ usr/libexec/vdsm/hooks/before_device_create/ovs_utils.py usr/libexec/vdsm/hooks/before_network_setup/50_ovs usr/libexec/vdsm/hooks/before_network_setup/ovs_utils.py +usr/libexec/vdsm/hooks/after_network_setup/50_ovs +usr/libexec/vdsm/hooks/after_network_setup/ovs_utils.py +usr/libexec/vdsm/hooks/after_network_setup_failture/50_ovs +usr/libexec/vdsm/hooks/after_network_setup_failture/ovs_utils.py diff --git a/tests/functional/networkTestsOVS.py b/tests/functional/networkTestsOVS.py index b26105d..d1fe95e 100644 --- a/tests/functional/networkTestsOVS.py +++ b/tests/functional/networkTestsOVS.py @@ -41,6 +41,7 @@ NetworkTest.__test__ = False
BRIDGE_NAME = 'ovsbr0' +ERR_HOOK_ERROR = 78
# Tests which are not supported by OVS hook (because of OVS hook or because of # tests themselves). Some of these tests should be inherited and 'repaired' diff --git a/vdsm.spec.in b/vdsm.spec.in index fe70e24..bdbd704 100644 --- a/vdsm.spec.in +++ b/vdsm.spec.in @@ -1180,6 +1180,10 @@ %{_libexecdir}/%{vdsm_name}/hooks/before_network_setup/ovs_setup_mtu.py* %{_libexecdir}/%{vdsm_name}/hooks/before_network_setup/ovs_setup_ovs.py* %{_libexecdir}/%{vdsm_name}/hooks/before_network_setup/ovs_utils.py* +%{_libexecdir}/%{vdsm_name}/hooks/after_network_setup/50_ovs +%{_libexecdir}/%{vdsm_name}/hooks/after_network_setup/ovs_utils.py* +%{_libexecdir}/%{vdsm_name}/hooks/after_network_setup_failture/50_ovs +%{_libexecdir}/%{vdsm_name}/hooks/after_network_setup_failture/ovs_utils.py*
%files hook-macspoof %defattr(-, root, root, -) diff --git a/vdsm_hooks/ovs/Makefile.am b/vdsm_hooks/ovs/Makefile.am index 4699ef5..48f1820 100644 --- a/vdsm_hooks/ovs/Makefile.am +++ b/vdsm_hooks/ovs/Makefile.am @@ -36,6 +36,8 @@ ovs_before_network_setup_ovs.py \ ovs_before_network_setup_ip.py \ ovs_before_network_setup_mtu.py \ + ovs_after_network_setup.py \ + ovs_after_network_setup_failture.py \ $(utilsfile) \ sudoers.in
@@ -59,6 +61,12 @@ $(DESTDIR)$(vdsmhooksdir)/before_network_setup/ovs_setup_ip.py $(INSTALL_SCRIPT) $(srcdir)/ovs_before_network_setup_mtu.py \ $(DESTDIR)$(vdsmhooksdir)/before_network_setup/ovs_setup_mtu.py + $(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/after_network_setup + $(INSTALL_SCRIPT) $(srcdir)/ovs_after_network_setup.py \ + $(DESTDIR)$(vdsmhooksdir)/after_network_setup/50_ovs + $(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture + $(INSTALL_SCRIPT) $(srcdir)/ovs_after_network_setup_failture.py \ + $(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture/50_ovs
uninstall-local: uninstall-data-utils uninstall-data-sudoers $(RM) $(DESTDIR)$(vdsmhooksdir)/after_get_caps/50_ovs @@ -68,6 +76,8 @@ $(RM) $(DESTDIR)$(vdsmhooksdir)/before_network_setup/ovs_setup_ovs.py $(RM) $(DESTDIR)$(vdsmhooksdir)/before_network_setup/ovs_setup_ip.py $(RM) $(DESTDIR)$(vdsmhooksdir)/before_network_setup/ovs_setup_mtu.py + $(RM) $(DESTDIR)$(vdsmhooksdir)/after_network_setup/50_ovs + $(RM) $(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture/50_ovs
install-data-utils: $(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/after_get_caps @@ -82,12 +92,20 @@ $(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/before_network_setup $(INSTALL_SCRIPT) $(srcdir)/$(utilsfile) \ $(DESTDIR)$(vdsmhooksdir)/before_network_setup/$(utilsfile) + $(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/after_network_setup + $(INSTALL_SCRIPT) $(srcdir)/$(utilsfile) \ + $(DESTDIR)$(vdsmhooksdir)/after_network_setup/$(utilsfile) + $(MKDIR_P) $(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture + $(INSTALL_SCRIPT) $(srcdir)/$(utilsfile) \ + $(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture/$(utilsfile)
uninstall-data-utils: $(RM) $(DESTDIR)$(vdsmhooksdir)/after_get_caps/$(utilsfile) $(RM) $(DESTDIR)$(vdsmhooksdir)/after_get_stats/$(utilsfile) $(RM) $(DESTDIR)$(vdsmhooksdir)/before_device_create/$(utilsfile) $(RM) $(DESTDIR)$(vdsmhooksdir)/before_network_setup/$(utilsfile) + $(RM) $(DESTDIR)$(vdsmhooksdir)/after_network_setup/$(utilsfile) + $(RM) $(DESTDIR)$(vdsmhooksdir)/after_network_setup_failture/$(utilsfile)
install-data-sudoers: $(MKDIR_P) $(DESTDIR)$(sysconfdir)/sudoers.d diff --git a/vdsm_hooks/ovs/README b/vdsm_hooks/ovs/README index 78e4fcb..8b7bba0 100644 --- a/vdsm_hooks/ovs/README +++ b/vdsm_hooks/ovs/README @@ -53,6 +53,22 @@ - All networks are bridged
+Rollback +-------- + +OVS rollback is a bit complex and *zaslouzisi* some explanation: + +1) After ovs_before_network_setup.py:configure() calls prepare_ovs() (which + is the last safe moment before we touch any system configuration) we save + initial network configuration into a temporary file. +2) If an error occurs during setupNetworks(), + ovs_after_network_setup_failture.py is executed. If there is an temporary + file containing initial network configuration, ovs_utils.py:rollback() + is called. +3) If there was no error, ovs_after_network_setup.py is executed and just + removes temporary initial config file. + + Backporting -----------
diff --git a/vdsm_hooks/ovs/ovs_after_network_setup.py b/vdsm_hooks/ovs/ovs_after_network_setup.py new file mode 100644 index 0000000..7f99db1 --- /dev/null +++ b/vdsm_hooks/ovs/ovs_after_network_setup.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# Copyright 2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Refer to the README and COPYING files for full details of the license +# +import traceback + +import hooking + +from ovs_utils import remove_init_config, log + + +def main(): + setup_nets_config = hooking.read_json() + + inRollback = setup_nets_config['request']['options'].get('_inRollback') + inOVSRollback = setup_nets_config['request']['options'].get( + '_inOVSRollback') + + if inRollback and not inOVSRollback: + log('Non-OVS rollback is done. Leaving OVS init_config for OVS ' + 'rollback.') + elif inRollback and inOVSRollback: + log('OVS rollback is done. Removing OVS init_config backup.') + remove_init_config() + else: + log('Network setup was successfull. Removing OVS init_config backup.') + remove_init_config() + + +if __name__ == '__main__': + try: + main() + except: + hooking.exit_hook(traceback.format_exc()) diff --git a/vdsm_hooks/ovs/ovs_after_network_setup_failture.py b/vdsm_hooks/ovs/ovs_after_network_setup_failture.py new file mode 100755 index 0000000..9935a13 --- /dev/null +++ b/vdsm_hooks/ovs/ovs_after_network_setup_failture.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# Copyright 2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Refer to the README and COPYING files for full details of the license +# +import traceback + +from vdsm import supervdsm + +import hooking + +from ovs_utils import log + + +def main(): + setup_nets_config = hooking.read_json() + in_rollback = setup_nets_config['request']['options'].get('_inRollback', + False) + if in_rollback: + log('Failed while trying to rollback.') + else: + log('Configuration failed. In this point, non-OVS rollback should be ' + 'done. Executing OVS rollback.') + supervdsm.getProxy().setupNetworks( + {}, {}, {'connectivityCheck': False, '_inRollback': True, + '_inOVSRollback': True}) + + +if __name__ == '__main__': + try: + main() + except: + hooking.exit_hook(traceback.format_exc()) diff --git a/vdsm_hooks/ovs/ovs_before_network_setup.py b/vdsm_hooks/ovs/ovs_before_network_setup.py index 54a941e..64a52c1 100755 --- a/vdsm_hooks/ovs/ovs_before_network_setup.py +++ b/vdsm_hooks/ovs/ovs_before_network_setup.py @@ -17,21 +17,27 @@ # # Refer to the README and COPYING files for full details of the license # -from contextlib import contextmanager from copy import deepcopy import sys import traceback + +from libvirt import libvirtError
from vdsm.netconfpersistence import RunningConfig
from hooking import execCmd import hooking
-from ovs_utils import (is_ovs_network, is_ovs_bond, rollback, log, EXT_IP, - EXT_OVS_VSCTL) +from ovs_utils import (is_ovs_network, is_ovs_bond, load_init_config, log, + save_init_config, iter_ovs_nets, suppress, + destroy_ovs_bridge, EXT_IP, EXT_OVS_VSCTL) from ovs_setup_ovs import configure_ovs, prepare_ovs from ovs_setup_ip import configure_ip from ovs_setup_mtu import configure_mtu + +# TODO: move required modules into vdsm/lib +sys.path.append('/usr/share/vdsm') +from network.configurators import libvirt
def _separate_ovs_nets_bonds(nets, bonds, running_config): @@ -59,29 +65,50 @@ return ovs_nets, non_ovs_nets, ovs_bonds, non_ovs_bonds
-@contextmanager -def _rollback(running_config, initial_config, in_rollback): - try: - yield - except: - if in_rollback: - log('Failed while trying to rollback:') - else: - log('Configuration failed. Entering rollback.') - rollback(running_config, initial_config) - log('Rollback finished. Initial error:') - raise +def _remove_ovs_nets(initial_config, running_config): + log('Remove OVS networks: %s %s' % (initial_config, running_config)) + for libvirt_ovs_nets in (iter_ovs_nets(running_config.networks), + iter_ovs_nets(initial_config.networks)): + for net, attrs in libvirt_ovs_nets: + with suppress(libvirtError): # network not found + libvirt.removeNetwork(net) + + destroy_ovs_bridge() + for net, attrs in running_config.networks.items(): + if is_ovs_network(attrs): + running_config.networks.pop(net) + for bond, attrs in running_config.bonds.items(): + if is_ovs_bond(attrs): + running_config.bonds.pop(bond) + running_config.save()
-def configure(nets, bonds, running_config, in_rollback): +def _rollback(running_config): + initial_config = load_init_config() + if initial_config is None: + log('No needed OVS changes to be done.') + else: + log('Remove OVS networks.') + _remove_ovs_nets(initial_config, running_config) + log('Reconfigure OVS networks according to initial_config.') + _configure(initial_config.networks, initial_config.bonds, + running_config, save_init=False) + + +def _configure(nets, bonds, running_config, save_init=True): initial_config = deepcopy(running_config)
commands, libvirt_create, libvirt_remove = prepare_ovs( nets, bonds, running_config) - with _rollback(running_config, initial_config, in_rollback): - configure_ovs(commands, libvirt_create, libvirt_remove, running_config) - configure_mtu(running_config) - configure_ip(nets, initial_config.networks) + + if save_init: + log('Saving initial configuration for optional rollback: %s' % + initial_config) + save_init_config(initial_config) + + configure_ovs(commands, libvirt_create, libvirt_remove, running_config) + configure_mtu(running_config) + configure_ip(nets, initial_config.networks)
log('Saving running configuration: %s %s' % (running_config.networks, running_config.bonds)) @@ -95,16 +122,29 @@ running_config = RunningConfig() networks = setup_nets_config['request']['networks'] bondings = setup_nets_config['request']['bondings'] - inRollback = setup_nets_config['request']['options'].get('_inRollback', - False) - ovs_nets, non_ovs_nets, ovs_bonds, non_ovs_bonds = \ - _separate_ovs_nets_bonds(networks, bondings, running_config) - configure(ovs_nets, ovs_bonds, running_config, inRollback)
- setup_nets_config['request']['bondings'] = non_ovs_bonds - setup_nets_config['request']['networks'] = non_ovs_nets - log('Hook finished, returning non-OVS networks and bondings back ' - 'to VDSM: %s' % setup_nets_config) + inRollback = setup_nets_config['request']['options'].get('_inRollback') + inOVSRollback = setup_nets_config['request']['options'].get( + '_inOVSRollback') + + if inRollback and not inOVSRollback: + log('Non-OVS rollback is to be done. Returning nets_config unchanged') + elif inOVSRollback: + log('OVS rollback is to be done.') + _rollback(running_config) + setup_nets_config['request']['bondings'] = {} + setup_nets_config['request']['networks'] = {} + log('OVS rollback finished, returning empty networks and bondings ' + 'configuration back to VDSM.') + else: + ovs_nets, non_ovs_nets, ovs_bonds, non_ovs_bonds = \ + _separate_ovs_nets_bonds(networks, bondings, running_config) + _configure(ovs_nets, ovs_bonds, running_config) + setup_nets_config['request']['bondings'] = non_ovs_bonds + setup_nets_config['request']['networks'] = non_ovs_nets + log('Hook finished, returning non-OVS networks and bondings back to ' + 'VDSM: %s' % setup_nets_config) + hooking.write_json(setup_nets_config)
diff --git a/vdsm_hooks/ovs/ovs_utils.py b/vdsm_hooks/ovs/ovs_utils.py index 7375d20..d592b8f 100644 --- a/vdsm_hooks/ovs/ovs_utils.py +++ b/vdsm_hooks/ovs/ovs_utils.py @@ -18,19 +18,14 @@ # Refer to the README and COPYING files for full details of the license # from contextlib import contextmanager -import sys - -from libvirt import libvirtError +import errno +import os +import pickle
from hooking import execCmd import hooking
from vdsm.utils import CommandPath -from vdsm import supervdsm - -# TODO: move required modules into vdsm/lib -sys.path.append('/usr/share/vdsm') -from network.configurators import libvirt
EXT_IP = CommandPath('ip', '/sbin/ip').cmd EXT_OVS_VSCTL = CommandPath('ovs-vsctl', @@ -40,6 +35,8 @@ '/usr/sbin/ovs-appctl', '/usr/bin/ovs-appctl').cmd BRIDGE_NAME = 'ovsbr0' + +INIT_CONFIG_FILE = '/tmp/ovs_init_config' # TODO: VDSM tmp folder
def rget(dict, keys, default=None): @@ -123,28 +120,30 @@ raise Exception('\n'.join(err))
-def rollback(running_config, initial_config): - diff = running_config.diffFrom(initial_config) - if diff: - for libvirt_ovs_nets in (iter_ovs_nets(running_config.networks), - iter_ovs_nets(initial_config.networks)): - for net, attrs in libvirt_ovs_nets: - with suppress(libvirtError): # network not found - libvirt.removeNetwork(net) - - destroy_ovs_bridge() - for net, attrs in running_config.networks.items(): - if is_ovs_network(attrs): - running_config.networks.pop(net) - for bond, attrs in running_config.bonds.items(): - if is_ovs_bond(attrs): - running_config.bonds.pop(bond) - running_config.save() - - supervdsm.getProxy().setupNetworks( - initial_config.networks, initial_config.bonds, - {'connectivityCheck': False, '_inRollback': True}) - - def log(message): hooking.log('OVS: %s' % message) + + +def load_init_config(): + try: + with open(INIT_CONFIG_FILE) as f: + init_config = pickle.load(f) + except IOError as e: + if e.errno != errno.ENOENT: + raise + return None + else: + return init_config + + +def save_init_config(init_config): + with open(INIT_CONFIG_FILE, 'w') as f: + pickle.dump(init_config, f) + + +def remove_init_config(): + try: + os.remove(INIT_CONFIG_FILE) + except OSError as e: + if e.errno != errno.ENOENT: + raise