Gitweb: http://git.fedorahosted.org/git/?p=fence-agents.git;a=commitdiff;h=01c534d3e... Commit: 01c534d3ecfb794ef7f3b5ea58478740f33c83fb Parent: d75b0ced9c203782a444bf51f4926d3c75b0fb5b Author: Marek 'marx' Grac mgrac@redhat.com AuthorDate: Wed Nov 19 10:51:57 2014 +0100 Committer: Marek 'marx' Grac mgrac@redhat.com CommitterDate: Wed Nov 19 10:54:01 2014 +0100
fence_mpath: new fence agent for dm-multipath based on mpathpersist
Previously, scenario with multipath and underlying SCSI devices was solved by using fence_scsi what works correctly but there are some limitation. The most important is that unfencing has to be done when all paths are available as it is executed only once. This new fence agent solve this situation properly as most of this situations are solved by mpathpersist which is part of dm-multipath. --- configure.ac | 4 + fence/agents/mpath/Makefile.am | 17 +++ fence/agents/mpath/fence_mpath.py | 244 +++++++++++++++++++++++++++++++++++ make/fencebuild.mk | 3 + tests/data/metadata/fence_mpath.xml | 86 ++++++++++++ 5 files changed, 354 insertions(+), 0 deletions(-)
diff --git a/configure.ac b/configure.ac index 530d64f..3ce4b06 100644 --- a/configure.ac +++ b/configure.ac @@ -175,6 +175,9 @@ AC_PATH_PROG([VGS_PATH], [vgs], [/usr/sbin/vgs]) AC_PATH_PROG([SUDO_PATH], [sudo], [/usr/bin/sudo]) AC_PATH_PROG([SSH_PATH], [ssh], [/usr/bin/ssh]) AC_PATH_PROG([TELNET_PATH], [telnet], [/usr/bin/telnet]) +AC_PATH_PROG([MPATH_PATH], [mpathpersist], [/usr/sbin/mpathpersist]) +AC_PATH_PROG([SUDO_PATH], [sudo], [/usr/bin/sudo]) + ## do subst
AC_SUBST([DEFAULT_CONFIG_DIR]) @@ -295,6 +298,7 @@ AC_CONFIG_FILES([Makefile fence/agents/lib/Makefile fence/agents/lpar/Makefile fence/agents/manual/Makefile + fence/agents/mpath/Makefile fence/agents/netio/Makefile fence/agents/ovh/Makefile fence/agents/pve/Makefile diff --git a/fence/agents/mpath/Makefile.am b/fence/agents/mpath/Makefile.am new file mode 100644 index 0000000..fd3d8d2 --- /dev/null +++ b/fence/agents/mpath/Makefile.am @@ -0,0 +1,17 @@ +MAINTAINERCLEANFILES = Makefile.in + +TARGET = fence_mpath + +SRC = $(TARGET).py + +EXTRA_DIST = $(SRC) + +sbin_SCRIPTS = $(TARGET) + +man_MANS = $(TARGET).8 + +FENCE_TEST_ARGS = -k 1 + +include $(top_srcdir)/make/fencebuild.mk +include $(top_srcdir)/make/fenceman.mk +include $(top_srcdir)/make/agentpycheck.mk diff --git a/fence/agents/mpath/fence_mpath.py b/fence/agents/mpath/fence_mpath.py new file mode 100644 index 0000000..8a4811c --- /dev/null +++ b/fence/agents/mpath/fence_mpath.py @@ -0,0 +1,244 @@ +#!/usr/bin/python -tt + +import sys +import stat +import re +import os +import logging +import atexit +sys.path.append("@FENCEAGENTSLIBDIR@") +from fencing import fail_usage, run_command, atexit_handler, check_input, process_input, show_docs +from fencing import fence_action, all_opt, run_delay + +#BEGIN_VERSION_GENERATION +RELEASE_VERSION="" +REDHAT_COPYRIGHT="" +BUILD_DATE="" +#END_VERSION_GENERATION + +def get_status(conn, options): + del conn + status = "off" + for dev in options["devices"]: + is_block_device(dev) + if options["--key"] in get_registration_keys(options, dev): + status = "on" + else: + logging.debug("No registration for key "\ + + options["--key"] + " on device " + dev + "\n") + return status + + +def set_status(conn, options): + del conn + count = 0 + if options["--action"] == "on": + for dev in options["devices"]: + is_block_device(dev) + + register_dev(options, dev) + if options["--key"] not in get_registration_keys(options, dev): + count += 1 + logging.debug("Failed to register key "\ + + options["--key"] + "on device " + dev + "\n") + continue + dev_write(options, dev) + + if get_reservation_key(options, dev) is None \ + and not reserve_dev(options, dev) \ + and get_reservation_key(options, dev) is None: + count += 1 + logging.debug("Failed to create reservation (key="\ + + options["--key"] + ", device=" + dev + ")\n") + + else: + dev_keys = dev_read(options) + + for dev in options["devices"]: + is_block_device(dev) + + if options["--key"] in get_registration_keys(options, dev): + preempt_abort(options, dev_keys[dev], dev) + + for dev in options["devices"]: + if options["--key"] in get_registration_keys(options, dev): + count += 1 + logging.debug("Failed to remove key "\ + + options["--key"] + " on device " + dev + "\n") + continue + + if not get_reservation_key(options, dev): + count += 1 + logging.debug("No reservation exists on device " + dev + "\n") + if count: + logging.error("Failed to verify " + str(count) + " device(s)") + sys.exit(1) + + +#run command, returns dict, ret["err"] = exit code; ret["out"] = output +def run_cmd(options, cmd): + ret = {} + + if options.has_key("--use-sudo"): + prefix = options["--sudo-path"] + " " + else: + prefix = "" + + (ret["err"], ret["out"], _) = run_command(options, prefix + cmd) + ret["out"] = "".join([i for i in ret["out"] if i is not None]) + return ret + + +# check if device exist and is block device +def is_block_device(dev): + if not os.path.exists(dev): + fail_usage("Failed: device "" + dev + "" does not exist") + if not stat.S_ISBLK(os.stat(dev).st_mode): + fail_usage("Failed: device "" + dev + "" is not a block device") + +# cancel registration +def preempt_abort(options, host, dev): + cmd = options["--mpathpersist-path"] + " -o --preempt-abort --prout-type=5 --param-rk=" + host +" --param-sark=" + options["--key"] +"-d " + dev + return not bool(run_cmd(options, cmd)["err"]) + +def register_dev(options, dev): + cmd = options["--mpathpersist-path"] + " -o --register --param-sark=" + options["--key"] + " -d " + dev + #cmd return code != 0 but registration can be successful + return not bool(run_cmd(options, cmd)["err"]) + +def reserve_dev(options, dev): + cmd = options["--mpathpersist-path"] + " -o --reserv --prout-type=5 --param-rk=" + options["--key"] + " -d " + dev + return not bool(run_cmd(options, cmd)["err"]) + +def get_reservation_key(options, dev): + cmd = options["--mpathpersist-path"] + " -i -r -d " + dev + out = run_cmd(options, cmd) + if out["err"]: + fail_usage("Cannot get reservation key") + match = re.search(r"\s+key\s*=\s*0x(\S+)\s+", out["out"], re.IGNORECASE) + return match.group(1) if match else None + +def get_registration_keys(options, dev): + keys = [] + cmd = options["--mpathpersist-path"] + " -i -k -d " + dev + out = run_cmd(options, cmd) + if out["err"]: + fail_usage("Cannot get registration keys") + for line in out["out"].split("\n"): + match = re.search(r"\s+0x(\S+)\s*", line) + if match: + keys.append(match.group(1)) + return keys + +def dev_write(options, dev): + file_path = options["--store-path"] + "/mpath.devices" + + if not os.path.isdir(os.path.dirname(options["--store-path"])): + os.makedirs(os.path.dirname(options["--store-path"])) + + try: + store_fh = open(file_path, "a+") + except IOError: + fail_usage("Failed: Cannot open file ""+ file_path + """) + out = store_fh.read() + if not re.search(r"^" + dev + r"\s+", out): + store_fh.write(dev + "\t" + options["--key"] + "\n") + store_fh.close() + +def dev_read(options): + dev_key = {} + file_path = options["--store-path"] + "/mpath.devices" + try: + store_fh = open(file_path, "r") + except IOError: + fail_usage("Failed: Cannot open file "" + file_path + """) + # get not empty lines from file + for (device, key) in [line.strip().split() for line in store_fh if line.strip()]: + dev_key[device] = key + store_fh.close() + return dev_key + +def define_new_opts(): + all_opt["devices"] = { + "getopt" : "d:", + "longopt" : "devices", + "help" : "-d, --devices=[devices] List of devices to use for current operation", + "required" : "0", + "shortdesc" : "List of devices to use for current operation. Devices can \ +be comma-separated list of device-mapper multipath devices (eg. /dev/dm-3). \ +Each device must support SCSI-3 persistent reservations.", + "order": 1 + } + all_opt["key"] = { + "getopt" : "k:", + "longopt" : "key", + "help" : "-k, --key=[key] Key to use for the current operation", + "required" : "1", + "shortdesc" : "Key to use for the current operation. This key should be \ +unique to a node and have to be written in /etc/multipath.conf. For the "on" action, the key specifies the key use to \ +register the local node. For the "off" action, this key specifies the key to \ +be removed from the device(s).", + "order": 1 + } + all_opt["mpathpersist_path"] = { + "getopt" : "X:", + "longopt" : "mpathpersist-path", + "help" : "--mpathpersist-path=[path] Path to mpathpersist binary", + "required" : "0", + "shortdesc" : "Path to mpathpersist binary", + "default" : "@MPATH_PATH@", + "order": 200 + } + all_opt["store_path"] = { + "getopt" : "X:", + "longopt" : "store-path", + "help" : "--store-path=[path] Path to directory containing cached keys", + "required" : "0", + "shortdesc" : "Path to directory where fence agent can store information", + "default" : "@STORE_PATH@", + "order": 200 + } + +def main(): + atexit.register(atexit_handler) + + device_opt = ["no_login", "no_password", "devices", "key", "sudo", \ + "fabric_fencing", "on_target", "store_path", "mpathpersist_path"] + + define_new_opts() + + options = check_input(device_opt, process_input(device_opt)) + + docs = {} + docs["shortdesc"] = "Fence agent for multipath persistent reservation" + docs["longdesc"] = "fence_mpath is an I/O fencing agent that uses SCSI-3 \ +persistent reservations to control access multipath devices. Underlying \ +devices must support SCSI-3 persistent reservations (SPC-3 or greater) as \ +well as the "preempt-and-abort" subcommand.\nThe fence_mpath agent works by \ +having an unique key for each pair of node and device that has to be set also \ +in /etc/multipath.conf. Once registered, a single node will become the reservation holder \ +by creating a "write exclusive, registrants only" reservation on the \ +device(s). The result is that only registered nodes may write to the \ +device(s). When a node failure occurs, the fence_mpath agent will remove the \ +key belonging to the failed node from the device(s). The failed node will no \ +longer be able to write to the device(s). A manual reboot is required." + docs["vendorurl"] = "https://www.sourceware.org/dm/" + show_docs(options, docs) + + run_delay(options) + + # Input control BEGIN + if not "--key" in options: + fail_usage("Failed: key is required") + + options["devices"] = options["--devices"].split(",") + + if not options["devices"]: + fail_usage("Failed: No devices found") + # Input control END + + result = fence_action(None, options, set_status, get_status) + sys.exit(result) + +if __name__ == "__main__": + main() diff --git a/make/fencebuild.mk b/make/fencebuild.mk index 3c14928..874ebeb 100644 --- a/make/fencebuild.mk +++ b/make/fencebuild.mk @@ -19,6 +19,9 @@ $(TARGET): $(SRC) -e 's#@''SUDO_PATH@#${SUDO_PATH}#g' \ -e 's#@''SSH_PATH@#${SSH_PATH}#g' \ -e 's#@''TELNET_PATH@#${TELNET_PATH}#g' \ + -e 's#@''MPATH_PATH@#${MPATH_PATH}#g' \ + -e 's#@''STORE_PATH@#${CLUSTERVARRUN}#g' \ + -e 's#@''SUDO_PATH@#${SUDO_PATH}#g' \ > $@
if [ 0 -eq `echo "$(SRC)" | grep fence_ &> /dev/null; echo $$?` ]; then \ diff --git a/tests/data/metadata/fence_mpath.xml b/tests/data/metadata/fence_mpath.xml new file mode 100644 index 0000000..bd8b141 --- /dev/null +++ b/tests/data/metadata/fence_mpath.xml @@ -0,0 +1,86 @@ +<?xml version="1.0" ?> +<resource-agent name="fence_mpath" shortdesc="Fence agent for multipath persistent reservation" > +<longdesc>...</longdesc> +<vendor-url>fence_mpath is an I/O fencing agent that uses SCSI-3 persistent reservations to control access to shared storage devices. These devices must be controlled by dm-multipath and underlying devices must support SCSI-3 persistent reservations (SPC-3 or greater) as well as the "preempt-and-abort" subcommand. + The fence_mpath agent works by having unique key for each pair: node and device. These keys have to be manually added to /etc/multipath.conf. Once registered, a single node will become the reservation holder by creating a "write exclusive, registrants only" reservation on the device(s). The result is that only registered nodes may write to the device(s). When a node failure occurs, the fence_scsi agent will remove the key belonging to the failed node from the device(s). The failed node will no longer be able to write to the device(s). A manual reboot is required.</vendor-url> +<parameters> + <parameter name="devices" unique="0" required="0"> + <getopt mixed="-d, --devices=[devices]" /> + <content type="string" /> + <shortdesc lang="en">List of devices to use for current operation. Devices can be comma-separated list of device-mapper multipath devices (eg. /dev/dm-3). Each device must support SCSI-3 persistent reservations.</shortdesc> + </parameter> + <parameter name="key" unique="0" required="1"> + <getopt mixed="-k, --key=[key]" /> + <content type="string" /> + <shortdesc lang="en">Key to use for the current operation. This key should be unique to a node and have to be written in /etc/multipath.conf. For the "on" action, the key specifies the key use to register the local node. For the "off" action, this key specifies the key to be removed from the device(s).</shortdesc> + </parameter> + <parameter name="action" unique="0" required="1"> + <getopt mixed="-o, --action=[action]" /> + <content type="string" default="off" /> + <shortdesc lang="en">Fencing Action</shortdesc> + </parameter> + <parameter name="verbose" unique="0" required="0"> + <getopt mixed="-v, --verbose" /> + <content type="boolean" /> + <shortdesc lang="en">Verbose mode</shortdesc> + </parameter> + <parameter name="debug" unique="0" required="0"> + <getopt mixed="-D, --debug-file=[debugfile]" /> + <content type="string" /> + <shortdesc lang="en">Write debug information to given file</shortdesc> + </parameter> + <parameter name="version" unique="0" required="0"> + <getopt mixed="-V, --version" /> + <content type="boolean" /> + <shortdesc lang="en">Display version information and exit</shortdesc> + </parameter> + <parameter name="help" unique="0" required="0"> + <getopt mixed="-h, --help" /> + <content type="boolean" /> + <shortdesc lang="en">Display help and exit</shortdesc> + </parameter> + <parameter name="shell_timeout" unique="0" required="0"> + <getopt mixed="--shell-timeout=[seconds]" /> + <content type="string" default="3" /> + <shortdesc lang="en">Wait X seconds for cmd prompt after issuing command</shortdesc> + </parameter> + <parameter name="power_timeout" unique="0" required="0"> + <getopt mixed="--power-timeout=[seconds]" /> + <content type="string" default="20" /> + <shortdesc lang="en">Test X seconds for status change after ON/OFF</shortdesc> + </parameter> + <parameter name="power_wait" unique="0" required="0"> + <getopt mixed="--power-wait=[seconds]" /> + <content type="string" default="0" /> + <shortdesc lang="en">Wait X seconds after issuing ON/OFF</shortdesc> + </parameter> + <parameter name="delay" unique="0" required="0"> + <getopt mixed="--delay=[seconds]" /> + <content type="string" default="0" /> + <shortdesc lang="en">Wait X seconds before fencing is started</shortdesc> + </parameter> + <parameter name="login_timeout" unique="0" required="0"> + <getopt mixed="--login-timeout=[seconds]" /> + <content type="string" default="5" /> + <shortdesc lang="en">Wait X seconds for cmd prompt after login</shortdesc> + </parameter> + <parameter name="mpathpersist_path" unique="0" required="0"> + <getopt mixed="--mpathpersist-path=[path]" /> + <content type="string" default="/usr/sbin/mpathpersist" /> + <shortdesc lang="en">Path to sg_persist binary</shortdesc> + </parameter> + <parameter name="retry_on" unique="0" required="0"> + <getopt mixed="--retry-on=[attempts]" /> + <content type="string" default="1" /> + <shortdesc lang="en">Count of attempts to retry power on</shortdesc> + </parameter> +</parameters> +<actions> + <action name="on" on_target="1" automatic="1"/> + <action name="off" /> + <action name="status" /> + <action name="list" /> + <action name="monitor" /> + <action name="metadata" /> +</actions> +</resource-agent>