Branch 'fence_sanlock' - 2 commits - fence_sanlock/fence_sanlock.in fence_sanlock/fence_sanlockd.c

[David Teigland]

 fence_sanlock/fence_sanlock.in |   48 +++++-------
 fence_sanlock/fence_sanlockd.c |  160 ++++++++++++++++++++++++++++++++++++-----
 2 files changed, 167 insertions(+), 41 deletions(-)

New commits:
commit 401d1830a7c0ba31789ee35d25d71aa5dc3a8889
Author: David Teigland <teigland(a)redhat.com&gt;
Date:   Thu Sep 20 17:16:06 2012 -0500

    fence_sanlockd: allow options to be delayed
    
    When fence_sanlockd is started with -w, it will
    wait for fence_sanlockd -s to send it the
    -i and -p options.
    
    Signed-off-by: David Teigland <teigland(a)redhat.com&gt;

diff --git a/fence_sanlock/fence_sanlockd.c b/fence_sanlock/fence_sanlockd.c
index e491ded..7f62e0a 100644
--- a/fence_sanlock/fence_sanlockd.c
+++ b/fence_sanlock/fence_sanlockd.c
@@ -60,8 +60,9 @@
 #define RUN_DIR "/var/run/fence_sanlockd"
 
 static char *prog_name = (char *)"fence_sanlockd";
-static int shutdown;
+
 static int we_are_victim;
+static int shutdown;
 static int daemon_debug;
 static int our_host_id;
 static char path[PATH_MAX];
@@ -70,6 +71,13 @@ static struct sanlk_resource *r;
 static struct sanlk_disk disk;
 static char rdbuf[sizeof(struct sanlk_resource) + sizeof(struct sanlk_disk)];
 static char lockfile_path[PATH_MAX];
+static char fifo_path[PATH_MAX];
+static char fifo_line[PATH_MAX];
+static char key1[PATH_MAX];
+static char key2[PATH_MAX];
+static char val1[PATH_MAX];
+static char val2[PATH_MAX];
+
 
 struct client {
 	int used;
@@ -258,17 +266,105 @@ static int setup_signals(void)
 	return 0;
 }
 
+static int wait_options(void)
+{
+	int fd, rv;
+
+	snprintf(fifo_path, PATH_MAX-1, "%s/prog_name.fifo", RUN_DIR);
+
+	rv = mkfifo(fifo_path, (S_IRUSR | S_IWUSR));
+	if (rv && errno != EEXIST) {
+		log_error("wait_options mkfifo error %d %s", errno, fifo_path);
+		return -1;
+	}
+
+	fd = open(fifo_path, O_RDONLY|O_CLOEXEC);
+	if (fd < 0) {
+		log_error("wait_options fifo open error %d %s", errno, fifo_path);
+		unlink(fifo_path);
+		return -1;
+	}
+
+ reread:
+	memset(fifo_line, 0, sizeof(fifo_line));
+
+	rv = read(fd, fifo_line, sizeof(fifo_line));
+	if (rv < 0) {
+		log_error("wait_options read error %d", errno);
+		goto out;
+	}
+
+	rv = sscanf(fifo_line, "%s %s %s %s", key1, val1, key2, val2);
+	if (rv != 4) {
+		log_error("wait_options scan error %d", rv);
+		goto reread;
+	}
+
+	if (strcmp(key1, "-p") || strcmp(key2, "-i")) {
+		log_error("wait_options args error");
+		goto reread;
+	}
+
+	strncpy(path, val1, PATH_MAX-1);
+	our_host_id = atoi(val2);
+
+	if (!our_host_id || our_host_id > MAX_HOSTS) {
+		log_error("wait_options invalid host_id");
+		goto reread;
+	}
+
+	if (!path[0]) {
+		log_error("wait_options invalid path");
+		goto reread;
+	}
+
+	log_debug("wait_options -p %s -i %d", path, our_host_id);
+ out:
+	close(fd);
+	unlink(fifo_path);
+	return 0;
+}
+
+static int send_options(void)
+{
+	int fd, rv;
+
+	snprintf(fifo_path, PATH_MAX-1, "%s/prog_name.fifo", RUN_DIR);
+
+	fd = open(fifo_path, O_WRONLY|O_CLOEXEC);
+	if (fd < 0) {
+		fprintf(stderr, "fifo open error %d %s\n", errno, fifo_path);
+		return -1;
+	}
+
+	memset(fifo_line, 0, sizeof(fifo_line));
+
+	snprintf(fifo_line, PATH_MAX-1, "-p %s -i %d", path, our_host_id);
+
+	rv = write(fd, fifo_line, sizeof(fifo_line));
+	if (rv < 0) {
+		fprintf(stderr, "fifo write error %d\n", errno);
+	} else {
+		rv = 0;
+	}
+
+	close(fd);
+	return rv;
+}
+
 static void print_usage(void)
 {
 	printf("Usage:\n");
-	printf("fence_sanlockd -p <path> -i <host_id>\n");
+	printf("fence_sanlockd [options]\n");
 	printf("\n");
 	printf("Options:\n");
-	printf("  -D           Enable debugging to stderr and don't fork\n");
-	printf("  -p <path>    Path to shared storage with sanlock leases\n");
-	printf("  -i <host_id> Local sanlock host_id\n");
-	printf("  -h           Print this help, then exit\n");
-	printf("  -V           Print program version information, then exit\n");
+	printf("  -D            Enable debugging to stderr and don't fork\n");
+	printf("  -p <path>     Path to shared storage with sanlock leases\n");
+	printf("  -i <host_id>  Local sanlock host_id (1-%d)\n", MAX_HOSTS);
+	printf("  -w            Wait for fence_sanlockd -s to send options (p,i)\n");
+	printf("  -s            Send options (p,i) to waiting fence_sanlockd -w\n");
+	printf("  -h            Print this help, then exit\n");
+	printf("  -V            Print program version information, then exit\n");
 }
 
 int main(int argc, char *argv[])
@@ -278,13 +374,14 @@ int main(int argc, char *argv[])
 	uint64_t live_time, now;
 	int poll_timeout;
 	int sleep_seconds;
+	int send_opts = 0, wait_opts = 0;
 	int cont = 1;
 	int optchar;
 	int sock, con, rv, i;
 	int align;
 
 	while (cont) {
-		optchar = getopt(argc, argv, "Dp:i:hV");
+		optchar = getopt(argc, argv, "Dp:i:hVws");
 
 		switch (optchar) {
 		case 'D':
@@ -295,6 +392,17 @@ int main(int argc, char *argv[])
 			break;
 		case 'i':
 			our_host_id = atoi(optarg);
+			if (our_host_id > MAX_HOSTS) {
+				fprintf(stderr, "invalid host_id %d, use 1-%d\n",
+					our_host_id, MAX_HOSTS);
+				exit(1);
+			}
+			break;
+		case 'w':
+			wait_opts = 1;
+			break;
+		case 's':
+			send_opts = 1;
 			break;
 		case 'h':
 			print_usage();
@@ -312,18 +420,21 @@ int main(int argc, char *argv[])
 		};
 	}
 
-	if (!our_host_id || our_host_id > MAX_HOSTS) {
-		daemon_debug = 1;
-		log_error("-i %d invalid, our_host_id must be between 1 and %d", our_host_id,
MAX_HOSTS);
+	if (wait_opts && send_opts) {
+		fprintf(stderr, "-w and -s options cannot be used together\n");
 		exit(1);
 	}
 
-	if (!path[0]) {
-		daemon_debug = 1;
-		log_error("path arg required");
+	if (!wait_opts && (!our_host_id || !path[0])) {
+		fprintf(stderr, "-i and -p options required\n");
 		exit(1);
 	}
 
+	if (send_opts) {
+		rv = send_options();
+		return rv;
+	}
+
 	if (!daemon_debug) {
 		if (daemon(0, 0) < 0) {
 			fprintf(stderr, "cannot fork daemon\n");
@@ -341,6 +452,12 @@ int main(int argc, char *argv[])
 	if (rv < 0)
 		goto out_lockfile;
 
+	if (wait_opts) {
+		rv = wait_options();
+		if (rv < 0)
+			goto out_lockfile;
+	}
+
 	con = wdmd_connect();
 	if (con < 0) {
 		log_error("wdmd connect error %d", con);


commit 0a80be08c4f85b9e1ab1c9db7911def51e213faf
Author: David Teigland <teigland(a)redhat.com&gt;
Date:   Thu Sep 20 14:06:00 2012 -0500

    fence_sanlock: variable alignment
    
    - make lease size depend on disk sector size
    - clarify fence_sanlock read_leader behavior
    
    Signed-off-by: David Teigland <teigland(a)redhat.com&gt;

diff --git a/fence_sanlock/fence_sanlock.in b/fence_sanlock/fence_sanlock.in
index a6f9178..f43acc2 100755
--- a/fence_sanlock/fence_sanlock.in
+++ b/fence_sanlock/fence_sanlock.in
@@ -165,35 +165,24 @@ EOF
 	return 0
 }
 
-get_leader_info() {
+read_leader() {
+	# verify storage has been initialized
+
 	leader=$(sanlock direct read_leader -r fence:h$host_id:$path:$offset 2>&1)
 	[ "$?" != 0 ] && {
-		echo "Unable to read $path"
+		echo "Error: unable to read $path"
 		return 1
 	}
 
-	timestamp="$(echo "$leader" | grep ^timestamp | awk '{print
$NF}')"
-
-	[ -z "$timestamp" ] && {
-		echo "Unable to determine timestamp"
-		return 1
-	}
-
-	return 0
-}
-
-verify_path() {
-	# verify storage has been initialized
-
-	get_leader_info || return 1	
-
 	magic="$(echo "$leader" | grep magic | awk '{print $NF}')"
+
 	[ -z "$magic" ] && {
-		echo "Unable to determine $path sanlock magic"
+		echo "Error: no sanlock magic number at $path:$offset"
 		return 1
 	}
+
 	[ "$magic" != "0x6152010" ] && {
-		echo "Error: $path magic $magic does not match sanlock magic 0x6152010"
+		echo "Error: invalid sanlock magic number $magic at $path:$offset"
 		return 1
 	}
 
@@ -201,7 +190,7 @@ verify_path() {
 }
 
 action_on() {
-	verify_path || return 1
+	read_leader || return 1
 
 	[ -z "$(pidof fence_sanlockd)" ] && {
 		daemonerr="$(fence_sanlockd -p $path -i $host_id 2>&1)"
@@ -245,11 +234,12 @@ action_on() {
 }
 
 action_off() {
-	verify_path || return 1
+	read_leader || return 1
 
 	owner_id="$(echo "$leader" | grep owner_id | awk '{print
$NF}')"
 	owner_gen="$(echo "$leader" | grep owner_gen | awk '{print
$NF}')"
 	ver="$(echo "$leader" | grep lver | awk '{print $NF}')"
+	timestamp="$(echo "$leader" | grep ^timestamp | awk '{print
$NF}')"
 
 	# lease is released, so host is off
 	[ "$timestamp" = 0 ] && {
@@ -308,7 +298,7 @@ action_off() {
 		# reacquired cleanly by the victim host (same host_id, new
 		# generation), we can quit with success
 
-		get_leader_info || return 1
+		read_leader || return 1
 
 		tmp_id="$(echo "$leader" | grep owner_id | awk '{print
$NF}')"
 		tmp_gen="$(echo "$leader" | grep owner_gen | awk '{print
$NF}')"
@@ -328,7 +318,9 @@ action_off() {
 }
 
 action_status() {
-	verify_path || return 1
+	read_leader || return 1
+
+	timestamp="$(echo "$leader" | grep ^timestamp | awk '{print
$NF}')"
 
 	# lease is released, so host is "off"
 	[ "$timestamp" = 0 ] && {
@@ -351,7 +343,7 @@ sanlock_init() {
 
 	echo -n "Initializing $max_hosts sanlock host leases on $path: "
 	for host_id in $(seq 1 $max_hosts); do
-		offset=$((host_id * 1048576))
+		offset=$((host_id * $align))
 		sanlock direct init -r fence:h$host_id:$path:$offset \
>  /dev/null 2>/dev/null || \ 
 			{ echo "error $? for
host $host_id" && return 1; }
@@ -365,16 +357,22 @@ sanlock_init() {
 
 # check actions and options compatibility
 # all actions beside metadata needs storage 
+
 [ "$action" != "metadata" ] && {
 	[ -z "$path" ] && {
 		echo "storage path argument required"
 		exit 1
 	}
+
 	# all actions beside sanlock_init needs host_id
 	[ "$action" != "sanlock_init" ] && [ -z "$host_id"
] && {
 		echo "host_id argument required"
 		exit 1
 	}
+
+	# FIXME: add direct align command to sanlock
+	# align="&(sanlock direct align $path)"
+	align=1048576
 }
 
 # verify host_id parameter
@@ -383,7 +381,7 @@ sanlock_init() {
 		echo "host_id must be between 1 and $max_hosts"
 		exit 1
 	fi
-	offset=$((host_id * 1048576))
+	offset=$((host_id * $align))
 }
 
 case "$action" in
diff --git a/fence_sanlock/fence_sanlockd.c b/fence_sanlock/fence_sanlockd.c
index ea7ffdc..e491ded 100644
--- a/fence_sanlock/fence_sanlockd.c
+++ b/fence_sanlock/fence_sanlockd.c
@@ -28,11 +28,11 @@
 #include "sanlock.h"
 #include "sanlock_admin.h"
 #include "sanlock_resource.h"
+#include "sanlock_direct.h"
 #include "wdmd.h"
 
 /*
  * TODO:
- * handle 4k disk blocks using sanlock_align()
  * variable i/o timeouts?
  *
  * shutdown: how/when does SIGTERM happen?
@@ -67,6 +67,7 @@ static int our_host_id;
 static char path[PATH_MAX];
 static struct sanlk_lockspace ls;
 static struct sanlk_resource *r;
+static struct sanlk_disk disk;
 static char rdbuf[sizeof(struct sanlk_resource) + sizeof(struct sanlk_disk)];
 static char lockfile_path[PATH_MAX];
 
@@ -280,6 +281,7 @@ int main(int argc, char *argv[])
 	int cont = 1;
 	int optchar;
 	int sock, con, rv, i;
+	int align;
 
 	while (cont) {
 		optchar = getopt(argc, argv, "Dp:i:hV");
@@ -369,6 +371,15 @@ int main(int argc, char *argv[])
 		goto out_refcount;
 	}
 
+	memset(&disk, 0, sizeof(disk));
+	sprintf(disk.path, "%s", path);
+
+	align = sanlock_direct_align(&disk);
+	if (align < 0) {
+		log_error("direct_align error %d", align);
+		goto out_refcount;
+	}
+
 	memset(&ls, 0, sizeof(ls));
 	sprintf(ls.host_id_disk.path, "%s", path);
 	strcpy(ls.name, "fence");
@@ -389,7 +400,7 @@ int main(int argc, char *argv[])
 	strcpy(r->lockspace_name, "fence");
 	sprintf(r->name, "h%d", our_host_id);
 	sprintf(r->disks[0].path, "%s", path);
-	r->disks[0].offset = our_host_id * 1048576;
+	r->disks[0].offset = our_host_id * align;
 	r->num_disks = 1;
 
 	log_debug("acquire begin");