fence_sanlock/fence_sanlock.8 fence_sanlock/fence_sanlockd.8 fence_sanlock/fence_sanlockd.c src/cmd.c src/helper.c src/resource.c src/sanlock_internal.h src/sanlock_resource.h

[David Teigland]

 fence_sanlock/fence_sanlock.8  |   85 ++++++++++++++++++++++++++++++++++++++---
 fence_sanlock/fence_sanlockd.8 |    4 +
 fence_sanlock/fence_sanlockd.c |   68 ++++++++++++++++++++++++++++++++
 src/cmd.c                      |    2 
 src/helper.c                   |    2 
 src/resource.c                 |   29 +++++--------
 src/sanlock_internal.h         |    4 +
 src/sanlock_resource.h         |   20 ++++++---
 8 files changed, 182 insertions(+), 32 deletions(-)

New commits:
commit 4c058f277ad2dbd852cd3087f48edf4562de0bb3
Author: David Teigland <teigland(a)redhat.com&gt;
Date:   Wed Apr 10 16:01:27 2013 -0500

    sanlock/fence_sanlock: request modes FORCE and GRACEFUL
    
    FORCE is new name for KILL_PID.
    GRACEFUL is killpath (or SIGTERM of no killpath is set)
    SIGUSR1 is removed; a killpath prog should send the signal
    
    fence_sanlock is converted to use killpath to send SIGUSR1.
    The killpath program is "fence_sanlockd -1" (-1 is new).
    
    Change sanlock helper to use execvp so we don't need to
    know the fence_sanlockd path.
    
    Signed-off-by: David Teigland <teigland(a)redhat.com&gt;

diff --git a/fence_sanlock/fence_sanlock.8 b/fence_sanlock/fence_sanlock.8
index d9e41be..66f0912 100644
--- a/fence_sanlock/fence_sanlock.8
+++ b/fence_sanlock/fence_sanlock.8
@@ -1,4 +1,4 @@
-.TH FENCE_SANLOCK 8 2012-09-26
+.TH FENCE_SANLOCK 8 2013-05-02
 
 .SH NAME
 fence_sanlock \- fence agent using watchdog and shared storage leases
@@ -166,7 +166,11 @@ See \-i
 
 .SH FILES 
 
-Example cluster.conf configuration for fence_sanlock:
+Example cluster.conf configuration for fence_sanlock.
+.br
+(For cman based clusters in which fenced runs agents.)
+.br
+Also see cluster.conf(5), fenced(8), fence_node(8).
 
 .nf
 <clusternode name="node01" nodeid="1">
@@ -194,10 +198,81 @@ Example cluster.conf configuration for fence_sanlock:
 <fencedevice name="wd" agent="fence_sanlock"
path="/dev/fence/leases"/>
 .fi
 
+.PP
+
+Example dlm.conf configuration for fence_sanlock.
+.br
+(For non-cman based clusters in which dlm_controld runs agents.)
+.br
+Also see dlm.conf(5), dlm_controld(8).
+
+.nf
+device wd /usr/sbin/fence_sanlock path=/dev/fence/leases
+connect wd node=1 host_id=1
+connect wd node=2 host_id=2
+unfence wd
+.fi
+
+.SH TEST
+
+To test fence_sanlock directly, without clustering:
+
+.nf
+1. Initialize storage
+
+node1: create 1G lv on shared storage /dev/fence/leases
+node1: fence_sanlock -o sanlock_init -p /dev/fence/leases
+
+2. Start services
+
+node1: service fence_sanlockd start
+node2: service fence_sanlockd start
+
+3. Enable fencing
+
+node1: fence_sanlock -o on -p /dev/fence/leases -i 1
+node2: fence_sanlock -o on -p /dev/fence/leases -i 2
+
+This "unfence" step may take a couple minutes.
+
+4. Verify hosts and leases
+
+node1: sanlock status
+s fence:1:/dev/fence/leases:0
+r fence:h1:/dev/fence/leases:1048576:1 p 2465
+
+node2: sanlock status
+s fence:2:/dev/fence/leases:0
+r fence:h2:/dev/fence/leases:2097152:1 p 2366
+
+node1: sanlock host_status
+lockspace fence
+1 timestamp 717
+2 timestamp 678
+
+node2: sanlock host_status
+lockspace fence
+1 timestamp 738
+2 timestamp 678
+
+5. Fence node2
+
+node1: fence_sanlock -o off -p /dev/fence/leases -i 2
+
+This may take a few minutes to return.
+
+When node2 is not dead before fencing, sanlock on node1 will log errors
+about failing to acquire the lease while node2 is still alive.  This is
+expected.
+
+6. Success
+
+node1 fence_sanlock should exit 0 after node2 is reset by its watchdog.
+.fi
+
+
 .SH SEE ALSO
 .BR fence_sanlockd (8),
 .BR sanlock (8),
-.BR wdmd (8),
-.BR fence_node (8),
-.BR fenced (8)
+.BR wdmd (8)
 
diff --git a/fence_sanlock/fence_sanlockd.8 b/fence_sanlock/fence_sanlockd.8
index 33ce051..33f2dc7 100644
--- a/fence_sanlock/fence_sanlockd.8
+++ b/fence_sanlock/fence_sanlockd.8
@@ -30,6 +30,10 @@ for full description.
 .B \-s
    Send options (p,i) to waiting fence_sanlockd -w.
 
+.B \-1
+   Send SIGUSR1 to running fence_sanlockd.
+
+
 .SH SEE ALSO
 .BR fence_sanlock (8),
 .BR sanlock (8),
diff --git a/fence_sanlock/fence_sanlockd.c b/fence_sanlock/fence_sanlockd.c
index aa45736..b52bd7c 100644
--- a/fence_sanlock/fence_sanlockd.c
+++ b/fence_sanlock/fence_sanlockd.c
@@ -149,6 +149,36 @@ static int client_add(int fd, void (*workfn)(int ci), void
(*deadfn)(int ci))
 	goto again;
 }
 
+static int read_lockfile(int *pid)
+{
+	char buf[16];
+	int fd, rv;
+
+	sprintf(lockfile_path, "%s/%s.pid", DAEMON_RUN_DIR, prog_name);
+
+	fd = open(lockfile_path, O_RDONLY);
+	if (fd < 0) {
+		log_error("lockfile open error %s: %s",
+			  lockfile_path, strerror(errno));
+		return -1;
+	}
+
+	memset(buf, 0, sizeof(buf));
+
+	rv = read(fd, buf, sizeof(buf));
+	if (rv < 0) {
+		log_error("lockfile read error %s: %s",
+			  lockfile_path, strerror(errno));
+		close(fd);
+		return -1;
+	}
+
+	*pid = atoi(buf);
+
+	close(fd);
+	return 0;
+}
+
 static int lockfile(void)
 {
 	char buf[16];
@@ -352,6 +382,26 @@ static int send_options(void)
 	return rv;
 }
 
+static int send_signal(int sig)
+{
+	int rv, pid;
+
+	openlog("fence_sanlockd-1", LOG_CONS | LOG_PID, LOG_DAEMON);
+
+	rv = read_lockfile(&pid);
+	if (rv < 0)
+		return rv;
+
+	rv = kill(pid, sig);
+	if (rv < 0) {
+		log_error("kill sig %d pid %d error %d", sig, pid, errno);
+	} else {
+		syslog(LOG_INFO, "sent signal %d to pid %d", sig, pid);
+	}
+
+	return rv;
+}
+
 /*
  * A running fence_sanlock agent has a pid file we can read.
  * We use this to check what host_id it's fencing, so we can
@@ -464,6 +514,7 @@ static void print_usage(void)
 	printf("  -i <host_id>  Local sanlock host_id (1-%d)\n", MAX_HOSTS);
 	printf("  -w            Wait for fence_sanlockd -s to send options (p,i)\n");
 	printf("  -s            Send options (p,i) to waiting fence_sanlockd -w\n");
+	printf("  -1            Send SIGUSR1 to running fence_sanlockd\n");
 	printf("  -h            Print this help, then exit\n");
 	printf("  -V            Print program version information, then exit\n");
 }
@@ -476,6 +527,7 @@ int main(int argc, char *argv[])
 	int poll_timeout;
 	int sleep_seconds;
 	int send_opts = 0, wait_opts = 0;
+	int send_sigusr1 = 0;
 	int cont = 1;
 	int optchar;
 	int sock, con, rv, i;
@@ -483,7 +535,7 @@ int main(int argc, char *argv[])
 	int victim_host_id;
 
 	while (cont) {
-		optchar = getopt(argc, argv, "Dp:i:hVws");
+		optchar = getopt(argc, argv, "Dp:i:hVws1");
 
 		switch (optchar) {
 		case 'D':
@@ -506,6 +558,9 @@ int main(int argc, char *argv[])
 		case 's':
 			send_opts = 1;
 			break;
+		case '1':
+			send_sigusr1 = 1;
+			break;
 		case 'h':
 			print_usage();
 			exit(0);
@@ -522,6 +577,11 @@ int main(int argc, char *argv[])
 		};
 	}
 
+	if (send_sigusr1) {
+		rv = send_signal(SIGUSR1);
+		return rv;
+	}
+
 	if (wait_opts && send_opts) {
 		fprintf(stderr, "-w and -s options cannot be used together\n");
 		exit(1);
@@ -584,6 +644,12 @@ int main(int argc, char *argv[])
 		goto out_refcount;
 	}
 
+	rv = sanlock_killpath(sock, 0, "fence_sanlockd", (char *)"-1");
+	if (rv < 0) {
+		log_error("killpath error %d", sock);
+		goto out_refcount;
+	}
+
 	rv = sanlock_restrict(sock, SANLK_RESTRICT_SIGKILL);
 	if (rv < 0) {
 		log_error("restrict error %d", sock);
diff --git a/src/cmd.c b/src/cmd.c
index 19416be..8ad72b7 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -310,6 +310,8 @@ static void cmd_acquire(struct task *task, struct cmd_args *ca)
 		token->io_timeout = spi.io_timeout;
 		if (cl->restricted & SANLK_RESTRICT_SIGKILL)
 			token->flags |= T_RESTRICT_SIGKILL;
+		if (cl->restricted & SANLK_RESTRICT_SIGTERM)
+			token->flags |= T_RESTRICT_SIGTERM;
 
 		/* save a record of what this token_id is for later debugging */
 		log_level(spi.space_id, token->token_id, NULL, LOG_WARNING,
diff --git a/src/helper.c b/src/helper.c
index 2fce221..27898d5 100644
--- a/src/helper.c
+++ b/src/helper.c
@@ -103,7 +103,7 @@ static void run_path(struct helper_msg *hm)
 		av[av_count++] = strdup(arg);
 	}
 
-	execv(av[0], av);
+	execvp(av[0], av);
 }
 
 static int read_hm(int fd, struct helper_msg *hm)
diff --git a/src/resource.c b/src/resource.c
index 2a60a16..226a6ce 100644
--- a/src/resource.c
+++ b/src/resource.c
@@ -564,6 +564,8 @@ static struct resource *new_resource(struct token *token)
 		r->pid = token->pid;
 		if (token->flags & T_RESTRICT_SIGKILL)
 			r->flags |= R_RESTRICT_SIGKILL;
+		if (token->flags & T_RESTRICT_SIGTERM)
+			r->flags |= R_RESTRICT_SIGTERM;
 	}
 
 	return r;
@@ -856,27 +858,20 @@ static void do_request(struct token *tt, int pid, uint32_t
force_mode)
 
 	memset(&hm, 0, sizeof(hm));
 
-	if (force_mode == SANLK_REQ_KILL_PID) {
+	if (force_mode == SANLK_REQ_FORCE) {
 		hm.type = HELPER_MSG_KILLPID;
 		hm.pid = pid;
 		hm.sig = (flags & R_RESTRICT_SIGKILL) ? SIGTERM : SIGKILL;
-	} else if (force_mode == SANLK_REQ_SIGUSR1) {
-		hm.type = HELPER_MSG_KILLPID;
-		hm.pid = pid;
-		hm.sig = SIGUSR1;
-	} else if (force_mode == SANLK_REQ_KILLPATH) {
-		if (!killpath[0]) {
-			log_error("do_request %d force_mode %d no killpath",
-				  pid, force_mode);
-			return;
+	} else if (force_mode == SANLK_REQ_GRACEFUL) {
+		if (killpath[0]) {
+			hm.type = HELPER_MSG_RUNPATH;
+			memcpy(hm.path, killpath, SANLK_HELPER_PATH_LEN);
+			memcpy(hm.args, killargs, SANLK_HELPER_ARGS_LEN);
+		} else {
+			hm.type = HELPER_MSG_KILLPID;
+			hm.pid = pid;
+			hm.sig = (flags & R_RESTRICT_SIGTERM) ? SIGKILL : SIGTERM;
 		}
-		/* there's no thread locking for the clients array so
-		   we can't go searching for killpath/killargs for this
-		   pid from here, so we copy the info into struct
-		   resource so we can use it here. */
-		hm.type = HELPER_MSG_RUNPATH;
-		memcpy(hm.path, killpath, SANLK_HELPER_PATH_LEN);
-		memcpy(hm.args, killargs, SANLK_HELPER_ARGS_LEN);
 	} else {
 		log_error("do_request %d unknown force_mode %d",
 			  pid, force_mode);
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index 34ce260..f0153c6 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -68,7 +68,8 @@ struct sync_disk {
  */
 
 #define T_RESTRICT_SIGKILL	0x00000001 /* inherited from client->restricted */
-#define T_LS_DEAD		0x00000002 /* don't bother trying to release if ls is dead */
+#define T_RESTRICT_SIGTERM	0x00000002 /* inherited from client->restricted */
+#define T_LS_DEAD		0x00000004 /* don't bother trying to release if ls is dead */
 
 struct token {
 	/* values copied from acquire res arg */
@@ -99,6 +100,7 @@ struct token {
 #define R_THREAD_EXAMINE    	0x00000002
 #define R_THREAD_RELEASE	0x00000004
 #define R_RESTRICT_SIGKILL	0x00000008 /* inherited from token */
+#define R_RESTRICT_SIGTERM	0x00000010 /* inherited from token */
 
 struct resource {
 	struct list_head list;
diff --git a/src/sanlock_resource.h b/src/sanlock_resource.h
index f68fc3e..3148b25 100644
--- a/src/sanlock_resource.h
+++ b/src/sanlock_resource.h
@@ -33,15 +33,21 @@
 
 /*
  * request force_mode
- * SANLK_REQ_KILL_PID: send SIGKILL to pid holding the resource, or
- *                     SIGTERM if SIGKILL is restricted
- * SANLK_REQ_SIGUSR1:  send SIGUSR1 to pid holding the resource
- * SANLK_REQ_KILLPATH: run configured killpath against the pid
+ *
+ * SANLK_REQ_FORCE (SANLK_REQ_KILL_PID deprecated)
+ * Send SIGKILL to the pid holding the resource
+ * (or SIGTERM if SIGKILL is restricted.)
+ *
+ * SANLK_REQ_GRACEFUL
+ * Run killpath against the pid if it is defined, otherwise
+ * send SIGTERM to the pid (or SIGKILL if SIGTERM is restricted).
  */
 
-#define SANLK_REQ_KILL_PID	0x00000001
-#define SANLK_REQ_SIGUSR1	0x00000002
-#define SANLK_REQ_KILLPATH	0x00000003
+#define SANLK_REQ_FORCE			0x00000001
+#define SANLK_REQ_GRACEFUL		0x00000002
+
+/* old name deprecated */
+#define SANLK_REQ_KILL_PID		SANLK_REQ_FORCE
 
 int sanlock_register(void);