2 commits - src/cmd.c src/lockspace.c src/lockspace.h src/resource.c src/resource.h src/sanlock_admin.h tests/sanlk_load.c

[David Teigland]

 src/cmd.c           |   57 ++++++++++++++++++++++++++++++++++++------
 src/lockspace.c     |   69 ++++++++++++++++++++++++++++++++++++++--------------
 src/lockspace.h     |    6 +++-
 src/resource.c      |   24 ++++++++++++++++++
 src/resource.h      |    2 +
 src/sanlock_admin.h |   10 ++++++-
 tests/sanlk_load.c  |   25 ++++++++++++++++++
 7 files changed, 163 insertions(+), 30 deletions(-)

New commits:
commit db2259e9ea5bdf00599db84b4818733d466c8667
Author: David Teigland <teigland(a)redhat.com&gt;
Date:   Mon Apr 30 14:31:10 2012 -0500

    sanlock: UNUSED flag for lockspace rem
    
    salock_rem_lockspace(UNUSED) will return -EBUSY
    if any resources exist for the given lockspace,
    instead of forcibly removing the lockspace by
    killing any pids using resources within the
    lockspace, and then removing the lockspace once
    all resources are gone.
    
    Signed-off-by: David Teigland <teigland(a)redhat.com&gt;

diff --git a/src/cmd.c b/src/cmd.c
index d5f5c3d..6417df0 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -982,6 +982,13 @@ static void cmd_rem_lockspace(struct cmd_args *ca)
 	log_debug("cmd_rem_lockspace %d,%d %.48s flags %x",
 		  ca->ci_in, fd, lockspace.name, ca->header.cmd_flags);
 
+	if (ca->header.cmd_flags & SANLK_REM_UNUSED) {
+		if (lockspace_is_used(&lockspace)) {
+			result = -EBUSY;
+			goto reply;
+		}
+	}
+
 	rv = rem_lockspace_start(&lockspace, &space_id);
 	if (rv < 0) {
 		result = rv;
diff --git a/src/resource.c b/src/resource.c
index 6e6de5d..cef0765 100644
--- a/src/resource.c
+++ b/src/resource.c
@@ -475,6 +475,30 @@ static struct resource *find_resource(struct token *token,
 	return NULL;
 }
 
+int lockspace_is_used(struct sanlk_lockspace *ls)
+{
+	struct resource *r;
+
+	pthread_mutex_lock(&resource_mutex);
+	list_for_each_entry(r, &resources_held, list) {
+		if (!strncmp(r->r.lockspace_name, ls->name, NAME_ID_SIZE))
+			goto yes;
+	}
+	list_for_each_entry(r, &resources_add, list) {
+		if (!strncmp(r->r.lockspace_name, ls->name, NAME_ID_SIZE))
+			goto yes;
+	}
+	list_for_each_entry(r, &resources_rem, list) {
+		if (!strncmp(r->r.lockspace_name, ls->name, NAME_ID_SIZE))
+			goto yes;
+	}
+	pthread_mutex_unlock(&resource_mutex);
+	return 0;
+ yes:
+	pthread_mutex_unlock(&resource_mutex);
+	return 1;
+}
+
 static void copy_disks(void *dst, void *src, int num_disks)
 {
 	struct sync_disk *d, *s;
diff --git a/src/resource.h b/src/resource.h
index 5393728..a75f1a1 100644
--- a/src/resource.h
+++ b/src/resource.h
@@ -11,6 +11,8 @@
 
 void send_state_resources(int fd);
 
+int lockspace_is_used(struct sanlk_lockspace *ls);
+
 void check_mode_block(struct token *token, int q, char *dblock);
 
 int acquire_token(struct task *task, struct token *token);
diff --git a/src/sanlock_admin.h b/src/sanlock_admin.h
index 64d654c..c5a3a0f 100644
--- a/src/sanlock_admin.h
+++ b/src/sanlock_admin.h
@@ -15,6 +15,7 @@
 
 /* rem flags */
 #define SANLK_REM_ASYNC		0x00000001
+#define SANLK_REM_UNUSED	0x00000002
 
 /*
  * add_lockspace returns:
@@ -40,9 +41,10 @@ int sanlock_inq_lockspace(struct sanlk_lockspace *ls, uint32_t flags);
  * 0: the lockspace has been removed successfully
  * -EINPROGRESS: the lockspace is already in the process of being removed
  * -ENOENT: lockspace not found
+ * -EBUSY: UNUSED was set and lockspace is being used
  *
  * The sanlock daemon will kill any pids using the lockspace when the
- * lockspace is removed.
+ * lockspace is removed (unless UNUSED is set).
  */
 
 int sanlock_rem_lockspace(struct sanlk_lockspace *ls, uint32_t flags);


commit 7f34244a35719de4ab7b753c9dc6bc6192574e90
Author: David Teigland <teigland(a)redhat.com&gt;
Date:   Mon Apr 30 12:18:18 2012 -0500

    sanlock: ASYNC flag for lockspace add and rem
    
    New SANLK_ADD_ASYNC for sanlock_add_lockspace(),
    and SANLK_REM_ASYNC for sanlock_rem_lockspace().
    
    The command will finish before the delta lease operation
    on disk.  With ASYNC, both return 0 if add/rem is
    successfully started.  sanlock_inq_lockspace() can be
    used to check the status of the lockspace after add
    or rem is started.
    
    sanlock_add_lockspace() still returns:
    
    0               add started (with ASYNC) or
                    completed successfully (sync)
    -EEXIST         lockspace already exists
    -EINPROGRESS    add lockspace in progress
    -EAGAIN         remove lockspace in progress
    -EINVAL         a lockspace exists with different properties
                    (same name but different disk locations, or
                    different names but same disk location)
    
    sanlock_rem_lockspace() still returns:
    
    0               remove started (with ASYNC) or
                    completed successfully (sync)
    -EINPROGRESS    remove lockspace in progress
    -ENOENT         no lockspace with matching name and
                    disk location is found
    
    sanlock_inq_lockspace() still returns:
    
    0               lockspace exists and is not being
                    added or removed
    -EINPROGRESS    add or rem is in progress
    -ENOENT         no lockspace with matching name and
                    disk location is found
    
    Signed-off-by: David Teigland <teigland(a)redhat.com&gt;

diff --git a/src/cmd.c b/src/cmd.c
index 7a44ce2..d5f5c3d 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -35,6 +35,7 @@
 #include <uuid/uuid.h>
 
 #include "sanlock_internal.h"
+#include "sanlock_admin.h"
 #include "sanlock_sock.h"
 #include "diskio.h"
 #include "log.h"
@@ -844,6 +845,8 @@ static void cmd_examine(struct task *task GNUC_UNUSED, struct cmd_args
*ca)
 static void cmd_add_lockspace(struct cmd_args *ca)
 {
 	struct sanlk_lockspace lockspace;
+	struct space *sp;
+	int async = ca->header.cmd_flags & SANLK_ADD_ASYNC;
 	int fd, rv, result;
 
 	fd = client[ca->ci_in].fd;
@@ -856,16 +859,31 @@ static void cmd_add_lockspace(struct cmd_args *ca)
 		goto reply;
 	}
 
-	log_debug("cmd_add_lockspace %d,%d %.48s:%llu:%s:%llu",
+	log_debug("cmd_add_lockspace %d,%d %.48s:%llu:%s:%llu flags %x",
 		  ca->ci_in, fd, lockspace.name,
 		  (unsigned long long)lockspace.host_id,
 		  lockspace.host_id_disk.path,
-		  (unsigned long long)lockspace.host_id_disk.offset);
+		  (unsigned long long)lockspace.host_id_disk.offset,
+		  ca->header.cmd_flags);
+
+	rv = add_lockspace_start(&lockspace, &sp);
+	if (rv < 0) {
+		result = rv;
+		goto reply;
+	}
+
+	if (async) {
+		result = rv;
+		log_debug("cmd_add_lockspace %d,%d async done %d", ca->ci_in, fd,
result);
+		send_result(fd, &ca->header, result);
+		client_resume(ca->ci_in);
+		add_lockspace_wait(sp);
+		return;
+	}
 
-	result = add_lockspace(&lockspace);
+	result = add_lockspace_wait(sp);
  reply:
 	log_debug("cmd_add_lockspace %d,%d done %d", ca->ci_in, fd, result);
-
 	send_result(fd, &ca->header, result);
 	client_resume(ca->ci_in);
 }
@@ -947,7 +965,9 @@ static void cmd_inq_lockspace(struct cmd_args *ca)
 static void cmd_rem_lockspace(struct cmd_args *ca)
 {
 	struct sanlk_lockspace lockspace;
+	int async = ca->header.cmd_flags & SANLK_REM_ASYNC;
 	int fd, rv, result;
+	unsigned int space_id;
 
 	fd = client[ca->ci_in].fd;
 
@@ -959,13 +979,27 @@ static void cmd_rem_lockspace(struct cmd_args *ca)
 		goto reply;
 	}
 
-	log_debug("cmd_rem_lockspace %d,%d %.48s",
-		  ca->ci_in, fd, lockspace.name);
+	log_debug("cmd_rem_lockspace %d,%d %.48s flags %x",
+		  ca->ci_in, fd, lockspace.name, ca->header.cmd_flags);
+
+	rv = rem_lockspace_start(&lockspace, &space_id);
+	if (rv < 0) {
+		result = rv;
+		goto reply;
+	}
 
-	result = rem_lockspace(&lockspace);
+	if (async) {
+		result = rv;
+		log_debug("cmd_rem_lockspace %d,%d async done %d", ca->ci_in, fd,
result);
+		send_result(fd, &ca->header, result);
+		client_resume(ca->ci_in);
+		rem_lockspace_wait(&lockspace, space_id);
+		return;
+	}
+
+	result = rem_lockspace_wait(&lockspace, space_id);
  reply:
 	log_debug("cmd_rem_lockspace %d,%d done %d", ca->ci_in, fd, result);
-
 	send_result(fd, &ca->header, result);
 	client_resume(ca->ci_in);
 }
diff --git a/src/lockspace.c b/src/lockspace.c
index 3432555..0e9ad87 100644
--- a/src/lockspace.c
+++ b/src/lockspace.c
@@ -574,16 +574,10 @@ static void free_sp(struct space *sp)
 	free(sp);
 }
 
-/*
- * When this function returns, it needs to be safe to being processing lease
- * requests and allowing pid's to run, so we need to own our host_id, and the
- * watchdog needs to be active watching our host_id renewals.
- */
-
-int add_lockspace(struct sanlk_lockspace *ls)
+int add_lockspace_start(struct sanlk_lockspace *ls, struct space **sp_out)
 {
 	struct space *sp, *sp2;
-	int rv, result;
+	int rv;
 
 	if (!ls->name[0] || !ls->host_id || !ls->host_id_disk.path[0]) {
 		log_error("add_lockspace bad args id %llu name %zu path %zu",
@@ -670,6 +664,22 @@ int add_lockspace(struct sanlk_lockspace *ls)
 		goto fail_del;
 	}
 
+	*sp_out = sp;
+	return 0;
+
+ fail_del:
+	pthread_mutex_lock(&spaces_mutex);
+	list_del(&sp->list);
+	pthread_mutex_unlock(&spaces_mutex);
+ fail_free:
+	free_sp(sp);
+	return rv;
+}
+
+int add_lockspace_wait(struct space *sp)
+{
+	int rv, result;
+
 	while (1) {
 		pthread_mutex_lock(&sp->mutex);
 		result = sp->lease_status.acquire_last_result;
@@ -703,7 +713,6 @@ int add_lockspace(struct sanlk_lockspace *ls)
 	pthread_mutex_lock(&spaces_mutex);
 	list_del(&sp->list);
 	pthread_mutex_unlock(&spaces_mutex);
- fail_free:
 	free_sp(sp);
 	return rv;
 }
@@ -736,11 +745,11 @@ int inq_lockspace(struct sanlk_lockspace *ls)
 	return rv;
 }
 
-int rem_lockspace(struct sanlk_lockspace *ls)
+int rem_lockspace_start(struct sanlk_lockspace *ls, unsigned int *space_id)
 {
-	struct space *sp, *sp2;
+	struct space *sp;
 	unsigned int id;
-	int rv, done;
+	int rv;
 
 	pthread_mutex_lock(&spaces_mutex);
 
@@ -769,15 +778,41 @@ int rem_lockspace(struct sanlk_lockspace *ls)
 		goto out;
 	}
 
+	/*
+	 * Removal happens in a round about way:
+	 * - we set external_remove
+	 * - main_loop sees external_remove and sets space_dead, killing_pids
+	 * - main_loop sees killing_pids and all pids dead, sets thread_stop,
+	 *   and moves sp from spaces to spaces_rem
+	 * - main_loop calls free_lockspaces(0), which joins any
+	 *   lockspace_thread that is done, and then frees sp
+	 *
+	 * Once we release spaces_mutex, the sp could be freed any time,
+	 * so we can't touch it.  Use its space_id to check for completion.
+	 */
+
 	sp->external_remove = 1;
 	id = sp->space_id;
 	pthread_mutex_unlock(&spaces_mutex);
 
+	*space_id = id;
+	rv = 0;
+ out:
+	return rv;
+}
+
+/* check for matching space_id in case the lockspace is added again */
+
+int rem_lockspace_wait(struct sanlk_lockspace *ls, unsigned int space_id)
+{
+	struct space *sp;
+	int done;
+
 	while (1) {
 		pthread_mutex_lock(&spaces_mutex);
-		sp2 = _search_space(ls->name, (struct sync_disk *)&ls->host_id_disk,
ls->host_id,
-			   	    &spaces, &spaces_rem, NULL);
-		if (sp2 && sp2->space_id == id)
+		sp = _search_space(ls->name, (struct sync_disk *)&ls->host_id_disk,
ls->host_id,
+			   	   &spaces, &spaces_rem, NULL);
+		if (sp && (sp->space_id == space_id))
 			done = 0;
 		else
 			done = 1;
@@ -787,9 +822,7 @@ int rem_lockspace(struct sanlk_lockspace *ls)
 			break;
 		sleep(1);
 	}
-	rv = 0;
- out:
-	return rv;
+	return 0;
 }
 
 /* 
diff --git a/src/lockspace.h b/src/lockspace.h
index af7d8c2..cf456ed 100644
--- a/src/lockspace.h
+++ b/src/lockspace.h
@@ -19,9 +19,11 @@ int test_id_bit(int host_id, char *bitmap);
 void set_id_bit(int host_id, char *bitmap, char *c);
 int check_our_lease(struct task *task, struct space *sp, int *check_all, char
*check_buf);
 void check_other_leases(struct task *task, struct space *sp, char *buf);
-int add_lockspace(struct sanlk_lockspace *ls);
+int add_lockspace_start(struct sanlk_lockspace *ls, struct space **sp_out);
+int add_lockspace_wait(struct space *sp);
 int inq_lockspace(struct sanlk_lockspace *ls);
-int rem_lockspace(struct sanlk_lockspace *ls);
+int rem_lockspace_start(struct sanlk_lockspace *ls, unsigned int *space_id);
+int rem_lockspace_wait(struct sanlk_lockspace *ls, unsigned int space_id);
 void free_lockspaces(int wait);
 
 #endif
diff --git a/src/sanlock_admin.h b/src/sanlock_admin.h
index 2ec2795..64d654c 100644
--- a/src/sanlock_admin.h
+++ b/src/sanlock_admin.h
@@ -10,6 +10,12 @@
 #ifndef __SANLOCK_ADMIN_H__
 #define __SANLOCK_ADMIN_H__
 
+/* add flags */
+#define SANLK_ADD_ASYNC		0x00000001
+
+/* rem flags */
+#define SANLK_REM_ASYNC		0x00000001
+
 /*
  * add_lockspace returns:
  * 0: the lockspace has been added successfully
diff --git a/tests/sanlk_load.c b/tests/sanlk_load.c
index cc511f2..8e87ab7 100644
--- a/tests/sanlk_load.c
+++ b/tests/sanlk_load.c
@@ -277,15 +277,20 @@ static int add_lockspace(int i)
 {
 	struct sanlk_lockspace ls;
 	int rv;
+	int async = !(i % 2);
+	uint32_t flags = 0;
 
 	memset(&ls, 0, sizeof(ls));
 	sprintf(ls.host_id_disk.path, "%s%d", lock_disk_base, i);
 	sprintf(ls.name, "lockspace%d", i);
 	ls.host_id = our_hostid;
 
+	if (async)
+		flags = SANLK_ADD_ASYNC;
+
 	printf("add lockspace%d...\n", i);
 
-	rv = sanlock_add_lockspace(&ls, 0);
+	rv = sanlock_add_lockspace(&ls, flags);
 	if (rv == -EEXIST)
 		return 0;
 
@@ -295,6 +300,24 @@ static int add_lockspace(int i)
 		return -1;
 	}
 
+	if (!async)
+		goto out;
+
+	while (1) {
+		rv = sanlock_inq_lockspace(&ls, 0);
+		if (!rv)
+			goto out;
+
+		if (rv == -EINPROGRESS) {
+			sleep(2);
+			continue;
+		}
+
+		log_error("sanlock_inq_lockspace error %d", rv);
+		return -1;
+	}
+
+ out:
 	printf("add done\n");
 	return 0;
 }