src/client.c src/host_id.c src/host_id.h src/main.c src/sanlock_internal.h src/sanlock_resource.h src/sanlock_sock.h src/token_manager.c src/token_manager.h

[David Teigland]

 src/client.c           |   39 ++++++++
 src/host_id.c          |   31 +++---
 src/host_id.h          |    1 
 src/main.c             |   78 ++++++++++++++--
 src/sanlock_internal.h |   15 +--
 src/sanlock_resource.h |   18 ++-
 src/sanlock_sock.h     |    2 
 src/token_manager.c    |  238 +++++++++++++++++++++++++++++++++++++++++--------
 src/token_manager.h    |    2 
 9 files changed, 359 insertions(+), 65 deletions(-)

New commits:
commit 5f333741f2093f92113e0f0e1adc452d1e20be4c
Author: David Teigland <teigland(a)redhat.com&gt;
Date:   Wed Aug 31 16:00:53 2011 -0500

    sanlock: examine resource requests
    
    when we find our host_id set in another host's delta lease,
    or when the examine api/cmd is called, scan all resource
    request blocks for any requests we should process.

diff --git a/src/client.c b/src/client.c
index 3115e07..4987867 100644
--- a/src/client.c
+++ b/src/client.c
@@ -690,6 +690,45 @@ int sanlock_request(uint32_t flags, uint32_t force_mode,
 	return rv;
 }
 
+int sanlock_examine(uint32_t flags, struct sanlk_lockspace *ls,
+		    struct sanlk_resource *res)
+{
+	char *data;
+	int rv, fd, cmd, datalen;
+
+	if (!ls && !res)
+		return -EINVAL;
+
+	rv = connect_socket(&fd);
+	if (rv < 0)
+		return rv;
+
+	if (ls && ls->host_id_disk.path[0]) {
+		cmd = SM_CMD_EXAMINE_LOCKSPACE;
+		datalen = sizeof(struct sanlk_lockspace);
+		data = (char *)ls;
+	} else {
+		cmd = SM_CMD_EXAMINE_RESOURCE;
+		datalen = sizeof(struct sanlk_resource);
+		data = (char *)res;
+	}
+
+	rv = send_header(fd, cmd, flags, datalen, 0, 0);
+	if (rv < 0)
+		goto out;
+
+	rv = send(fd, data, datalen, 0);
+	if (rv < 0) {
+		rv = -errno;
+		goto out;
+	}
+
+	rv = recv_result(fd);
+ out:
+	close(fd);
+	return rv;
+}
+
 /*
  * convert from struct sanlk_resource to string with format:
  *
<lockspace_name>:<resource_name>:<path>:<offset>[:<path>:<offset>...]:<lver>
diff --git a/src/host_id.c b/src/host_id.c
index a1cc468..badccd9 100644
--- a/src/host_id.c
+++ b/src/host_id.c
@@ -32,6 +32,7 @@
 #include "watchdog.h"
 #include "task.h"
 #include "direct.h"
+#include "token_manager.h"
 
 static unsigned int space_id_counter = 1;
 
@@ -146,6 +147,17 @@ int get_space_info(char *space_name, struct space *sp_out)
 	return rv;
 }
 
+void block_watchdog_updates(char *space_name)
+{
+	struct space *sp;
+
+	pthread_mutex_lock(&spaces_mutex);
+	sp = _search_space(space_name, NULL, 0, &spaces, NULL, NULL);
+	if (sp)
+		sp->block_watchdog_updates = 1;
+	pthread_mutex_unlock(&spaces_mutex);
+}
+
 int host_id_disk_info(char *name, struct sync_disk *disk)
 {
 	struct space space;
@@ -269,7 +281,7 @@ void check_other_leases(struct task *task, struct space *sp, char
*buf)
 	struct host_info *info;
 	char *bitmap;
 	uint64_t now;
-	int i, new = 0;
+	int i, new;
 
 	disk = &sp->host_id_disk;
 
@@ -309,18 +321,11 @@ void check_other_leases(struct task *task, struct space *sp, char
*buf)
 
 		log_space(sp, "request from host_id %d", i+1);
 		info->last_req = now;
-		new++;
+		new = 1;
 	}
 
-	/* TODO: add a thread that will periodically scan spaces and
-	   for any with req_count > req_check, scan request blocks for
-	   all locally held paxos leases in that lockspace. */
-
-	if (new) {
-		pthread_mutex_lock(&sp->mutex);
-		sp->req_count++;
-		pthread_mutex_unlock(&sp->mutex);
-	}
+	if (new)
+		set_resource_examine(sp->space_name, NULL);
 }
 
 /*
@@ -546,7 +551,9 @@ static void *lockspace_thread(void *arg_in)
 		 * pet the watchdog
 		 */
 
-		if (delta_result == SANLK_OK && !sp->thread_stop)
+		if (delta_result == SANLK_OK &&
+		    !sp->thread_stop &&
+		    !sp->block_watchdog_updates)
 			update_watchdog_file(sp, last_success);
 
 		pthread_mutex_unlock(&sp->mutex);
diff --git a/src/host_id.h b/src/host_id.h
index 3a289c2..ea5c225 100644
--- a/src/host_id.h
+++ b/src/host_id.h
@@ -12,6 +12,7 @@
 int print_space_state(struct space *sp, char *str);
 int _get_space_info(char *space_name, struct space *sp_out);
 int get_space_info(char *space_name, struct space *sp_out);
+void block_watchdog_updates(char *space_name);
 int host_id_disk_info(char *name, struct sync_disk *disk);
 int host_info_set_bit(char *space_name, uint64_t host_id);
 int host_info_clear_bit(char *space_name, uint64_t host_id);
diff --git a/src/main.c b/src/main.c
index 8b354b2..ef98a45 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1438,6 +1438,55 @@ static void cmd_request(struct task *task, struct cmd_args *ca)
 	client_resume(ca->ci_in);
 }
 
+static void cmd_examine(struct task *task GNUC_UNUSED, struct cmd_args *ca)
+{
+	union {
+		struct sanlk_resource r;
+		struct sanlk_lockspace s;
+	} buf;
+	struct sanlk_resource *res = NULL;
+	struct sanlk_lockspace *ls = NULL;
+	char *space_name = NULL;
+	char *res_name = NULL;
+	int fd, rv, result, count = 0, datalen;
+
+	fd = client[ca->ci_in].fd;
+
+	if (ca->header.cmd == SM_CMD_EXAMINE_RESOURCE) {
+		datalen = sizeof(struct sanlk_resource);
+		res = &buf.r;
+	} else {
+		datalen = sizeof(struct sanlk_lockspace);
+		ls = &buf.s;
+	}
+
+	rv = recv(fd, &buf, datalen, MSG_WAITALL);
+	if (rv != datalen) {
+		log_error("cmd_examine %d,%d recv %d %d",
+			  ca->ci_in, fd, rv, errno);
+		result = -ENOTCONN;
+		goto reply;
+	}
+
+	if (res) {
+		space_name = res->lockspace_name;
+		res_name = res->name;
+	} else {
+		space_name = ls->name;
+	}
+
+	log_debug("cmd_examine %d,%d %.48s %.48s",
+		  ca->ci_in, fd, space_name, res_name ? res_name : "");
+
+	count = set_resource_examine(space_name, res_name);
+	result = 0;
+ reply:
+	log_debug("cmd_examine %d,%d done %d", ca->ci_in, fd, count);
+
+	send_result(fd, &ca->header, result);
+	client_resume(ca->ci_in);
+}
+
 static void cmd_add_lockspace(struct cmd_args *ca)
 {
 	struct sanlk_lockspace lockspace;
@@ -1698,6 +1747,10 @@ static void call_cmd(struct task *task, struct cmd_args *ca)
 	case SM_CMD_INIT_RESOURCE:
 		cmd_init_resource(task, ca);
 		break;
+	case SM_CMD_EXAMINE_LOCKSPACE:
+	case SM_CMD_EXAMINE_RESOURCE:
+		cmd_examine(task, ca);
+		break;
 	};
 }
 
@@ -2277,6 +2330,8 @@ static void process_connection(int ci)
 	case SM_CMD_ADD_LOCKSPACE:
 	case SM_CMD_REM_LOCKSPACE:
 	case SM_CMD_REQUEST:
+	case SM_CMD_EXAMINE_RESOURCE:
+	case SM_CMD_EXAMINE_LOCKSPACE:
 	case SM_CMD_ALIGN:
 	case SM_CMD_INIT_LOCKSPACE:
 	case SM_CMD_INIT_RESOURCE:
@@ -2741,8 +2796,7 @@ static void print_usage(void)
 	printf("sanlock client status [-D]\n");
 	printf("sanlock client log_dump\n");
 	printf("sanlock client shutdown\n");
-	printf("sanlock client init -s LOCKSPACE\n");
-	printf("sanlock client init -r RESOURCE\n");
+	printf("sanlock client init -s LOCKSPACE | -r RESOURCE\n");
 	printf("sanlock client align -s LOCKSPACE\n");
 	printf("sanlock client add_lockspace -s LOCKSPACE\n");
 	printf("sanlock client rem_lockspace -s LOCKSPACE\n");
@@ -2750,13 +2804,12 @@ static void print_usage(void)
 	printf("sanlock client acquire -r RESOURCE -p <pid>\n");
 	printf("sanlock client release -r RESOURCE -p <pid>\n");
 	printf("sanlock client inquire -p <pid>\n");
-	printf("sanlock client request -r RESOURCE\n");
+	printf("sanlock client request -r RESOURCE -f <force_mode>\n");
+	printf("sanlock client examine -r RESOURCE | -s LOCKSPACE\n");
 	printf("\n");
 	printf("sanlock direct <action> [-a 0|1] [-o 0|1]\n");
-	printf("sanlock direct init -s LOCKSPACE\n");
-	printf("sanlock direct init -r RESOURCE\n");
-	printf("sanlock direct read_leader -s LOCKSPACE\n");
-	printf("sanlock direct read_leader -r RESOURCE\n");
+	printf("sanlock direct init -s LOCKSPACE | -r RESOURCE\n");
+	printf("sanlock direct read_leader -s LOCKSPACE | -r RESOURCE\n");
 	printf("sanlock direct read_id -s LOCKSPACE\n");
 	printf("sanlock direct live_id -s LOCKSPACE\n");
 	printf("sanlock direct dump <path>[:<offset>]\n");
@@ -2856,6 +2909,8 @@ static int read_command_line(int argc, char *argv[])
 			com.action = ACT_INQUIRE;
 		else if (!strcmp(act, "request"))
 			com.action = ACT_REQUEST;
+		else if (!strcmp(act, "examine"))
+			com.action = ACT_EXAMINE;
 		else if (!strcmp(act, "align"))
 			com.action = ACT_CLIENT_ALIGN;
 		else if (!strcmp(act, "init"))
@@ -3177,6 +3232,15 @@ static int do_client(void)
 		log_tool("request done %d", rv);
 		break;
 
+	case ACT_EXAMINE:
+		log_tool("examine");
+		if (com.lockspace.host_id_disk.path[0])
+			rv = sanlock_examine(0, &com.lockspace, NULL);
+		else
+			rv = sanlock_examine(0, NULL, com.res_args[0]);
+		log_tool("examine done %d", rv);
+		break;
+
 	case ACT_CLIENT_ALIGN:
 		log_tool("align");
 		rv = sanlock_align(&com.lockspace.host_id_disk);
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index 4e69988..3d78a0c 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -24,6 +24,7 @@
 
 #include "sanlock.h"
 #include "sanlock_rv.h"
+#include "sanlock_resource.h"
 #include "leader.h"
 #include "list.h"
 #include "monotime.h"
@@ -60,9 +61,11 @@ struct sync_disk {
 	int fd;			/* sanlk_disk pad2 */
 };
 
-/* Once token and token->disks are initialized by the main loop, the only
-   fields that are modified are disk fd's by open_disks() in the lease
-   threads. */
+/*
+ * There are two different wrappers around a sanlk_resource:
+ * 'struct token' keeps track of resources per-client, client.tokens[]
+ * 'struct resource' keeps track of resources globally, resources list
+ */
 
 struct token {
 	/* values copied from acquire res arg */
@@ -121,13 +124,12 @@ struct space {
 	int space_dead;
 	int killing_pids;
 	int external_remove;
+	int block_watchdog_updates;
 	int thread_stop;
 	pthread_t thread;
 	pthread_mutex_t mutex; /* protects lease_status, thread_stop  */
 	struct lease_status lease_status;
 	int wd_fd;
-	uint32_t req_count;
-	uint32_t req_check;
 	struct host_info host_info[DEFAULT_MAX_HOSTS];
 };
 
@@ -420,7 +422,7 @@ struct space {
 #define HOSTID_AIO_CB_SIZE 64
 #define WORKER_AIO_CB_SIZE 8
 #define DIRECT_AIO_CB_SIZE 8
-#define RELEASE_AIO_CB_SIZE 64
+#define RESOURCE_AIO_CB_SIZE 64
 #define LIB_AIO_CB_SIZE 8
 
 struct aicb {
@@ -520,6 +522,7 @@ enum {
 	ACT_READ_LEADER,
 	ACT_CLIENT_INIT,
 	ACT_CLIENT_ALIGN,
+	ACT_EXAMINE,
 };
 
 EXTERN int external_shutdown;
diff --git a/src/sanlock_resource.h b/src/sanlock_resource.h
index 2005022..4998c16 100644
--- a/src/sanlock_resource.h
+++ b/src/sanlock_resource.h
@@ -20,9 +20,17 @@
  * registered pid
  */
 
-int sanlock_register(void);
+/* restrict flags */
+#define SANLK_RESTRICT_ALL	0x00000001
+
+/* release flags */
+#define SANLK_REL_ALL		0x00000001
 
-#define SANLK_RESTRICT_ALL 0x1
+/* request flags */
+#define SANLK_REQ_KILL_PID	0x00000001
+#define SANLK_REQ_BLOCK_WD	0x00000002
+
+int sanlock_register(void);
 
 int sanlock_restrict(int sock, uint32_t flags);
 
@@ -30,8 +38,6 @@ int sanlock_acquire(int sock, int pid, uint32_t flags, int res_count,
 		    struct sanlk_resource *res_args[],
 		    struct sanlk_options *opt_in);
 
-#define SANLK_REL_ALL 0x1
-
 int sanlock_release(int sock, int pid, uint32_t flags, int res_count,
 		    struct sanlk_resource *res_args[]);
 
@@ -41,12 +47,14 @@ int sanlock_inquire(int sock, int pid, uint32_t flags, int
*res_count,
 int sanlock_request(uint32_t flags, uint32_t force_mode,
 		    struct sanlk_resource *res);
 
+int sanlock_examine(uint32_t flags, struct sanlk_lockspace *ls,
+		    struct sanlk_resource *res);
+
 /*
  * Functions to convert between string and struct resource formats.
  * All allocate space for returned data that the caller must free.
  */
 
-
 /*
  * convert from struct sanlk_resource to string with format:
  *
<lockspace_name>:<resource_name>:<path>:<offset>[:<path>:<offset>...]:<lver>
diff --git a/src/sanlock_sock.h b/src/sanlock_sock.h
index 32c3511..5c6a004 100644
--- a/src/sanlock_sock.h
+++ b/src/sanlock_sock.h
@@ -32,6 +32,8 @@ enum {
 	SM_CMD_ALIGN		= 12,
 	SM_CMD_INIT_LOCKSPACE	= 13,
 	SM_CMD_INIT_RESOURCE	= 14,
+	SM_CMD_EXAMINE_LOCKSPACE = 15,
+	SM_CMD_EXAMINE_RESOURCE	 = 16,
 };
 
 struct sm_header {
diff --git a/src/token_manager.c b/src/token_manager.c
index b786e12..4d447a6 100644
--- a/src/token_manager.c
+++ b/src/token_manager.c
@@ -19,6 +19,7 @@
 #include <pthread.h>
 #include <time.h>
 #include <syslog.h>
+#include <signal.h>
 #include <sys/types.h>
 #include <sys/time.h>
 
@@ -28,41 +29,92 @@
 #include "paxos_lease.h"
 #include "token_manager.h"
 #include "task.h"
+#include "host_id.h"
 
 static struct list_head resources;
 static struct list_head dispose_resources;
 static pthread_mutex_t resource_mutex = PTHREAD_MUTEX_INITIALIZER;
 static pthread_cond_t resource_cond = PTHREAD_COND_INITIALIZER;
-static pthread_t release_thread;
-static int release_thread_stop;
+static pthread_t resource_pt;
+static int resource_thread_stop;
+static int resource_examine;
 
 struct resource {
 	struct list_head list;
-	char space_name[NAME_ID_SIZE+1];
-	char resource_name[NAME_ID_SIZE+1];
 	struct token *token;
 	int pid;
+	int examine;
+	uint64_t lver;
+	struct sanlk_resource r;
 };
 
+int set_resource_examine(char *space_name, char *res_name)
+{
+	struct resource *r;
+	int count = 0;
+
+	pthread_mutex_lock(&resource_mutex);
+	list_for_each_entry(r, &resources, list) {
+		if (strncmp(r->r.lockspace_name, space_name, NAME_ID_SIZE))
+			continue;
+		if (res_name && strncmp(r->r.name, res_name, NAME_ID_SIZE))
+			continue;
+		r->examine = 1;
+		resource_examine = 1;
+		count++;
+	}
+	if (count)
+		pthread_cond_signal(&resource_cond);
+	pthread_mutex_unlock(&resource_mutex);
+
+	return count;
+}
+
+static struct resource *find_resource_examine(void)
+{
+	struct resource *r;
+
+	list_for_each_entry(r, &resources, list) {
+		if (r->examine)
+			return r;
+	}
+	return NULL;
+}
+
 static struct resource *find_resource(struct token *token,
 				      struct list_head *head)
 {
 	struct resource *r;
 
 	list_for_each_entry(r, head, list) {
-		if (strncmp(r->space_name, token->r.lockspace_name, NAME_ID_SIZE))
+		if (strncmp(r->r.lockspace_name, token->r.lockspace_name, NAME_ID_SIZE))
 			continue;
-		if (strncmp(r->resource_name, token->r.name, NAME_ID_SIZE))
+		if (strncmp(r->r.name, token->r.name, NAME_ID_SIZE))
 			continue;
 		return r;
 	}
 	return NULL;
 }
 
+static void save_resource_lver(struct token *token, uint64_t lver)
+{
+	struct resource *r;
+
+	pthread_mutex_lock(&resource_mutex);
+	r = find_resource(token, &resources);
+	if (r)
+		r->lver = lver;
+	pthread_mutex_unlock(&resource_mutex);
+
+	if (!r)
+		log_errot(token, "save_resource_lver no r");
+
+}
+
 int add_resource(struct token *token, int pid)
 {
 	struct resource *r;
-	int rv;
+	int rv, disks_len, r_len;
 
 	pthread_mutex_lock(&resource_mutex);
 
@@ -82,15 +134,17 @@ int add_resource(struct token *token, int pid)
 		goto out;
 	}
 
-	r = malloc(sizeof(struct resource));
+	disks_len = token->r.num_disks * sizeof(struct sync_disk);
+	r_len = sizeof(struct resource) + disks_len;
+
+	r = malloc(r_len);
 	if (!r) {
 		rv = -ENOMEM;
 		goto out;
 	}
-
-	memset(r, 0, sizeof(struct resource));
-	strncpy(r->space_name, token->r.lockspace_name, NAME_ID_SIZE);
-	strncpy(r->resource_name, token->r.name, NAME_ID_SIZE);
+	memset(r, 0, r_len);
+	memcpy(&r->r, &token->r, sizeof(struct sanlk_resource));
+	memcpy(&r->r.disks, &token->r.disks, disks_len);
 	r->token = token;
 	r->pid = pid;
 	list_add_tail(&r->list, &resources);
@@ -152,6 +206,8 @@ int acquire_token(struct task *task, struct token *token,
 	if (rv < 0)
 		return rv;
 
+	save_resource_lver(token, token->leader.lver);
+
 	memcpy(&token->leader, &leader_ret, sizeof(struct leader_record));
 	token->r.lver = token->leader.lver;
 	return rv; /* SANLK_OK */
@@ -246,7 +302,7 @@ int request_token(struct task *task, struct token *token, uint32_t
force_mode,
 		goto out;
 	}
 
-req_write:
+ req_write:
 	req.version = REQ_DISK_VERSION_MAJOR | REQ_DISK_VERSION_MINOR;
 	req.lver = token->acquire_lver;
 	req.force_mode = force_mode;
@@ -262,46 +318,158 @@ req_write:
 	return rv;
 }
 
-/* thread that releases tokens of pid's that die */
+static int examine_token(struct task *task, struct token *token,
+			 struct request_record *req_out)
+{
+	struct request_record req;
+	int rv;
+
+	memset(&req, 0, sizeof(req));
+
+	rv = open_disks(token->disks, token->r.num_disks);
+	if (!majority_disks(token, rv)) {
+		log_debug("request open_disk error %s", token->disks[0].path);
+		return -ENODEV;
+	}
+
+	rv = paxos_lease_request_read(task, token, &req);
+	if (rv < 0)
+		goto out;
 
-static void *async_release_thread(void *arg GNUC_UNUSED)
+	if (req.magic != REQ_DISK_MAGIC) {
+		rv = SANLK_REQUEST_MAGIC;
+		goto out;
+	}
+
+	if ((req.version & 0xFFFF0000) != REQ_DISK_VERSION_MAJOR) {
+		rv = SANLK_REQUEST_VERSION;
+		goto out;
+	}
+
+	memcpy(req_out, &req, sizeof(struct request_record));
+ out:
+	close_disks(token->disks, token->r.num_disks);
+
+	log_debug("examine rv %d lver %llu mode %u",
+		  rv, (unsigned long long)req.lver, req.force_mode);
+
+	return rv;
+}
+
+/*
+ * - releases tokens of pid's that die
+ * - examines request blocks of resources
+ */
+
+static void *resource_thread(void *arg GNUC_UNUSED)
 {
 	struct task task;
 	struct resource *r;
-	struct token *token;
+	struct token *token, *tt = NULL;
+	struct request_record req;
+	uint64_t lver;
+	int rv, j, pid, tt_len;
 
 	memset(&task, 0, sizeof(struct task));
 	setup_task_timeouts(&task, main_task.io_timeout_seconds);
-	setup_task_aio(&task, main_task.use_aio, RELEASE_AIO_CB_SIZE);
-	sprintf(task.name, "%s", "release");
+	setup_task_aio(&task, main_task.use_aio, RESOURCE_AIO_CB_SIZE);
+	sprintf(task.name, "%s", "resource");
+
+	/* a fake/tmp token struct we copy necessary res info into,
+	   because other functions take a token struct arg */
+
+	tt_len = sizeof(struct token) + (SANLK_MAX_DISKS * sizeof(struct sync_disk));
+	tt = malloc(tt_len);
+	if (!tt) {
+		log_error("resource_thread tt malloc error");
+		goto out;
+	}
+	memset(tt, 0, tt_len);
+	tt->disks = (struct sync_disk *)&tt->r.disks[0];
 
 	while (1) {
 		pthread_mutex_lock(&resource_mutex);
-		while (list_empty(&dispose_resources)) {
-			if (release_thread_stop) {
+		while (list_empty(&dispose_resources) && !resource_examine) {
+			if (resource_thread_stop) {
 				pthread_mutex_unlock(&resource_mutex);
 				goto out;
 			}
 			pthread_cond_wait(&resource_cond, &resource_mutex);
 		}
 
-		r = list_first_entry(&dispose_resources, struct resource, list);
-		pthread_mutex_unlock(&resource_mutex);
+		if (!list_empty(&dispose_resources)) {
+			r = list_first_entry(&dispose_resources, struct resource, list);
+			pthread_mutex_unlock(&resource_mutex);
 
-		token = r->token;
-		release_token(&task, token);
+			token = r->token;
+			release_token(&task, token);
 
-		/* we don't want to remove r from dispose_list until after the
-		   lease is released because we don't want a new token for
-		   the same resource to be added and attempt to acquire
-		   the lease until after it's been released */
+			/* we don't want to remove r from dispose_list until after the
+		   	   lease is released because we don't want a new token for
+		   	   the same resource to be added and attempt to acquire
+		   	   the lease until after it's been released */
 
-		pthread_mutex_lock(&resource_mutex);
-		_del_resource(r);
-		pthread_mutex_unlock(&resource_mutex);
-		free(token);
+			pthread_mutex_lock(&resource_mutex);
+			_del_resource(r);
+			pthread_mutex_unlock(&resource_mutex);
+			free(token);
+
+		} else if (resource_examine) {
+			r = find_resource_examine();
+			if (!r) {
+				resource_examine = 0;
+				pthread_mutex_unlock(&resource_mutex);
+				continue;
+			}
+			r->examine = 0;
+
+			/* we can't safely access r->token here, and
+			   r may be freed after we release mutex, so copy
+			   everything we need before unlocking mutex */
+
+			pid = r->pid;
+			lver = r->lver;
+			memcpy(&tt->r, &r->r, sizeof(struct sanlk_resource));
+			memcpy(&tt->r.disks, &r->r.disks, r->r.num_disks * sizeof(struct
sync_disk));
+			pthread_mutex_unlock(&resource_mutex);
+
+			for (j = 0; j < tt->r.num_disks; j++) {
+				tt->disks[j].sector_size = 0;
+				tt->disks[j].fd = -1;
+			}
+
+			rv = examine_token(&task, tt, &req);
+
+			if (rv != SANLK_OK)
+				continue;
+
+			if (!req.force_mode || !req.lver)
+				continue;
+
+			if (req.lver <= lver) {
+				log_debug("examine req lver %llu our lver %llu",
+					  (unsigned long long)req.lver,
+					  (unsigned long long)lver);
+				continue;
+			}
+
+			if (req.force_mode == SANLK_REQ_KILL_PID) {
+				/* look up r again to check it still exists and
+				   pid is same? */
+
+				log_error("req_kill_pid %d %.48s:%.48s", pid,
+					  tt->r.lockspace_name, tt->r.name);
+				kill(pid, SIGKILL);
+
+			} else if (req.force_mode == SANLK_REQ_BLOCK_WD) {
+				log_error("req_block_wd %.48s", tt->r.lockspace_name);
+				block_watchdog_updates(tt->r.lockspace_name);
+			}
+		}
 	}
  out:
+	if (tt)
+		free(tt);
 	close_task_aio(&task);
 	return NULL;
 }
@@ -332,7 +500,7 @@ int setup_token_manager(void)
 	INIT_LIST_HEAD(&resources);
 	INIT_LIST_HEAD(&dispose_resources);
 
-	rv = pthread_create(&release_thread, NULL, async_release_thread, NULL);
+	rv = pthread_create(&resource_pt, NULL, resource_thread, NULL);
 	if (rv)
 		return -1;
 	return 0;
@@ -341,9 +509,9 @@ int setup_token_manager(void)
 void close_token_manager(void)
 {
 	pthread_mutex_lock(&resource_mutex);
-	release_thread_stop = 1;
+	resource_thread_stop = 1;
 	pthread_cond_signal(&resource_cond);
 	pthread_mutex_unlock(&resource_mutex);
-	pthread_join(release_thread, NULL);
+	pthread_join(resource_pt, NULL);
 }
 
diff --git a/src/token_manager.h b/src/token_manager.h
index 7825642..da34baa 100644
--- a/src/token_manager.h
+++ b/src/token_manager.h
@@ -22,6 +22,8 @@ int request_token(struct task *task, struct token *token, uint32_t
force_mode,
 int add_resource(struct token *token, int pid);
 void del_resource(struct token *token);
 
+int set_resource_examine(char *space_name, char *res_name);
+
 int setup_token_manager(void);
 void close_token_manager(void);