src/main.c | 23 +++++++++-----
src/sanlock_internal.h | 11 +++---
src/sanlock_resource.h | 1
src/token_manager.c | 80 ++++++++++++++++++++++++++++++++-----------------
src/token_manager.h | 2 -
tests/devcount.c | 25 +++++++++++++++
6 files changed, 101 insertions(+), 41 deletions(-)
New commits:
commit e3077bf925432aa852f6eb4f3eb552c21a09d353
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Sep 14 13:50:33 2011 -0500
sanlock: setup fewer aio events
since there is a system limit on these, we'd unnecessarily
cause io_setup to fail and not use aio at all. We really
shouldn't need many anyway.
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index 0bd3d4c..6e7aae6 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -431,11 +431,11 @@ struct space {
* The delta lease algorithm expects real io timeouts.
*/
-#define HOSTID_AIO_CB_SIZE 64
-#define WORKER_AIO_CB_SIZE 8
-#define DIRECT_AIO_CB_SIZE 8
-#define RESOURCE_AIO_CB_SIZE 64
-#define LIB_AIO_CB_SIZE 8
+#define HOSTID_AIO_CB_SIZE 4
+#define WORKER_AIO_CB_SIZE 2
+#define DIRECT_AIO_CB_SIZE 1
+#define RESOURCE_AIO_CB_SIZE 2
+#define LIB_AIO_CB_SIZE 1
struct aicb {
int used;
commit dd6791b3844e16144fbe81330371f5d4f4847757
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Sep 14 13:24:51 2011 -0500
sanlock: add flag RESTRICT_SIGKILL
which causes sanlock to only use SIGTERM to kill pid,
not to escalate to SIGKILL. A process may be holding
a lease based on state outside itself; the fact that
the pid exits does not imply it's safe to release
the lease. The pid would choose to ignore the SIGTERM
if it's not safe to release the lease, and using
SIGKILL makes it impossible for the pid to make that
choice.
diff --git a/src/main.c b/src/main.c
index d3d0c4f..2d6eeee 100644
--- a/src/main.c
+++ b/src/main.c
@@ -66,7 +66,7 @@ struct client {
int suspend;
int need_free;
int killing;
- uint32_t cmd_flags;
+ uint32_t restrict;
char owner_name[SANLK_NAME_LEN+1];
pthread_mutex_t mutex;
void *workfn;
@@ -183,7 +183,7 @@ static void _client_free(int ci)
cl->suspend = 0;
cl->need_free = 0;
cl->killing = 0;
- cl->cmd_flags = 0;
+ cl->restrict = 0;
memset(cl->owner_name, 0, sizeof(cl->owner_name));
cl->workfn = NULL;
cl->deadfn = NULL;
@@ -432,7 +432,7 @@ static void kill_pids(struct space *sp)
{
struct client *cl;
int ci, pid;
- int sig = SIGTERM;
+ int sig;
int found = 0;
log_space(sp, "kill_pids %d", sp->killing_pids);
@@ -447,8 +447,6 @@ static void kill_pids(struct space *sp)
goto do_dump;
}
- if (sp->killing_pids > 1)
- sig = SIGKILL;
sp->killing_pids++;
for (ci = 0; ci <= client_maxi; ci++) {
@@ -464,6 +462,11 @@ static void kill_pids(struct space *sp)
if (!client_using_space(cl, sp))
goto unlock;
+ if (!(cl->restrict & SANLK_RESTRICT_SIGKILL) && cl->killing > 1)
+ sig = SIGKILL;
+ else
+ sig = SIGTERM;
+
pid = cl->pid;
cl->killing++;
found++;
@@ -777,6 +780,7 @@ static void cmd_acquire(struct task *task, struct cmd_args *ca)
char *opt_str;
uint64_t acquire_lver = 0;
uint32_t new_num_hosts = 0;
+ uint32_t cl_restrict;
int token_len, disks_len;
int fd, rv, i, j, empty_slots;
int alloc_count = 0, add_count = 0, acquire_count = 0;
@@ -809,6 +813,7 @@ static void cmd_acquire(struct task *task, struct cmd_args *ca)
pthread_mutex_unlock(&cl->mutex);
goto done;
}
+ cl_restrict = cl->restrict;
empty_slots = 0;
for (i = 0; i < SANLK_MAX_RESOURCES; i++) {
@@ -962,7 +967,7 @@ static void cmd_acquire(struct task *task, struct cmd_args *ca)
for (i = 0; i < new_tokens_count; i++) {
token = new_tokens[i];
- rv = add_resource(token, cl_pid);
+ rv = add_resource(token, cl_pid, cl_restrict);
if (rv < 0) {
if (!com.quiet_fail)
log_errot(token, "cmd_acquire %d,%d,%d add_resource %d",
@@ -1967,6 +1972,7 @@ static int print_state_client(struct client *cl, int ci, char *str)
"ci=%d "
"fd=%d "
"pid=%d "
+ "restrict=%x "
"cmd_active=%d "
"cmd_last=%d "
"pid_dead=%d "
@@ -1976,6 +1982,7 @@ static int print_state_client(struct client *cl, int ci, char *str)
ci,
cl->fd,
cl->pid,
+ cl->restrict,
cl->cmd_active,
cl->cmd_last,
cl->pid_dead,
@@ -2297,7 +2304,7 @@ static void cmd_restrict(int ci, int fd, struct sm_header *h_recv)
log_debug("cmd_restrict ci %d fd %d pid %d flags %x",
ci, fd, client[ci].pid, h_recv->cmd_flags);
- client[ci].cmd_flags = h_recv->cmd_flags;
+ client[ci].restrict = h_recv->cmd_flags;
send_result(fd, h_recv, 0);
}
@@ -2532,7 +2539,7 @@ static void process_connection(int ci)
ci, rv, h.magic, SM_MAGIC);
goto dead;
}
- if (client[ci].cmd_flags & SANLK_RESTRICT_ALL) {
+ if (client[ci].restrict & SANLK_RESTRICT_ALL) {
log_error("ci %d fd %d pid %d cmd %d restrict all",
ci, client[ci].fd, client[ci].pid, h.cmd);
goto dead;
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index 0b39934..0bd3d4c 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -90,6 +90,7 @@ struct token {
};
#define R_EXAMINE 0x00000001
+#define R_NO_SIGKILL 0x00000002
struct resource {
struct list_head list;
diff --git a/src/sanlock_resource.h b/src/sanlock_resource.h
index 770ebfc..ae8dfa4 100644
--- a/src/sanlock_resource.h
+++ b/src/sanlock_resource.h
@@ -22,6 +22,7 @@
/* restrict flags */
#define SANLK_RESTRICT_ALL 0x00000001
+#define SANLK_RESTRICT_SIGKILL 0x00000002
/* release flags */
#define SANLK_REL_ALL 0x00000001
diff --git a/src/token_manager.c b/src/token_manager.c
index 33de66a..824a202 100644
--- a/src/token_manager.c
+++ b/src/token_manager.c
@@ -98,7 +98,7 @@ static void save_resource_lver(struct token *token, uint64_t lver)
}
-int add_resource(struct token *token, int pid)
+int add_resource(struct token *token, int pid, uint32_t cl_restrict)
{
struct resource *r;
int rv, disks_len, r_len;
@@ -135,6 +135,8 @@ int add_resource(struct token *token, int pid)
r->token_id = token->token_id;
r->token = token;
r->pid = pid;
+ if (cl_restrict & SANLK_RESTRICT_SIGKILL)
+ r->flags |= R_NO_SIGKILL;
list_add_tail(&r->list, &resources);
rv = 0;
out:
@@ -343,6 +345,55 @@ static int examine_token(struct task *task, struct token *token,
return rv;
}
+static void do_req_kill_pid(struct token *tt, int pid)
+{
+ struct resource *r;
+ uint32_t flags;
+ int found = 0;
+
+ pthread_mutex_lock(&resource_mutex);
+ r = find_resource(tt, &resources);
+ if (r && r->pid == pid) {
+ found = 1;
+ flags = r->flags;
+ }
+ pthread_mutex_unlock(&resource_mutex);
+
+ if (!found) {
+ log_error("req pid %d %.48s:%.48s not found",
+ pid, tt->r.lockspace_name, tt->r.name);
+ return;
+ }
+
+ log_debug("do_req_kill_pid %d flags %x %.48s:%.48s",
+ pid, flags, tt->r.lockspace_name, tt->r.name);
+
+ /* TODO: share code with kill_pids() to gradually
+ * escalate from killscript, SIGTERM, SIGKILL */
+
+ kill(pid, SIGTERM);
+
+ if (flags & R_NO_SIGKILL)
+ return;
+
+ sleep(1);
+ kill(pid, SIGKILL);
+}
+
+/*
+ * TODO? add force_mode SANLK_REQ_KILL_PID_OR_RESET
+ * which would attempt to kill the pid like KILL_PID,
+ * but if the pid doesn't exit will block watchdog
+ * updates to reset the host.
+ * Here set r->block_wd_time = now + pid_exit_time,
+ * In renewal check for any r in resources with
+ * block_wd_time <= now, and if found will not
+ * update the watchdog. If the pid continues to
+ * not exit, the wd will fire and reset the machine.
+ * If the pid exits before pid_exit_time, no wd
+ * updates will be skipped.
+ */
+
/*
* - releases tokens of pid's that die
* - examines request blocks of resources
@@ -440,33 +491,8 @@ static void *resource_thread(void *arg GNUC_UNUSED)
continue;
}
- /*
- * TODO: add force_mode SANLK_REQ_KILL_PID_OR_RESET
- * which would attempt to kill the pid like KILL_PID,
- * but if the pid doesn't exit will block watchdog
- * updates to reset the host.
- * Here set r->block_wd_time = now + pid_exit_time,
- * In renewal check for any r in resources with
- * block_wd_time <= now, and if found will not
- * update the watchdog. If the pid continues to
- * not exit, the wd will fire and reset the machine.
- * If the pid exits before pid_exit_time, no wd
- * updates will be skipped.
- */
-
if (req.force_mode == SANLK_REQ_KILL_PID) {
- /* look up r again? verify it exists and pid same */
-
- log_debug("req force_mode %u pid %d %.48s:%.48s",
- req.force_mode, pid,
- tt->r.lockspace_name, tt->r.name);
-
- /* TODO: share code with kill_pids() to gradually
- * escalate from killscript, SIGTERM, SIGKILL */
-
- kill(pid, SIGTERM);
- sleep(1);
- kill(pid, SIGKILL);
+ do_req_kill_pid(tt, pid);
} else {
log_error("req force_mode %u unknown", req.force_mode);
}
diff --git a/src/token_manager.h b/src/token_manager.h
index da34baa..751b636 100644
--- a/src/token_manager.h
+++ b/src/token_manager.h
@@ -19,7 +19,7 @@ void release_token_async(struct token *token);
int request_token(struct task *task, struct token *token, uint32_t force_mode,
uint64_t *owner_id);
-int add_resource(struct token *token, int pid);
+int add_resource(struct token *token, int pid, uint32_t cl_restrict);
void del_resource(struct token *token);
int set_resource_examine(char *space_name, char *res_name);
diff --git a/tests/devcount.c b/tests/devcount.c
index 9db5a43..6c51c6c 100644
--- a/tests/devcount.c
+++ b/tests/devcount.c
@@ -58,6 +58,18 @@ do { \
printf("ERROR %llu " fmt "\n", (unsigned long long)time(NULL),
##args); \
} while (0)
+static void sigterm_handler(int sig)
+{
+ log_debug("sigterm_handler %d", sig);
+}
+
+static void setup_sigterm(void)
+{
+ struct sigaction act;
+ memset(&act, 0, sizeof(act));
+ act.sa_handler = sigterm_handler;
+ sigaction(SIGTERM, &act, NULL);
+}
static int kill_pid(int pid)
{
@@ -836,6 +848,13 @@ static int do_wrap(int argc, char *argv[])
exit(-1);
}
+ rv = sanlock_restrict(sock, SANLK_RESTRICT_SIGKILL);
+ if (rv < 0) {
+ log_error("%s c %d sanlock_restrict error %d",
+ count_path, pid, sock);
+ exit(-1);
+ }
+
rv = sanlock_acquire(sock, -1, 0, 1, &res, NULL);
if (rv < 0) {
log_error("%s c %d sanlock_acquire error %d",
@@ -1489,6 +1508,12 @@ int main(int argc, char *argv[])
else if (!strcmp(argv[1], "rw") || !strcmp(argv[1], "wr"))
rv = do_count(argc, argv);
+ else if (!strcmp(argv[1], "rwsig")) {
+ setup_sigterm();
+ argv[1] = "rw";
+ rv = do_count(argc, argv);
+ }
+
else if (!strcmp(argv[1], "lock"))
rv = do_lock(argc, argv);