Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=e3a... Commit: e3a0d33dddf556dc698f31ff6ae0ce2fa66afbd3 Parent: 7579b0ae42a63f2fc2bea8d07b3da823b311cd78 Author: David Teigland teigland@redhat.com AuthorDate: Tue Mar 27 12:04:13 2012 -0500 Committer: David Teigland teigland@redhat.com CommitterDate: Tue Mar 27 12:04:13 2012 -0500
dlm_controld: fencing delay
- split enable_startup_fencing and post_join_delay values - set fence time after victim rejoins daemon cpg cleanly - use post_join_delay to avoid fencing a node in the short time between it joining the cluster (giving cluster quorum) and joining the daemon cpg, which allows it to bypass fencing - reassign some command line options
Signed-off-by: David Teigland teigland@redhat.com --- dlm_controld/config.c | 6 ++++++ dlm_controld/daemon_cpg.c | 44 +++++++++++++++++++++++++++++++++++--------- dlm_controld/dlm_daemon.h | 10 +++++++--- dlm_controld/main.c | 42 ++++++++++++++++++++++-------------------- dlm_controld/member.c | 4 +++- 5 files changed, 73 insertions(+), 33 deletions(-)
diff --git a/dlm_controld/config.c b/dlm_controld/config.c index f668b86..1066fbe 100644 --- a/dlm_controld/config.c +++ b/dlm_controld/config.c @@ -89,9 +89,15 @@ void setup_config(int update) else if (!optk_timewarn && !strncmp(line, "timewarn", strlen("timewarn")) && !update) set_val(line, &cfgk_timewarn);
+ else if (!optd_post_join_delay && !strncmp(line, "post_join_delay", strlen("post_join_delay"))) + set_val(line, &cfgd_post_join_delay); + else if (!optd_enable_fencing && !strncmp(line, "enable_fencing", strlen("enable_fencing")) && !update) set_val(line, &cfgd_enable_fencing);
+ else if (!optd_enable_startup_fencing && !strncmp(line, "enable_startup_fencing", strlen("enable_startup_fencing")) && !update) + set_val(line, &cfgd_enable_startup_fencing); + else if (!optd_enable_quorum_fencing && !strncmp(line, "enable_quorum_fencing", strlen("enable_quorum_fencing")) && !update) set_val(line, &cfgd_enable_quorum_fencing);
diff --git a/dlm_controld/daemon_cpg.c b/dlm_controld/daemon_cpg.c index e5e9270..d1f0e19 100644 --- a/dlm_controld/daemon_cpg.c +++ b/dlm_controld/daemon_cpg.c @@ -70,6 +70,7 @@ struct node_daemon { int recover_setup; int need_fence_clear; int need_fencing; + int delay_fencing; int fence_pid; int fence_pid_wait; int fence_actor_done; @@ -98,7 +99,7 @@ static int daemon_ringid_wait; static struct cpg_ring_id daemon_ringid; static int daemon_clear_nodeid; static int daemon_clear_pid; -static uint64_t last_join_monotime; +static uint64_t daemon_last_join_monotime; static uint32_t last_join_seq; static uint32_t send_fipu_seq; static int fence_in_progress_unknown = 1; @@ -736,13 +737,13 @@ static void daemon_fence_work(void) continue; }
- if (!cfgd_startup_fence) + if (!cfgd_enable_startup_fencing) continue;
- if (monotime() - last_join_monotime < cfgd_startup_fence) { + if (monotime() - daemon_last_join_monotime < cfgd_post_join_delay) { log_debug("fence startup %d delay %d from %llu", - node->nodeid, cfgd_startup_fence, - (unsigned long long)last_join_monotime); + node->nodeid, cfgd_post_join_delay, + (unsigned long long)daemon_last_join_monotime); poll_fencing++; continue; } @@ -792,8 +793,30 @@ static void daemon_fence_work(void) /* node rejoined cleanly, doesn't need fencing */ log_debug("fence request %d member skip", node->nodeid); node->need_fencing = 0; + node->fence_walltime = time(NULL); + node->fence_monotime = monotime(); + continue; + } + + /* + if (daemon_pid_wait) { + log_debug("fence request %d delay for other pid %d", + node->nodeid, daemon_pid_wait); + node->delay_fencing = 1; + poll_fencing++; continue; } + */ + + if (monotime() - cluster_last_join_monotime < cfgd_post_join_delay) { + log_debug("fence request %d delay %d from %llu", + node->nodeid, cfgd_post_join_delay, + (unsigned long long)cluster_last_join_monotime); + node->delay_fencing = 1; + poll_fencing++; + continue; + } + node->delay_fencing = 0;
/* get_fence_actor picks the low nodeid that existed when node failed and is still around. if the current @@ -836,6 +859,9 @@ static void daemon_fence_work(void) if (!node->need_fencing) continue;
+ if (node->delay_fencing) + continue; + if (!node->fence_pid_wait) { /* * another node is the actor, or we were actor, @@ -873,7 +899,7 @@ static void daemon_fence_work(void) if (rv < 0) { /* shouldn't happen */ log_error("fence wait %d pid %d error %d", - node->nodeid, node->fence_pid_wait, rv); + node->nodeid, node->fence_pid, rv); node->fence_pid_wait = 0; continue; } @@ -913,7 +939,7 @@ static void daemon_fence_work(void) out_fipu: need = nodes_need_fencing();
- if (cfgd_startup_fence && fence_in_progress_unknown && !need && list_empty(&startup_nodes)) { + if (cfgd_enable_startup_fencing && fence_in_progress_unknown && !need && list_empty(&startup_nodes)) { /* * case A in comment above * all nodes are starting and have fipu set, they all do @@ -969,7 +995,7 @@ static void daemon_fence_work(void) } }
- if (!cfgd_startup_fence && fence_in_progress_unknown) { + if (!cfgd_enable_startup_fencing && fence_in_progress_unknown) { /* * case C in comment above * all nodes are starting and have fipu set. All expect a @@ -1712,7 +1738,7 @@ static void confchg_cb_daemon(cpg_handle_t handle, node->daemon_member = 1; node->daemon_add_time = now;
- last_join_monotime = now; + daemon_last_join_monotime = now; last_join_seq++;
/* a joining node shows prev members in joined list */ diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h index 000ba22..da53627 100644 --- a/dlm_controld/dlm_daemon.h +++ b/dlm_controld/dlm_daemon.h @@ -81,7 +81,9 @@ #define DEFAULT_LOGFILE LOG_FILE_PATH
#define DEFAULT_DEBUG_LOGFILE 0 +#define DEFAULT_POST_JOIN_DELAY 30 #define DEFAULT_ENABLE_FENCING 1 +#define DEFAULT_ENABLE_STARTUP_FENCING 1 #define DEFAULT_ENABLE_QUORUM_FENCING 1 #define DEFAULT_ENABLE_QUORUM_LOCKSPACE 0 #define DEFAULT_ENABLE_FSCONTROL 0 @@ -93,7 +95,6 @@ #define DEFAULT_DROP_RESOURCES_COUNT 10 #define DEFAULT_DROP_RESOURCES_AGE 10000 /* 10 sec */ #define DEFAULT_FENCE_ALL_AGENT "dlm_stonith" -#define DEFAULT_STARTUP_FENCE 30
/* DLM_LOCKSPACE_LEN: maximum lockspace name length, from linux/dlmconstants.h. Copied in libdlm.h so apps don't need to include the kernel header. @@ -132,6 +133,7 @@ EXTERN int plock_fd; EXTERN int plock_ci; EXTERN struct list_head lockspaces; EXTERN int cluster_quorate; +EXTERN uint64_t cluster_last_join_monotime; EXTERN uint64_t cluster_quorate_monotime; EXTERN uint64_t cluster_joined_monotime; EXTERN uint64_t cluster_joined_walltime; @@ -147,7 +149,9 @@ EXTERN int optk_debug; EXTERN int optk_timewarn; EXTERN int optk_protocol; EXTERN int optd_debug_logfile; +EXTERN int optd_post_join_delay; EXTERN int optd_enable_fencing; +EXTERN int optd_enable_startup_fencing; EXTERN int optd_enable_quorum_fencing; EXTERN int optd_enable_quorum_lockspace; EXTERN int optd_enable_fscontrol; @@ -158,14 +162,15 @@ EXTERN int optd_plock_ownership; EXTERN int optd_drop_resources_time; EXTERN int optd_drop_resources_count; EXTERN int optd_drop_resources_age; -EXTERN int optd_startup_fence; EXTERN int optd_fence_all_agent;
EXTERN int cfgk_debug; EXTERN int cfgk_timewarn; EXTERN int cfgk_protocol; EXTERN int cfgd_debug_logfile; +EXTERN int cfgd_post_join_delay; EXTERN int cfgd_enable_fencing; +EXTERN int cfgd_enable_startup_fencing; EXTERN int cfgd_enable_quorum_fencing; EXTERN int cfgd_enable_quorum_lockspace; EXTERN int cfgd_enable_fscontrol; @@ -176,7 +181,6 @@ EXTERN int cfgd_plock_ownership; EXTERN int cfgd_drop_resources_time; EXTERN int cfgd_drop_resources_count; EXTERN int cfgd_drop_resources_age; -EXTERN int cfgd_startup_fence; EXTERN char fence_all_agent[PATH_MAX];
#define LOG_DUMP_SIZE DLMC_DUMP_SIZE diff --git a/dlm_controld/main.c b/dlm_controld/main.c index e623a6f..8e63bb3 100644 --- a/dlm_controld/main.c +++ b/dlm_controld/main.c @@ -1113,16 +1113,16 @@ static void print_usage(void) printf(" -r <num> dlm kernel lowcomms protocol, 0 tcp, 1 sctp, 2 detect\n"); printf(" 2 selects tcp if corosync rrp_mode is "none", otherwise sctp\n"); printf(" Default is 2\n"); - printf(" -f <num> Enable (1) or disable (0) fencing recovery dependency\n"); + + printf(" -j <sec> Seconds to delay fencing after cluster join\n"); + printf(" Default is %d\n", DEFAULT_POST_JOIN_DELAY); + printf(" -f <num> Enable (1) or disable (0) fencing\n"); printf(" Default is %d\n", DEFAULT_ENABLE_FENCING); - printf(" -q <num> Enable (1) or disable (0) quorum recovery dependency\n"); + printf(" -s <num> Enable (1) or disable (0) startup fencing\n"); + printf(" Default is %d\n", DEFAULT_ENABLE_STARTUP_FENCING); + printf(" -q <num> Enable (1) or disable (0) quorum wait before fencing\n"); printf(" Default is %d\n", DEFAULT_ENABLE_QUORUM_FENCING); - printf(" -s <num> Enable (1) or disable (0) fs_controld recovery coordination\n"); - printf(" Default is %d\n", DEFAULT_ENABLE_FSCONTROL); -#if 0 - printf(" -d <num> Enable (1) or disable (0) deadlock detection code\n"); - printf(" Default is %d\n", DEFAULT_ENABLE_DEADLK); -#endif + printf(" -p <num> Enable (1) or disable (0) plock code for cluster fs\n"); printf(" Default is %d\n", DEFAULT_ENABLE_PLOCK); printf(" -P Enable plock debugging\n"); @@ -1136,11 +1136,12 @@ static void print_usage(void) printf(" Default is %u\n", DEFAULT_DROP_RESOURCES_COUNT); printf(" -a <ms> plock ownership drop resources age (milliseconds)\n"); printf(" Default is %u\n", DEFAULT_DROP_RESOURCES_AGE); + printf(" -h Print this help, then exit\n"); printf(" -V Print program version information, then exit\n"); }
-#define OPTION_STRING "LDKf:q:p:Pl:o:t:c:a:hVr:s:e:d:" +#define OPTION_STRING "LDKf:q:p:Pl:o:t:c:a:hVr:s:e:j:"
static void read_arguments(int argc, char **argv) { @@ -1170,18 +1171,22 @@ static void read_arguments(int argc, char **argv) cfgk_protocol = atoi(optarg); break;
- case 's': - optd_enable_fscontrol = 1; - cfgd_enable_fscontrol = atoi(optarg); - break; - /* fencing options */
+ case 'j': + optd_post_join_delay = 1; + cfgd_post_join_delay = atoi(optarg); + break; + case 'f': optd_enable_fencing = 1; cfgd_enable_fencing = atoi(optarg); break;
+ case 's': + optd_enable_startup_fencing = 1; + cfgd_enable_startup_fencing = atoi(optarg); + case 'q': optd_enable_quorum_fencing = 1; cfgd_enable_quorum_fencing = atoi(optarg); @@ -1192,10 +1197,6 @@ static void read_arguments(int argc, char **argv) strcpy(fence_all_agent, optarg); break;
- case 'd': - optd_startup_fence = 1; - cfgd_startup_fence = atoi(optarg); -
/* plock options */
@@ -1306,10 +1307,11 @@ int main(int argc, char **argv) cfgd_drop_resources_count = DEFAULT_DROP_RESOURCES_COUNT; cfgd_drop_resources_age = DEFAULT_DROP_RESOURCES_AGE;
+ cfgd_post_join_delay = DEFAULT_POST_JOIN_DELAY; cfgd_enable_fencing = DEFAULT_ENABLE_FENCING; - cfgd_enable_quorum_lockspace= DEFAULT_ENABLE_QUORUM_LOCKSPACE; + cfgd_enable_startup_fencing = DEFAULT_ENABLE_STARTUP_FENCING; cfgd_enable_quorum_fencing = DEFAULT_ENABLE_QUORUM_FENCING; - cfgd_startup_fence = DEFAULT_STARTUP_FENCE; + cfgd_enable_quorum_lockspace= DEFAULT_ENABLE_QUORUM_LOCKSPACE;
strcpy(fence_all_agent, DEFAULT_FENCE_ALL_AGENT); memset(&fence_all_device, 0, sizeof(struct fence_device)); diff --git a/dlm_controld/member.c b/dlm_controld/member.c index 841e5e3..7222b9e 100644 --- a/dlm_controld/member.c +++ b/dlm_controld/member.c @@ -150,6 +150,8 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate, quorum_nodes[i], cluster_ringid_seq); add_cluster_node(quorum_nodes[i], now);
+ cluster_last_join_monotime = now; + err = corosync_cfg_get_node_addrs(ch, quorum_nodes[i], MAX_NODE_ADDRESSES, &num_addrs, addrs); @@ -354,7 +356,7 @@ int setup_node_config(void)
log_debug("node_config %d", nodeid);
- if (cfgd_enable_fencing && cfgd_startup_fence) + if (cfgd_enable_fencing && cfgd_enable_startup_fencing) add_startup_node(nodeid); }