cluster: master - qdisk: Implement I/O timeouts in qdiskd

[Lon Hohberger]

Gitweb:       
http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit:        6e91a44cfb2d6baa1a639a2f6e6023bf82ab3cb7
Parent:        5577bb82ea7a0ed3d1b27f2a6acdb1926e2b6a78
Author:        Eduardo Damato <edamato(a)redhat.com&gt;
AuthorDate:    Tue Sep 29 09:58:18 2009 -0400
Committer:     Lon Hohberger <lhh(a)redhat.com&gt;
CommitterDate: Tue Sep 29 10:08:11 2009 -0400

qdisk: Implement I/O timeouts in qdiskd

This allows administrators to make qdiskd reboot the
system if it can not write its status out for interval*tko
seconds.

Signed-off-by: Eduardo Damato <edamato(a)redhat.com&gt;
Signed-off-by: Lon Hohberger <lhh(a)redhat.com&gt;
---
 cman/man/qdisk.5  |    7 +++++++
 cman/qdisk/disk.h |    3 ++-
 cman/qdisk/main.c |   35 +++++++++++++++++++++++++++++++++--
 3 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/cman/man/qdisk.5 b/cman/man/qdisk.5
index 8ac9ad5..b28390e 100644
--- a/cman/man/qdisk.5
+++ b/cman/man/qdisk.5
@@ -294,6 +294,13 @@ pass.  The default for this value is 0 (off).  This option can be
changed
 while qdiskd is running.
 
 .in 9
+\fIio_timeout\fP\fB="\fP0\fB"\fP
+.in 12
+If set to 1 (on), qdiskd will watch internal timers and reboot the node
+if qdisk is not able to write to disk after (interval * tko) seconds.
+The default for this value is 0 (off).
+
+.in 9
 \fIscheduler\fP\fB="\fPrr\fB"\fP
 .in 12
 Valid values are 'rr', 'fifo', and 'other'.  Selects the
scheduling queue
diff --git a/cman/qdisk/disk.h b/cman/qdisk/disk.h
index 9704b8e..b00c580 100644
--- a/cman/qdisk/disk.h
+++ b/cman/qdisk/disk.h
@@ -53,7 +53,8 @@ typedef enum {
 	RF_PARANOID = 0x8,
 	RF_ALLOW_KILL = 0x10,
 	RF_UPTIME = 0x20,
-	RF_CMAN_LABEL = 0x40
+	RF_CMAN_LABEL = 0x40,
+	RF_IOTIMEOUT = 0x80
 } run_flag_t;
 
 
diff --git a/cman/qdisk/main.c b/cman/qdisk/main.c
index 5fe6c11..e4bcb60 100644
--- a/cman/qdisk/main.c
+++ b/cman/qdisk/main.c
@@ -910,7 +910,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 	int low_id, bid_pending = 0, score, score_max, score_req,
 	    upgrade = 0, count, errors, error_cycles = 0;
 	memb_mask_t mask, master_mask;
-	struct timeval maxtime, oldtime, newtime, diff, sleeptime, interval;
+	struct timeval maxtime, oldtime, newtime, diff, sleeptime, interval, lastok;
 
 	ctx->qc_status = S_NONE;
 	
@@ -920,6 +920,9 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 	interval.tv_usec = 0;
 	interval.tv_sec = ctx->qc_interval;
 	
+	lastok.tv_usec = 0;
+	lastok.tv_sec = 0;
+	
 	get_my_score(&score, &score_max);
 	if (score_max < ctx->qc_scoremin) {
 		logt_print(LOG_WARNING, "Minimum score (%d) is impossible to "
@@ -1113,6 +1116,8 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 			logt_print(LOG_ERR, "Error writing to quorum disk\n");
 			errors++; /* this value isn't really used 
 				     at this point */
+ 		} else {
+ 			get_time(&lastok, ctx->qc_flags&RF_UPTIME);
 		}
 
 		/* write out our local status */
@@ -1121,11 +1126,27 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 		/* Cycle. We could time the loop and sleep
 		   (interval-looptime), but this is fine for now.*/
 		get_time(&newtime, ctx->qc_flags&RF_UPTIME);
-		_diff_tv(&diff, &oldtime, &newtime);
 		
+ 		/*
+		 * Reboot if the last successful hearbeat was longer ago than interval*TKO_COUNT
+		 */
+		_diff_tv(&diff, &lastok, &newtime);
+		if (_cmp_tv(&maxtime, &diff) == 1 &&
+		    ctx->qc_flags & RF_IOTIMEOUT) {
+			logt_print(LOG_EMERG, "Failed to send a heartbeat "
+				   "within %d second%s (%d.%06d) - REBOOTING\n",
+				   (int)maxtime.tv_sec,
+				   maxtime.tv_sec==1?"":"s",
+				   (int)diff.tv_sec,
+				   (int)diff.tv_usec);
+			if (!(ctx->qc_flags & RF_DEBUG)) 
+				reboot(RB_AUTOBOOT);
+		}
+	
 		/*
 		 * Reboot if we didn't send a heartbeat in interval*TKO_COUNT
 		 */
+		_diff_tv(&diff, &oldtime, &newtime);
 		if (_cmp_tv(&maxtime, &diff) == 1 &&
 		    ctx->qc_flags & RF_PARANOID) {
 			logt_print(LOG_EMERG, "Failed to complete a cycle within "
@@ -1369,6 +1390,16 @@ get_dynamic_config_data(qd_ctx *ctx, int ccsfd)
 		free(val);
 	}
 	
+	/* default = off, so, 1 to turn on */
+	snprintf(query, sizeof(query), "/cluster/quorumd/@io_timeout");
+	if (ccs_get(ccsfd, query, &val) == 0) {
+		if (!atoi(val))
+			ctx->qc_flags &= ~RF_IOTIMEOUT;
+		else
+			ctx->qc_flags |= RF_IOTIMEOUT;
+		free(val);
+	}
+  	
 	/*
 	 * Get flag to see if we're supposed to reboot if we can't complete
 	 * a pass in failure time