Gitweb:
http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 66747c23e6791b7a3360c85f36ad4c865ebea7d4
Parent: 7001f065adda29e9df3f0933fb94587f6387d6c9
Author: Eduardo Damato <edamato(a)redhat.com>
AuthorDate: Tue Sep 29 09:58:18 2009 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Tue Sep 29 10:04:32 2009 -0400
qdisk: Implement I/O timeouts in qdiskd
This allows administrators to make qdiskd reboot the
system if it can not write its status out for interval*tko
seconds.
Signed-off-by: Eduardo Damato <edamato(a)redhat.com>
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
cman/man/qdisk.5 | 7 +++++++
cman/qdisk/disk.h | 3 ++-
cman/qdisk/main.c | 35 +++++++++++++++++++++++++++++++++--
3 files changed, 42 insertions(+), 3 deletions(-)
diff --git a/cman/man/qdisk.5 b/cman/man/qdisk.5
index 8ac9ad5..b28390e 100644
--- a/cman/man/qdisk.5
+++ b/cman/man/qdisk.5
@@ -294,6 +294,13 @@ pass. The default for this value is 0 (off). This option can be
changed
while qdiskd is running.
.in 9
+\fIio_timeout\fP\fB="\fP0\fB"\fP
+.in 12
+If set to 1 (on), qdiskd will watch internal timers and reboot the node
+if qdisk is not able to write to disk after (interval * tko) seconds.
+The default for this value is 0 (off).
+
+.in 9
\fIscheduler\fP\fB="\fPrr\fB"\fP
.in 12
Valid values are 'rr', 'fifo', and 'other'. Selects the
scheduling queue
diff --git a/cman/qdisk/disk.h b/cman/qdisk/disk.h
index 9704b8e..b00c580 100644
--- a/cman/qdisk/disk.h
+++ b/cman/qdisk/disk.h
@@ -53,7 +53,8 @@ typedef enum {
RF_PARANOID = 0x8,
RF_ALLOW_KILL = 0x10,
RF_UPTIME = 0x20,
- RF_CMAN_LABEL = 0x40
+ RF_CMAN_LABEL = 0x40,
+ RF_IOTIMEOUT = 0x80
} run_flag_t;
diff --git a/cman/qdisk/main.c b/cman/qdisk/main.c
index fb29e8d..3b09b28 100644
--- a/cman/qdisk/main.c
+++ b/cman/qdisk/main.c
@@ -907,7 +907,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
int low_id, bid_pending = 0, score, score_max, score_req,
upgrade = 0, count, errors, error_cycles = 0;
memb_mask_t mask, master_mask;
- struct timeval maxtime, oldtime, newtime, diff, sleeptime, interval;
+ struct timeval maxtime, oldtime, newtime, diff, sleeptime, interval, lastok;
ctx->qc_status = S_NONE;
@@ -917,6 +917,9 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
interval.tv_usec = 0;
interval.tv_sec = ctx->qc_interval;
+ lastok.tv_usec = 0;
+ lastok.tv_sec = 0;
+
get_my_score(&score, &score_max);
if (score_max < ctx->qc_scoremin) {
logt_print(LOG_WARNING, "Minimum score (%d) is impossible to "
@@ -1110,6 +1113,8 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
logt_print(LOG_ERR, "Error writing to quorum disk\n");
errors++; /* this value isn't really used
at this point */
+ } else {
+ get_time(&lastok, ctx->qc_flags&RF_UPTIME);
}
/* write out our local status */
@@ -1118,11 +1123,27 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
/* Cycle. We could time the loop and sleep
(interval-looptime), but this is fine for now.*/
get_time(&newtime, ctx->qc_flags&RF_UPTIME);
- _diff_tv(&diff, &oldtime, &newtime);
+ /*
+ * Reboot if the last successful hearbeat was longer ago than interval*TKO_COUNT
+ */
+ _diff_tv(&diff, &lastok, &newtime);
+ if (_cmp_tv(&maxtime, &diff) == 1 &&
+ ctx->qc_flags & RF_IOTIMEOUT) {
+ logt_print(LOG_EMERG, "Failed to send a heartbeat "
+ "within %d second%s (%d.%06d) - REBOOTING\n",
+ (int)maxtime.tv_sec,
+ maxtime.tv_sec==1?"":"s",
+ (int)diff.tv_sec,
+ (int)diff.tv_usec);
+ if (!(ctx->qc_flags & RF_DEBUG))
+ reboot(RB_AUTOBOOT);
+ }
+
/*
* Reboot if we didn't send a heartbeat in interval*TKO_COUNT
*/
+ _diff_tv(&diff, &oldtime, &newtime);
if (_cmp_tv(&maxtime, &diff) == 1 &&
ctx->qc_flags & RF_PARANOID) {
logt_print(LOG_EMERG, "Failed to complete a cycle within "
@@ -1366,6 +1387,16 @@ get_dynamic_config_data(qd_ctx *ctx, int ccsfd)
free(val);
}
+ /* default = off, so, 1 to turn on */
+ snprintf(query, sizeof(query), "/cluster/quorumd/@io_timeout");
+ if (ccs_get(ccsfd, query, &val) == 0) {
+ if (!atoi(val))
+ ctx->qc_flags &= ~RF_IOTIMEOUT;
+ else
+ ctx->qc_flags |= RF_IOTIMEOUT;
+ free(val);
+ }
+
/*
* Get flag to see if we're supposed to reboot if we can't complete
* a pass in failure time