cluster: RHEL57 - rgmanager: Initial commit of central proc + migration support
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 963d04336736b0ad7b4582b100048ea89ba48fad
Parent: 640f9211a325d1e093fd3c108a97fedae646e86e
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Wed Oct 21 13:38:42 2009 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Apr 8 09:31:15 2011 -0400
rgmanager: Initial commit of central proc + migration support
This adds preliminary migration support of virtual
machines when using central processing.
Resolves: rhbz#525271
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/daemons/service_op.c | 85 ++++++++++++++++++++++-
rgmanager/src/daemons/slang_event.c | 59 ++++++++++++++++
rgmanager/src/resources/default_event_script.sl | 8 +-
3 files changed, 146 insertions(+), 6 deletions(-)
diff --git a/rgmanager/src/daemons/service_op.c b/rgmanager/src/daemons/service_op.c
index 6b0db24..2ea59f9 100644
--- a/rgmanager/src/daemons/service_op.c
+++ b/rgmanager/src/daemons/service_op.c
@@ -291,7 +291,88 @@ service_op_convalesce(char *svcName)
/*
- TODO
- service_op_migrate()
+ service_op_migrate() - send a virtual machine to another host
+ in the cluster
*/
+int
+service_op_migrate(char *svcName,
+ int target_node)
+{
+ SmMessageSt msg;
+ int msg_ret;
+ msgctx_t ctx;
+ rg_state_t svcStatus;
+ int msgtarget = my_id();
+
+ /* Build the message header */
+ msg.sm_hdr.gh_magic = GENERIC_HDR_MAGIC;
+ msg.sm_hdr.gh_command = RG_ACTION_REQUEST;
+ msg.sm_hdr.gh_arg1 = RG_ACTION_MASTER;
+ msg.sm_hdr.gh_length = sizeof (SmMessageSt);
+
+ msg.sm_data.d_action = RG_MIGRATE;
+
+ strncpy(msg.sm_data.d_svcName, svcName,
+ sizeof(msg.sm_data.d_svcName));
+
+ msg.sm_data.d_ret = 0;
+ msg.sm_data.d_svcOwner = target_node;
+ /* Open a connection to the local node - it will decide what to
+ do in this case. XXX inefficient; should queue requests
+ locally and immediately forward requests otherwise */
+
+ if (get_service_state_internal(svcName, &svcStatus) < 0)
+ return RG_EFAIL;
+ if (svcStatus.rs_owner > 0) {
+ if (member_online(svcStatus.rs_owner)) {
+ msgtarget = svcStatus.rs_owner;
+ }
+
+ if (msgtarget <= 0) {
+ return RG_EFAIL;
+ }
+ }
+
+ if (msg_open(MSG_CLUSTER, msgtarget, RG_PORT, &ctx, 2)< 0) {
+ clulog(LOG_ERR,
+ "#58: Failed opening connection to member #%d\n",
+ my_id());
+ return -1;
+ }
+
+ /* Encode */
+ swab_SmMessageSt(&msg);
+
+ /* Send stop message to the other node */
+ if (msg_send(&ctx, &msg, sizeof (SmMessageSt)) <
+ (int)sizeof (SmMessageSt)) {
+ clulog(LOG_ERR, "Failed to send complete message\n");
+ msg_close(&ctx);
+ return -1;
+ }
+
+ /* Check the response */
+ do {
+ msg_ret = msg_receive(&ctx, &msg,
+ sizeof (SmMessageSt), 10);
+ if ((msg_ret == -1 && errno != ETIMEDOUT) ||
+ (msg_ret > 0)) {
+ break;
+ }
+ } while(1);
+
+ if (msg_ret != sizeof (SmMessageSt)) {
+ clulog(LOG_WARNING, "Strange response size: %d vs %d\n",
+ msg_ret, (int)sizeof(SmMessageSt));
+ return 0; /* XXX really UNKNOWN */
+ }
+
+ /* Got a valid response from other node. */
+ msg_close(&ctx);
+
+ /* Decode */
+ swab_SmMessageSt(&msg);
+
+ return msg.sm_data.d_ret;
+}
diff --git a/rgmanager/src/daemons/slang_event.c b/rgmanager/src/daemons/slang_event.c
index 08974ad..468706f 100644
--- a/rgmanager/src/daemons/slang_event.c
+++ b/rgmanager/src/daemons/slang_event.c
@@ -619,6 +619,63 @@ out:
}
+static int
+sl_migrate_service(void)
+{
+ char *svcname = NULL;
+ int target_node = 0;
+ int nargs, t, newowner = 0, ret = -1;
+
+ nargs = SLang_Num_Function_Args;
+
+ /* Takes one, two, or three */
+ if (nargs != 2) {
+ SLang_verror(SL_Syntax_Error,
+ (char *)"%s: Wrong # of args (%d), must be 2: service_name target_node\n",
+ __FUNCTION__, nargs);
+ return -1;
+ }
+
+ t = SLang_peek_at_stack();
+ if (t != SLANG_INT_TYPE) {
+ SLang_verror(SL_Syntax_Error,
+ (char *)"%s: expected type %d got %d\n",
+ __FUNCTION__, SLANG_INT_TYPE, t);
+ goto out;
+ }
+
+ if (SLang_pop_integer(&target_node) < 0) {
+ SLang_verror(SL_Syntax_Error,
+ (char *)"%s: Failed to pop integer from stack!\n",
+ __FUNCTION__);
+ goto out;
+ }
+
+ t = SLang_peek_at_stack();
+ if (t != SLANG_STRING_TYPE) {
+ SLang_verror(SL_Syntax_Error,
+ (char *)"%s: expected type %d got %d\n",
+ __FUNCTION__,
+ SLANG_STRING_TYPE, t);
+ goto out;
+ }
+
+ if (SLpop_string(&svcname) < 0) {
+ goto out;
+ }
+
+ ret = service_op_migrate(svcname, target_node);
+
+ if (ret == 0)
+ ret = target_node;
+out:
+ if (svcname)
+ free(svcname);
+ _user_return = ret;
+ return ret;
+}
+
+
/* Take an array of integers given its length and
push it on to the S/Lang stack */
void
@@ -1016,6 +1073,8 @@ SLang_Intrin_Fun_Type rgmanager_slang[] =
MAKE_INTRINSIC_S((char *)"service_convalesce", sl_convalesce_service,
SLANG_INT_TYPE),
MAKE_INTRINSIC_0("service_start", sl_start_service, SLANG_INT_TYPE),
+ MAKE_INTRINSIC_0((char *)"service_migrate", sl_migrate_service,
+ SLANG_INT_TYPE),
MAKE_INTRINSIC_S("service_status", sl_service_status,
SLANG_VOID_TYPE),
MAKE_INTRINSIC_S("service_freeze", sl_service_freeze,
diff --git a/rgmanager/src/resources/default_event_script.sl b/rgmanager/src/resources/default_event_script.sl
index da6bc1f..db6e789 100644
--- a/rgmanager/src/resources/default_event_script.sl
+++ b/rgmanager/src/resources/default_event_script.sl
@@ -597,11 +597,11 @@ define default_user_event_handler()
ret = service_convalesce(service_name);
- }
+ } else if (user_request == USER_MIGRATE) {
- %
- % todo - migrate
- %
+ ret = service_migrate(service_name, user_target);
+
+ }
return ret;
}
13 years, 1 month
cluster: RHEL57 - rgmanager: Allow non-root clustat
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 640f9211a325d1e093fd3c108a97fedae646e86e
Parent: d4dd514faa27f4f7c91205d7525c7f3bc1885aa3
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Mon Mar 28 15:56:08 2011 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Apr 8 09:31:15 2011 -0400
rgmanager: Allow non-root clustat
This allows non-root users access to the running cluster
and service states.
This requires you to add a user to the 'root' group,
matching cman_tool's requirements.
Simple backport from RHEL6 / STABLE31
Resolves: rhbz#510300
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/include/message.h | 1 +
rgmanager/include/resgroup.h | 3 +++
rgmanager/src/clulib/msg_socket.c | 16 +++++++++++++++-
rgmanager/src/clulib/rg_strings.c | 2 ++
rgmanager/src/daemons/main.c | 33 ++++++++++++++++++++++++++++++++-
5 files changed, 53 insertions(+), 2 deletions(-)
diff --git a/rgmanager/include/message.h b/rgmanager/include/message.h
index f987ec6..4b9ae3a 100644
--- a/rgmanager/include/message.h
+++ b/rgmanager/include/message.h
@@ -89,6 +89,7 @@ typedef struct _msgctx {
} cluster_info;
struct {
int sockfd;
+ struct ucred cred;
int pad;
} local_info;
} u;
diff --git a/rgmanager/include/resgroup.h b/rgmanager/include/resgroup.h
index ded11f4..793ad3b 100644
--- a/rgmanager/include/resgroup.h
+++ b/rgmanager/include/resgroup.h
@@ -187,6 +187,9 @@ int rg_unlock(struct dlm_lksb *p);
/* Return codes */
+#define RG_EPERM -18 /* Permission denied */
+#define RG_ERELO -17 /* Relocation failure; service running
+ on original node */
#define RG_EEXCL -16 /* Service not runnable due to
the fact that it is tagged
exclusive and there are no
diff --git a/rgmanager/src/clulib/msg_socket.c b/rgmanager/src/clulib/msg_socket.c
index c392757..4fc9f88 100644
--- a/rgmanager/src/clulib/msg_socket.c
+++ b/rgmanager/src/clulib/msg_socket.c
@@ -330,6 +330,8 @@ static int
sock_msg_accept(msgctx_t *listenctx, msgctx_t *acceptctx)
{
errno = EINVAL;
+ struct ucred cred;
+ socklen_t credlen = sizeof(cred);
if (!listenctx || !acceptctx)
return -1;
@@ -345,6 +347,18 @@ sock_msg_accept(msgctx_t *listenctx, msgctx_t *acceptctx)
if (acceptctx->u.local_info.sockfd < 0)
return -1;
+ memset(&cred, 0, sizeof(cred));
+ if (getsockopt(acceptctx->u.local_info.sockfd, SOL_SOCKET,
+ SO_PEERCRED, (void *)&cred, &credlen) < 0) {
+ perror("getsockopt");
+ cred.uid = (uid_t)-1;
+ cred.gid = (gid_t)-1;
+ cred.pid = (pid_t)-1;
+ }
+
+ memcpy(&acceptctx->u.local_info.cred, &cred,
+ sizeof(cred));
+
set_cloexec(acceptctx->u.local_info.sockfd);
acceptctx->flags = (SKF_READ | SKF_WRITE);
@@ -372,7 +386,7 @@ sock_msg_listen(int me, void *portp, msgctx_t **listen_ctx)
set_cloexec(sock);
unlink(RGMGR_SOCK);
- om = umask(077);
+ om = umask(0117);
su.sun_family = PF_LOCAL;
snprintf(su.sun_path, sizeof(su.sun_path), "%s", path);
diff --git a/rgmanager/src/clulib/rg_strings.c b/rgmanager/src/clulib/rg_strings.c
index 4d1eccb..47b06e0 100644
--- a/rgmanager/src/clulib/rg_strings.c
+++ b/rgmanager/src/clulib/rg_strings.c
@@ -26,6 +26,8 @@ struct string_val {
const struct string_val rg_error_strings[] = {
+ { RG_EPERM, "Permission denied" },
+ { RG_ERELO, "Failed; service running on original owner" },
{ RG_EEXCL, "Service not runnable: cannot run exclusive" },
{ RG_EDOMAIN, "Service not runnable: restricted failover domain offline" },
{ RG_ESCRIPT, "S/Lang Script Error" },
diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c
index 52e38bc..aa78cef 100644
--- a/rgmanager/src/daemons/main.c
+++ b/rgmanager/src/daemons/main.c
@@ -32,6 +32,8 @@
#include <msgsimple.h>
#include <vf.h>
#include <lock.h>
+#include <sys/socket.h>
+#include <message.h>
#include <rg_queue.h>
#include <malloc.h>
#include <cman-private.h>
@@ -428,11 +430,17 @@ do_lockreq(msgctx_t *ctx, int req)
int
dispatch_msg(msgctx_t *ctx, int nodeid, int need_close)
{
- int ret = 0, sz = -1, nid;
+ int ret = 0, sz = -1, nid, read_only = 1;
char msgbuf[4096];
generic_msg_hdr *msg_hdr = (generic_msg_hdr *)msgbuf;
SmMessageSt *msg_sm = (SmMessageSt *)msgbuf;
+ if (ctx->type == MSG_CLUSTER) {
+ read_only = 0;
+ } else if (ctx->u.local_info.cred.uid == 0) {
+ read_only = 0;
+ }
+
memset(msgbuf, 0, sizeof(msgbuf));
/* Peek-a-boo */
@@ -486,6 +494,10 @@ dispatch_msg(msgctx_t *ctx, int nodeid, int need_close)
case RG_LOCK:
case RG_UNLOCK:
+ if (read_only) {
+ msg_send_simple(ctx, RG_FAIL, RG_EPERM, 0);
+ goto out;
+ }
if (rg_quorate())
do_lockreq(ctx, msg_hdr->gh_command);
break;
@@ -498,6 +510,10 @@ dispatch_msg(msgctx_t *ctx, int nodeid, int need_close)
break;
case RG_ACTION_REQUEST:
+ if (read_only) {
+ msg_send_simple(ctx, RG_FAIL, RG_EPERM, 0);
+ goto out;
+ }
if (sz < (int)sizeof(msg_sm)) {
clulog(LOG_ERR,
@@ -557,6 +573,11 @@ dispatch_msg(msgctx_t *ctx, int nodeid, int need_close)
return 0;
case RG_EVENT:
+ if (read_only) {
+ msg_send_simple(ctx, RG_FAIL, RG_EPERM, 0);
+ goto out;
+ }
+
/* Service event. Run a dependency check */
if (sz < (int)sizeof(msg_sm)) {
clulog(LOG_ERR,
@@ -579,6 +600,11 @@ dispatch_msg(msgctx_t *ctx, int nodeid, int need_close)
break;
case RG_EXITING:
+ if (read_only) {
+ msg_send_simple(ctx, RG_FAIL, RG_EPERM, 0);
+ goto out;
+ }
+
if (!member_online(msg_hdr->gh_arg1))
break;
@@ -589,6 +615,11 @@ dispatch_msg(msgctx_t *ctx, int nodeid, int need_close)
break;
case VF_MESSAGE:
+ if (read_only) {
+ msg_send_simple(ctx, RG_FAIL, RG_EPERM, 0);
+ goto out;
+ }
+
/* Ignore; our VF thread handles these
- except for VF_CURRENT XXX (bad design) */
if (msg_hdr->gh_arg1 == VF_CURRENT)
13 years, 1 month
cluster: RHEL57 - resource-agents: Try force-unmount before fuser for netfs.sh
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: d4dd514faa27f4f7c91205d7525c7f3bc1885aa3
Parent: 8d94f0871db6236ac3caa366560578976477d0eb
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Mon Mar 21 20:58:11 2011 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Apr 8 09:31:15 2011 -0400
resource-agents: Try force-unmount before fuser for netfs.sh
Resolves: rhbz#678494
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
Reviewed-by: Fabio M. Di Nitto <fdinitto(a)redhat.com>
---
rgmanager/src/resources/netfs.sh | 7 ++++++-
1 files changed, 6 insertions(+), 1 deletions(-)
diff --git a/rgmanager/src/resources/netfs.sh b/rgmanager/src/resources/netfs.sh
index 1a669d3..837a4c4 100755
--- a/rgmanager/src/resources/netfs.sh
+++ b/rgmanager/src/resources/netfs.sh
@@ -486,6 +486,7 @@ stopNFSFilesystem() {
typeset no_umount=""
typeset force_umount=""
typeset fstype=""
+ typeset umount_flag=""
#
@@ -558,7 +559,7 @@ stopNFSFilesystem() {
sync; sync; sync
ocf_log info "unmounting $mp"
- umount $mp
+ umount $umount_flag $mp
if [ $? -eq 0 ]; then
umount_failed=
done=$YES
@@ -568,6 +569,10 @@ stopNFSFilesystem() {
umount_failed=yes
if [ "$force_umount" ]; then
+ if [ -z "$umount_flag" ]; then
+ umount_flag="-f"
+ continue
+ fi
if [ $try -eq 1 ]; then
fuser -TERM -kvm "$mp"
else
13 years, 1 month
cluster: RHEL57 - cman: Don't increment on LEAVING->MEMBER transition
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 8d94f0871db6236ac3caa366560578976477d0eb
Parent: 1837d87a8548f2244db2634897ad831d90217a44
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Mon Dec 6 15:01:55 2010 -0500
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Apr 8 09:31:15 2011 -0400
cman: Don't increment on LEAVING->MEMBER transition
If a node left the cluster prior using 'cman_tool leave', its
internal state is set to NODESTATE_LEAVING.
Ordinarily, during the transition from NODESTATE_LEAVING to
NODESTATE_DEAD, the member count is decremented. However, it
is possible to rejoin the cluster membership from the
NODESTATE_LEAVING state.
If this occurs, the cluster member count is incremented, but
since the node never transitioned from LEAVING->DEAD, it was
not previously decremented, causing an ever-increasing node
count in 'cman_tool status'.
This patch resolves the issue by not incrementing the member
count during a LEAVING->MEMBER transition.
Resolves: rhbz#649533
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
Reviewed-by: Fabio M. Di Nitto <fdinitto(a)redhat.com>
Reviewed-by: Christine Caulfield <ccaulfie(a)redhat.com>
---
cman/daemon/commands.c | 5 ++++-
1 files changed, 4 insertions(+), 1 deletions(-)
diff --git a/cman/daemon/commands.c b/cman/daemon/commands.c
index 6ffe9e5..6c91d7a 100644
--- a/cman/daemon/commands.c
+++ b/cman/daemon/commands.c
@@ -1994,9 +1994,12 @@ void add_ais_node(int nodeid, uint64_t incarnation, int total_members)
if (node->state == NODESTATE_DEAD || node->state == NODESTATE_LEAVING) {
gettimeofday(&node->join_time, NULL);
+ /* If a node rejoins before it completes a leave,
+ * we should not increment cluster_members */
+ if (node->state != NODESTATE_LEAVING)
+ cluster_members++;
node->incarnation = incarnation;
node->state = NODESTATE_MEMBER;
- cluster_members++;
recalculate_quorum(0, 0);
}
}
13 years, 1 month
cluster: RHEL57 - cman: Clarify 'expected' option
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 1837d87a8548f2244db2634897ad831d90217a44
Parent: 8a36ef759c422205d282e2f5543e8a76b00eb847
Author: Shane Bradley <sbradley(a)redhat.com>
AuthorDate: Thu Apr 7 09:59:51 2011 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Apr 8 09:31:15 2011 -0400
cman: Clarify 'expected' option
Resolves: rhbz#688701
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
Reviewed-by: Fabio M. Di Nitto <fdinitto(a)redhat.com>
---
cman/man/cman_tool.8 | 6 ++++++
1 files changed, 6 insertions(+), 0 deletions(-)
diff --git a/cman/man/cman_tool.8 b/cman/man/cman_tool.8
index 554a316..71e1841 100644
--- a/cman/man/cman_tool.8
+++ b/cman/man/cman_tool.8
@@ -58,6 +58,12 @@ version of "leave force" so only use if if you really know what you are doing.
Tells CMAN a new value of expected votes and instructs it to recalculate
quorum based on this value.
.br
+The recalculation takes into account the number of currently
+active nodes, so this option should not be used in an attempt to
+artificially lower the quorum value in advance of a planned shutdown of
+cluster nodes. Instead, the 'cman_tool leave remove' command should be
+used (see the 'leave' subcommand above).
+.br
Use this option if your cluster has lost quorum due to nodes failing and
you need to get it running again in a hurry.
13 years, 1 month
cluster: STABLE31 - rgmanager: Pause during exit if we stopped services
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 079f56ecb56b0d53b44b1871fa8937c22981fc1e
Parent: 1d56164f2dea0772188696a8578a4144c8379885
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Thu Apr 7 16:01:58 2011 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Thu Apr 7 16:51:00 2011 -0400
rgmanager: Pause during exit if we stopped services
A difference between rgmanager 1.9.x and later versions is that
they rely on openais/corosync for messaging. This messaging is
quite reliable and has proved useful.
However, one drawback is that if you very quickly stop rgmanager
and corosync/cman, the other nodes in the cluster can not restart
services because message traffic is interrupted for the duration
of the token timeout.
There is no simple solution to this problem. Rgmanager could
(in theory) find new placements for services prior to stopping,
but this is a large amount of design work; it was never designed
to run policies in the exit path.
A far simpler idea is to simply give the other nodes time to
restart services.
NOTE: This solution does not and can not work with central
processing mode.
Resolves: rhbz#619468
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/include/event.h | 1 +
rgmanager/include/groups.h | 2 +-
rgmanager/src/daemons/groups.c | 12 +++++++++++-
rgmanager/src/daemons/main.c | 15 ++++++++++++---
rgmanager/src/daemons/rg_event.c | 7 +++++++
5 files changed, 32 insertions(+), 5 deletions(-)
diff --git a/rgmanager/include/event.h b/rgmanager/include/event.h
index ad3fe39..9fc9521 100644
--- a/rgmanager/include/event.h
+++ b/rgmanager/include/event.h
@@ -119,6 +119,7 @@ int slang_process_event(event_table_t *event_table, event_t *ev);
/* For distributed events. */
void set_transition_throttling(int nsecs);
+int get_transition_throttling(void);
/* Simplified service start. */
int service_op_start(char *svcName, int *target_list, int target_list_len,
diff --git a/rgmanager/include/groups.h b/rgmanager/include/groups.h
index f776a31..f02f0e2 100644
--- a/rgmanager/include/groups.h
+++ b/rgmanager/include/groups.h
@@ -25,7 +25,7 @@ void kill_resource_groups(void);
/* do this op on all resource groups. The handler for the request
will sort out whether or not it's a valid request given the state */
-void rg_doall(int request, int block, const char *debugfmt);
+int rg_doall(int request, int block, const char *debugfmt);
void do_status_checks(void); /* Queue status checks for locally running
services */
diff --git a/rgmanager/src/daemons/groups.c b/rgmanager/src/daemons/groups.c
index ec8ab63..e9413cd 100644
--- a/rgmanager/src/daemons/groups.c
+++ b/rgmanager/src/daemons/groups.c
@@ -1276,13 +1276,21 @@ svc_exists(const char *svcname)
}
-void
+/*
+ * Perform an operation on all resources groups.
+ *
+ * Returns the number of requests queued. This value is
+ * only used during shutdown, where we queue RG_STOP_EXITING
+ * only for services we have running locally as an optimization.
+ */
+int
rg_doall(int request, int block,
const char __attribute__ ((unused)) *debugfmt)
{
resource_node_t *curr;
rg_state_t svcblk;
char rg[64];
+ int queued = 0;
pthread_rwlock_rdlock(&resource_lock);
list_do(&_tree, curr) {
@@ -1307,6 +1315,7 @@ rg_doall(int request, int block,
rt_enqueue_request(rg, request, NULL, 0,
0, 0, 0);
+ ++queued;
} while (!list_done(&_tree, curr));
pthread_rwlock_unlock(&resource_lock);
@@ -1316,6 +1325,7 @@ rg_doall(int request, int block,
other rgmanagers to complete. */
if (block)
rg_wait_threads();
+ return queued;
}
diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c
index 02cd2dc..a3ab7c4 100644
--- a/rgmanager/src/daemons/main.c
+++ b/rgmanager/src/daemons/main.c
@@ -29,7 +29,6 @@
void dump_thread_states(FILE *);
#endif
static int configure_rgmanager(int ccsfd, int debug, int *cluster_timeout);
-void set_transition_throttling(int);
void flag_shutdown(int sig);
@@ -43,6 +42,7 @@ static int signalled = 0;
static uint8_t ALIGNED port = RG_PORT;
static char *rgmanager_lsname = (char *)"rgmanager"; /* XXX default */
static int status_poll_interval = DEFAULT_CHECK_INTERVAL;
+static int stops_queued = 0;
static void
segfault(int __attribute__ ((unused)) sig)
@@ -930,7 +930,7 @@ static void *
shutdown_thread(void __attribute__ ((unused)) *arg)
{
rg_lockall(L_SYS|L_SHUTDOWN);
- rg_doall(RG_STOP_EXITING, 1, NULL);
+ stops_queued = rg_doall(RG_STOP_EXITING, 1, NULL);
running = 0;
pthread_exit(NULL);
@@ -1119,8 +1119,17 @@ out_ls:
out:
rgm_dbus_release();
- logt_print(LOG_NOTICE, "Shutdown complete, exiting\n");
+ logt_print(LOG_DEBUG, "Stopped %d services\n", stops_queued);
+ logt_print(LOG_NOTICE, "Disconnecting from CMAN\n");
cman_finish(clu);
+
+ if (stops_queued && !central_events_enabled()) {
+ logt_print(LOG_DEBUG, "Pausing to allow services to "
+ "start on other node(s)\n");
+ sleep(get_transition_throttling() * 3);
+ }
+
+ logt_print(LOG_NOTICE, "Exiting\n");
close_logging();
/*malloc_stats();*/
diff --git a/rgmanager/src/daemons/rg_event.c b/rgmanager/src/daemons/rg_event.c
index d03f4e3..7048bc6 100644
--- a/rgmanager/src/daemons/rg_event.c
+++ b/rgmanager/src/daemons/rg_event.c
@@ -56,6 +56,13 @@ set_transition_throttling(int nsecs)
}
+int
+get_transition_throttling(void)
+{
+ return transition_throttling;
+}
+
+
void
set_central_events(int flag)
{
13 years, 1 month
cluster: STABLE31 - cman: Clarify 'expected' option
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 1d56164f2dea0772188696a8578a4144c8379885
Parent: 00c3dbbbde98f51ddd99994f88c7a1e339ff6130
Author: Shane Bradley <sbradley(a)redhat.com>
AuthorDate: Thu Apr 7 09:59:51 2011 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Thu Apr 7 10:04:13 2011 -0400
cman: Clarify 'expected' option
Resolves: rhbz#688701
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
cman/man/cman_tool.8 | 6 ++++++
1 files changed, 6 insertions(+), 0 deletions(-)
diff --git a/cman/man/cman_tool.8 b/cman/man/cman_tool.8
index a5b9f23..2f402a6 100644
--- a/cman/man/cman_tool.8
+++ b/cman/man/cman_tool.8
@@ -58,6 +58,12 @@ version of "leave force" so only use if if you really know what you are doing.
Tells CMAN a new value of expected votes and instructs it to recalculate
quorum based on this value.
.br
+The recalculation takes into account the number of currently
+active nodes, so this option should not be used in an attempt to
+artificially lower the quorum value in advance of a planned shutdown of
+cluster nodes. Instead, the 'cman_tool leave remove' command should be
+used (see the 'leave' subcommand above).
+.br
Use this option if your cluster has lost quorum due to nodes failing and
you need to get it running again in a hurry.
13 years, 1 month
cluster: RHEL57 - rgmanager: Fix quotaoff handling
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 8a36ef759c422205d282e2f5543e8a76b00eb847
Parent: 975a371ae56179be9f0d806bad7482db8541a4fc
Author: Ryan Mitchell <rmitchel(a)redhat.com>
AuthorDate: Thu Apr 7 09:55:36 2011 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Thu Apr 7 09:55:36 2011 -0400
rgmanager: Fix quotaoff handling
This is a simple port from the RHEL4 branch performed by
Ryan Mitchell.
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/resources/fs.sh | 49 ++++++++++++++++++++++++++++++-----------
1 files changed, 36 insertions(+), 13 deletions(-)
diff --git a/rgmanager/src/resources/fs.sh b/rgmanager/src/resources/fs.sh
index ce3e983..49912c2 100755
--- a/rgmanager/src/resources/fs.sh
+++ b/rgmanager/src/resources/fs.sh
@@ -699,19 +699,11 @@ isAlive()
# Decide which quota options are enabled and return a string
# which we can pass to quotaon
#
-enable_fs_quotas()
+quota_opts()
{
- declare -i need_check=0
- declare -i rv
declare quotaopts=""
- declare mopt
declare opts=$1
- declare mp=$2
-
- if [ -z "`which quotaon`" ]; then
- ocf_log err "quotaon not found in $PATH"
- return $OCF_ERR_GENERIC
- fi
+ declare mopt
for mopt in `echo $opts | sed -e s/,/\ /g`; do
case $mopt in
@@ -730,6 +722,33 @@ enable_fs_quotas()
esac
done
+ echo $quotaopts
+ return 0
+}
+
+
+
+#
+# Enable quotas on the mount point if the user requested them
+#
+enable_fs_quotas()
+{
+ declare -i need_check=0
+ declare -i rv
+ declare quotaopts=""
+ declare mopt
+ declare opts=$1
+ declare mp=$2
+
+ if [ -z "`which quotaon`" ]; then
+ ocf_log err "quotaon not found in $PATH"
+ return $OCF_ERR_GENERIC
+ fi
+
+ quotaopts=$(quota_opts $opts)
+
+ ocf_log info "quotaopts = $quotaopts"
+
[ -z "$quotaopts" ] && return 0
# Ok, create quota files if they don't exist
@@ -991,6 +1010,7 @@ stopFilesystem() {
typeset force_umount=""
typeset self_fence=""
typeset fstype=""
+ typeset quotaopts=""
#
@@ -1057,10 +1077,13 @@ stop: Could not match $OCF_RESKEY_device with a real device"
;;
$YES)
sync; sync; sync
- ocf_log info "unmounting $mp"
-
+ quotaopts=$(quota_opts $OCF_RESKEY_options)
+ if [ -n "$quotaopts" ]; then
+ ocf_log debug "Turning off quotas for $mp"
+ quotaoff -$quotaopts $mp &> /dev/null
+ fi
- quotaoff -gu $mp &> /dev/null
+ ocf_log info "unmounting $mp"
umount $mp
if [ $? -eq 0 ]; then
umount_failed=
13 years, 1 month
cluster: STABLE31 - Improve rgmanager's exclusive prioritization handling
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 00c3dbbbde98f51ddd99994f88c7a1e339ff6130
Parent: 62cc660f462f8672fe75ebc8e53b184075ff3d5b
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Mon Mar 21 21:16:25 2011 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Thu Apr 7 09:42:58 2011 -0400
Improve rgmanager's exclusive prioritization handling
Resolves: rhbz#680256
Resolves: rhbz#690191
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/resources/default_event_script.sl | 72 ++++++++++++++++++-----
1 files changed, 56 insertions(+), 16 deletions(-)
diff --git a/rgmanager/src/resources/default_event_script.sl b/rgmanager/src/resources/default_event_script.sl
index 87c8a1c..759215c 100644
--- a/rgmanager/src/resources/default_event_script.sl
+++ b/rgmanager/src/resources/default_event_script.sl
@@ -77,7 +77,7 @@ define separate_nodes(node_list)
define exclusive_prioritize(svc, node_list)
{
variable services = service_list();
- variable len, x, y, owner, state, preferred_owner;
+ variable len, x, y, owner, nowner, state, preferred_owner;
variable svc_excl, other_excl;
variable nodes_x, nodes_s, nodes_e;
@@ -86,7 +86,8 @@ define exclusive_prioritize(svc, node_list)
%
svc_excl = atoi(service_property(svc, "exclusive"));
if (svc_excl == 0) {
- return node_list;
+ notice("Starting ", svc, " on ", node_list);
+ return service_start(svc, node_list);
}
(nodes_e, nodes_s, nodes_x) = separate_nodes(node_list);
@@ -96,7 +97,11 @@ define exclusive_prioritize(svc, node_list)
% If we've got an exclusive service, only allow it to start on
% empty nodes.
%
- return nodes_e;
+ notice("Starting ", svc, " on ", nodes_e);
+ nowner = service_start(svc, nodes_e);
+ if ((nowner > 0) or (nowner != FAIL)) {
+ return nowner;
+ }
}
if (length(nodes_x) == 0) {
@@ -105,7 +110,7 @@ define exclusive_prioritize(svc, node_list)
% and no empty nodes, the service can not be started
%
notice("No empty / exclusive nodes available; cannot restart ", svc);
- return nodes_x;
+ return ERR_DOMAIN;
}
%
@@ -149,14 +154,16 @@ define exclusive_prioritize(svc, node_list)
() = service_stop(services[x]);
%
- % Return just the one node.
+ % Try just the one node.
%
- node_list = subtract([0], 0);
- node_list = union(node_list, owner);
- return node_list;
+ notice("Starting ", svc, " on ", owner);
+ nowner = service_start(svc, owner);
+ if ((nowner > 0) or (nowner != FAIL)) {
+ return nowner;
+ }
}
- return node_list;
+ return ERR_DOMAIN;
}
@@ -220,13 +227,13 @@ define move_or_start(service, node_list)
return ERR_ABORT;
}
} else {
- node_list = exclusive_prioritize(service, node_list);
- notice("Starting ", service, " on ", node_list);
+ return exclusive_prioritize(service, node_list);
}
if (length(node_list) == 0) {
return ERR_DOMAIN;
}
+ notice("Starting ", service, " on ", node_list);
return service_start(service, node_list);
}
@@ -448,7 +455,7 @@ define default_node_event_handler()
define default_service_event_handler()
{
variable services = service_list();
- variable x;
+ variable x, excl, len;
variable depends;
variable depend_mode;
variable policy;
@@ -492,15 +499,16 @@ define default_service_event_handler()
return;
}
- for (x = 0; x < length(services); x++) {
+ %
+ % Simplistic dependency handling
+ %
+ len = length(services);
+ for (x = 0; x < len; x++) {
if (service_name == services[x]) {
% don't do anything to ourself!
continue;
}
- %
- % Simplistic dependency handling
- %
depends = service_property(services[x], "depend");
depend_mode = service_property(services[x], "depend_mode");
@@ -524,6 +532,38 @@ define default_service_event_handler()
()=service_stop(services[x]);
}
}
+
+ %
+ % Try to restart exclusive service which might have been recently
+ % stopped in order to make room for other exclusive services.
+ %
+ % Note that as a side effect, exclusive services (>=2) can't be
+ % stopped with clusvcadm -s; they will just pop right back - you
+ % must disable them if want them to stay stopped.
+ %
+ % This code has a side effect of brute-forcing the lowest-priority
+ % exclusive service offline in a cascaded fashion.
+ %
+ for (x = 0; x < len; x++) {
+ if (service_name == services[x]) {
+ % don't do anything to ourself!
+ continue;
+ }
+
+ excl = atoi(service_property(services[x], "exclusive"));
+ % non-exclusive or highest-prio (1) shouldn't get here
+ if ((excl == 0) or (excl == 1)) {
+ continue;
+ }
+
+ (,,, owner, state) = service_status(services[x]);
+ if (state == "stopped") {
+ info("Restarting stopped exclusive priority ",
+ excl, " service ", services[x]);
+ nodes = allowed_nodes(services[x]);
+ ()=move_or_start(services[x], nodes);
+ }
+ }
}
define default_config_event_handler()
13 years, 1 month
cluster: RHEL6 - Fix bug 683213 - mirror dev failure in HA LVM can cause service failure
by Jonathan Brassow
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: fe9bef6e01c83e87bee845caa6d763deb33da431
Parent: e51fbc090cb3b7a343aae3ff7d1ffcd94f7fbb5b
Author: Jonathan Brassow <jbrassow(a)f14.(none)>
AuthorDate: Wed Apr 6 11:23:29 2011 -0500
Committer: Jonathan Brassow <jbrassow(a)redhat.com>
CommitterDate: Wed Apr 6 11:36:11 2011 -0500
Fix bug 683213 - mirror dev failure in HA LVM can cause service failure
A new option was added to 'vgreduce', --force. This argument is now
required.
Additionally, repair is attempted when deactivating a device as well as
when trying to activate. This is because a shutdown of a service can
race against a device failure. Adding the repair option to the shutdown
process adds resilience.
---
rgmanager/src/resources/lvm_by_lv.sh | 63 +++++++++++++++------------------
rgmanager/src/resources/lvm_by_vg.sh | 4 +-
2 files changed, 31 insertions(+), 36 deletions(-)
diff --git a/rgmanager/src/resources/lvm_by_lv.sh b/rgmanager/src/resources/lvm_by_lv.sh
index 759062a..4aead58 100644
--- a/rgmanager/src/resources/lvm_by_lv.sh
+++ b/rgmanager/src/resources/lvm_by_lv.sh
@@ -289,45 +289,40 @@ lv_activate()
if ! lv_activate_and_tag $1 $my_name $lv_path; then
ocf_log err "Failed to $1 $lv_path"
- if [ "$1" == "start" ]; then
- ocf_log notice "Attempting cleanup of $OCF_RESKEY_vg_name"
-
- if vgreduce --removemissing --config \
- "activation { volume_list = \"$OCF_RESKEY_vg_name\" }" \
- $OCF_RESKEY_vg_name; then
- ocf_log notice "$OCF_RESKEY_vg_name now consistent"
- owner=`lvs -o tags --noheadings $lv_path`
- if [ ! -z $owner ] && [ $owner != $my_name ]; then
- if is_node_member_clustat $owner ; then
- ocf_log err "$owner owns $lv_path unable to $1"
- return $OCF_ERR_GENERIC
- fi
- ocf_log notice "Owner of $lv_path is not in the cluster"
- ocf_log notice "Stealing $lv_path"
-
- lvchange --deltag $owner $lv_path
- if [ $? -ne 0 ]; then
- ocf_log err "Failed to steal $lv_path from $owner"
- return $OCF_ERR_GENERIC
- fi
-
- # Warning --deltag doesn't always result in failure
- if [ ! -z `lvs -o tags --noheadings $lv_path` ]; then
- ocf_log err "Failed to steal $lv_path from $owner."
- return $OCF_ERR_GENERIC
- fi
+ ocf_log notice "Attempting cleanup of $OCF_RESKEY_vg_name"
+
+ if vgreduce --removemissing --force --config \
+ "activation { volume_list = \"$OCF_RESKEY_vg_name\" }" \
+ $OCF_RESKEY_vg_name; then
+ ocf_log notice "$OCF_RESKEY_vg_name now consistent"
+ owner=`lvs -o tags --noheadings $lv_path`
+ if [ ! -z $owner ] && [ $owner != $my_name ]; then
+ if is_node_member_clustat $owner ; then
+ ocf_log err "$owner owns $lv_path unable to $1"
+ return $OCF_ERR_GENERIC
fi
+ ocf_log notice "Owner of $lv_path is not in the cluster"
+ ocf_log notice "Stealing $lv_path"
- if ! lv_activate_and_tag $1 $my_name $lv_path; then
- ocf_log err "Failed second attempt to $1 $lv_path"
+ lvchange --deltag $owner $lv_path
+ if [ $? -ne 0 ]; then
+ ocf_log err "Failed to steal $lv_path from $owner"
return $OCF_ERR_GENERIC
- else
- ocf_log notice "Second attempt to $1 $lv_path successful"
- return $OCF_SUCCESS
fi
- else
- ocf_log err "Failed to make $OCF_RESKEY_vg_name consistent"
+
+ # Warning --deltag doesn't always result in failure
+ if [ ! -z `lvs -o tags --noheadings $lv_path` ]; then
+ ocf_log err "Failed to steal $lv_path from $owner."
+ return $OCF_ERR_GENERIC
+ fi
+ fi
+
+ if ! lv_activate_and_tag $1 $my_name $lv_path; then
+ ocf_log err "Failed second attempt to $1 $lv_path"
return $OCF_ERR_GENERIC
+ else
+ ocf_log notice "Second attempt to $1 $lv_path successful"
+ return $OCF_SUCCESS
fi
else
ocf_log err "Failed to $1 $lv_path"
diff --git a/rgmanager/src/resources/lvm_by_vg.sh b/rgmanager/src/resources/lvm_by_vg.sh
index f0ca671..2cce7c9 100644
--- a/rgmanager/src/resources/lvm_by_vg.sh
+++ b/rgmanager/src/resources/lvm_by_vg.sh
@@ -160,7 +160,7 @@ function vg_start_clustered
ocf_log err "Failed to activate volume group, $OCF_RESKEY_vg_name"
ocf_log notice "Attempting cleanup of $OCF_RESKEY_vg_name"
- if ! vgreduce --removemissing $OCF_RESKEY_vg_name; then
+ if ! vgreduce --removemissing --force $OCF_RESKEY_vg_name; then
ocf_log err "Failed to make $OCF_RESKEY_vg_name consistent"
return $OCF_ERR_GENERIC
fi
@@ -238,7 +238,7 @@ function vg_start_single
ocf_log err "Failed to activate volume group, $OCF_RESKEY_vg_name"
ocf_log notice "Attempting cleanup of $OCF_RESKEY_vg_name"
- if ! vgreduce --removemissing --config \
+ if ! vgreduce --removemissing --force --config \
"activation { volume_list = \"$OCF_RESKEY_vg_name\" }" \
$OCF_RESKEY_vg_name; then
13 years, 1 month