Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=a59... Commit: a59ecbb74af3e4385497ea84bfaa1219f6a5cab6 Parent: e48f97cd32ecb5b595449fd946226e47f69880d3 Author: Fabio M. Di Nitto fdinitto@redhat.com AuthorDate: Wed Nov 30 12:57:30 2011 +0100 Committer: Fabio M. Di Nitto fdinitto@redhat.com CommitterDate: Wed Mar 7 14:51:34 2012 +0100
config: drastically improve cman RRP configuration handling
- don't allow configuration of more than 2 rings - allow overrided of alternate mcast address and port via envars - when using broadcast, set different ports on second ring. this also required a substantial change in transport handling - add support for <cman> <multicast addr= port= ttl=/> <altmulticast addr= port= ttl=/> </cman> - don't allow overlap of addresses/ports - remove redundant port settings in cman - change relaxng schema to reflect above changes
Resolves: rhbz#733298
Reviewed-by: Jan Friesse jfriesse@redhat.com Signed-off-by: Fabio M. Di Nitto fdinitto@redhat.com --- cman/daemon/cman-preconfig.c | 214 ++++++++++++++++++++++------------ config/tools/xml/cluster.rng.in.head | 64 +++++++--- 2 files changed, 182 insertions(+), 96 deletions(-)
diff --git a/cman/daemon/cman-preconfig.c b/cman/daemon/cman-preconfig.c index 13949ec..c8f69e5 100644 --- a/cman/daemon/cman-preconfig.c +++ b/cman/daemon/cman-preconfig.c @@ -50,14 +50,19 @@ static char nodename[MAX_CLUSTER_MEMBER_NAME_LEN]; static int nodeid; static int two_node; static unsigned int disable_openais; -static unsigned int portnum; static int num_nodenames; static char *key_filename=NULL; -static char *mcast_name; static char *cluster_name; static char error_reason[1024] = { '\0' }; static hdb_handle_t cluster_parent_handle; static int use_hashed_cluster_id = 1; +static unsigned int portnum = 0; +static unsigned int altportnum = 0; +static char *mcast_name = NULL; +static char *altmcast_name = NULL; +static unsigned int ttl = 1; +static unsigned int altttl = 1; +
/* * Exports the interface for the service @@ -292,7 +297,7 @@ static int add_udpu_members(struct objdb_iface_ver0 *objdb, hdb_handle_t interfa #define PRIMARY_IFACE 0 #define ALT_IFACE 1
-static int add_ifaddr(struct objdb_iface_ver0 *objdb, char *mcast, char *ifaddr, int port, int ttl, int altiface, enum tx_mech transport) +static int add_ifaddr(struct objdb_iface_ver0 *objdb, char *mcast, char *ifaddr, int port, int intttl, int altiface, enum tx_mech transport) { hdb_handle_t totem_object_handle; hdb_handle_t find_handle; @@ -308,6 +313,11 @@ static int add_ifaddr(struct objdb_iface_ver0 *objdb, char *mcast, char *ifaddr, [TX_MECH_RDMA] = "iba", };
+ if (num_interfaces >= 2) { + snprintf(error_reason, sizeof(error_reason) - 1, "Configuration of more than 2 rings is not supported"); + return -1; + } + /* Check the families match */ if (address_family(mcast, &mcast_addr, 0) != address_family(ifaddr, &if_addr, mcast_addr.ss_family)) { @@ -379,14 +389,14 @@ static int add_ifaddr(struct objdb_iface_ver0 *objdb, char *mcast, char *ifaddr, tmp, strlen(tmp)+1, OBJDB_VALUETYPE_STRING);
/* paranoia check. corosync already does it */ - if ((ttl < 0) || (ttl > 255)) { - sprintf(error_reason, "TTL value (%u) out of range (0 - 255)", ttl); + if ((intttl < 0) || (intttl > 255)) { + sprintf(error_reason, "TTL value (%u) out of range (0 - 255)", intttl); return -1; }
/* add the key to the objdb only if value is not default */ - if (ttl != 1) { - sprintf(tmp, "%d", ttl); + if (intttl != 1) { + sprintf(tmp, "%d", intttl); objdb->object_key_create_typed(interface_object_handle, "ttl", tmp, strlen(tmp)+1, OBJDB_VALUETYPE_STRING); } @@ -433,11 +443,11 @@ static char *default_mcast(char *node, int altiface)
if (family == AF_INET) { snprintf(addr, sizeof(addr), "239.192.%d.%d", clusterid >> 8, clusterid % 0xFF); - return addr; + return strdup(addr); } if (family == AF_INET6) { snprintf(addr, sizeof(addr), "ff15::%x", clusterid); - return addr; + return strdup(addr); }
return NULL; @@ -583,6 +593,11 @@ static int get_env_overrides(void) portnum = atoi(getenv("CMAN_IP_PORT")); }
+ /* optional alternate port */ + if (getenv("CMAN_IP_ALTPORT")) { + altportnum = atoi(getenv("CMAN_IP_ALTPORT")); + } + /* optional security key filename */ if (getenv("CMAN_KEYFILE")) { key_filename = strdup(getenv("CMAN_KEYFILE")); @@ -606,6 +621,10 @@ static int get_env_overrides(void) mcast_name = getenv("CMAN_MCAST_ADDR"); }
+ if (getenv("CMAN_ALTMCAST_ADDR")) { + altmcast_name = getenv("CMAN_ALTMCAST_ADDR"); + } + if (getenv("CMAN_2NODE")) { two_node = 1; expected_votes = 1; @@ -630,11 +649,15 @@ static int get_nodename(struct objdb_iface_ver0 *objdb) hdb_handle_t object_handle; hdb_handle_t find_handle; hdb_handle_t node_object_handle; + hdb_handle_t mcast_handle; hdb_handle_t alt_object; enum tx_mech transport = TX_MECH_UDP; char *str; int error; - unsigned int ttl = 1; + unsigned int mcast_portnum = DEFAULT_PORT; + unsigned int altmcast_portnum = DEFAULT_PORT; + char *altmcast_name_tmp = NULL; + int broadcast = 0;
if (!getenv("CMAN_NOCONFIG")) { /* our nodename */ @@ -683,7 +706,7 @@ static int get_nodename(struct objdb_iface_ver0 *objdb) }
/* Add <cman> bits to pass down to the main module*/ - if ( (node_object_handle = nodelist_byname(objdb, cluster_parent_handle, nodename))) { + if ((node_object_handle = nodelist_byname(objdb, cluster_parent_handle, nodename))) { if (objdb_get_string(objdb, node_object_handle, "nodeid", &nodeid_str)) { sprintf(error_reason, "This node has no nodeid in cluster.conf"); write_cman_pipe("This node has no nodeid in cluster.conf"); @@ -692,84 +715,105 @@ static int get_nodename(struct objdb_iface_ver0 *objdb) }
objdb->object_find_create(cluster_parent_handle, "cman", strlen("cman"), &find_handle); - - if (objdb->object_find_next(find_handle, &object_handle) == 0) { - - hdb_handle_t mcast_handle; - hdb_handle_t find_handle2; - - if (!mcast_name) { - - objdb->object_find_create(object_handle, "multicast", strlen("multicast"), &find_handle2); - if (objdb->object_find_next(find_handle2, &mcast_handle) == 0) { - - objdb_get_string(objdb, mcast_handle, "addr", &mcast_name); - objdb_get_int(objdb, mcast_handle, "ttl", &ttl, ttl); - } - objdb->object_find_destroy(find_handle2); - } - - if (!mcast_name) { - mcast_name = default_mcast(nodename, PRIMARY_IFACE); - - } - if (!mcast_name) - return -1; - - objdb->object_key_create_typed(object_handle, "nodename", - nodename, strlen(nodename)+1, OBJDB_VALUETYPE_STRING); + if (objdb->object_find_next(find_handle, &object_handle)) { + sprintf(error_reason, "Unable to find cman in config db"); + write_cman_pipe(error_reason); + return -1; } objdb->object_find_destroy(find_handle);
- nodeid = atoi(nodeid_str); - error = 0; - - /* optional port */ - if (!portnum) { - objdb_get_int(objdb, object_handle, "port", &portnum, DEFAULT_PORT); - } - /* Check for broadcast */ if (!objdb_get_string(objdb, object_handle, "broadcast", &str)) { if (strcmp(str, "yes") == 0) { - mcast_name = strdup("255.255.255.255"); - if (!mcast_name) - return -1; + broadcast = 1; transport = TX_MECH_UDPB; } }
/* Check for transport */ if (!objdb_get_string(objdb, object_handle, "transport", &str)) { + if ((broadcast) && (strcmp(str, "udpb"))) { + sprintf(error_reason, "Transport and broadcast option are mutually exclusive"); + write_cman_pipe(error_reason); + return -1; + } if (strcmp(str, "udp") == 0) { - if (transport != TX_MECH_UDPB) { - transport = TX_MECH_UDP; - } + transport = TX_MECH_UDP; } else if (strcmp(str, "udpb") == 0) { + broadcast = 1; transport = TX_MECH_UDPB; } else if (strcmp(str, "udpu") == 0) { - if (transport != TX_MECH_UDPB) { - transport = TX_MECH_UDPU; - } else { - sprintf(error_reason, "Transport and broadcast option are mutually exclusive"); - write_cman_pipe("Transport and broadcast option are mutually exclusive"); - return -1; - } + transport = TX_MECH_UDPU; } else if (strcmp(str, "rdma") == 0) { - if (transport != TX_MECH_UDPB) { - transport = TX_MECH_RDMA; - } else { - sprintf(error_reason, "Transport and broadcast option are mutually exclusive"); - write_cman_pipe("Transport and broadcast option are mutually exclusive"); - return -1; - } + transport = TX_MECH_RDMA; } else { sprintf(error_reason, "Transport option value can be one of udp, udpb, udpu, rdma"); - write_cman_pipe("Transport option value can be one of udp, udpb, udpu, rdma"); + write_cman_pipe(error_reason); return -1; } }
+ if (broadcast) { + mcast_name = strdup("255.255.255.255"); + if (!mcast_name) { + sprintf(error_reason, "Unable to set mcast_name"); + write_cman_pipe(error_reason); + return -1; + } + altmcast_name = strdup("255.255.255.255"); + if (!altmcast_name) { + sprintf(error_reason, "Unable to set altmcast_name"); + write_cman_pipe(error_reason); + return -1; + } + altmcast_portnum = DEFAULT_PORT + 2; + } + + objdb->object_find_create(object_handle, "multicast", strlen("multicast"), &find_handle); + if (objdb->object_find_next(find_handle, &mcast_handle) == 0) { + if (!mcast_name) + objdb_get_string(objdb, mcast_handle, "addr", &mcast_name); + objdb_get_int(objdb, mcast_handle, "ttl", &ttl, ttl); + objdb_get_int(objdb, mcast_handle, "port", &mcast_portnum, DEFAULT_PORT); + } + objdb->object_find_destroy(find_handle); + + if (!mcast_name) { + mcast_name = default_mcast(nodename, PRIMARY_IFACE); + } + + if (!mcast_name) { + sprintf(error_reason, "Unable to set mcast_name"); + write_cman_pipe(error_reason); + return -1; + } + + objdb->object_find_create(object_handle, "altmulticast", strlen("altmulticast"), &find_handle); + if (objdb->object_find_next(find_handle, &mcast_handle) == 0) { + objdb_get_string(objdb, mcast_handle, "addr", &altmcast_name_tmp); + objdb_get_int(objdb, mcast_handle, "ttl", &altttl, ttl); + if (!broadcast) { + objdb_get_int(objdb, mcast_handle, "port", &altmcast_portnum, DEFAULT_PORT); + } else { + objdb_get_int(objdb, mcast_handle, "port", &altmcast_portnum, DEFAULT_PORT + 2); + } + } + objdb->object_find_destroy(find_handle); + + /* See if the user wants our default set of openais services (default=yes) */ + objdb_get_int(objdb, object_handle, "disable_openais", &disable_openais, 0); + + objdb->object_key_create_typed(object_handle, "nodename", + nodename, strlen(nodename)+1, OBJDB_VALUETYPE_STRING); + + nodeid = atoi(nodeid_str); + error = 0; + + /* optional port */ + if (!portnum) { + objdb_get_int(objdb, object_handle, "port", &portnum, mcast_portnum); + } + if (add_ifaddr(objdb, mcast_name, nodename, portnum, ttl, PRIMARY_IFACE, transport)) { write_cman_pipe(error_reason); @@ -780,24 +824,43 @@ static int get_nodename(struct objdb_iface_ver0 *objdb) num_nodenames = 1; objdb->object_find_create(node_object_handle,"altname", strlen("altname"), &find_handle); while (objdb->object_find_next(find_handle, &alt_object) == 0) { - unsigned int port; - unsigned int altttl = 1; char *node; - char *mcast;
if (objdb_get_string(objdb, alt_object, "name", &node)) { continue; }
- objdb_get_int(objdb, alt_object, "port", &port, portnum); + objdb_get_int(objdb, alt_object, "port", &altportnum, altmcast_portnum);
- objdb_get_int(objdb, alt_object, "ttl", &altttl, ttl); + objdb_get_int(objdb, alt_object, "ttl", &altttl, altttl);
- if (objdb_get_string(objdb, alt_object, "mcast", &mcast)) { - mcast = default_mcast(nodename, ALT_IFACE); + if (!altmcast_name) { + if (objdb_get_string(objdb, alt_object, "mcast", &altmcast_name)) { + if (altmcast_name_tmp) { + altmcast_name = altmcast_name_tmp; + } else { + altmcast_name = default_mcast(nodename, ALT_IFACE); + } + } + } + + if (!altmcast_name) { + sprintf(error_reason, "Unable to determine alternate multicast name"); + write_cman_pipe(error_reason); + return -1; }
- if (add_ifaddr(objdb, mcast, node, portnum, altttl, + if (!strcmp(altmcast_name, mcast_name) && + ((altportnum == portnum) || (altportnum == portnum - 1) || (portnum == altportnum - 1))) { + sprintf(error_reason, "Alternate communication channel (mcast: %s ports: %d,%d) cannot use\n" + "same address and ports of primary channel (mcast: %s ports: %d,%d)", + altmcast_name, altportnum, altportnum - 1, + mcast_name, portnum, portnum - 1); + write_cman_pipe(error_reason); + return -1; + } + + if (add_ifaddr(objdb, altmcast_name, node, altportnum, altttl, ALT_IFACE, transport)) { write_cman_pipe(error_reason); return -1; @@ -1326,9 +1389,6 @@ static int get_cman_globals(struct objdb_iface_ver0 *objdb) /* Get the <cman> bits that override <totem> bits */ objdb->object_find_create(cluster_parent_handle, "cman", strlen("cman"), &find_handle); if (objdb->object_find_next(find_handle, &object_handle) == 0) { - if (!portnum) - objdb_get_int(objdb, object_handle, "port", &portnum, DEFAULT_PORT); - if (!key_filename) objdb_get_string(objdb, object_handle, "keyfile", &key_filename);
diff --git a/config/tools/xml/cluster.rng.in.head b/config/tools/xml/cluster.rng.in.head index 11d5052..a669c98 100644 --- a/config/tools/xml/cluster.rng.in.head +++ b/config/tools/xml/cluster.rng.in.head @@ -144,26 +144,25 @@ To validate your cluster.conf against this schema, run: </optional> <optional> <element name="multicast" rha:description="The multicast element - provides the ability for a user to specify a multicast address - instead of using the multicast address generated by cman. If + provides the ability for a user to specify a multicast address, + port and TTL (Time To Live) instead of using cman defaults. If a user does not specify a multicast address, cman creates one. It - forms the upper 16 bits of the multicast address with 239.192 and - forms the lower 16 bits based on the cluster ID."> - <optional> - <attribute name="addr" rha:description="A multicast address specified - by a user. If you do specify a multicast address, you should - use the 239.192.x.x series that cman uses. Otherwise, using a - multicast address outside that range may cause unpredictable - results. For example, using 224.0.0.x (All hosts on the network) - may not be routed correctly, or even routed at all by some - hardware." rha:sample="239.192.0.1"/> - </optional> - <optional> - <attribute name="ttl" rha:description="Define the TTL (time to live) of - a multicast packets. Useful only if nodes are on different subnets and - a multicast router is available in between." rha:default="1" - rha:sample="24"/> - </optional> + forms the upper 16 bits of the multicast address with 239.192 for + IPv4 and ff15:: for IPv6 and forms the lower 16 bits based on + the cluster ID."> + <ref name="MULTICASTOPTS"/> + </element> + </optional> + <optional> + <element name="altmulticast" rha:description="The altmulticast element + provides the ability for a user to specify a multicast address, + port and TTL (Time To Live) instead of using cman defaults for + the alternate communication channel (RRP). If a user does not + specify an alternate multicast address, cman creates one. It + forms the upper 16 bits of the multicast address with 239.192 for + IPv4 and ff15:: for IPv6 and forms the lower 16 bits based on + the cluster ID + 1."> + <ref name="MULTICASTOPTS"/> </element> </optional> </element> @@ -1013,6 +1012,33 @@ To validate your cluster.conf against this schema, run: </element> <!-- cluster end --> </start>
+<!-- begin mcast definitions --> + + <define name="MULTICASTOPTS"> + <optional> + <attribute name="addr" rha:description="A multicast address specified + by a user. If you do specify a multicast address, you should + use the 239.192.x.x series that cman uses. Otherwise, using a + multicast address outside that range may cause unpredictable + results. For example, using 224.0.0.x (All hosts on the network) + may not be routed correctly, or even routed at all by some + hardware." rha:sample="239.192.0.1"/> + </optional> + <optional> + <attribute name="ttl" rha:description="Define the TTL (time to live) of + a multicast packets. Useful only if nodes are on different subnets and + a multicast router is available in between." rha:default="1" + rha:sample="24"/> + </optional> + <optional> + <attribute name="port"> + <data type="nonNegativeInteger"/> + </attribute> + </optional> + </define> + +<!-- end mcast definitions --> + <!-- begin node altname definitions -->
<define name="ALTNAME">
cluster-commits@lists.fedorahosted.org