[389-commits] 2 commits - ldap/servers

[Noriko Hosoi]

 ldap/servers/plugins/replication/repl5.h          |    6 ++++
 ldap/servers/plugins/replication/repl5_protocol.c |   28 ++++++++++++++++++++++
 ldap/servers/plugins/replication/repl_extop.c     |   24 ++++++++++++++++++
 3 files changed, 58 insertions(+)

New commits:
commit f2a04fdc45cc8a408267019990504354282c4303
Merge: f4b90ed... 0b95451...
Author: Noriko Hosoi <nhosoi at redhat.com>
Date:   Fri Mar 5 15:06:43 2010 -0800

    Merge branch '547503'



commit 0b95451c7e50cb6b2d0cb310dddca18336e1b2ac
Author: Noriko Hosoi <nhosoi at redhat.com>
Date:   Fri Mar 5 10:07:38 2010 -0800

    570667 - MMR: simultaneous total updates on the masters cause
             deadlock and data loss
    
    https://bugzilla.redhat.com/show_bug.cgi?id=570667
    
    Description: In the MMR topology, if a master receives a total
    update request to initialize the other master and being initialized
    by the other master at the same time, the 2 replication threads hang
    and the replicated backend instance could be wiped out.
    
    To prevent the server running the total update supplier and the
    consumer at the same time, REPLICA_TOTAL_EXCL_SEND and _RECV bits
    have been introduced.  If the server is sending the total update
    to other replicas, the server rejects the total update request
    on the backend.  But the server can send multiple total updates
    to other replicas at the same time.  If the total update from
    other master is in progress on the server, the server rejects
    another total update from yet another master as well as a request
    to initialize other replicas.

diff --git a/ldap/servers/plugins/replication/repl5.h b/ldap/servers/plugins/replication/repl5.h
index 97ce556..c6859dd 100644
--- a/ldap/servers/plugins/replication/repl5.h
+++ b/ldap/servers/plugins/replication/repl5.h
@@ -521,6 +521,12 @@ void replica_write_ruv (Replica *r);
 #define REPLICA_INCREMENTAL_IN_PROGRESS 2 /* Set only between start and stop inc */
 #define REPLICA_TOTAL_IN_PROGRESS 4 /* Set only between start and stop total */
 #define REPLICA_AGREEMENTS_DISABLED 8 /* Replica is offline */
+#define REPLICA_TOTAL_EXCL_SEND 16 /* The server is either sending or receiving
+                                      the total update.  Introducing it if SEND
+                                      is active, RECV should back off. And
+                                      vice versa.  But SEND can coexist. */
+#define REPLICA_TOTAL_EXCL_RECV 32 /* ditto */
+
 PRBool replica_is_state_flag_set(Replica *r, PRInt32 flag);
 void replica_set_state_flag (Replica *r, PRUint32 flag, PRBool clear); 
 void replica_set_tombstone_reap_stop(Replica *r, PRBool val);
diff --git a/ldap/servers/plugins/replication/repl5_protocol.c b/ldap/servers/plugins/replication/repl5_protocol.c
index 927c450..efb3271 100644
--- a/ldap/servers/plugins/replication/repl5_protocol.c
+++ b/ldap/servers/plugins/replication/repl5_protocol.c
@@ -317,6 +317,28 @@ prot_thread_main(void *arg)
 		dev_debug("prot_thread_main(STATE_PERFORMING_INCREMENTAL_UPDATE): end");
 		break;
 	      case STATE_PERFORMING_TOTAL_UPDATE:
+		{
+		Slapi_DN *dn = agmt_get_replarea(agmt);
+		Replica *replica = NULL;
+		Object *replica_obj = replica_get_replica_from_dn(dn);
+		if (replica_obj)
+		{
+		    replica = (Replica*) object_get_data (replica_obj);
+		    /* If total update against this replica is in progress,
+		     * we should not initiate the total update to other replicas. */
+		    if (replica_is_state_flag_set(replica, REPLICA_TOTAL_EXCL_RECV))
+		    {
+		        object_release(replica_obj);
+                slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name,
+                    "%s: total update on the replica is in progress.  Cannot initiate the total update.\n", agmt_get_long_name(rp->agmt));
+		        break;
+		    }
+		    else
+		    {
+		        replica_set_state_flag (replica, REPLICA_TOTAL_EXCL_SEND, 0);
+		    }
+		}
+
 		PR_Lock(rp->lock);
     
 		/* stop incremental protocol if running */
@@ -332,7 +354,13 @@ prot_thread_main(void *arg)
 		   replica initialization is completed. */
 		agmt_replica_init_done (agmt);
     
+		if (replica_obj)
+		{
+		    replica_set_state_flag (replica, REPLICA_TOTAL_EXCL_SEND, 1);
+		    object_release(replica_obj);
+		}
 		break;
+		}
 	      case STATE_FINISHED:
 		dev_debug("prot_thread_main(STATE_FINISHED): exiting prot_thread_main");
 		done = 1;
diff --git a/ldap/servers/plugins/replication/repl_extop.c b/ldap/servers/plugins/replication/repl_extop.c
index b65c6c8..c47ea93 100644
--- a/ldap/servers/plugins/replication/repl_extop.c
+++ b/ldap/servers/plugins/replication/repl_extop.c
@@ -678,6 +678,25 @@ multimaster_extop_StartNSDS50ReplicationRequest(Slapi_PBlock *pb)
 		goto send_response;
 	}
 
+	if (REPL_PROTOCOL_50_TOTALUPDATE == connext->repl_protocol_version)
+	{
+		/* If total update has been initiated against other replicas or
+		 * this replica is already being initialized, we should return
+		 * an error immediately. */
+		if (replica_is_state_flag_set(replica,
+							REPLICA_TOTAL_EXCL_SEND|REPLICA_TOTAL_EXCL_RECV))
+		{
+			slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name,
+				"%s: total update on is initiated on the replica.  Cannot execute the total update from other master.\n", repl_root);
+			response = NSDS50_REPL_REPLICA_BUSY;
+			goto send_response;
+		}
+		else
+		{
+			replica_set_state_flag (replica, REPLICA_TOTAL_EXCL_RECV, 0);
+		}
+	}
+
     /* check that this replica is not a 4.0 consumer */
     if (replica_is_legacy_consumer (replica))
     {
@@ -861,6 +880,11 @@ multimaster_extop_StartNSDS50ReplicationRequest(Slapi_PBlock *pb)
 	slapi_pblock_get(pb, SLAPI_CONNECTION, &connext->connection);
 
 send_response:
+	if (connext && replica &&
+		(REPL_PROTOCOL_50_TOTALUPDATE == connext->repl_protocol_version))
+	{
+		replica_set_state_flag (replica, REPLICA_TOTAL_EXCL_RECV, 1);
+	}
     if (response != NSDS50_REPL_REPLICA_READY)
     {
 		int resp_log_level = SLAPI_LOG_FATAL;