[389-ds-base] branch master updated (ab81048 -> 486584d)

[git repository hosting]

This is an automated email from the git hooks/post-receive script.

spichugi pushed a change to branch master
in repository 389-ds-base.

      from  ab81048   Ticket 49291 - slapi_search_internal_callback_pb may SIGSEV if
related pblock has not operation set
       new  486584d   Issue 623 - Add test case and refactor the cleanallruv suite

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "adds" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../tests/suites/replication/cleanallruv_test.py   | 762 ++++++++++-----------
 1 file changed, 355 insertions(+), 407 deletions(-)

-- 
To stop receiving notification emails like this one, please contact
the administrator of this repository.

[389-ds-base] 01/01: Issue 623 - Add test case and refactor the cleanallruv suite

[git repository hosting]

This is an automated email from the git hooks/post-receive script.

spichugi pushed a commit to branch master
in repository 389-ds-base.

commit 486584dd2e1887765ab96a5354add31a844840b5
Author: Simon Pichugin <spichugi(a)redhat.com&gt;
Date:   Fri Jun 16 11:14:59 2017 +0200

    Issue 623 - Add test case and refactor the cleanallruv suite
    
    Description: Port test case from TET, rewrite it and adopt/clean up
    the cleanallruv test suite for the changes, add test plans
    for the suite.
    
    https://pagure.io/389-ds-base/issue/623
    
    Reviewed by: mreynolds (Thanks!)
---
 .../tests/suites/replication/cleanallruv_test.py   | 762 ++++++++++-----------
 1 file changed, 355 insertions(+), 407 deletions(-)

diff --git a/dirsrvtests/tests/suites/replication/cleanallruv_test.py
b/dirsrvtests/tests/suites/replication/cleanallruv_test.py
index 22e81b7..f56dc41 100644
--- a/dirsrvtests/tests/suites/replication/cleanallruv_test.py
+++ b/dirsrvtests/tests/suites/replication/cleanallruv_test.py
@@ -83,65 +83,24 @@ def check_ruvs(msg, topology_m4):
     while not clean and count < 10:
         clean = True
 
-        # Check master 1
-        try:
-            entry = topology_m4.ms["master1"].search_s(DEFAULT_SUFFIX,
-                                                       ldap.SCOPE_SUBTREE,
-                                                       REPLICA_RUV_FILTER)
-            if not entry:
-                log.error('%s: Failed to find db tombstone entry from master' %
-                          msg)
-            elements = entry[0].getValues('nsds50ruv')
-            for ruv in elements:
-                if 'replica 4' in ruv:
-                    # Not cleaned
-                    log.error('%s: Master 1 not cleaned!' % msg)
-                    clean = False
-            if clean:
-                log.info('%s: Master 1 is cleaned.' % msg)
-        except ldap.LDAPError as e:
-            log.fatal('%s: Unable to search master 1 for db tombstone: %s' %
-                      (msg, str(e)))
-
-        # Check master 2
-        try:
-            entry = topology_m4.ms["master2"].search_s(DEFAULT_SUFFIX,
-                                                       ldap.SCOPE_SUBTREE,
-                                                       REPLICA_RUV_FILTER)
-            if not entry:
-                log.error('%s: Failed to find tombstone entry from master' %
-                          msg)
-            elements = entry[0].getValues('nsds50ruv')
-            for ruv in elements:
-                if 'replica 4' in ruv:
-                    # Not cleaned
-                    log.error('%s: Master 2 not cleaned!' % msg)
-                    clean = False
-            if clean:
-                log.info('%s: Master 2 is cleaned.', msg)
-        except ldap.LDAPError as e:
-            log.fatal('Unable to search master 2 for db tombstone: ' +
-                      e.message['desc'])
-
-        # Check master 3
-        try:
-            entry = topology_m4.ms["master3"].search_s(DEFAULT_SUFFIX,
-                                                       ldap.SCOPE_SUBTREE,
-                                                       REPLICA_RUV_FILTER)
-            if not entry:
-                log.error('%s: Failed to find db tombstone entry from master' %
-                          msg)
-            elements = entry[0].getValues('nsds50ruv')
-            for ruv in elements:
-                if 'replica 4' in ruv:
-                    # Not cleaned
-                    log.error('%s: Master 3 not cleaned!' % msg)
-                    clean = False
-            if clean:
-                log.info('%s: Master 3 is cleaned.' % msg)
-        except ldap.LDAPError as e:
-            log.fatal('%s: Unable to search master 3 for db tombstone: %s' %
-                      (msg, str(e)))
+        for num in range(1, 4):
+            try:
+                entry =
topology_m4.ms["master{}".format(num)].search_s(DEFAULT_SUFFIX,
+                                                                       
ldap.SCOPE_SUBTREE,
+                                                                       
REPLICA_RUV_FILTER)
+                if not entry:
+                    log.error('%s: Failed to find db tombstone entry from master'
%
+                              msg)
+                elements = entry[0].getValues('nsds50ruv')
+                for ruv in elements:
+                    if 'replica 4' in ruv:
+                        # Not cleaned
+                        log.error('{}: Master {} is not cleaned!'.format(msg,
num))
+                        clean = False
+                if clean:
+                    log.info('{}: Master {} is cleaned!'.format(msg, num))
+            except ldap.LDAPError as e:
+                log.fatal('{}: Unable to search master {} for db tombstone:
{}'.format(msg, num, str(e)))
         # Sleep a bit and give it chance to clean up...
         time.sleep(5)
         count += 1
@@ -151,6 +110,7 @@ def check_ruvs(msg, topology_m4):
 
 def task_done(topology_m4, task_dn, timeout=60):
     """Check if the task is complete"""
+
     attrlist = ['nsTaskLog', 'nsTaskStatus', 'nsTaskExitCode',
                 'nsTaskCurrentItem', 'nsTaskTotalItems']
     done = False
@@ -174,10 +134,9 @@ def task_done(topology_m4, task_dn, timeout=60):
 
 
 def restore_master4(topology_m4):
-    '''
-    In our tests will always be removing master 4, so we need a common
+    """In our tests will always be removing master 4, so we need a common
     way to restore it for another test
-    '''
+    """
 
     log.info('Restoring master 4...')
 
@@ -185,139 +144,59 @@ def restore_master4(topology_m4):
     topology_m4.ms["master4"].replica.enableReplication(suffix=SUFFIX,
role=REPLICAROLE_MASTER,
                                                         replicaId=REPLICAID_MASTER_4)
 
-    #
-    # Create agreements from master 4 -> m1, m2 ,m3
-    #
-    # Creating agreement from master 4 to master 1
-    properties = {RA_NAME: 'meTo_%s:%s' %
(topology_m4.ms["master1"].host, topology_m4.ms["master1"].port),
-                  RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
-                  RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
-                  RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
-                  RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
-    m4_m1_agmt = topology_m4.ms["master4"].agreement.create(suffix=SUFFIX,
host=topology_m4.ms["master1"].host,
-                                                           
port=topology_m4.ms["master1"].port, properties=properties)
-    if not m4_m1_agmt:
-        log.fatal("Fail to create a master -> master replica agreement")
-        sys.exit(1)
-    log.debug("%s created" % m4_m1_agmt)
-
-    # Creating agreement from master 4 to master 2
-    properties = {RA_NAME: 'meTo_%s:%s' %
(topology_m4.ms["master2"].host, topology_m4.ms["master2"].port),
-                  RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
-                  RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
-                  RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
-                  RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
-    m4_m2_agmt = topology_m4.ms["master4"].agreement.create(suffix=SUFFIX,
host=topology_m4.ms["master2"].host,
-                                                           
port=topology_m4.ms["master2"].port, properties=properties)
-    if not m4_m2_agmt:
-        log.fatal("Fail to create a master -> master replica agreement")
-        sys.exit(1)
-    log.debug("%s created" % m4_m2_agmt)
-
-    # Creating agreement from master 4 to master 3
-    properties = {RA_NAME: 'meTo_%s:%s' %
(topology_m4.ms["master3"].host, topology_m4.ms["master3"].port),
-                  RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
-                  RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
-                  RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
-                  RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
-    m4_m3_agmt = topology_m4.ms["master4"].agreement.create(suffix=SUFFIX,
host=topology_m4.ms["master3"].host,
-                                                           
port=topology_m4.ms["master3"].port, properties=properties)
-    if not m4_m3_agmt:
-        log.fatal("Fail to create a master -> master replica agreement")
-        sys.exit(1)
-    log.debug("%s created" % m4_m3_agmt)
-
-    #
-    # Create agreements from m1, m2, m3 to master 4
-    #
-    # Creating agreement from master 1 to master 4
-    properties = {RA_NAME: 'meTo_%s:%s' %
(topology_m4.ms["master4"].host, topology_m4.ms["master4"].port),
-                  RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
-                  RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
-                  RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
-                  RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
-    m1_m4_agmt = topology_m4.ms["master1"].agreement.create(suffix=SUFFIX,
host=topology_m4.ms["master4"].host,
-                                                           
port=topology_m4.ms["master4"].port, properties=properties)
-    if not m1_m4_agmt:
-        log.fatal("Fail to create a master -> master replica agreement")
-        sys.exit(1)
-    log.debug("%s created" % m1_m4_agmt)
-
-    # Creating agreement from master 2 to master 4
-    properties = {RA_NAME: 'meTo_%s:%s' %
(topology_m4.ms["master4"].host, topology_m4.ms["master4"].port),
-                  RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
-                  RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
-                  RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
-                  RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
-    m2_m4_agmt = topology_m4.ms["master2"].agreement.create(suffix=SUFFIX,
host=topology_m4.ms["master4"].host,
-                                                           
port=topology_m4.ms["master4"].port, properties=properties)
-    if not m2_m4_agmt:
-        log.fatal("Fail to create a master -> master replica agreement")
-        sys.exit(1)
-    log.debug("%s created" % m2_m4_agmt)
-
-    # Creating agreement from master 3 to master 4
-    properties = {RA_NAME: 'meTo_%s:%s' %
(topology_m4.ms["master4"].host, topology_m4.ms["master4"].port),
-                  RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
-                  RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
-                  RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
-                  RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
-    m3_m4_agmt = topology_m4.ms["master3"].agreement.create(suffix=SUFFIX,
host=topology_m4.ms["master4"].host,
-                                                           
port=topology_m4.ms["master4"].port, properties=properties)
-    if not m3_m4_agmt:
-        log.fatal("Fail to create a master -> master replica agreement")
-        sys.exit(1)
-    log.debug("%s created" % m3_m4_agmt)
+    for num in range(1, 4):
+        host_to = topology_m4.ms["master{}".format(num)].host
+        port_to = topology_m4.ms["master{}".format(num)].port
+        properties = {RA_NAME: 'meTo_{}:{}'.format(host_to, port_to),
+                      RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
+                      RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
+                      RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
+                      RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
+        agmt = topology_m4.ms["master4"].agreement.create(suffix=SUFFIX,
host=host_to,
+                                                          port=port_to,
properties=properties)
+        if not agmt:
+            log.fatal("Fail to create a master -> master replica
agreement")
+            assert False
+        log.debug("%s created" % agmt)
+
+        host_to = topology_m4.ms["master4"].host
+        port_to = topology_m4.ms["master4"].port
+        properties = {RA_NAME: 'meTo_{}:{}'.format(host_to, port_to),
+                      RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
+                      RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
+                      RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
+                      RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
+        agmt =
topology_m4.ms["master{}".format(num)].agreement.create(suffix=SUFFIX,
host=host_to,
+                                                                       port=port_to,
properties=properties)
+        if not agmt:
+            log.fatal("Fail to create a master -> master replica
agreement")
+            assert False
+        log.debug("%s created" % agmt)
 
-    #
     # Stop the servers - this allows the rid(for master4) to be used again
-    #
-    topology_m4.ms["master1"].stop(timeout=30)
-    topology_m4.ms["master2"].stop(timeout=30)
-    topology_m4.ms["master3"].stop(timeout=30)
-    topology_m4.ms["master4"].stop(timeout=30)
+    for num in range(1, 5):
+        topology_m4.ms["master{}".format(num)].stop(timeout=30)
 
-    #
     # Initialize the agreements
-    #
-    # m1 -> m2
     topology_m4.ms["master1"].start(timeout=30)
-    topology_m4.ms["master2"].start(timeout=30)
-    time.sleep(5)
-    topology_m4.ms["master1"].agreement.init(SUFFIX, HOST_MASTER_2,
PORT_MASTER_2)
-   
topology_m4.ms["master1"].waitForReplInit(topology_m4.ms["master1_agmts"]["m1_m2"])
+    for num in range(2, 5):
+        host_to = topology_m4.ms["master{}".format(num)].host
+        port_to = topology_m4.ms["master{}".format(num)].port
+        topology_m4.ms["master{}".format(num)].start(timeout=30)
+        time.sleep(5)
+        topology_m4.ms["master1"].agreement.init(SUFFIX, host_to, port_to)
+       
topology_m4.ms["master1"].waitForReplInit(topology_m4.ms["master1_agmts"]["m1_m{}".format(num)])
 
-    # m1 -> m3
-    topology_m4.ms["master3"].start(timeout=30)
     time.sleep(5)
-    topology_m4.ms["master1"].agreement.init(SUFFIX, HOST_MASTER_3,
PORT_MASTER_3)
-   
topology_m4.ms["master1"].waitForReplInit(topology_m4.ms["master1_agmts"]["m1_m3"])
 
-    # m1 -> m4
-    time.sleep(5)
-    topology_m4.ms["master4"].start(timeout=30)
-    topology_m4.ms["master1"].agreement.init(SUFFIX, HOST_MASTER_4,
PORT_MASTER_4)
-   
topology_m4.ms["master1"].waitForReplInit(topology_m4.ms["master1_agmts"]["m1_m4"])
-    time.sleep(5)
-
-    #
     # Test Replication is working
-    #
-    # Check replication is working with previous working master(m1 -> m2)
-    if topology_m4.ms["master1"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master2"]):
-        log.info('Replication is working m1 -> m2.')
-    else:
-        log.fatal('restore_master4: Replication is not working from m1 ->
m2.')
-        assert False
-    time.sleep(1)
-
-    # Check replication is working from master 1 to  master 4...
-    if topology_m4.ms["master1"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master4"]):
-        log.info('Replication is working m1 -> m4.')
-    else:
-        log.fatal('restore_master4: Replication is not working from m1 ->
m4.')
-        assert False
-    time.sleep(1)
+    for num in range(2, 5):
+        if topology_m4.ms["master1"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master{}".format(num)]):
+            log.info('Replication is working m1 -> m{}.'.format(num))
+        else:
+            log.fatal('restore_master4: Replication is not working from m1 ->
m{}.'.format(num))
+            assert False
+        time.sleep(1)
 
     # Check replication is working from master 4 to master1...
     if topology_m4.ms["master4"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master1"]):
@@ -330,136 +209,104 @@ def restore_master4(topology_m4):
     log.info('Master 4 has been successfully restored.')
 
 
-def test_cleanallruv_init(topology_m4):
-    '''
-    Make updates on each master to make sure we have the all master RUVs on
-    each master.
-    '''
-
-    log.info('Initializing cleanAllRUV test suite...')
-
-    # Master 1
-    if not topology_m4.ms["master1"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master2"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 1
and master 2.')
-        assert False
-
-    if not topology_m4.ms["master1"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master3"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 1
and master 3.')
-        assert False
-
-    if not topology_m4.ms["master1"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master4"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 1
and master 4.')
-        assert False
-
-    # Master 2
-    if not topology_m4.ms["master2"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master1"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 2
and master 1.')
-        assert False
-
-    if not topology_m4.ms["master2"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master3"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 2
and master 3.')
-        assert False
-
-    if not topology_m4.ms["master2"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master4"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 2
and master 4.')
-        assert False
-
-    # Master 3
-    if not topology_m4.ms["master3"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master1"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 2
and master 1.')
-        assert False
-
-    if not topology_m4.ms["master3"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master2"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 2
and master 2.')
-        assert False
-
-    if not topology_m4.ms["master3"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master4"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 2
and master 4.')
-        assert False
-
-    # Master 4
-    if not topology_m4.ms["master4"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master1"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 2
and master 1.')
-        assert False
-
-    if not topology_m4.ms["master4"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master2"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 2
and master 2.')
-        assert False
-
-    if not topology_m4.ms["master4"].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master3"]):
-        log.fatal('test_cleanallruv_init: Replication is not working between master 2
and master 3.')
-        assert False
-
-    log.info('Initialized cleanAllRUV test suite.')
-
-
-def test_cleanallruv_clean(topology_m4):
-    '''
-    Disable a master, remove agreements to that master, and clean the RUVs on
-    the remaining replicas
-    '''
-
-    log.info('Running test_cleanallruv_clean...')
+def test_clean(topology_m4):
+    """Check that cleanallruv task works properly
+
+    :ID: e9b3ce5c-e17c-409e-aafc-e97d630f2878
+    :feature: CleanAllRUV
+    :setup: Replication setup with four masters
+    :steps: 1. Check that replication works on all masters
+            2. Disable replication on master 4
+            3. Remove agreements to master 4 from other masters
+            4. Run a cleanallruv task on master 1 with a 'force' option
'on'
+            5. Check that everything was cleaned and no hanging tasks left
+    :expectedresults: Everything was cleaned and no hanging tasks left
+    """
+
+    log.info('Running test_clean...')
+
+    log.info('Check that replication works properly on all masters')
+    agmt_nums = {"master1": ("2", "3", "4"),
+                 "master2": ("1", "3", "4"),
+                 "master3": ("1", "2", "4"),
+                 "master4": ("1", "2", "3")}
+
+    for inst_name, agmts in agmt_nums.items():
+        for num in agmts:
+            if not topology_m4.ms[inst_name].testReplication(DEFAULT_SUFFIX,
topology_m4.ms["master{}".format(num)]):
+                log.fatal(
+                    'test_replication: Replication is not working between {} and
master {}.'.format(inst_name,
+                                                                                         
          num))
+                assert False
 
     # Disable master 4
-    log.info('test_cleanallruv_clean: disable master 4...')
+    log.info('test_clean: disable master 4...')
     topology_m4.ms["master4"].replica.disableReplication(DEFAULT_SUFFIX)
 
     # Remove the agreements from the other masters that point to master 4
-    remove_master4_agmts("test_cleanallruv_clean", topology_m4)
+    remove_master4_agmts("test_clean", topology_m4)
 
     # Run the task
-    log.info('test_cleanallruv_clean: run the cleanAllRUV task...')
+    log.info('test_clean: run the cleanAllRUV task...')
     try:
         topology_m4.ms["master1"].tasks.cleanAllRUV(suffix=DEFAULT_SUFFIX,
replicaid='4',
                                                     args={TASK_WAIT: True})
     except ValueError as e:
-        log.fatal('test_cleanallruv_clean: Problem running cleanAllRuv task: ' +
+        log.fatal('test_clean: Problem running cleanAllRuv task: ' +
                   e.message('desc'))
         assert False
 
     # Check the other master's RUV for 'replica 4'
-    log.info('test_cleanallruv_clean: check all the masters have been
cleaned...')
-    clean = check_ruvs("test_cleanallruv_clean", topology_m4)
+    log.info('test_clean: check all the masters have been cleaned...')
+    clean = check_ruvs("test_clean", topology_m4)
 
     if not clean:
-        log.fatal('test_cleanallruv_clean: Failed to clean replicas')
+        log.fatal('test_clean: Failed to clean replicas')
         assert False
 
-    log.info('test_cleanallruv_clean PASSED, restoring master 4...')
+    log.info('test_clean PASSED, restoring master 4...')
 
-    #
     # Cleanup - restore master 4
-    #
     restore_master4(topology_m4)
 
 
-def test_cleanallruv_clean_restart(topology_m4):
-    '''
-    Test that if a master istopped during the clean process, that it
-    resumes and finishes when its started.
-    '''
+def test_clean_restart(topology_m4):
+    """Check that cleanallruv task works properly after a restart
 
-    log.info('Running test_cleanallruv_clean_restart...')
+    :ID: c6233bb3-092c-4919-9ac9-80dd02cc6e02
+    :feature: CleanAllRUV
+    :setup: Replication setup with four masters
+    :steps: 1. Disable replication on master 4
+            2. Remove agreements to master 4 from other masters
+            3. Stop master 3
+            4. Run a cleanallruv task on master 1
+            5. Stop master 1
+            6. Start master 3 and make sure that no crash happened
+            7. Start master 1 and make sure that no crash happened
+            8. Check that everything was cleaned and no hanging tasks left
+    :expectedresults: Everything was cleaned and no hanging tasks left
+    """
+
+    log.info('Running test_clean_restart...')
 
     # Disable master 4
-    log.info('test_cleanallruv_clean_restart: disable master 4...')
+    log.info('test_clean_restart: disable master 4...')
     topology_m4.ms["master4"].replica.disableReplication(DEFAULT_SUFFIX)
 
     # Remove the agreements from the other masters that point to master 4
-    log.info('test_cleanallruv_clean: remove all the agreements to master 4...')
-    remove_master4_agmts("test_cleanallruv_clean restart", topology_m4)
+    log.info('test_clean: remove all the agreements to master 4...')
+    remove_master4_agmts("test_clean restart", topology_m4)
 
     # Stop master 3 to keep the task running, so we can stop master 1...
     topology_m4.ms["master3"].stop(timeout=30)
 
     # Run the task
-    log.info('test_cleanallruv_clean_restart: run the cleanAllRUV task...')
+    log.info('test_clean_restart: run the cleanAllRUV task...')
     try:
         (task_dn, rc) = topology_m4.ms["master1"].tasks.cleanAllRUV(
             suffix=DEFAULT_SUFFIX, replicaid='4', args={TASK_WAIT: False})
     except ValueError as e:
-        log.fatal('test_cleanallruv_clean_restart: Problem running cleanAllRuv task:
' +
+        log.fatal('test_clean_restart: Problem running cleanAllRuv task: ' +
                   e.message('desc'))
         assert False
 
@@ -470,41 +317,49 @@ def test_cleanallruv_clean_restart(topology_m4):
     # Now start master 3 & 1, and make sure we didn't crash
     topology_m4.ms["master3"].start(timeout=30)
     if topology_m4.ms["master3"].detectDisorderlyShutdown():
-        log.fatal('test_cleanallruv_clean_restart: Master 3 previously
crashed!')
+        log.fatal('test_clean_restart: Master 3 previously crashed!')
         assert False
 
     topology_m4.ms["master1"].start(timeout=30)
     if topology_m4.ms["master1"].detectDisorderlyShutdown():
-        log.fatal('test_cleanallruv_clean_restart: Master 1 previously
crashed!')
+        log.fatal('test_clean_restart: Master 1 previously crashed!')
         assert False
 
     # Wait a little for agmts/cleanallruv to wake up
     if not task_done(topology_m4, task_dn):
-        log.fatal('test_cleanallruv_clean_restart: cleanAllRUV task did not
finish')
+        log.fatal('test_clean_restart: cleanAllRUV task did not finish')
         assert False
 
     # Check the other master's RUV for 'replica 4'
-    log.info('test_cleanallruv_clean_restart: check all the masters have been
cleaned...')
-    clean = check_ruvs("test_cleanallruv_clean_restart", topology_m4)
+    log.info('test_clean_restart: check all the masters have been cleaned...')
+    clean = check_ruvs("test_clean_restart", topology_m4)
     if not clean:
         log.fatal('Failed to clean replicas')
         assert False
 
-    log.info('test_cleanallruv_clean_restart PASSED, restoring master 4...')
+    log.info('test_clean_restart PASSED, restoring master 4...')
 
-    #
     # Cleanup - restore master 4
-    #
     restore_master4(topology_m4)
 
 
-def test_cleanallruv_clean_force(topology_m4):
-    '''
-    Disable a master, remove agreements to that master, and clean the RUVs on
-    the remaining replicas
-    '''
+def test_clean_force(topology_m4):
+    """Check that multiple tasks with a 'force' option work
properly
+
+    :ID: eb76a93d-8d1c-405e-9f25-6e8d5a781098
+    :feature: CleanAllRUV
+    :setup: Replication setup with four masters
+    :steps: 1. Stop master 3
+            2. Add a bunch of updates to master 4
+            3. Disable replication on master 4
+            4. Start master 3
+            5. Remove agreements to master 4 from other masters
+            6. Run a cleanallruv task on master 1 with a 'force' option
'on'
+            7. Check that everything was cleaned and no hanging tasks left
+    :expectedresults: Everything was cleaned and no hanging tasks left
+    """
 
-    log.info('Running test_cleanallruv_clean_force...')
+    log.info('Running test_clean_force...')
 
     # Stop master 3, while we update master 4, so that 3 is behind the other masters
     topology_m4.ms["master3"].stop(timeout=10)
@@ -515,73 +370,75 @@ def test_cleanallruv_clean_force(topology_m4):
     m4_add_users.join()
 
     # Disable master 4
-    log.info('test_cleanallruv_clean_force: disable master 4...')
+    log.info('test_clean_force: disable master 4...')
     topology_m4.ms["master4"].replica.disableReplication(DEFAULT_SUFFIX)
 
     # Start master 3, it should be out of sync with the other replicas...
     topology_m4.ms["master3"].start(timeout=30)
 
     # Remove the agreements from the other masters that point to master 4
-    remove_master4_agmts("test_cleanallruv_clean_force", topology_m4)
+    remove_master4_agmts("test_clean_force", topology_m4)
 
     # Run the task, use "force" because master 3 is not in sync with the other
replicas
     # in regards to the replica 4 RUV
-    log.info('test_cleanallruv_clean_force: run the cleanAllRUV task...')
+    log.info('test_clean_force: run the cleanAllRUV task...')
     try:
         topology_m4.ms["master1"].tasks.cleanAllRUV(suffix=DEFAULT_SUFFIX,
replicaid='4',
                                                     force=True, args={TASK_WAIT: True})
     except ValueError as e:
-        log.fatal('test_cleanallruv_clean_force: Problem running cleanAllRuv task:
' +
+        log.fatal('test_clean_force: Problem running cleanAllRuv task: ' +
                   e.message('desc'))
         assert False
 
     # Check the other master's RUV for 'replica 4'
-    log.info('test_cleanallruv_clean_force: check all the masters have been
cleaned...')
-    clean = check_ruvs("test_cleanallruv_clean_force", topology_m4)
+    log.info('test_clean_force: check all the masters have been cleaned...')
+    clean = check_ruvs("test_clean_force", topology_m4)
     if not clean:
-        log.fatal('test_cleanallruv_clean_force: Failed to clean replicas')
+        log.fatal('test_clean_force: Failed to clean replicas')
         assert False
 
-    log.info('test_cleanallruv_clean_force PASSED, restoring master 4...')
+    log.info('test_clean_force PASSED, restoring master 4...')
 
-    #
     # Cleanup - restore master 4
-    #
     restore_master4(topology_m4)
 
 
-def test_cleanallruv_abort(topology_m4):
-    '''
-    Test the abort task.
+def test_abort(topology_m4):
+    """Test the abort task basic functionality
 
-    DIsable master 4
-    Stop master 2 so that it can not be cleaned
-    Run the clean task
-    Wait a bit
-    Abort the task
-    Verify task is aborted
-    '''
+    :ID: b09a6887-8de0-4fac-8e41-73ccbaaf7a08
+    :feature: CleanAllRUV
+    :setup: Replication setup with four masters
+    :steps: 1. Disable replication on master 4
+            2. Remove agreements to master 4 from other masters
+            3. Stop master 2
+            4. Run a cleanallruv task on master 1
+            5. Run a cleanallruv abort task on master 1
+            6. Wait for the task to be done
+            7. Check that no hanging tasks left
+    :expectedresults: No hanging tasks left
+    """
 
-    log.info('Running test_cleanallruv_abort...')
+    log.info('Running test_abort...')
 
     # Disable master 4
-    log.info('test_cleanallruv_abort: disable replication on master 4...')
+    log.info('test_abort: disable replication on master 4...')
     topology_m4.ms["master4"].replica.disableReplication(DEFAULT_SUFFIX)
 
     # Remove the agreements from the other masters that point to master 4
-    remove_master4_agmts("test_cleanallruv_abort", topology_m4)
+    remove_master4_agmts("test_abort", topology_m4)
 
     # Stop master 2
-    log.info('test_cleanallruv_abort: stop master 2 to freeze the cleanAllRUV
task...')
+    log.info('test_abort: stop master 2 to freeze the cleanAllRUV task...')
     topology_m4.ms["master2"].stop(timeout=30)
 
     # Run the task
-    log.info('test_cleanallruv_abort: add the cleanAllRUV task...')
+    log.info('test_abort: add the cleanAllRUV task...')
     try:
         (clean_task_dn, rc) =
topology_m4.ms["master1"].tasks.cleanAllRUV(suffix=DEFAULT_SUFFIX,
                                                                          
replicaid='4', args={TASK_WAIT: False})
     except ValueError as e:
-        log.fatal('test_cleanallruv_abort: Problem running cleanAllRuv task: ' +
+        log.fatal('test_abort: Problem running cleanAllRuv task: ' +
                   e.message('desc'))
         assert False
 
@@ -589,70 +446,82 @@ def test_cleanallruv_abort(topology_m4):
     time.sleep(5)
 
     # Abort the task
-    log.info('test_cleanallruv_abort: abort the cleanAllRUV task...')
+    log.info('test_abort: abort the cleanAllRUV task...')
     try:
         topology_m4.ms["master1"].tasks.abortCleanAllRUV(suffix=DEFAULT_SUFFIX,
replicaid='4',
                                                          args={TASK_WAIT: True})
     except ValueError as e:
-        log.fatal('test_cleanallruv_abort: Problem running abortCleanAllRuv task:
' +
+        log.fatal('test_abort: Problem running abortCleanAllRuv task: ' +
                   e.message('desc'))
         assert False
 
     # Check master 1 does not have the clean task running
-    log.info('test_cleanallruv_abort: check master 1 no longer has a cleanAllRUV
task...')
+    log.info('test_abort: check master 1 no longer has a cleanAllRUV task...')
     if not task_done(topology_m4, clean_task_dn):
-        log.fatal('test_cleanallruv_abort: CleanAllRUV task was not aborted')
+        log.fatal('test_abort: CleanAllRUV task was not aborted')
         assert False
 
     # Start master 2
-    log.info('test_cleanallruv_abort: start master 2 to begin the restore
process...')
+    log.info('test_abort: start master 2 to begin the restore process...')
     topology_m4.ms["master2"].start(timeout=30)
 
     #
     # Now run the clean task task again to we can properly restore master 4
     #
-    log.info('test_cleanallruv_abort: run cleanAllRUV task so we can properly restore
master 4...')
+    log.info('test_abort: run cleanAllRUV task so we can properly restore master
4...')
     try:
         topology_m4.ms["master1"].tasks.cleanAllRUV(suffix=DEFAULT_SUFFIX,
                                                     replicaid='4',
args={TASK_WAIT: True})
     except ValueError as e:
-        log.fatal('test_cleanallruv_abort: Problem running cleanAllRuv task: ' +
e.message('desc'))
+        log.fatal('test_abort: Problem running cleanAllRuv task: ' +
e.message('desc'))
         assert False
 
-    log.info('test_cleanallruv_abort PASSED, restoring master 4...')
+    log.info('test_abort PASSED, restoring master 4...')
 
-    #
     # Cleanup - Restore master 4
-    #
     restore_master4(topology_m4)
 
 
-def test_cleanallruv_abort_restart(topology_m4):
-    '''
-    Test the abort task can handle a restart, and then resume
-    '''
+def test_abort_restart(topology_m4):
+    """Test the abort task can handle a restart, and then resume
+
+    :ID: b66e33d4-fe85-4e1c-b882-75da80f70ab3
+    :feature: CleanAllRUV
+    :setup: Replication setup with four masters
+    :steps: 1. Disable replication on master 4
+            2. Remove agreements to master 4 from other masters
+            3. Stop master 3
+            4. Run a cleanallruv task on master 1
+            5. Run a cleanallruv abort task on master 1
+            6. Restart master 1 and make sure that no crash happened
+            7. Start master 3
+            8. Check that abort task was resumed on master 1
+               and errors log doesn't have 'Aborting abort task' message
+    :expectedresults: Abort task was resumed on master 1
+                      and errors log doesn't have 'Aborting abort task'
message
+    """
 
-    log.info('Running test_cleanallruv_abort_restart...')
+    log.info('Running test_abort_restart...')
 
     # Disable master 4
-    log.info('test_cleanallruv_abort_restart: disable replication on master
4...')
+    log.info('test_abort_restart: disable replication on master 4...')
     topology_m4.ms["master4"].replica.disableReplication(DEFAULT_SUFFIX)
 
     # Remove the agreements from the other masters that point to master 4
-    log.info('test_cleanallruv_abort_restart: remove all the agreements to master
4...)')
-    remove_master4_agmts("test_cleanallruv_abort_restart", topology_m4)
+    log.info('test_abort_restart: remove all the agreements to master 4...)')
+    remove_master4_agmts("test_abort_restart", topology_m4)
 
     # Stop master 3
-    log.info('test_cleanallruv_abort_restart: stop master 3 to freeze the cleanAllRUV
task...')
+    log.info('test_abort_restart: stop master 3 to freeze the cleanAllRUV
task...')
     topology_m4.ms["master3"].stop()
 
     # Run the task
-    log.info('test_cleanallruv_abort_restart: add the cleanAllRUV task...')
+    log.info('test_abort_restart: add the cleanAllRUV task...')
     try:
         (clean_task_dn, rc) =
topology_m4.ms["master1"].tasks.cleanAllRUV(suffix=DEFAULT_SUFFIX,
                                                                          
replicaid='4', args={TASK_WAIT: False})
     except ValueError as e:
-        log.fatal('test_cleanallruv_abort_restart: Problem running cleanAllRuv task:
' +
+        log.fatal('test_abort_restart: Problem running cleanAllRuv task: ' +
                   e.message('desc'))
         assert False
 
@@ -660,12 +529,12 @@ def test_cleanallruv_abort_restart(topology_m4):
     time.sleep(5)
 
     # Abort the task
-    log.info('test_cleanallruv_abort_restart: abort the cleanAllRUV task...')
+    log.info('test_abort_restart: abort the cleanAllRUV task...')
     try:
         topology_m4.ms["master1"].tasks.abortCleanAllRUV(suffix=DEFAULT_SUFFIX,
replicaid='4',
                                                          certify=True, args={TASK_WAIT:
False})
     except ValueError as e:
-        log.fatal('test_cleanallruv_abort_restart: Problem running
test_cleanallruv_abort_restart task: ' +
+        log.fatal('test_abort_restart: Problem running test_abort_restart task: '
+
                   e.message('desc'))
         assert False
 
@@ -673,16 +542,16 @@ def test_cleanallruv_abort_restart(topology_m4):
     time.sleep(5)
 
     # Check master 1 does not have the clean task running
-    log.info('test_cleanallruv_abort: check master 1 no longer has a cleanAllRUV
task...')
+    log.info('test_abort: check master 1 no longer has a cleanAllRUV task...')
 
     if not task_done(topology_m4, clean_task_dn):
-        log.fatal('test_cleanallruv_abort_restart: CleanAllRUV task was not
aborted')
+        log.fatal('test_abort_restart: CleanAllRUV task was not aborted')
         assert False
 
     # Now restart master 1, and make sure the abort process completes
     topology_m4.ms["master1"].restart()
     if topology_m4.ms["master1"].detectDisorderlyShutdown():
-        log.fatal('test_cleanallruv_abort_restart: Master 1 previously
crashed!')
+        log.fatal('test_abort_restart: Master 1 previously crashed!')
         assert False
 
     # Start master 3
@@ -690,61 +559,61 @@ def test_cleanallruv_abort_restart(topology_m4):
 
     # Check master 1 tried to run abort task.  We expect the abort task to be aborted.
     if not topology_m4.ms["master1"].searchErrorsLog('Aborting abort
task'):
-        log.fatal('test_cleanallruv_abort_restart: Abort task did not restart')
+        log.fatal('test_abort_restart: Abort task did not restart')
         assert False
 
-    #
     # Now run the clean task task again to we can properly restore master 4
-    #
-    log.info('test_cleanallruv_abort_restart: run cleanAllRUV task so we can properly
restore master 4...')
+    log.info('test_abort_restart: run cleanAllRUV task so we can properly restore
master 4...')
     try:
         topology_m4.ms["master1"].tasks.cleanAllRUV(suffix=DEFAULT_SUFFIX,
                                                     replicaid='4',
args={TASK_WAIT: True})
     except ValueError as e:
-        log.fatal('test_cleanallruv_abort_restart: Problem running cleanAllRuv task:
' +
+        log.fatal('test_abort_restart: Problem running cleanAllRuv task: ' +
                   e.message('desc'))
         assert False
 
-    log.info('test_cleanallruv_abort_restart PASSED, restoring master 4...')
+    log.info('test_abort_restart PASSED, restoring master 4...')
 
-    #
     # Cleanup - Restore master 4
-    #
     restore_master4(topology_m4)
 
 
-def test_cleanallruv_abort_certify(topology_m4):
-    '''
-    Test the abort task.
+def test_abort_certify(topology_m4):
+    """Test the abort task with a replica-certify-all option
 
-    Disable master 4
-    Stop master 2 so that it can not be cleaned
-    Run the clean task
-    Wait a bit
-    Abort the task
-    Verify task is aborted
-    '''
+    :ID: 78959966-d644-44a8-b98c-1fcf21b45eb0
+    :feature: CleanAllRUV
+    :setup: Replication setup with four masters
+    :steps: 1. Disable replication on master 4
+            2. Remove agreements to master 4 from other masters
+            3. Stop master 2
+            4. Run a cleanallruv task on master 1
+            5. Run a cleanallruv abort task on master 1 with a replica-certify-all
option
+            6. Wait for the task to be done
+            7. Check that no hanging tasks left
+    :expectedresults: No hanging tasks left
+    """
 
-    log.info('Running test_cleanallruv_abort_certify...')
+    log.info('Running test_abort_certify...')
 
     # Disable master 4
-    log.info('test_cleanallruv_abort_certify: disable replication on master
4...')
+    log.info('test_abort_certify: disable replication on master 4...')
     topology_m4.ms["master4"].replica.disableReplication(DEFAULT_SUFFIX)
 
     # Remove the agreements from the other masters that point to master 4
-    remove_master4_agmts("test_cleanallruv_abort_certify", topology_m4)
+    remove_master4_agmts("test_abort_certify", topology_m4)
 
     # Stop master 2
-    log.info('test_cleanallruv_abort_certify: stop master 2 to freeze the cleanAllRUV
task...')
+    log.info('test_abort_certify: stop master 2 to freeze the cleanAllRUV
task...')
     topology_m4.ms["master2"].stop()
 
     # Run the task
-    log.info('test_cleanallruv_abort_certify: add the cleanAllRUV task...')
+    log.info('test_abort_certify: add the cleanAllRUV task...')
     try:
         (clean_task_dn, rc) =
topology_m4.ms["master1"].tasks.cleanAllRUV(suffix=DEFAULT_SUFFIX,
                                                                          
replicaid='4', args={TASK_WAIT: False})
     except ValueError as e:
-        log.fatal('test_cleanallruv_abort_certify: Problem running cleanAllRuv task:
' +
+        log.fatal('test_abort_certify: Problem running cleanAllRuv task: ' +
                   e.message('desc'))
         assert False
 
@@ -752,69 +621,77 @@ def test_cleanallruv_abort_certify(topology_m4):
     time.sleep(5)
 
     # Abort the task
-    log.info('test_cleanallruv_abort_certify: abort the cleanAllRUV task...')
+    log.info('test_abort_certify: abort the cleanAllRUV task...')
     try:
         (abort_task_dn, rc) =
topology_m4.ms["master1"].tasks.abortCleanAllRUV(suffix=DEFAULT_SUFFIX,
                                                                               
replicaid='4', certify=True,
                                                                               
args={TASK_WAIT: False})
     except ValueError as e:
-        log.fatal('test_cleanallruv_abort_certify: Problem running abortCleanAllRuv
task: ' +
+        log.fatal('test_abort_certify: Problem running abortCleanAllRuv task: '
+
                   e.message('desc'))
         assert False
 
     # Wait a while and make sure the abort task is still running
-    log.info('test_cleanallruv_abort_certify: sleep for 5 seconds')
+    log.info('test_abort_certify: sleep for 5 seconds')
     time.sleep(5)
 
     if task_done(topology_m4, abort_task_dn, 60):
-        log.fatal('test_cleanallruv_abort_certify: abort task incorrectly
finished')
+        log.fatal('test_abort_certify: abort task incorrectly finished')
         assert False
 
     # Now start master 2 so it can be aborted
-    log.info('test_cleanallruv_abort_certify: start master 2 to allow the abort task
to finish...')
+    log.info('test_abort_certify: start master 2 to allow the abort task to
finish...')
     topology_m4.ms["master2"].start()
 
     # Wait for the abort task to stop
     if not task_done(topology_m4, abort_task_dn, 60):
-        log.fatal('test_cleanallruv_abort_certify: The abort CleanAllRUV task was not
aborted')
+        log.fatal('test_abort_certify: The abort CleanAllRUV task was not
aborted')
         assert False
 
     # Check master 1 does not have the clean task running
-    log.info('test_cleanallruv_abort_certify: check master 1 no longer has a
cleanAllRUV task...')
+    log.info('test_abort_certify: check master 1 no longer has a cleanAllRUV
task...')
     if not task_done(topology_m4, clean_task_dn):
-        log.fatal('test_cleanallruv_abort_certify: CleanAllRUV task was not
aborted')
+        log.fatal('test_abort_certify: CleanAllRUV task was not aborted')
         assert False
 
     # Start master 2
-    log.info('test_cleanallruv_abort_certify: start master 2 to begin the restore
process...')
+    log.info('test_abort_certify: start master 2 to begin the restore
process...')
     topology_m4.ms["master2"].start()
 
-    #
     # Now run the clean task task again to we can properly restore master 4
-    #
-    log.info('test_cleanallruv_abort_certify: run cleanAllRUV task so we can properly
restore master 4...')
+    log.info('test_abort_certify: run cleanAllRUV task so we can properly restore
master 4...')
     try:
         topology_m4.ms["master1"].tasks.cleanAllRUV(suffix=DEFAULT_SUFFIX,
                                                     replicaid='4',
args={TASK_WAIT: True})
     except ValueError as e:
-        log.fatal('test_cleanallruv_abort_certify: Problem running cleanAllRuv task:
' +
+        log.fatal('test_abort_certify: Problem running cleanAllRuv task: ' +
                   e.message('desc'))
         assert False
 
-    log.info('test_cleanallruv_abort_certify PASSED, restoring master 4...')
+    log.info('test_abort_certify PASSED, restoring master 4...')
 
-    #
     # Cleanup - Restore master 4
-    #
     restore_master4(topology_m4)
 
 
-def test_cleanallruv_stress_clean(topology_m4):
-    '''
-    Put each server(m1 - m4) under stress, and perform the entire clean process
-    '''
-    log.info('Running test_cleanallruv_stress_clean...')
-    log.info('test_cleanallruv_stress_clean: put all the masters under load...')
+def test_stress_clean(topology_m4):
+    """Put each server(m1 - m4) under a stress, and perform the entire
clean process
+
+    :ID: a8263cd6-f068-4357-86e0-e7c34504c8c5
+    :feature: CleanAllRUV
+    :setup: Replication setup with four masters
+    :steps: 1. Add a bunch of updates to all masters
+            2. Put master 4 to read-only mode
+            3. Disable replication on master 4 and wait for the changes
+            4. Start master 3
+            5. Remove agreements to master 4 from other masters
+            6. Run a cleanallruv task on master 1
+            7. Check that everything was cleaned and no hanging tasks left
+    :expectedresults: Everything was cleaned and no hanging tasks left
+    """
+
+    log.info('Running test_stress_clean...')
+    log.info('test_stress_clean: put all the masters under load...')
 
     # Put all the masters under load
     m1_add_users = AddUsers(topology_m4.ms["master1"], 2000)
@@ -827,63 +704,60 @@ def test_cleanallruv_stress_clean(topology_m4):
     m4_add_users.start()
 
     # Allow sometime to get replication flowing in all directions
-    log.info('test_cleanallruv_stress_clean: allow some time for replication to get
flowing...')
+    log.info('test_stress_clean: allow some time for replication to get
flowing...')
     time.sleep(5)
 
     # Put master 4 into read only mode
-    log.info('test_cleanallruv_stress_clean: put master 4 into read-only
mode...')
+    log.info('test_stress_clean: put master 4 into read-only mode...')
     try:
         topology_m4.ms["master4"].modify_s(DN_CONFIG, [(ldap.MOD_REPLACE,
'nsslapd-readonly', 'on')])
     except ldap.LDAPError as e:
-        log.fatal('test_cleanallruv_stress_clean: Failed to put master 4 into
read-only mode: error ' +
+        log.fatal('test_stress_clean: Failed to put master 4 into read-only mode:
error ' +
                   e.message['desc'])
         assert False
 
     # We need to wait for master 4 to push its changes out
-    log.info('test_cleanallruv_stress_clean: allow some time for master 4 to push
changes out (60 seconds)...')
+    log.info('test_stress_clean: allow some time for master 4 to push changes out (60
seconds)...')
     time.sleep(60)
 
     # Disable master 4
-    log.info('test_cleanallruv_stress_clean: disable replication on master
4...')
+    log.info('test_stress_clean: disable replication on master 4...')
     try:
         topology_m4.ms["master4"].replica.disableReplication(DEFAULT_SUFFIX)
     except:
-        log.fatal('test_cleanallruv_stress_clean: failed to diable replication')
+        log.fatal('test_stress_clean: failed to diable replication')
         assert False
 
     # Remove the agreements from the other masters that point to master 4
-    remove_master4_agmts("test_cleanallruv_stress_clean", topology_m4)
+    remove_master4_agmts("test_stress_clean", topology_m4)
 
     # Run the task
-    log.info('test_cleanallruv_stress_clean: Run the cleanAllRUV task...')
+    log.info('test_stress_clean: Run the cleanAllRUV task...')
     try:
         topology_m4.ms["master1"].tasks.cleanAllRUV(suffix=DEFAULT_SUFFIX,
replicaid='4',
                                                     args={TASK_WAIT: True})
     except ValueError as e:
-        log.fatal('test_cleanallruv_stress_clean: Problem running cleanAllRuv task:
' +
+        log.fatal('test_stress_clean: Problem running cleanAllRuv task: ' +
                   e.message('desc'))
         assert False
 
     # Wait for the update to finish
-    log.info('test_cleanallruv_stress_clean: wait for all the updates to
finish...')
+    log.info('test_stress_clean: wait for all the updates to finish...')
     m1_add_users.join()
     m2_add_users.join()
     m3_add_users.join()
     m4_add_users.join()
 
     # Check the other master's RUV for 'replica 4'
-    log.info('test_cleanallruv_stress_clean: check if all the replicas have been
cleaned...')
-    clean = check_ruvs("test_cleanallruv_stress_clean", topology_m4)
+    log.info('test_stress_clean: check if all the replicas have been
cleaned...')
+    clean = check_ruvs("test_stress_clean", topology_m4)
     if not clean:
-        log.fatal('test_cleanallruv_stress_clean: Failed to clean replicas')
+        log.fatal('test_stress_clean: Failed to clean replicas')
         assert False
 
-    log.info('test_cleanallruv_stress_clean:  PASSED, restoring master 4...')
+    log.info('test_stress_clean:  PASSED, restoring master 4...')
 
-    #
     # Cleanup - restore master 4
-    #
-
     # Sleep for a bit to replication complete
     log.info("Sleep for 120 seconds to allow replication to complete...")
     time.sleep(120)
@@ -892,13 +766,87 @@ def test_cleanallruv_stress_clean(topology_m4):
     try:
         topology_m4.ms["master4"].modify_s(DN_CONFIG, [(ldap.MOD_REPLACE,
'nsslapd-readonly', 'off')])
     except ldap.LDAPError as e:
-        log.fatal('test_cleanallruv_stress_clean: Failed to put master 4 into
read-only mode: error ' +
+        log.fatal('test_stress_clean: Failed to put master 4 into read-only mode:
error ' +
                   e.message['desc'])
         assert False
 
     restore_master4(topology_m4)
 
 
+def test_multiple_tasks_with_force(topology_m4):
+    """Check that multiple tasks with a 'force' option work
properly
+
+    :ID: eb76a93d-8d1c-405e-9f25-6e8d5a781098
+    :feature: CleanAllRUV
+    :setup: Replication setup with four masters
+    :steps: 1. Stop master 3
+            2. Add a bunch of updates to master 4
+            3. Disable replication on master 4
+            4. Start master 3
+            5. Remove agreements to master 4 from other masters
+            6. Run a cleanallruv task on master 1 with a 'force' option
'on'
+            7. Run one more cleanallruv task on master 1 with a 'force' option
'off'
+            8. Check that everything was cleaned and no hanging tasks left
+    :expectedresults: Everything was cleaned and no hanging tasks left
+    """
+
+    log.info('Running test_multiple_tasks_with_force...')
+
+    # Stop master 3, while we update master 4, so that 3 is behind the other masters
+    topology_m4.ms["master3"].stop(timeout=10)
+
+    # Add a bunch of updates to master 4
+    m4_add_users = AddUsers(topology_m4.ms["master4"], 1500)
+    m4_add_users.start()
+    m4_add_users.join()
+
+    # Disable master 4
+    log.info('test_multiple_tasks_with_force: disable master 4...')
+    topology_m4.ms["master4"].replica.disableReplication(DEFAULT_SUFFIX)
+
+    # Start master 3, it should be out of sync with the other replicas...
+    topology_m4.ms["master3"].start(timeout=30)
+
+    # Remove the agreements from the other masters that point to master 4
+    remove_master4_agmts("test_multiple_tasks_with_force", topology_m4)
+
+    # Run the task, use "force" because master 3 is not in sync with the other
replicas
+    # in regards to the replica 4 RUV
+    log.info('test_multiple_tasks_with_force: run the cleanAllRUV task with
"force" on...')
+    try:
+        (clean_task_dn, rc) =
topology_m4.ms["master1"].tasks.cleanAllRUV(suffix=DEFAULT_SUFFIX,
replicaid='4',
+                                                                          force=True,
args={TASK_WAIT: False})
+    except ValueError as e:
+        log.fatal('test_multiple_tasks_with_force: Problem running cleanAllRuv task:
' +
+                  e.message('desc'))
+        assert False
+
+    log.info('test_multiple_tasks_with_force: run the cleanAllRUV task with
"force" off...')
+    try:
+        topology_m4.ms["master1"].tasks.cleanAllRUV(suffix=DEFAULT_SUFFIX,
replicaid='4',
+                                                    args={TASK_WAIT: True})
+    except ValueError as e:
+        log.fatal('test_multiple_tasks_with_force: Problem running cleanAllRuv task:
' +
+                  e.message('desc'))
+        assert False
+
+    # Check the other master's RUV for 'replica 4'
+    log.info('test_multiple_tasks_with_force: check all the masters have been
cleaned...')
+    clean = check_ruvs("test_clean_force", topology_m4)
+    if not clean:
+        log.fatal('test_multiple_tasks_with_force: Failed to clean replicas')
+        assert False
+
+    # Check master 1 does not have the clean task running
+    log.info('test_abort: check master 1 no longer has a cleanAllRUV task...')
+    if not task_done(topology_m4, clean_task_dn):
+        log.fatal('test_abort: CleanAllRUV task was not aborted')
+        assert False
+
+    # Cleanup - restore master 4
+    restore_master4(topology_m4)
+
+
 if __name__ == '__main__':
     # Run isolated
     # -s for DEBUG mode

-- 
To stop receiving notification emails like this one, please contact
the administrator of this repository.